{ "best_global_step": null, "best_metric": null, "best_model_checkpoint": null, "epoch": 0.5481798715203426, "eval_steps": 500, "global_step": 1024, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.5, "completions/max_length": 768.0, "completions/max_terminated_length": 753.0, "completions/mean_length": 664.84375, "completions/mean_terminated_length": 561.6875, "completions/min_length": 322.0, "completions/min_terminated_length": 322.0, "entropy": 0.48598330840468407, "epoch": 0.0005353319057815846, "frac_reward_zero_std": 0.25, "grad_norm": 0.003244089661166072, "learning_rate": 1e-05, "loss": 0.0473, "num_tokens": 25147.0, "reward": 0.46875, "reward_std": 0.2651650309562683, "rewards/accuracy_reward/mean": 0.46875, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.2930827140808105, "sampling/importance_sampling_ratio/mean": 0.9998305439949036, "sampling/importance_sampling_ratio/min": 0.6979114413261414, "sampling/sampling_logp_difference/max": 0.3596630096435547, "sampling/sampling_logp_difference/mean": 0.012516415677964687, "step": 1 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00011034102863050066, "clip_ratio/low_min": 0.00011034102863050066, "clip_ratio/region_mean": 0.00011034102863050066, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 762.0, "completions/mean_length": 615.625, "completions/mean_terminated_length": 564.8333740234375, "completions/min_length": 349.0, "completions/min_terminated_length": 349.0, "entropy": 0.4234888218343258, "epoch": 0.0010706638115631692, "frac_reward_zero_std": 0.0, "grad_norm": 0.011267696507275105, "learning_rate": 1e-05, "loss": 0.0936, "num_tokens": 48695.0, "reward": 0.59375, "reward_std": 0.4355708956718445, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.3123857975006104, "sampling/importance_sampling_ratio/mean": 0.9996686577796936, "sampling/importance_sampling_ratio/min": 0.6920886039733887, "sampling/sampling_logp_difference/max": 0.3680412769317627, "sampling/sampling_logp_difference/mean": 0.011322839185595512, "step": 2 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 763.0, "completions/mean_length": 525.0, "completions/mean_terminated_length": 468.923095703125, "completions/min_length": 314.0, "completions/min_terminated_length": 314.0, "entropy": 0.450862105935812, "epoch": 0.0016059957173447537, "frac_reward_zero_std": 0.5, "grad_norm": 0.005509702488780022, "learning_rate": 1e-05, "loss": 0.0714, "num_tokens": 68967.0, "reward": 0.78125, "reward_std": 0.2630178928375244, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.2685112953186035, "sampling/importance_sampling_ratio/mean": 0.9999655485153198, "sampling/importance_sampling_ratio/min": 0.7146701812744141, "sampling/sampling_logp_difference/max": 0.3359341621398926, "sampling/sampling_logp_difference/mean": 0.012161738239228725, "step": 3 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 743.0, "completions/mean_length": 599.4375, "completions/mean_terminated_length": 560.5384521484375, "completions/min_length": 353.0, "completions/min_terminated_length": 353.0, "entropy": 0.40837138518691063, "epoch": 0.0021413276231263384, "frac_reward_zero_std": 0.5, "grad_norm": 0.009048921056091785, "learning_rate": 1e-05, "loss": 0.0159, "num_tokens": 92461.0, "reward": 0.46875, "reward_std": 0.2630178928375244, "rewards/accuracy_reward/mean": 0.46875, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.3900072574615479, "sampling/importance_sampling_ratio/mean": 0.9997454285621643, "sampling/importance_sampling_ratio/min": 0.6160321831703186, "sampling/sampling_logp_difference/max": 0.48445606231689453, "sampling/sampling_logp_difference/mean": 0.011591978371143341, "step": 4 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.4375, "completions/max_length": 768.0, "completions/max_terminated_length": 768.0, "completions/mean_length": 671.46875, "completions/mean_terminated_length": 596.388916015625, "completions/min_length": 427.0, "completions/min_terminated_length": 427.0, "entropy": 0.41664107516407967, "epoch": 0.0026766595289079227, "frac_reward_zero_std": 0.25, "grad_norm": 0.00951086264103651, "learning_rate": 1e-05, "loss": 0.0529, "num_tokens": 117740.0, "reward": 0.34375, "reward_std": 0.378745436668396, "rewards/accuracy_reward/mean": 0.34375, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.2843331098556519, "sampling/importance_sampling_ratio/mean": 1.0001472234725952, "sampling/importance_sampling_ratio/min": 0.5767552852630615, "sampling/sampling_logp_difference/max": 0.5503373146057129, "sampling/sampling_logp_difference/mean": 0.012003127485513687, "step": 5 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.263157800072804e-05, "clip_ratio/low_min": 5.263157800072804e-05, "clip_ratio/region_mean": 5.263157800072804e-05, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 753.0, "completions/mean_length": 534.1875, "completions/mean_terminated_length": 468.7200012207031, "completions/min_length": 322.0, "completions/min_terminated_length": 322.0, "entropy": 0.4959532096982002, "epoch": 0.0032119914346895075, "frac_reward_zero_std": 0.0, "grad_norm": 0.014368094503879547, "learning_rate": 1e-05, "loss": 0.0858, "num_tokens": 138738.0, "reward": 0.625, "reward_std": 0.4082317352294922, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.3477996587753296, "sampling/importance_sampling_ratio/mean": 1.0001568794250488, "sampling/importance_sampling_ratio/min": 0.7303528189659119, "sampling/sampling_logp_difference/max": 0.3142275810241699, "sampling/sampling_logp_difference/mean": 0.013215973973274231, "step": 6 }, { "clip_ratio/high_max": 5.781683648820035e-05, "clip_ratio/high_mean": 5.781683648820035e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 5.781683648820035e-05, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 745.0, "completions/mean_length": 557.65625, "completions/mean_terminated_length": 535.8965454101562, "completions/min_length": 254.0, "completions/min_terminated_length": 254.0, "entropy": 0.3992920517921448, "epoch": 0.003747323340471092, "frac_reward_zero_std": 0.25, "grad_norm": 0.0207853764295578, "learning_rate": 1e-05, "loss": 0.0173, "num_tokens": 160207.0, "reward": 0.78125, "reward_std": 0.3608423173427582, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.5296844244003296, "sampling/importance_sampling_ratio/mean": 0.9999474287033081, "sampling/importance_sampling_ratio/min": 0.7430385947227478, "sampling/sampling_logp_difference/max": 0.4250614643096924, "sampling/sampling_logp_difference/mean": 0.011352009139955044, "step": 7 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.5126354962121695e-05, "clip_ratio/low_min": 4.5126354962121695e-05, "clip_ratio/region_mean": 4.5126354962121695e-05, "completions/clipped_ratio": 0.28125, "completions/max_length": 768.0, "completions/max_terminated_length": 749.0, "completions/mean_length": 666.65625, "completions/mean_terminated_length": 627.0, "completions/min_length": 391.0, "completions/min_terminated_length": 391.0, "entropy": 0.32646026462316513, "epoch": 0.004282655246252677, "frac_reward_zero_std": 0.5, "grad_norm": 0.006821369286626577, "learning_rate": 1e-05, "loss": 0.0193, "num_tokens": 185060.0, "reward": 0.5, "reward_std": 0.26726123690605164, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5080004930496216, "sampling/importance_sampling_ratio/max": 1.3118391036987305, "sampling/importance_sampling_ratio/mean": 0.99983811378479, "sampling/importance_sampling_ratio/min": 0.7614741921424866, "sampling/sampling_logp_difference/max": 0.27249908447265625, "sampling/sampling_logp_difference/mean": 0.009400330483913422, "step": 8 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.375, "completions/max_length": 768.0, "completions/max_terminated_length": 733.0, "completions/mean_length": 618.90625, "completions/mean_terminated_length": 529.4500122070312, "completions/min_length": 297.0, "completions/min_terminated_length": 297.0, "entropy": 0.35843971744179726, "epoch": 0.004817987152034261, "frac_reward_zero_std": 0.0, "grad_norm": 0.02422148361802101, "learning_rate": 1e-05, "loss": 0.1226, "num_tokens": 208801.0, "reward": 0.625, "reward_std": 0.3945523500442505, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.3372341394424438, "sampling/importance_sampling_ratio/mean": 0.9997857213020325, "sampling/importance_sampling_ratio/min": 0.6132058501243591, "sampling/sampling_logp_difference/max": 0.48905467987060547, "sampling/sampling_logp_difference/mean": 0.010105549357831478, "step": 9 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.747126488131471e-05, "clip_ratio/low_min": 5.747126488131471e-05, "clip_ratio/region_mean": 5.747126488131471e-05, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 700.0, "completions/mean_length": 520.53125, "completions/mean_terminated_length": 485.1785888671875, "completions/min_length": 300.0, "completions/min_terminated_length": 300.0, "entropy": 0.34622709080576897, "epoch": 0.0053533190578158455, "frac_reward_zero_std": 0.5, "grad_norm": 0.00984555296599865, "learning_rate": 1e-05, "loss": 0.0782, "num_tokens": 228490.0, "reward": 0.8125, "reward_std": 0.2587745785713196, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.2665451765060425, "sampling/importance_sampling_ratio/mean": 0.9997326731681824, "sampling/importance_sampling_ratio/min": 0.7133243083953857, "sampling/sampling_logp_difference/max": 0.33781909942626953, "sampling/sampling_logp_difference/mean": 0.009415730834007263, "step": 10 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00012798634998034686, "clip_ratio/low_min": 0.00012798634998034686, "clip_ratio/region_mean": 0.00012798634998034686, "completions/clipped_ratio": 0.6875, "completions/max_length": 768.0, "completions/max_terminated_length": 758.0, "completions/mean_length": 724.15625, "completions/mean_terminated_length": 627.7000122070312, "completions/min_length": 450.0, "completions/min_terminated_length": 450.0, "entropy": 0.6496313735842705, "epoch": 0.005888650963597431, "frac_reward_zero_std": 0.25, "grad_norm": 0.005864573642611504, "learning_rate": 1e-05, "loss": -0.0007, "num_tokens": 255631.0, "reward": 0.15625, "reward_std": 0.3061639964580536, "rewards/accuracy_reward/mean": 0.15625, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.6560299396514893, "sampling/importance_sampling_ratio/mean": 1.000302791595459, "sampling/importance_sampling_ratio/min": 0.7081769704818726, "sampling/sampling_logp_difference/max": 0.5044231414794922, "sampling/sampling_logp_difference/mean": 0.01602434553205967, "step": 11 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.34375, "completions/max_length": 768.0, "completions/max_terminated_length": 765.0, "completions/mean_length": 610.59375, "completions/mean_terminated_length": 528.1428833007812, "completions/min_length": 315.0, "completions/min_terminated_length": 315.0, "entropy": 0.4252280630171299, "epoch": 0.006423982869379015, "frac_reward_zero_std": 0.25, "grad_norm": 0.013786358758807182, "learning_rate": 1e-05, "loss": 0.0442, "num_tokens": 278674.0, "reward": 0.625, "reward_std": 0.2925041913986206, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.5340704917907715, "sampling/importance_sampling_ratio/mean": 0.9997371435165405, "sampling/importance_sampling_ratio/min": 0.7066288590431213, "sampling/sampling_logp_difference/max": 0.42792463302612305, "sampling/sampling_logp_difference/mean": 0.011518296785652637, "step": 12 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.28125, "completions/max_length": 768.0, "completions/max_terminated_length": 765.0, "completions/mean_length": 553.90625, "completions/mean_terminated_length": 470.13043212890625, "completions/min_length": 229.0, "completions/min_terminated_length": 229.0, "entropy": 0.4396296590566635, "epoch": 0.006959314775160599, "frac_reward_zero_std": 0.0, "grad_norm": 0.011722812429070473, "learning_rate": 1e-05, "loss": -0.0626, "num_tokens": 299823.0, "reward": 0.53125, "reward_std": 0.4807935357093811, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.475489854812622, "sampling/importance_sampling_ratio/mean": 0.9998274445533752, "sampling/importance_sampling_ratio/min": 0.5904674530029297, "sampling/sampling_logp_difference/max": 0.5268406867980957, "sampling/sampling_logp_difference/mean": 0.012197126634418964, "step": 13 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00016838656665640883, "clip_ratio/low_min": 0.00016838656665640883, "clip_ratio/region_mean": 0.00016838656665640883, "completions/clipped_ratio": 0.28125, "completions/max_length": 768.0, "completions/max_terminated_length": 765.0, "completions/mean_length": 594.40625, "completions/mean_terminated_length": 526.478271484375, "completions/min_length": 313.0, "completions/min_terminated_length": 313.0, "entropy": 0.3696550354361534, "epoch": 0.007494646680942184, "frac_reward_zero_std": 0.0, "grad_norm": 0.013173367828130722, "learning_rate": 1e-05, "loss": 0.0506, "num_tokens": 322396.0, "reward": 0.65625, "reward_std": 0.4355708956718445, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.2909218072891235, "sampling/importance_sampling_ratio/mean": 1.0002373456954956, "sampling/importance_sampling_ratio/min": 0.6608690619468689, "sampling/sampling_logp_difference/max": 0.4141995906829834, "sampling/sampling_logp_difference/mean": 0.010886088944971561, "step": 14 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 759.0, "completions/mean_length": 613.46875, "completions/mean_terminated_length": 591.3928833007812, "completions/min_length": 304.0, "completions/min_terminated_length": 304.0, "entropy": 0.3582111708819866, "epoch": 0.008029978586723769, "frac_reward_zero_std": 0.25, "grad_norm": 0.008435110561549664, "learning_rate": 1e-05, "loss": -0.0, "num_tokens": 345475.0, "reward": 0.8125, "reward_std": 0.3104073107242584, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.2862032651901245, "sampling/importance_sampling_ratio/mean": 1.0000219345092773, "sampling/importance_sampling_ratio/min": 0.7341545224189758, "sampling/sampling_logp_difference/max": 0.3090357780456543, "sampling/sampling_logp_difference/mean": 0.010146892629563808, "step": 15 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0001517675700597465, "clip_ratio/low_min": 0.0001517675700597465, "clip_ratio/region_mean": 0.0001517675700597465, "completions/clipped_ratio": 0.28125, "completions/max_length": 768.0, "completions/max_terminated_length": 742.0, "completions/mean_length": 628.65625, "completions/mean_terminated_length": 574.1304321289062, "completions/min_length": 378.0, "completions/min_terminated_length": 378.0, "entropy": 0.7098877020180225, "epoch": 0.008565310492505354, "frac_reward_zero_std": 0.0, "grad_norm": 0.018570125102996826, "learning_rate": 1e-05, "loss": 0.0919, "num_tokens": 370344.0, "reward": 0.59375, "reward_std": 0.512376070022583, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.6732138395309448, "sampling/importance_sampling_ratio/mean": 0.9998745322227478, "sampling/importance_sampling_ratio/min": 0.7282438278198242, "sampling/sampling_logp_difference/max": 0.5147461891174316, "sampling/sampling_logp_difference/mean": 0.0150149529799819, "step": 16 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.829984391108155e-05, "clip_ratio/low_min": 4.829984391108155e-05, "clip_ratio/region_mean": 4.829984391108155e-05, "completions/clipped_ratio": 0.46875, "completions/max_length": 768.0, "completions/max_terminated_length": 765.0, "completions/mean_length": 666.34375, "completions/mean_terminated_length": 576.6470336914062, "completions/min_length": 335.0, "completions/min_terminated_length": 335.0, "entropy": 0.5932197459042072, "epoch": 0.009100642398286937, "frac_reward_zero_std": 0.25, "grad_norm": 0.00621638773009181, "learning_rate": 1e-05, "loss": 0.0107, "num_tokens": 395475.0, "reward": 0.15625, "reward_std": 0.3198433816432953, "rewards/accuracy_reward/mean": 0.15625, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.3349889516830444, "sampling/importance_sampling_ratio/mean": 0.9999136924743652, "sampling/importance_sampling_ratio/min": 0.6694192290306091, "sampling/sampling_logp_difference/max": 0.40134477615356445, "sampling/sampling_logp_difference/mean": 0.01411344762891531, "step": 17 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.530973430722952e-05, "clip_ratio/low_min": 5.530973430722952e-05, "clip_ratio/region_mean": 5.530973430722952e-05, "completions/clipped_ratio": 0.5, "completions/max_length": 768.0, "completions/max_terminated_length": 762.0, "completions/mean_length": 671.96875, "completions/mean_terminated_length": 575.9375, "completions/min_length": 279.0, "completions/min_terminated_length": 279.0, "entropy": 0.47273609787225723, "epoch": 0.009635974304068522, "frac_reward_zero_std": 0.0, "grad_norm": 0.016480356454849243, "learning_rate": 1e-05, "loss": 0.0461, "num_tokens": 420858.0, "reward": 0.53125, "reward_std": 0.4218915104866028, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.3640564680099487, "sampling/importance_sampling_ratio/mean": 1.000091552734375, "sampling/importance_sampling_ratio/min": 0.5932015180587769, "sampling/sampling_logp_difference/max": 0.5222210884094238, "sampling/sampling_logp_difference/mean": 0.012002711184322834, "step": 18 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 9.925135964294896e-05, "clip_ratio/low_min": 9.925135964294896e-05, "clip_ratio/region_mean": 9.925135964294896e-05, "completions/clipped_ratio": 0.59375, "completions/max_length": 768.0, "completions/max_terminated_length": 764.0, "completions/mean_length": 677.96875, "completions/mean_terminated_length": 546.3846435546875, "completions/min_length": 353.0, "completions/min_terminated_length": 353.0, "entropy": 0.4563947506248951, "epoch": 0.010171306209850108, "frac_reward_zero_std": 0.5, "grad_norm": 0.012845396064221859, "learning_rate": 1e-05, "loss": 0.0329, "num_tokens": 446721.0, "reward": 0.4375, "reward_std": 0.249358132481575, "rewards/accuracy_reward/mean": 0.4375, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.48965322971344, "sampling/importance_sampling_ratio/mean": 1.0000492334365845, "sampling/importance_sampling_ratio/min": 0.6992079019546509, "sampling/sampling_logp_difference/max": 0.3985433578491211, "sampling/sampling_logp_difference/mean": 0.012940243817865849, "step": 19 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00013168069563107565, "clip_ratio/low_min": 0.00013168069563107565, "clip_ratio/region_mean": 0.00013168069563107565, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 687.0, "completions/mean_length": 529.21875, "completions/mean_terminated_length": 449.625, "completions/min_length": 244.0, "completions/min_terminated_length": 244.0, "entropy": 0.489932082593441, "epoch": 0.010706638115631691, "frac_reward_zero_std": 0.0, "grad_norm": 0.015450346283614635, "learning_rate": 1e-05, "loss": 0.0744, "num_tokens": 467336.0, "reward": 0.40625, "reward_std": 0.4628904461860657, "rewards/accuracy_reward/mean": 0.40625, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.7126078605651855, "sampling/importance_sampling_ratio/mean": 0.9998895525932312, "sampling/importance_sampling_ratio/min": 0.7210950255393982, "sampling/sampling_logp_difference/max": 0.5380172729492188, "sampling/sampling_logp_difference/mean": 0.012707538902759552, "step": 20 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.526083259610459e-05, "clip_ratio/low_min": 5.526083259610459e-05, "clip_ratio/region_mean": 5.526083259610459e-05, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 699.0, "completions/mean_length": 591.96875, "completions/mean_terminated_length": 533.2916870117188, "completions/min_length": 369.0, "completions/min_terminated_length": 369.0, "entropy": 0.4202882684767246, "epoch": 0.011241970021413276, "frac_reward_zero_std": 0.25, "grad_norm": 0.01361797470599413, "learning_rate": 1e-05, "loss": 0.0766, "num_tokens": 490311.0, "reward": 0.71875, "reward_std": 0.3608423173427582, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.2968697547912598, "sampling/importance_sampling_ratio/mean": 0.9999016523361206, "sampling/importance_sampling_ratio/min": 0.7421221733093262, "sampling/sampling_logp_difference/max": 0.29824137687683105, "sampling/sampling_logp_difference/mean": 0.010941424407064915, "step": 21 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0001045044045895338, "clip_ratio/low_min": 0.0001045044045895338, "clip_ratio/region_mean": 0.0001045044045895338, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 732.0, "completions/mean_length": 589.90625, "completions/mean_terminated_length": 530.5416870117188, "completions/min_length": 274.0, "completions/min_terminated_length": 274.0, "entropy": 0.4353359416127205, "epoch": 0.011777301927194861, "frac_reward_zero_std": 0.0, "grad_norm": 0.006856705527752638, "learning_rate": 1e-05, "loss": 0.0813, "num_tokens": 512564.0, "reward": 0.53125, "reward_std": 0.5038893818855286, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.4417986869812012, "sampling/importance_sampling_ratio/mean": 1.0003200769424438, "sampling/importance_sampling_ratio/min": 0.7698214650154114, "sampling/sampling_logp_difference/max": 0.3658914566040039, "sampling/sampling_logp_difference/mean": 0.012175213545560837, "step": 22 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 751.0, "completions/mean_length": 592.125, "completions/mean_terminated_length": 551.5384521484375, "completions/min_length": 347.0, "completions/min_terminated_length": 347.0, "entropy": 0.4204316474497318, "epoch": 0.012312633832976445, "frac_reward_zero_std": 0.5, "grad_norm": 0.01653178781270981, "learning_rate": 1e-05, "loss": 0.023, "num_tokens": 535624.0, "reward": 0.6875, "reward_std": 0.249358132481575, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.393133521080017, "sampling/importance_sampling_ratio/mean": 1.0003411769866943, "sampling/importance_sampling_ratio/min": 0.7714617848396301, "sampling/sampling_logp_difference/max": 0.3315556049346924, "sampling/sampling_logp_difference/mean": 0.01182654406875372, "step": 23 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.463286652229726e-05, "clip_ratio/low_min": 5.463286652229726e-05, "clip_ratio/region_mean": 5.463286652229726e-05, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 753.0, "completions/mean_length": 532.75, "completions/mean_terminated_length": 508.4137878417969, "completions/min_length": 130.0, "completions/min_terminated_length": 130.0, "entropy": 0.41935884580016136, "epoch": 0.01284796573875803, "frac_reward_zero_std": 0.5, "grad_norm": 0.00237714103423059, "learning_rate": 1e-05, "loss": 0.0965, "num_tokens": 556744.0, "reward": 0.65625, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.4122968912124634, "sampling/importance_sampling_ratio/mean": 0.9998799562454224, "sampling/importance_sampling_ratio/min": 0.7301450967788696, "sampling/sampling_logp_difference/max": 0.3452174663543701, "sampling/sampling_logp_difference/mean": 0.01174243539571762, "step": 24 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.431052730069496e-05, "clip_ratio/low_min": 4.431052730069496e-05, "clip_ratio/region_mean": 4.431052730069496e-05, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 762.0, "completions/mean_length": 602.75, "completions/mean_terminated_length": 564.6154174804688, "completions/min_length": 308.0, "completions/min_terminated_length": 308.0, "entropy": 0.5643983520567417, "epoch": 0.013383297644539615, "frac_reward_zero_std": 0.25, "grad_norm": 0.0036444501020014286, "learning_rate": 1e-05, "loss": 0.0616, "num_tokens": 579528.0, "reward": 0.65625, "reward_std": 0.3377465009689331, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.3097071647644043, "sampling/importance_sampling_ratio/mean": 1.0000592470169067, "sampling/importance_sampling_ratio/min": 0.7228518724441528, "sampling/sampling_logp_difference/max": 0.324550986289978, "sampling/sampling_logp_difference/mean": 0.01465004775673151, "step": 25 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 9.006520849652588e-05, "clip_ratio/low_min": 9.006520849652588e-05, "clip_ratio/region_mean": 9.006520849652588e-05, "completions/clipped_ratio": 0.46875, "completions/max_length": 768.0, "completions/max_terminated_length": 766.0, "completions/mean_length": 676.3125, "completions/mean_terminated_length": 595.4117431640625, "completions/min_length": 449.0, "completions/min_terminated_length": 449.0, "entropy": 0.455068938434124, "epoch": 0.013918629550321198, "frac_reward_zero_std": 0.5, "grad_norm": 0.007197745610028505, "learning_rate": 1e-05, "loss": 0.0294, "num_tokens": 605210.0, "reward": 0.21875, "reward_std": 0.2630178928375244, "rewards/accuracy_reward/mean": 0.21875, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.5281137228012085, "sampling/importance_sampling_ratio/mean": 0.9997848272323608, "sampling/importance_sampling_ratio/min": 0.6987742781639099, "sampling/sampling_logp_difference/max": 0.42403411865234375, "sampling/sampling_logp_difference/mean": 0.012703796848654747, "step": 26 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00013512342411559075, "clip_ratio/low_min": 0.00013512342411559075, "clip_ratio/region_mean": 0.00013512342411559075, "completions/clipped_ratio": 0.5625, "completions/max_length": 768.0, "completions/max_terminated_length": 758.0, "completions/mean_length": 673.3125, "completions/mean_terminated_length": 551.5714721679688, "completions/min_length": 381.0, "completions/min_terminated_length": 381.0, "entropy": 0.6482994183897972, "epoch": 0.014453961456102784, "frac_reward_zero_std": 0.0, "grad_norm": 0.016385406255722046, "learning_rate": 1e-05, "loss": 0.0969, "num_tokens": 630612.0, "reward": 0.40625, "reward_std": 0.4218915104866028, "rewards/accuracy_reward/mean": 0.40625, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.6067651510238647, "sampling/importance_sampling_ratio/mean": 1.000059962272644, "sampling/importance_sampling_ratio/min": 0.6817489266395569, "sampling/sampling_logp_difference/max": 0.47422289848327637, "sampling/sampling_logp_difference/mean": 0.014250493608415127, "step": 27 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0001526857631688472, "clip_ratio/low_min": 0.0001526857631688472, "clip_ratio/region_mean": 0.0001526857631688472, "completions/clipped_ratio": 0.5, "completions/max_length": 768.0, "completions/max_terminated_length": 755.0, "completions/mean_length": 693.84375, "completions/mean_terminated_length": 619.6875, "completions/min_length": 331.0, "completions/min_terminated_length": 331.0, "entropy": 0.4106770195066929, "epoch": 0.014989293361884369, "frac_reward_zero_std": 0.25, "grad_norm": 0.010193257592618465, "learning_rate": 1e-05, "loss": 0.0151, "num_tokens": 657095.0, "reward": 0.25, "reward_std": 0.3514062464237213, "rewards/accuracy_reward/mean": 0.25, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.3577656745910645, "sampling/importance_sampling_ratio/mean": 1.0001087188720703, "sampling/importance_sampling_ratio/min": 0.7057578563690186, "sampling/sampling_logp_difference/max": 0.3484830856323242, "sampling/sampling_logp_difference/mean": 0.011250431649386883, "step": 28 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 767.0, "completions/mean_length": 622.375, "completions/mean_terminated_length": 588.7692260742188, "completions/min_length": 331.0, "completions/min_terminated_length": 331.0, "entropy": 0.44800711050629616, "epoch": 0.015524625267665952, "frac_reward_zero_std": 0.25, "grad_norm": 0.005607192870229483, "learning_rate": 1e-05, "loss": 0.0299, "num_tokens": 681251.0, "reward": 0.59375, "reward_std": 0.3198433816432953, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.3808902502059937, "sampling/importance_sampling_ratio/mean": 1.0000520944595337, "sampling/importance_sampling_ratio/min": 0.658728837966919, "sampling/sampling_logp_difference/max": 0.41744327545166016, "sampling/sampling_logp_difference/mean": 0.0115430923178792, "step": 29 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.625, "completions/max_length": 768.0, "completions/max_terminated_length": 761.0, "completions/mean_length": 715.875, "completions/mean_terminated_length": 629.0, "completions/min_length": 494.0, "completions/min_terminated_length": 494.0, "entropy": 0.6344446279108524, "epoch": 0.016059957173447537, "frac_reward_zero_std": 0.5, "grad_norm": 0.0017212615348398685, "learning_rate": 1e-05, "loss": 0.0028, "num_tokens": 708455.0, "reward": 0.3125, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.3125, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.3901278972625732, "sampling/importance_sampling_ratio/mean": 0.9994648694992065, "sampling/importance_sampling_ratio/min": 0.5709463357925415, "sampling/sampling_logp_difference/max": 0.560460090637207, "sampling/sampling_logp_difference/mean": 0.01577536202967167, "step": 30 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.375, "completions/max_length": 768.0, "completions/max_terminated_length": 711.0, "completions/mean_length": 613.96875, "completions/mean_terminated_length": 521.5499877929688, "completions/min_length": 341.0, "completions/min_terminated_length": 341.0, "entropy": 0.34220656752586365, "epoch": 0.016595289079229122, "frac_reward_zero_std": 0.25, "grad_norm": 0.00600267481058836, "learning_rate": 1e-05, "loss": 0.0678, "num_tokens": 731782.0, "reward": 0.53125, "reward_std": 0.3377464711666107, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.2601256370544434, "sampling/importance_sampling_ratio/mean": 0.9999644160270691, "sampling/importance_sampling_ratio/min": 0.7214158773422241, "sampling/sampling_logp_difference/max": 0.3265395164489746, "sampling/sampling_logp_difference/mean": 0.009856891818344593, "step": 31 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.158894055057317e-05, "clip_ratio/low_min": 5.158894055057317e-05, "clip_ratio/region_mean": 5.158894055057317e-05, "completions/clipped_ratio": 0.28125, "completions/max_length": 768.0, "completions/max_terminated_length": 768.0, "completions/mean_length": 600.375, "completions/mean_terminated_length": 534.7825927734375, "completions/min_length": 267.0, "completions/min_terminated_length": 267.0, "entropy": 0.38323255628347397, "epoch": 0.017130620985010708, "frac_reward_zero_std": 0.0, "grad_norm": 0.012019449844956398, "learning_rate": 1e-05, "loss": 0.0308, "num_tokens": 754738.0, "reward": 0.53125, "reward_std": 0.512376070022583, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.6615517139434814, "sampling/importance_sampling_ratio/mean": 1.0002702474594116, "sampling/importance_sampling_ratio/min": 0.7132049798965454, "sampling/sampling_logp_difference/max": 0.5077519416809082, "sampling/sampling_logp_difference/mean": 0.010766430757939816, "step": 32 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.969436460873112e-05, "clip_ratio/low_min": 5.969436460873112e-05, "clip_ratio/region_mean": 5.969436460873112e-05, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 747.0, "completions/mean_length": 582.875, "completions/mean_terminated_length": 540.1538696289062, "completions/min_length": 335.0, "completions/min_terminated_length": 335.0, "entropy": 0.5091266073286533, "epoch": 0.017665952890792293, "frac_reward_zero_std": 0.25, "grad_norm": 0.01115038525313139, "learning_rate": 1e-05, "loss": 0.0964, "num_tokens": 777014.0, "reward": 0.59375, "reward_std": 0.3061639666557312, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.586151123046875, "sampling/importance_sampling_ratio/mean": 0.9998085498809814, "sampling/importance_sampling_ratio/min": 0.6728514432907104, "sampling/sampling_logp_difference/max": 0.46131038665771484, "sampling/sampling_logp_difference/mean": 0.013545024208724499, "step": 33 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.069010537932627e-05, "clip_ratio/low_min": 4.069010537932627e-05, "clip_ratio/region_mean": 4.069010537932627e-05, "completions/clipped_ratio": 0.40625, "completions/max_length": 768.0, "completions/max_terminated_length": 765.0, "completions/mean_length": 685.46875, "completions/mean_terminated_length": 629.0, "completions/min_length": 273.0, "completions/min_terminated_length": 273.0, "entropy": 0.4806034788489342, "epoch": 0.018201284796573874, "frac_reward_zero_std": 0.5, "grad_norm": 0.012209653854370117, "learning_rate": 1e-05, "loss": 0.0312, "num_tokens": 802837.0, "reward": 0.40625, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.40625, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.41061532497406, "sampling/importance_sampling_ratio/mean": 0.9998142719268799, "sampling/importance_sampling_ratio/min": 0.716969907283783, "sampling/sampling_logp_difference/max": 0.34402596950531006, "sampling/sampling_logp_difference/mean": 0.012602795846760273, "step": 34 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.3125, "completions/max_length": 768.0, "completions/max_terminated_length": 763.0, "completions/mean_length": 624.9375, "completions/mean_terminated_length": 559.9091186523438, "completions/min_length": 343.0, "completions/min_terminated_length": 343.0, "entropy": 0.5183686465024948, "epoch": 0.01873661670235546, "frac_reward_zero_std": 0.5, "grad_norm": 0.005360551178455353, "learning_rate": 1e-05, "loss": 0.0511, "num_tokens": 826731.0, "reward": 0.46875, "reward_std": 0.2630178928375244, "rewards/accuracy_reward/mean": 0.46875, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.5687141418457031, "sampling/importance_sampling_ratio/mean": 0.9999753832817078, "sampling/importance_sampling_ratio/min": 0.7450584173202515, "sampling/sampling_logp_difference/max": 0.45025634765625, "sampling/sampling_logp_difference/mean": 0.013193635269999504, "step": 35 }, { "clip_ratio/high_max": 6.695232877973467e-05, "clip_ratio/high_mean": 6.695232877973467e-05, "clip_ratio/low_mean": 4.41852243966423e-05, "clip_ratio/low_min": 4.41852243966423e-05, "clip_ratio/region_mean": 0.00011113755317637697, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 758.0, "completions/mean_length": 581.6875, "completions/mean_terminated_length": 547.1851806640625, "completions/min_length": 383.0, "completions/min_terminated_length": 383.0, "entropy": 0.39413612335920334, "epoch": 0.019271948608137045, "frac_reward_zero_std": 0.0, "grad_norm": 0.01678226888179779, "learning_rate": 1e-05, "loss": 0.112, "num_tokens": 849001.0, "reward": 0.65625, "reward_std": 0.3808925747871399, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.382460594177246, "sampling/importance_sampling_ratio/mean": 0.9995832443237305, "sampling/importance_sampling_ratio/min": 0.6742712259292603, "sampling/sampling_logp_difference/max": 0.394122838973999, "sampling/sampling_logp_difference/mean": 0.01092858798801899, "step": 36 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 701.0, "completions/mean_length": 553.0, "completions/mean_terminated_length": 530.7586059570312, "completions/min_length": 245.0, "completions/min_terminated_length": 245.0, "entropy": 0.38594045862555504, "epoch": 0.01980728051391863, "frac_reward_zero_std": 0.5, "grad_norm": 0.005036050453782082, "learning_rate": 1e-05, "loss": 0.0155, "num_tokens": 870657.0, "reward": 0.71875, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.3651607036590576, "sampling/importance_sampling_ratio/mean": 1.0000442266464233, "sampling/importance_sampling_ratio/min": 0.7367088794708252, "sampling/sampling_logp_difference/max": 0.31127214431762695, "sampling/sampling_logp_difference/mean": 0.011241357773542404, "step": 37 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.5, "completions/max_length": 768.0, "completions/max_terminated_length": 739.0, "completions/mean_length": 622.15625, "completions/mean_terminated_length": 476.3125, "completions/min_length": 149.0, "completions/min_terminated_length": 149.0, "entropy": 0.4367636665701866, "epoch": 0.020342612419700215, "frac_reward_zero_std": 0.0, "grad_norm": 0.010261477902531624, "learning_rate": 1e-05, "loss": 0.0506, "num_tokens": 894446.0, "reward": 0.40625, "reward_std": 0.4628904461860657, "rewards/accuracy_reward/mean": 0.40625, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.3593658208847046, "sampling/importance_sampling_ratio/mean": 1.0002293586730957, "sampling/importance_sampling_ratio/min": 0.7748647332191467, "sampling/sampling_logp_difference/max": 0.3070182800292969, "sampling/sampling_logp_difference/mean": 0.012136059813201427, "step": 38 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.607445589499548e-05, "clip_ratio/low_min": 4.607445589499548e-05, "clip_ratio/region_mean": 4.607445589499548e-05, "completions/clipped_ratio": 0.28125, "completions/max_length": 768.0, "completions/max_terminated_length": 747.0, "completions/mean_length": 591.90625, "completions/mean_terminated_length": 523.0, "completions/min_length": 319.0, "completions/min_terminated_length": 319.0, "entropy": 0.3763354830443859, "epoch": 0.0208779443254818, "frac_reward_zero_std": 0.25, "grad_norm": 0.007283422164618969, "learning_rate": 1e-05, "loss": 0.0209, "num_tokens": 916947.0, "reward": 0.4375, "reward_std": 0.3514062464237213, "rewards/accuracy_reward/mean": 0.4375, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.4077885150909424, "sampling/importance_sampling_ratio/mean": 1.000258445739746, "sampling/importance_sampling_ratio/min": 0.6510267853736877, "sampling/sampling_logp_difference/max": 0.42920446395874023, "sampling/sampling_logp_difference/mean": 0.0101675596088171, "step": 39 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.553734106593765e-05, "clip_ratio/low_min": 4.553734106593765e-05, "clip_ratio/region_mean": 4.553734106593765e-05, "completions/clipped_ratio": 0.46875, "completions/max_length": 768.0, "completions/max_terminated_length": 750.0, "completions/mean_length": 639.375, "completions/mean_terminated_length": 525.8823852539062, "completions/min_length": 358.0, "completions/min_terminated_length": 358.0, "entropy": 0.6507696583867073, "epoch": 0.021413276231263382, "frac_reward_zero_std": 0.5, "grad_norm": 0.006135659758001566, "learning_rate": 1e-05, "loss": 0.0579, "num_tokens": 941271.0, "reward": 0.34375, "reward_std": 0.22201895713806152, "rewards/accuracy_reward/mean": 0.34375, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.2908605337142944, "sampling/importance_sampling_ratio/mean": 0.9999883770942688, "sampling/importance_sampling_ratio/min": 0.7787885665893555, "sampling/sampling_logp_difference/max": 0.2553091049194336, "sampling/sampling_logp_difference/mean": 0.016175610944628716, "step": 40 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00011416822235332802, "clip_ratio/low_min": 0.00011416822235332802, "clip_ratio/region_mean": 0.00011416822235332802, "completions/clipped_ratio": 0.3125, "completions/max_length": 768.0, "completions/max_terminated_length": 754.0, "completions/mean_length": 580.96875, "completions/mean_terminated_length": 495.9545593261719, "completions/min_length": 336.0, "completions/min_terminated_length": 336.0, "entropy": 0.44963598996400833, "epoch": 0.021948608137044967, "frac_reward_zero_std": 0.25, "grad_norm": 0.006392901297658682, "learning_rate": 1e-05, "loss": 0.0577, "num_tokens": 963374.0, "reward": 0.40625, "reward_std": 0.3377464711666107, "rewards/accuracy_reward/mean": 0.40625, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.2743258476257324, "sampling/importance_sampling_ratio/mean": 1.0001462697982788, "sampling/importance_sampling_ratio/min": 0.788698673248291, "sampling/sampling_logp_difference/max": 0.2424173355102539, "sampling/sampling_logp_difference/mean": 0.012541564181447029, "step": 41 }, { "clip_ratio/high_max": 6.28456546110101e-05, "clip_ratio/high_mean": 6.28456546110101e-05, "clip_ratio/low_mean": 0.00010341246161260642, "clip_ratio/low_min": 0.00010341246161260642, "clip_ratio/region_mean": 0.00016625811622361653, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 741.0, "completions/mean_length": 553.0625, "completions/mean_terminated_length": 522.357177734375, "completions/min_length": 288.0, "completions/min_terminated_length": 288.0, "entropy": 0.3921343423426151, "epoch": 0.022483940042826552, "frac_reward_zero_std": 0.25, "grad_norm": 0.008824815042316914, "learning_rate": 1e-05, "loss": 0.0089, "num_tokens": 984496.0, "reward": 0.6875, "reward_std": 0.38298875093460083, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.3491756916046143, "sampling/importance_sampling_ratio/mean": 1.0000351667404175, "sampling/importance_sampling_ratio/min": 0.611833393573761, "sampling/sampling_logp_difference/max": 0.49129533767700195, "sampling/sampling_logp_difference/mean": 0.01122310571372509, "step": 42 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.3125, "completions/max_length": 768.0, "completions/max_terminated_length": 768.0, "completions/mean_length": 593.25, "completions/mean_terminated_length": 513.8181762695312, "completions/min_length": 367.0, "completions/min_terminated_length": 367.0, "entropy": 0.372965756803751, "epoch": 0.023019271948608137, "frac_reward_zero_std": 0.5, "grad_norm": 0.01738766022026539, "learning_rate": 1e-05, "loss": 0.009, "num_tokens": 1007360.0, "reward": 0.5625, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.5648531913757324, "sampling/importance_sampling_ratio/mean": 0.9999549984931946, "sampling/importance_sampling_ratio/min": 0.5807793736457825, "sampling/sampling_logp_difference/max": 0.543384313583374, "sampling/sampling_logp_difference/mean": 0.010912786237895489, "step": 43 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 9.434629828319885e-05, "clip_ratio/low_min": 9.434629828319885e-05, "clip_ratio/region_mean": 9.434629828319885e-05, "completions/clipped_ratio": 0.3125, "completions/max_length": 768.0, "completions/max_terminated_length": 725.0, "completions/mean_length": 613.5, "completions/mean_terminated_length": 543.2727661132812, "completions/min_length": 306.0, "completions/min_terminated_length": 306.0, "entropy": 0.3671180475503206, "epoch": 0.023554603854389723, "frac_reward_zero_std": 0.75, "grad_norm": 0.011241797357797623, "learning_rate": 1e-05, "loss": 0.0035, "num_tokens": 1030720.0, "reward": 0.8125, "reward_std": 0.1157275140285492, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.35966956615448, "sampling/importance_sampling_ratio/mean": 0.9995852112770081, "sampling/importance_sampling_ratio/min": 0.728817343711853, "sampling/sampling_logp_difference/max": 0.3163321018218994, "sampling/sampling_logp_difference/mean": 0.009871450252830982, "step": 44 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.3614792957669124e-05, "clip_ratio/low_min": 4.3614792957669124e-05, "clip_ratio/region_mean": 4.3614792957669124e-05, "completions/clipped_ratio": 0.3125, "completions/max_length": 768.0, "completions/max_terminated_length": 752.0, "completions/mean_length": 643.15625, "completions/mean_terminated_length": 586.4091186523438, "completions/min_length": 400.0, "completions/min_terminated_length": 400.0, "entropy": 0.3872305490076542, "epoch": 0.024089935760171308, "frac_reward_zero_std": 0.25, "grad_norm": 0.007765679154545069, "learning_rate": 1e-05, "loss": 0.0361, "num_tokens": 1055429.0, "reward": 0.5625, "reward_std": 0.3514062464237213, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.9600664377212524, "sampling/importance_sampling_ratio/mean": 0.9999384880065918, "sampling/importance_sampling_ratio/min": 0.6967722177505493, "sampling/sampling_logp_difference/max": 0.672978401184082, "sampling/sampling_logp_difference/mean": 0.011130640283226967, "step": 45 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.46875, "completions/max_length": 768.0, "completions/max_terminated_length": 757.0, "completions/mean_length": 672.6875, "completions/mean_terminated_length": 588.5882568359375, "completions/min_length": 434.0, "completions/min_terminated_length": 434.0, "entropy": 0.48364455066621304, "epoch": 0.02462526766595289, "frac_reward_zero_std": 0.0, "grad_norm": 0.019248412922024727, "learning_rate": 1e-05, "loss": 0.0493, "num_tokens": 1080971.0, "reward": 0.5, "reward_std": 0.4355512857437134, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5080004930496216, "sampling/importance_sampling_ratio/max": 1.3275351524353027, "sampling/importance_sampling_ratio/mean": 0.9994391798973083, "sampling/importance_sampling_ratio/min": 0.7093425393104553, "sampling/sampling_logp_difference/max": 0.343416690826416, "sampling/sampling_logp_difference/mean": 0.01235866080969572, "step": 46 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 711.0, "completions/mean_length": 534.1875, "completions/mean_terminated_length": 500.7857360839844, "completions/min_length": 225.0, "completions/min_terminated_length": 225.0, "entropy": 0.5136398896574974, "epoch": 0.025160599571734475, "frac_reward_zero_std": 0.25, "grad_norm": 0.018716780468821526, "learning_rate": 1e-05, "loss": 0.0673, "num_tokens": 1101825.0, "reward": 0.71875, "reward_std": 0.3471629321575165, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.299971580505371, "sampling/importance_sampling_ratio/mean": 0.9997742772102356, "sampling/importance_sampling_ratio/min": 0.7442049980163574, "sampling/sampling_logp_difference/max": 0.2954387664794922, "sampling/sampling_logp_difference/mean": 0.013368945568799973, "step": 47 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.494505603564903e-05, "clip_ratio/low_min": 5.494505603564903e-05, "clip_ratio/region_mean": 5.494505603564903e-05, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 756.0, "completions/mean_length": 534.5625, "completions/mean_terminated_length": 501.21429443359375, "completions/min_length": 295.0, "completions/min_terminated_length": 295.0, "entropy": 0.3989657089114189, "epoch": 0.02569593147751606, "frac_reward_zero_std": 0.25, "grad_norm": 0.008748093619942665, "learning_rate": 1e-05, "loss": 0.0894, "num_tokens": 1122067.0, "reward": 0.6875, "reward_std": 0.3745020925998688, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.3007398843765259, "sampling/importance_sampling_ratio/mean": 0.9996334910392761, "sampling/importance_sampling_ratio/min": 0.765166699886322, "sampling/sampling_logp_difference/max": 0.26766157150268555, "sampling/sampling_logp_difference/mean": 0.011446605436503887, "step": 48 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.698820652673021e-05, "clip_ratio/low_min": 6.698820652673021e-05, "clip_ratio/region_mean": 6.698820652673021e-05, "completions/clipped_ratio": 0.46875, "completions/max_length": 768.0, "completions/max_terminated_length": 596.0, "completions/mean_length": 586.5625, "completions/mean_terminated_length": 426.4705810546875, "completions/min_length": 236.0, "completions/min_terminated_length": 236.0, "entropy": 0.6239189095795155, "epoch": 0.026231263383297645, "frac_reward_zero_std": 0.25, "grad_norm": 0.01232483796775341, "learning_rate": 1e-05, "loss": 0.02, "num_tokens": 1144773.0, "reward": 0.59375, "reward_std": 0.3061639964580536, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.4282665252685547, "sampling/importance_sampling_ratio/mean": 0.9999468922615051, "sampling/importance_sampling_ratio/min": 0.7118720412254333, "sampling/sampling_logp_difference/max": 0.3564615249633789, "sampling/sampling_logp_difference/mean": 0.016130099073052406, "step": 49 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.6023564209463075e-05, "clip_ratio/low_min": 4.6023564209463075e-05, "clip_ratio/region_mean": 4.6023564209463075e-05, "completions/clipped_ratio": 0.28125, "completions/max_length": 768.0, "completions/max_terminated_length": 714.0, "completions/mean_length": 605.1875, "completions/mean_terminated_length": 541.478271484375, "completions/min_length": 409.0, "completions/min_terminated_length": 409.0, "entropy": 0.43123210966587067, "epoch": 0.02676659528907923, "frac_reward_zero_std": 0.25, "grad_norm": 0.01060446910560131, "learning_rate": 1e-05, "loss": 0.046, "num_tokens": 1167691.0, "reward": 0.625, "reward_std": 0.3104073107242584, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.429684042930603, "sampling/importance_sampling_ratio/mean": 1.000015377998352, "sampling/importance_sampling_ratio/min": 0.7132202982902527, "sampling/sampling_logp_difference/max": 0.35745346546173096, "sampling/sampling_logp_difference/mean": 0.011622624471783638, "step": 50 }, { "clip_ratio/high_max": 4.822530900128186e-05, "clip_ratio/high_mean": 4.822530900128186e-05, "clip_ratio/low_mean": 0.0001036484245560132, "clip_ratio/low_min": 0.0001036484245560132, "clip_ratio/region_mean": 0.00015187373355729505, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 739.0, "completions/mean_length": 614.28125, "completions/mean_terminated_length": 585.8148193359375, "completions/min_length": 358.0, "completions/min_terminated_length": 358.0, "entropy": 0.3837196100503206, "epoch": 0.027301927194860815, "frac_reward_zero_std": 0.25, "grad_norm": 0.009614828042685986, "learning_rate": 1e-05, "loss": 0.0479, "num_tokens": 1191244.0, "reward": 0.65625, "reward_std": 0.3061639964580536, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.337222933769226, "sampling/importance_sampling_ratio/mean": 0.9998320937156677, "sampling/importance_sampling_ratio/min": 0.7234455943107605, "sampling/sampling_logp_difference/max": 0.3237299919128418, "sampling/sampling_logp_difference/mean": 0.010459691286087036, "step": 51 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 715.0, "completions/mean_length": 523.34375, "completions/mean_terminated_length": 478.03704833984375, "completions/min_length": 284.0, "completions/min_terminated_length": 284.0, "entropy": 0.3651442211121321, "epoch": 0.027837259100642397, "frac_reward_zero_std": 0.5, "grad_norm": 0.01307887677103281, "learning_rate": 1e-05, "loss": 0.0541, "num_tokens": 1211647.0, "reward": 0.46875, "reward_std": 0.2630178928375244, "rewards/accuracy_reward/mean": 0.46875, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.2707027196884155, "sampling/importance_sampling_ratio/mean": 1.0000180006027222, "sampling/importance_sampling_ratio/min": 0.7189217209815979, "sampling/sampling_logp_difference/max": 0.33000290393829346, "sampling/sampling_logp_difference/mean": 0.01065903715789318, "step": 52 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.2130097426706925e-05, "clip_ratio/low_min": 4.2130097426706925e-05, "clip_ratio/region_mean": 4.2130097426706925e-05, "completions/clipped_ratio": 0.3125, "completions/max_length": 768.0, "completions/max_terminated_length": 726.0, "completions/mean_length": 627.09375, "completions/mean_terminated_length": 563.0454711914062, "completions/min_length": 338.0, "completions/min_terminated_length": 338.0, "entropy": 0.4228546507656574, "epoch": 0.028372591006423982, "frac_reward_zero_std": 0.0, "grad_norm": 0.013921543024480343, "learning_rate": 1e-05, "loss": 0.0994, "num_tokens": 1235306.0, "reward": 0.6875, "reward_std": 0.4671337604522705, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.3246186971664429, "sampling/importance_sampling_ratio/mean": 1.0000325441360474, "sampling/importance_sampling_ratio/min": 0.6458011269569397, "sampling/sampling_logp_difference/max": 0.43726372718811035, "sampling/sampling_logp_difference/mean": 0.01152092032134533, "step": 53 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00024238396872533485, "clip_ratio/low_min": 0.00024238396872533485, "clip_ratio/region_mean": 0.00024238396872533485, "completions/clipped_ratio": 0.375, "completions/max_length": 768.0, "completions/max_terminated_length": 727.0, "completions/mean_length": 643.28125, "completions/mean_terminated_length": 568.4500122070312, "completions/min_length": 362.0, "completions/min_terminated_length": 362.0, "entropy": 0.4607427045702934, "epoch": 0.028907922912205567, "frac_reward_zero_std": 0.0, "grad_norm": 0.017727605998516083, "learning_rate": 1e-05, "loss": 0.0639, "num_tokens": 1259675.0, "reward": 0.46875, "reward_std": 0.4628904461860657, "rewards/accuracy_reward/mean": 0.46875, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.6147571802139282, "sampling/importance_sampling_ratio/mean": 0.999921441078186, "sampling/importance_sampling_ratio/min": 0.6753311157226562, "sampling/sampling_logp_difference/max": 0.479184627532959, "sampling/sampling_logp_difference/mean": 0.012804209254682064, "step": 54 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.4626918679568917e-05, "clip_ratio/low_min": 4.4626918679568917e-05, "clip_ratio/region_mean": 4.4626918679568917e-05, "completions/clipped_ratio": 0.34375, "completions/max_length": 768.0, "completions/max_terminated_length": 755.0, "completions/mean_length": 616.46875, "completions/mean_terminated_length": 537.0952758789062, "completions/min_length": 331.0, "completions/min_terminated_length": 331.0, "entropy": 0.44960571825504303, "epoch": 0.029443254817987152, "frac_reward_zero_std": 0.0, "grad_norm": 0.006553029175847769, "learning_rate": 1e-05, "loss": 0.1434, "num_tokens": 1283146.0, "reward": 0.625, "reward_std": 0.48503684997558594, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.2985329627990723, "sampling/importance_sampling_ratio/mean": 1.0000635385513306, "sampling/importance_sampling_ratio/min": 0.699820339679718, "sampling/sampling_logp_difference/max": 0.3569316864013672, "sampling/sampling_logp_difference/mean": 0.012360668741166592, "step": 55 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.5236410844372585e-05, "clip_ratio/low_min": 5.5236410844372585e-05, "clip_ratio/region_mean": 5.5236410844372585e-05, "completions/clipped_ratio": 0.375, "completions/max_length": 768.0, "completions/max_terminated_length": 756.0, "completions/mean_length": 598.34375, "completions/mean_terminated_length": 496.5500183105469, "completions/min_length": 262.0, "completions/min_terminated_length": 262.0, "entropy": 0.6260490566492081, "epoch": 0.029978586723768737, "frac_reward_zero_std": 0.5, "grad_norm": 0.0034410294611006975, "learning_rate": 1e-05, "loss": 0.0055, "num_tokens": 1306053.0, "reward": 0.46875, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.46875, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.3745598793029785, "sampling/importance_sampling_ratio/mean": 1.0001850128173828, "sampling/importance_sampling_ratio/min": 0.7050579190254211, "sampling/sampling_logp_difference/max": 0.3494753837585449, "sampling/sampling_logp_difference/mean": 0.01465124636888504, "step": 56 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 694.0, "completions/mean_length": 483.21875, "completions/mean_terminated_length": 464.2333679199219, "completions/min_length": 276.0, "completions/min_terminated_length": 276.0, "entropy": 0.4522119015455246, "epoch": 0.030513918629550323, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": -0.0114, "num_tokens": 1324756.0, "reward": 0.84375, "reward_std": 0.1293872892856598, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.4202548265457153, "sampling/importance_sampling_ratio/mean": 1.0000139474868774, "sampling/importance_sampling_ratio/min": 0.6987296342849731, "sampling/sampling_logp_difference/max": 0.3584914207458496, "sampling/sampling_logp_difference/mean": 0.012116029858589172, "step": 57 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.6957175072748214e-05, "clip_ratio/low_min": 4.6957175072748214e-05, "clip_ratio/region_mean": 4.6957175072748214e-05, "completions/clipped_ratio": 0.34375, "completions/max_length": 768.0, "completions/max_terminated_length": 746.0, "completions/mean_length": 646.40625, "completions/mean_terminated_length": 582.7142944335938, "completions/min_length": 386.0, "completions/min_terminated_length": 386.0, "entropy": 0.4879292882978916, "epoch": 0.031049250535331904, "frac_reward_zero_std": 0.5, "grad_norm": 0.008696580305695534, "learning_rate": 1e-05, "loss": -0.0096, "num_tokens": 1349505.0, "reward": 0.5625, "reward_std": 0.2587745785713196, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.5511821508407593, "sampling/importance_sampling_ratio/mean": 0.9999637007713318, "sampling/importance_sampling_ratio/min": 0.6396264433860779, "sampling/sampling_logp_difference/max": 0.4468710422515869, "sampling/sampling_logp_difference/mean": 0.012860474176704884, "step": 58 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 727.0, "completions/mean_length": 520.75, "completions/mean_terminated_length": 463.69232177734375, "completions/min_length": 228.0, "completions/min_terminated_length": 228.0, "entropy": 0.40757014974951744, "epoch": 0.03158458244111349, "frac_reward_zero_std": 0.5, "grad_norm": 0.009540090337395668, "learning_rate": 1e-05, "loss": 0.0508, "num_tokens": 1370033.0, "reward": 0.53125, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.4402263164520264, "sampling/importance_sampling_ratio/mean": 1.000462293624878, "sampling/importance_sampling_ratio/min": 0.5509796738624573, "sampling/sampling_logp_difference/max": 0.5960574150085449, "sampling/sampling_logp_difference/mean": 0.01166108250617981, "step": 59 }, { "clip_ratio/high_max": 4.752851600642316e-05, "clip_ratio/high_mean": 4.752851600642316e-05, "clip_ratio/low_mean": 4.07298794016242e-05, "clip_ratio/low_min": 4.07298794016242e-05, "clip_ratio/region_mean": 8.825839540804736e-05, "completions/clipped_ratio": 0.375, "completions/max_length": 768.0, "completions/max_terminated_length": 765.0, "completions/mean_length": 623.84375, "completions/mean_terminated_length": 537.3500366210938, "completions/min_length": 397.0, "completions/min_terminated_length": 397.0, "entropy": 0.5110281445086002, "epoch": 0.032119914346895075, "frac_reward_zero_std": 0.25, "grad_norm": 0.008212021552026272, "learning_rate": 1e-05, "loss": 0.0801, "num_tokens": 1394372.0, "reward": 0.59375, "reward_std": 0.3061639666557312, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.3844656944274902, "sampling/importance_sampling_ratio/mean": 1.0003094673156738, "sampling/importance_sampling_ratio/min": 0.70673006772995, "sampling/sampling_logp_difference/max": 0.3471064567565918, "sampling/sampling_logp_difference/mean": 0.014087519608438015, "step": 60 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.28125, "completions/max_length": 768.0, "completions/max_terminated_length": 704.0, "completions/mean_length": 563.875, "completions/mean_terminated_length": 484.0, "completions/min_length": 289.0, "completions/min_terminated_length": 289.0, "entropy": 0.42480451986193657, "epoch": 0.032655246252676656, "frac_reward_zero_std": 0.25, "grad_norm": 0.004322219640016556, "learning_rate": 1e-05, "loss": -0.0194, "num_tokens": 1416256.0, "reward": 0.53125, "reward_std": 0.2651650309562683, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.3357380628585815, "sampling/importance_sampling_ratio/mean": 0.9996961951255798, "sampling/importance_sampling_ratio/min": 0.7725263833999634, "sampling/sampling_logp_difference/max": 0.28948402404785156, "sampling/sampling_logp_difference/mean": 0.01185667235404253, "step": 61 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.028157829656266e-05, "clip_ratio/low_min": 5.028157829656266e-05, "clip_ratio/region_mean": 5.028157829656266e-05, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 749.0, "completions/mean_length": 512.8125, "completions/mean_terminated_length": 453.923095703125, "completions/min_length": 169.0, "completions/min_terminated_length": 169.0, "entropy": 0.5509180948138237, "epoch": 0.033190578158458245, "frac_reward_zero_std": 0.0, "grad_norm": 0.009267719462513924, "learning_rate": 1e-05, "loss": 0.0678, "num_tokens": 1436114.0, "reward": 0.6875, "reward_std": 0.4355512857437134, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.281874418258667, "sampling/importance_sampling_ratio/mean": 0.9997726678848267, "sampling/importance_sampling_ratio/min": 0.434792697429657, "sampling/sampling_logp_difference/max": 0.8328859806060791, "sampling/sampling_logp_difference/mean": 0.014345284551382065, "step": 62 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00011029374581994489, "clip_ratio/low_min": 0.00011029374581994489, "clip_ratio/region_mean": 0.00011029374581994489, "completions/clipped_ratio": 0.34375, "completions/max_length": 768.0, "completions/max_terminated_length": 746.0, "completions/mean_length": 588.0625, "completions/mean_terminated_length": 493.8095397949219, "completions/min_length": 247.0, "completions/min_terminated_length": 247.0, "entropy": 0.5057032033801079, "epoch": 0.03372591006423983, "frac_reward_zero_std": 0.5, "grad_norm": 0.005740536376833916, "learning_rate": 1e-05, "loss": 0.0254, "num_tokens": 1458724.0, "reward": 0.40625, "reward_std": 0.22201895713806152, "rewards/accuracy_reward/mean": 0.40625, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.5643742084503174, "sampling/importance_sampling_ratio/mean": 1.0002732276916504, "sampling/importance_sampling_ratio/min": 0.5843526124954224, "sampling/sampling_logp_difference/max": 0.5372506380081177, "sampling/sampling_logp_difference/mean": 0.013789980672299862, "step": 63 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00019218432862544432, "clip_ratio/low_min": 0.00019218432862544432, "clip_ratio/region_mean": 0.00019218432862544432, "completions/clipped_ratio": 0.34375, "completions/max_length": 768.0, "completions/max_terminated_length": 762.0, "completions/mean_length": 618.875, "completions/mean_terminated_length": 540.7619018554688, "completions/min_length": 311.0, "completions/min_terminated_length": 311.0, "entropy": 0.5076218545436859, "epoch": 0.034261241970021415, "frac_reward_zero_std": 0.25, "grad_norm": 0.008714552037417889, "learning_rate": 1e-05, "loss": 0.015, "num_tokens": 1482432.0, "reward": 0.46875, "reward_std": 0.3377464711666107, "rewards/accuracy_reward/mean": 0.46875, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.4101232290267944, "sampling/importance_sampling_ratio/mean": 1.0002740621566772, "sampling/importance_sampling_ratio/min": 0.5970253348350525, "sampling/sampling_logp_difference/max": 0.5157957077026367, "sampling/sampling_logp_difference/mean": 0.013612430542707443, "step": 64 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 9.223867891705595e-05, "clip_ratio/low_min": 9.223867891705595e-05, "clip_ratio/region_mean": 9.223867891705595e-05, "completions/clipped_ratio": 0.28125, "completions/max_length": 768.0, "completions/max_terminated_length": 759.0, "completions/mean_length": 632.71875, "completions/mean_terminated_length": 579.7825927734375, "completions/min_length": 275.0, "completions/min_terminated_length": 275.0, "entropy": 0.38875191286206245, "epoch": 0.034796573875803, "frac_reward_zero_std": 0.25, "grad_norm": 0.009033304639160633, "learning_rate": 1e-05, "loss": 0.0495, "num_tokens": 1506431.0, "reward": 0.6875, "reward_std": 0.3924052119255066, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.4655523300170898, "sampling/importance_sampling_ratio/mean": 1.000010371208191, "sampling/importance_sampling_ratio/min": 0.682938814163208, "sampling/sampling_logp_difference/max": 0.3822321891784668, "sampling/sampling_logp_difference/mean": 0.011259174905717373, "step": 65 }, { "clip_ratio/high_max": 4.552075915853493e-05, "clip_ratio/high_mean": 4.552075915853493e-05, "clip_ratio/low_mean": 5.34645005245693e-05, "clip_ratio/low_min": 5.34645005245693e-05, "clip_ratio/region_mean": 9.898525968310423e-05, "completions/clipped_ratio": 0.3125, "completions/max_length": 768.0, "completions/max_terminated_length": 751.0, "completions/mean_length": 606.21875, "completions/mean_terminated_length": 532.6818237304688, "completions/min_length": 231.0, "completions/min_terminated_length": 231.0, "entropy": 0.42484213784337044, "epoch": 0.035331905781584586, "frac_reward_zero_std": 0.25, "grad_norm": 0.02042962983250618, "learning_rate": 1e-05, "loss": 0.0479, "num_tokens": 1529622.0, "reward": 0.75, "reward_std": 0.2925041913986206, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.4566490650177002, "sampling/importance_sampling_ratio/mean": 1.0000078678131104, "sampling/importance_sampling_ratio/min": 0.5184912085533142, "sampling/sampling_logp_difference/max": 0.656832218170166, "sampling/sampling_logp_difference/mean": 0.011515411548316479, "step": 66 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 7.318500865949318e-05, "clip_ratio/low_min": 7.318500865949318e-05, "clip_ratio/region_mean": 7.318500865949318e-05, "completions/clipped_ratio": 0.4375, "completions/max_length": 768.0, "completions/max_terminated_length": 678.0, "completions/mean_length": 590.625, "completions/mean_terminated_length": 452.6666564941406, "completions/min_length": 207.0, "completions/min_terminated_length": 207.0, "entropy": 0.5239551886916161, "epoch": 0.03586723768736617, "frac_reward_zero_std": 0.5, "grad_norm": 0.003302964847534895, "learning_rate": 1e-05, "loss": 0.0211, "num_tokens": 1552554.0, "reward": 0.15625, "reward_std": 0.22201895713806152, "rewards/accuracy_reward/mean": 0.15625, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.4207777976989746, "sampling/importance_sampling_ratio/mean": 0.9995875954627991, "sampling/importance_sampling_ratio/min": 0.3964710533618927, "sampling/sampling_logp_difference/max": 0.9251523017883301, "sampling/sampling_logp_difference/mean": 0.013529600575566292, "step": 67 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 658.0, "completions/mean_length": 539.6875, "completions/mean_terminated_length": 475.7599792480469, "completions/min_length": 298.0, "completions/min_terminated_length": 298.0, "entropy": 0.5009765848517418, "epoch": 0.03640256959314775, "frac_reward_zero_std": 0.25, "grad_norm": 0.0059877983294427395, "learning_rate": 1e-05, "loss": 0.0903, "num_tokens": 1573376.0, "reward": 0.75, "reward_std": 0.3514062464237213, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.4059853553771973, "sampling/importance_sampling_ratio/mean": 1.000280499458313, "sampling/importance_sampling_ratio/min": 0.581082820892334, "sampling/sampling_logp_difference/max": 0.5428619384765625, "sampling/sampling_logp_difference/mean": 0.013365201652050018, "step": 68 }, { "clip_ratio/high_max": 4.892368087894283e-05, "clip_ratio/high_mean": 4.892368087894283e-05, "clip_ratio/low_mean": 6.33874224149622e-05, "clip_ratio/low_min": 6.33874224149622e-05, "clip_ratio/region_mean": 0.00011231110329390503, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 766.0, "completions/mean_length": 551.25, "completions/mean_terminated_length": 511.1111145019531, "completions/min_length": 279.0, "completions/min_terminated_length": 279.0, "entropy": 0.3620293475687504, "epoch": 0.03693790149892934, "frac_reward_zero_std": 0.25, "grad_norm": 0.0056429170072078705, "learning_rate": 1e-05, "loss": 0.0131, "num_tokens": 1594616.0, "reward": 0.875, "reward_std": 0.292504221200943, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 1.4040403366088867, "sampling/importance_sampling_ratio/mean": 1.000301718711853, "sampling/importance_sampling_ratio/min": 0.7337977290153503, "sampling/sampling_logp_difference/max": 0.3393540382385254, "sampling/sampling_logp_difference/mean": 0.010295838117599487, "step": 69 }, { "clip_ratio/high_max": 6.225099787116051e-05, "clip_ratio/high_mean": 6.225099787116051e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 6.225099787116051e-05, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 736.0, "completions/mean_length": 522.4375, "completions/mean_terminated_length": 514.51611328125, "completions/min_length": 356.0, "completions/min_terminated_length": 356.0, "entropy": 0.28916994854807854, "epoch": 0.03747323340471092, "frac_reward_zero_std": 0.75, "grad_norm": 0.002398902550339699, "learning_rate": 1e-05, "loss": 0.0202, "num_tokens": 1614702.0, "reward": 0.96875, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.96875, "rewards/accuracy_reward/std": 0.1767766922712326, "sampling/importance_sampling_ratio/max": 1.4625073671340942, "sampling/importance_sampling_ratio/mean": 1.0001654624938965, "sampling/importance_sampling_ratio/min": 0.7339086532592773, "sampling/sampling_logp_difference/max": 0.3801523447036743, "sampling/sampling_logp_difference/mean": 0.008633803576231003, "step": 70 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 737.0, "completions/mean_length": 519.28125, "completions/mean_terminated_length": 502.70001220703125, "completions/min_length": 295.0, "completions/min_terminated_length": 295.0, "entropy": 0.4071941450238228, "epoch": 0.03800856531049251, "frac_reward_zero_std": 0.5, "grad_norm": 0.017797451466321945, "learning_rate": 1e-05, "loss": 0.0142, "num_tokens": 1634791.0, "reward": 0.75, "reward_std": 0.2314550280570984, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.303678274154663, "sampling/importance_sampling_ratio/mean": 0.9998772740364075, "sampling/importance_sampling_ratio/min": 0.7757813334465027, "sampling/sampling_logp_difference/max": 0.26518964767456055, "sampling/sampling_logp_difference/mean": 0.011396494694054127, "step": 71 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.0792361435014755e-05, "clip_ratio/low_min": 5.0792361435014755e-05, "clip_ratio/region_mean": 5.0792361435014755e-05, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 671.0, "completions/mean_length": 524.65625, "completions/mean_terminated_length": 468.5000305175781, "completions/min_length": 257.0, "completions/min_terminated_length": 257.0, "entropy": 0.47270843386650085, "epoch": 0.03854389721627409, "frac_reward_zero_std": 0.5, "grad_norm": 0.006420954596251249, "learning_rate": 1e-05, "loss": 0.0073, "num_tokens": 1655068.0, "reward": 0.40625, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.40625, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.2891714572906494, "sampling/importance_sampling_ratio/mean": 0.999759316444397, "sampling/importance_sampling_ratio/min": 0.5663257241249084, "sampling/sampling_logp_difference/max": 0.5685858726501465, "sampling/sampling_logp_difference/mean": 0.012472325935959816, "step": 72 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.17812768521253e-05, "clip_ratio/low_min": 5.17812768521253e-05, "clip_ratio/region_mean": 5.17812768521253e-05, "completions/clipped_ratio": 0.40625, "completions/max_length": 768.0, "completions/max_terminated_length": 649.0, "completions/mean_length": 615.8125, "completions/mean_terminated_length": 511.6842041015625, "completions/min_length": 349.0, "completions/min_terminated_length": 349.0, "entropy": 0.4182238429784775, "epoch": 0.03907922912205567, "frac_reward_zero_std": 0.25, "grad_norm": 0.012820947915315628, "learning_rate": 1e-05, "loss": 0.038, "num_tokens": 1678062.0, "reward": 0.46875, "reward_std": 0.378745436668396, "rewards/accuracy_reward/mean": 0.46875, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.8784372806549072, "sampling/importance_sampling_ratio/mean": 0.9999639391899109, "sampling/importance_sampling_ratio/min": 0.7614421844482422, "sampling/sampling_logp_difference/max": 0.6304402351379395, "sampling/sampling_logp_difference/mean": 0.012042717076838017, "step": 73 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00017125568774645217, "clip_ratio/low_min": 0.00017125568774645217, "clip_ratio/region_mean": 0.00017125568774645217, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 722.0, "completions/mean_length": 505.3125, "completions/mean_terminated_length": 478.137939453125, "completions/min_length": 281.0, "completions/min_terminated_length": 281.0, "entropy": 0.43436403200030327, "epoch": 0.03961456102783726, "frac_reward_zero_std": 0.25, "grad_norm": 0.003982620779424906, "learning_rate": 1e-05, "loss": 0.0692, "num_tokens": 1698096.0, "reward": 0.84375, "reward_std": 0.3198433816432953, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.486291527748108, "sampling/importance_sampling_ratio/mean": 0.9996992349624634, "sampling/importance_sampling_ratio/min": 0.5546742081642151, "sampling/sampling_logp_difference/max": 0.589374303817749, "sampling/sampling_logp_difference/mean": 0.011237709783017635, "step": 74 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.9058086005970836e-05, "clip_ratio/low_min": 4.9058086005970836e-05, "clip_ratio/region_mean": 4.9058086005970836e-05, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 751.0, "completions/mean_length": 560.59375, "completions/mean_terminated_length": 522.1851806640625, "completions/min_length": 290.0, "completions/min_terminated_length": 290.0, "entropy": 0.41883276030421257, "epoch": 0.04014989293361884, "frac_reward_zero_std": 0.5, "grad_norm": 0.010470947250723839, "learning_rate": 1e-05, "loss": 0.0191, "num_tokens": 1719403.0, "reward": 0.84375, "reward_std": 0.22201895713806152, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.940041184425354, "sampling/importance_sampling_ratio/mean": 0.9999339580535889, "sampling/importance_sampling_ratio/min": 0.7019378542900085, "sampling/sampling_logp_difference/max": 0.6627092361450195, "sampling/sampling_logp_difference/mean": 0.010889444500207901, "step": 75 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.069010537932627e-05, "clip_ratio/low_min": 4.069010537932627e-05, "clip_ratio/region_mean": 4.069010537932627e-05, "completions/clipped_ratio": 0.34375, "completions/max_length": 768.0, "completions/max_terminated_length": 768.0, "completions/mean_length": 581.96875, "completions/mean_terminated_length": 484.5238037109375, "completions/min_length": 256.0, "completions/min_terminated_length": 256.0, "entropy": 0.4762725979089737, "epoch": 0.04068522483940043, "frac_reward_zero_std": 0.25, "grad_norm": 0.014987333677709103, "learning_rate": 1e-05, "loss": 0.0228, "num_tokens": 1742426.0, "reward": 0.5625, "reward_std": 0.3471825420856476, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.3577488660812378, "sampling/importance_sampling_ratio/mean": 0.9997811913490295, "sampling/importance_sampling_ratio/min": 0.6076112389564514, "sampling/sampling_logp_difference/max": 0.49821996688842773, "sampling/sampling_logp_difference/mean": 0.013569815084338188, "step": 76 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00014673568512080237, "clip_ratio/low_min": 0.00014673568512080237, "clip_ratio/region_mean": 0.00014673568512080237, "completions/clipped_ratio": 0.4375, "completions/max_length": 768.0, "completions/max_terminated_length": 739.0, "completions/mean_length": 648.84375, "completions/mean_terminated_length": 556.1666870117188, "completions/min_length": 380.0, "completions/min_terminated_length": 380.0, "entropy": 0.5342629104852676, "epoch": 0.04122055674518201, "frac_reward_zero_std": 0.0, "grad_norm": 0.013919983990490437, "learning_rate": 1e-05, "loss": 0.0549, "num_tokens": 1767157.0, "reward": 0.59375, "reward_std": 0.4534739851951599, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.386790156364441, "sampling/importance_sampling_ratio/mean": 0.9998939037322998, "sampling/importance_sampling_ratio/min": 0.5932039618492126, "sampling/sampling_logp_difference/max": 0.5222170352935791, "sampling/sampling_logp_difference/mean": 0.013663308694958687, "step": 77 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.046427031629719e-05, "clip_ratio/low_min": 5.046427031629719e-05, "clip_ratio/region_mean": 5.046427031629719e-05, "completions/clipped_ratio": 0.28125, "completions/max_length": 768.0, "completions/max_terminated_length": 724.0, "completions/mean_length": 528.90625, "completions/mean_terminated_length": 435.34783935546875, "completions/min_length": 218.0, "completions/min_terminated_length": 218.0, "entropy": 0.5320987552404404, "epoch": 0.0417558886509636, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0025, "num_tokens": 1788234.0, "reward": 0.53125, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.416995644569397, "sampling/importance_sampling_ratio/mean": 0.9996212720870972, "sampling/importance_sampling_ratio/min": 0.5916980504989624, "sampling/sampling_logp_difference/max": 0.5247588157653809, "sampling/sampling_logp_difference/mean": 0.014332974329590797, "step": 78 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00019893924036296085, "clip_ratio/low_min": 0.00019893924036296085, "clip_ratio/region_mean": 0.00019893924036296085, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 722.0, "completions/mean_length": 624.8125, "completions/mean_terminated_length": 577.0833740234375, "completions/min_length": 398.0, "completions/min_terminated_length": 398.0, "entropy": 0.3578733876347542, "epoch": 0.04229122055674518, "frac_reward_zero_std": 0.25, "grad_norm": 0.006287885829806328, "learning_rate": 1e-05, "loss": 0.0442, "num_tokens": 1811620.0, "reward": 0.53125, "reward_std": 0.3471629321575165, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.2843456268310547, "sampling/importance_sampling_ratio/mean": 0.9999666810035706, "sampling/importance_sampling_ratio/min": 0.775160014629364, "sampling/sampling_logp_difference/max": 0.2546858787536621, "sampling/sampling_logp_difference/mean": 0.010287775658071041, "step": 79 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.3125, "completions/max_length": 768.0, "completions/max_terminated_length": 688.0, "completions/mean_length": 622.03125, "completions/mean_terminated_length": 555.6818237304688, "completions/min_length": 411.0, "completions/min_terminated_length": 411.0, "entropy": 0.5975942797958851, "epoch": 0.042826552462526764, "frac_reward_zero_std": 0.75, "grad_norm": 0.011447020806372166, "learning_rate": 1e-05, "loss": -0.0076, "num_tokens": 1835797.0, "reward": 0.6875, "reward_std": 0.1157275140285492, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.3627763986587524, "sampling/importance_sampling_ratio/mean": 1.0000898838043213, "sampling/importance_sampling_ratio/min": 0.4534623622894287, "sampling/sampling_logp_difference/max": 0.7908430099487305, "sampling/sampling_logp_difference/mean": 0.013825246132910252, "step": 80 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00010936133185168728, "clip_ratio/low_min": 0.00010936133185168728, "clip_ratio/region_mean": 0.00010936133185168728, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 684.0, "completions/mean_length": 543.21875, "completions/mean_terminated_length": 491.3461608886719, "completions/min_length": 299.0, "completions/min_terminated_length": 299.0, "entropy": 0.4437571093440056, "epoch": 0.04336188436830835, "frac_reward_zero_std": 0.25, "grad_norm": 0.010027516633272171, "learning_rate": 1e-05, "loss": 0.1184, "num_tokens": 1857156.0, "reward": 0.84375, "reward_std": 0.3061639666557312, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.3923063278198242, "sampling/importance_sampling_ratio/mean": 1.000060796737671, "sampling/importance_sampling_ratio/min": 0.6629504561424255, "sampling/sampling_logp_difference/max": 0.4110550880432129, "sampling/sampling_logp_difference/mean": 0.011959006078541279, "step": 81 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00019894931028829888, "clip_ratio/low_min": 0.00019894931028829888, "clip_ratio/region_mean": 0.00019894931028829888, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 745.0, "completions/mean_length": 516.21875, "completions/mean_terminated_length": 458.1153869628906, "completions/min_length": 173.0, "completions/min_terminated_length": 173.0, "entropy": 0.378199927508831, "epoch": 0.043897216274089934, "frac_reward_zero_std": 0.0, "grad_norm": 0.011769967153668404, "learning_rate": 1e-05, "loss": -0.0326, "num_tokens": 1877275.0, "reward": 0.5625, "reward_std": 0.4671337604522705, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.4055572748184204, "sampling/importance_sampling_ratio/mean": 1.0005459785461426, "sampling/importance_sampling_ratio/min": 0.7398213744163513, "sampling/sampling_logp_difference/max": 0.34043383598327637, "sampling/sampling_logp_difference/mean": 0.010453764349222183, "step": 82 }, { "clip_ratio/high_max": 4.6125460357870907e-05, "clip_ratio/high_mean": 4.6125460357870907e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 4.6125460357870907e-05, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 762.0, "completions/mean_length": 648.78125, "completions/mean_terminated_length": 615.3999633789062, "completions/min_length": 351.0, "completions/min_terminated_length": 351.0, "entropy": 0.3781026266515255, "epoch": 0.04443254817987152, "frac_reward_zero_std": 0.25, "grad_norm": 0.010206066071987152, "learning_rate": 1e-05, "loss": 0.0189, "num_tokens": 1901892.0, "reward": 0.59375, "reward_std": 0.3966485261917114, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.3806746006011963, "sampling/importance_sampling_ratio/mean": 1.000369668006897, "sampling/importance_sampling_ratio/min": 0.6630758047103882, "sampling/sampling_logp_difference/max": 0.41086602210998535, "sampling/sampling_logp_difference/mean": 0.01089530996978283, "step": 83 }, { "clip_ratio/high_max": 5.763024455518462e-05, "clip_ratio/high_mean": 5.763024455518462e-05, "clip_ratio/low_mean": 4.48028658865951e-05, "clip_ratio/low_min": 4.48028658865951e-05, "clip_ratio/region_mean": 0.00010243311044177972, "completions/clipped_ratio": 0.375, "completions/max_length": 768.0, "completions/max_terminated_length": 759.0, "completions/mean_length": 649.71875, "completions/mean_terminated_length": 578.75, "completions/min_length": 258.0, "completions/min_terminated_length": 258.0, "entropy": 0.5005024299025536, "epoch": 0.044967880085653104, "frac_reward_zero_std": 0.0, "grad_norm": 0.008646410889923573, "learning_rate": 1e-05, "loss": 0.0632, "num_tokens": 1926651.0, "reward": 0.4375, "reward_std": 0.49022960662841797, "rewards/accuracy_reward/mean": 0.4375, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.4742728471755981, "sampling/importance_sampling_ratio/mean": 0.9998708367347717, "sampling/importance_sampling_ratio/min": 0.5484101176261902, "sampling/sampling_logp_difference/max": 0.6007318496704102, "sampling/sampling_logp_difference/mean": 0.012708599679172039, "step": 84 }, { "clip_ratio/high_max": 6.589351687580347e-05, "clip_ratio/high_mean": 6.589351687580347e-05, "clip_ratio/low_mean": 6.713211769238114e-05, "clip_ratio/low_min": 6.713211769238114e-05, "clip_ratio/region_mean": 0.00013302563456818461, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 744.0, "completions/mean_length": 524.90625, "completions/mean_terminated_length": 499.75860595703125, "completions/min_length": 303.0, "completions/min_terminated_length": 303.0, "entropy": 0.35518455505371094, "epoch": 0.045503211991434686, "frac_reward_zero_std": 0.25, "grad_norm": 0.003262939862906933, "learning_rate": 1e-05, "loss": 0.0258, "num_tokens": 1947264.0, "reward": 0.6875, "reward_std": 0.2925041913986206, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.3010053634643555, "sampling/importance_sampling_ratio/mean": 0.9998657703399658, "sampling/importance_sampling_ratio/min": 0.7846524715423584, "sampling/sampling_logp_difference/max": 0.2631373405456543, "sampling/sampling_logp_difference/mean": 0.00970697682350874, "step": 85 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00011831519077531993, "clip_ratio/low_min": 0.00011831519077531993, "clip_ratio/region_mean": 0.00011831519077531993, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 722.0, "completions/mean_length": 515.53125, "completions/mean_terminated_length": 498.70001220703125, "completions/min_length": 241.0, "completions/min_terminated_length": 241.0, "entropy": 0.41699234768748283, "epoch": 0.046038543897216275, "frac_reward_zero_std": 0.25, "grad_norm": 0.01781095378100872, "learning_rate": 1e-05, "loss": 0.023, "num_tokens": 1967001.0, "reward": 0.59375, "reward_std": 0.3608423173427582, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.3767662048339844, "sampling/importance_sampling_ratio/mean": 1.0002977848052979, "sampling/importance_sampling_ratio/min": 0.6516874432563782, "sampling/sampling_logp_difference/max": 0.4281902313232422, "sampling/sampling_logp_difference/mean": 0.012085186317563057, "step": 86 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00014681524044135585, "clip_ratio/low_min": 0.00014681524044135585, "clip_ratio/region_mean": 0.00014681524044135585, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 756.0, "completions/mean_length": 597.625, "completions/mean_terminated_length": 540.8333740234375, "completions/min_length": 244.0, "completions/min_terminated_length": 244.0, "entropy": 0.3625175543129444, "epoch": 0.046573875802997856, "frac_reward_zero_std": 0.25, "grad_norm": 0.0166816096752882, "learning_rate": 1e-05, "loss": 0.0258, "num_tokens": 1990149.0, "reward": 0.59375, "reward_std": 0.3608423173427582, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.3373417854309082, "sampling/importance_sampling_ratio/mean": 1.0002481937408447, "sampling/importance_sampling_ratio/min": 0.7186197638511658, "sampling/sampling_logp_difference/max": 0.3304229974746704, "sampling/sampling_logp_difference/mean": 0.010592046193778515, "step": 87 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00016476913879159838, "clip_ratio/low_min": 0.00016476913879159838, "clip_ratio/region_mean": 0.00016476913879159838, "completions/clipped_ratio": 0.46875, "completions/max_length": 768.0, "completions/max_terminated_length": 767.0, "completions/mean_length": 716.65625, "completions/mean_terminated_length": 671.3529663085938, "completions/min_length": 510.0, "completions/min_terminated_length": 510.0, "entropy": 0.4938277080655098, "epoch": 0.047109207708779445, "frac_reward_zero_std": 0.25, "grad_norm": 0.011431061662733555, "learning_rate": 1e-05, "loss": 0.028, "num_tokens": 2017066.0, "reward": 0.46875, "reward_std": 0.3198433816432953, "rewards/accuracy_reward/mean": 0.46875, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.6422104835510254, "sampling/importance_sampling_ratio/mean": 1.0000323057174683, "sampling/importance_sampling_ratio/min": 0.7556983828544617, "sampling/sampling_logp_difference/max": 0.49604320526123047, "sampling/sampling_logp_difference/mean": 0.012017695233225822, "step": 88 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.4404972868505865e-05, "clip_ratio/low_min": 4.4404972868505865e-05, "clip_ratio/region_mean": 4.4404972868505865e-05, "completions/clipped_ratio": 0.4375, "completions/max_length": 768.0, "completions/max_terminated_length": 767.0, "completions/mean_length": 667.71875, "completions/mean_terminated_length": 589.7222290039062, "completions/min_length": 446.0, "completions/min_terminated_length": 446.0, "entropy": 0.5489422045648098, "epoch": 0.04764453961456103, "frac_reward_zero_std": 0.25, "grad_norm": 0.014813964255154133, "learning_rate": 1e-05, "loss": 0.0558, "num_tokens": 2042441.0, "reward": 0.5, "reward_std": 0.3745020925998688, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5080004930496216, "sampling/importance_sampling_ratio/max": 1.3826590776443481, "sampling/importance_sampling_ratio/mean": 1.000160574913025, "sampling/importance_sampling_ratio/min": 0.14336563646793365, "sampling/sampling_logp_difference/max": 1.942357063293457, "sampling/sampling_logp_difference/mean": 0.014316913671791553, "step": 89 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 8.70278527145274e-05, "clip_ratio/low_min": 8.70278527145274e-05, "clip_ratio/region_mean": 8.70278527145274e-05, "completions/clipped_ratio": 0.5, "completions/max_length": 768.0, "completions/max_terminated_length": 694.0, "completions/mean_length": 651.3125, "completions/mean_terminated_length": 534.625, "completions/min_length": 342.0, "completions/min_terminated_length": 342.0, "entropy": 0.5815541483461857, "epoch": 0.048179871520342615, "frac_reward_zero_std": 0.25, "grad_norm": 0.00825833436101675, "learning_rate": 1e-05, "loss": 0.0972, "num_tokens": 2067611.0, "reward": 0.375, "reward_std": 0.3745020925998688, "rewards/accuracy_reward/mean": 0.375, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.7591688632965088, "sampling/importance_sampling_ratio/mean": 1.0003857612609863, "sampling/importance_sampling_ratio/min": 0.6370046734809875, "sampling/sampling_logp_difference/max": 0.5648415088653564, "sampling/sampling_logp_difference/mean": 0.013535372912883759, "step": 90 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.28125, "completions/max_length": 768.0, "completions/max_terminated_length": 702.0, "completions/mean_length": 558.21875, "completions/mean_terminated_length": 476.13043212890625, "completions/min_length": 168.0, "completions/min_terminated_length": 168.0, "entropy": 0.40279893949627876, "epoch": 0.0487152034261242, "frac_reward_zero_std": 0.0, "grad_norm": 0.01722664013504982, "learning_rate": 1e-05, "loss": -0.0101, "num_tokens": 2089202.0, "reward": 0.5625, "reward_std": 0.4261348247528076, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.4850578308105469, "sampling/importance_sampling_ratio/mean": 0.9998269081115723, "sampling/importance_sampling_ratio/min": 0.607376754283905, "sampling/sampling_logp_difference/max": 0.49860596656799316, "sampling/sampling_logp_difference/mean": 0.011388967745006084, "step": 91 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00024040970674832352, "clip_ratio/low_min": 0.00024040970674832352, "clip_ratio/region_mean": 0.00024040970674832352, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 762.0, "completions/mean_length": 526.375, "completions/mean_terminated_length": 491.857177734375, "completions/min_length": 252.0, "completions/min_terminated_length": 252.0, "entropy": 0.513542901724577, "epoch": 0.04925053533190578, "frac_reward_zero_std": 0.0, "grad_norm": 0.00837272685021162, "learning_rate": 1e-05, "loss": 0.0441, "num_tokens": 2109454.0, "reward": 0.78125, "reward_std": 0.4218915104866028, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.8132972717285156, "sampling/importance_sampling_ratio/mean": 0.9997177720069885, "sampling/importance_sampling_ratio/min": 0.5944162607192993, "sampling/sampling_logp_difference/max": 0.595146894454956, "sampling/sampling_logp_difference/mean": 0.013096220791339874, "step": 92 }, { "clip_ratio/high_max": 9.506726928520948e-05, "clip_ratio/high_mean": 9.506726928520948e-05, "clip_ratio/low_mean": 4.1145489376503974e-05, "clip_ratio/low_min": 4.1145489376503974e-05, "clip_ratio/region_mean": 0.00013621275866171345, "completions/clipped_ratio": 0.3125, "completions/max_length": 768.0, "completions/max_terminated_length": 734.0, "completions/mean_length": 634.53125, "completions/mean_terminated_length": 573.8636474609375, "completions/min_length": 265.0, "completions/min_terminated_length": 265.0, "entropy": 0.4839530698955059, "epoch": 0.04978586723768737, "frac_reward_zero_std": 0.0, "grad_norm": 0.014939776621758938, "learning_rate": 1e-05, "loss": 0.0643, "num_tokens": 2133583.0, "reward": 0.53125, "reward_std": 0.4944729208946228, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.6251133680343628, "sampling/importance_sampling_ratio/mean": 0.9999118447303772, "sampling/importance_sampling_ratio/min": 0.7049896121025085, "sampling/sampling_logp_difference/max": 0.4855775833129883, "sampling/sampling_logp_difference/mean": 0.012979834340512753, "step": 93 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.5, "completions/max_length": 768.0, "completions/max_terminated_length": 730.0, "completions/mean_length": 678.375, "completions/mean_terminated_length": 588.75, "completions/min_length": 281.0, "completions/min_terminated_length": 281.0, "entropy": 0.5444022342562675, "epoch": 0.05032119914346895, "frac_reward_zero_std": 0.25, "grad_norm": 0.005601354409009218, "learning_rate": 1e-05, "loss": 0.0001, "num_tokens": 2159131.0, "reward": 0.46875, "reward_std": 0.3471629321575165, "rewards/accuracy_reward/mean": 0.46875, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.460326075553894, "sampling/importance_sampling_ratio/mean": 1.0002961158752441, "sampling/importance_sampling_ratio/min": 0.7336817383766174, "sampling/sampling_logp_difference/max": 0.378659725189209, "sampling/sampling_logp_difference/mean": 0.01323297806084156, "step": 94 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 737.0, "completions/mean_length": 547.9375, "completions/mean_terminated_length": 525.1724243164062, "completions/min_length": 283.0, "completions/min_terminated_length": 283.0, "entropy": 0.35648392140865326, "epoch": 0.05085653104925054, "frac_reward_zero_std": 0.0, "grad_norm": 0.005154947284609079, "learning_rate": 1e-05, "loss": 0.1319, "num_tokens": 2179857.0, "reward": 0.8125, "reward_std": 0.3945523500442505, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.26478111743927, "sampling/importance_sampling_ratio/mean": 0.9999009966850281, "sampling/importance_sampling_ratio/min": 0.61456698179245, "sampling/sampling_logp_difference/max": 0.48683738708496094, "sampling/sampling_logp_difference/mean": 0.009781748987734318, "step": 95 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 7.002801430644467e-05, "clip_ratio/low_min": 7.002801430644467e-05, "clip_ratio/region_mean": 7.002801430644467e-05, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 690.0, "completions/mean_length": 534.8125, "completions/mean_terminated_length": 491.629638671875, "completions/min_length": 243.0, "completions/min_terminated_length": 243.0, "entropy": 0.5257068872451782, "epoch": 0.05139186295503212, "frac_reward_zero_std": 0.0, "grad_norm": 0.02136640064418316, "learning_rate": 1e-05, "loss": 0.1841, "num_tokens": 2200491.0, "reward": 0.71875, "reward_std": 0.4397946000099182, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.2795007228851318, "sampling/importance_sampling_ratio/mean": 1.0003798007965088, "sampling/importance_sampling_ratio/min": 0.725347101688385, "sampling/sampling_logp_difference/max": 0.3211050033569336, "sampling/sampling_logp_difference/mean": 0.01373940147459507, "step": 96 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 762.0, "completions/mean_length": 617.6875, "completions/mean_terminated_length": 567.5833740234375, "completions/min_length": 206.0, "completions/min_terminated_length": 206.0, "entropy": 0.3262052573263645, "epoch": 0.05192719486081371, "frac_reward_zero_std": 0.0, "grad_norm": 0.006677660625427961, "learning_rate": 1e-05, "loss": 0.0107, "num_tokens": 2223961.0, "reward": 0.59375, "reward_std": 0.3987956643104553, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.4547582864761353, "sampling/importance_sampling_ratio/mean": 0.9997563362121582, "sampling/importance_sampling_ratio/min": 0.6356496214866638, "sampling/sampling_logp_difference/max": 0.4531078338623047, "sampling/sampling_logp_difference/mean": 0.009383311495184898, "step": 97 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 747.0, "completions/mean_length": 547.15625, "completions/mean_terminated_length": 532.433349609375, "completions/min_length": 324.0, "completions/min_terminated_length": 324.0, "entropy": 0.32807236537337303, "epoch": 0.05246252676659529, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0381, "num_tokens": 2245238.0, "reward": 0.71875, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.3241428136825562, "sampling/importance_sampling_ratio/mean": 0.9998565316200256, "sampling/importance_sampling_ratio/min": 0.7468260526657104, "sampling/sampling_logp_difference/max": 0.29192304611206055, "sampling/sampling_logp_difference/mean": 0.009362924844026566, "step": 98 }, { "clip_ratio/high_max": 5.383290408644825e-05, "clip_ratio/high_mean": 5.383290408644825e-05, "clip_ratio/low_mean": 9.745195711730048e-05, "clip_ratio/low_min": 9.745195711730048e-05, "clip_ratio/region_mean": 0.00015128486120374873, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 761.0, "completions/mean_length": 604.59375, "completions/mean_terminated_length": 558.8399658203125, "completions/min_length": 321.0, "completions/min_terminated_length": 321.0, "entropy": 0.42305028066039085, "epoch": 0.05299785867237687, "frac_reward_zero_std": 0.25, "grad_norm": 0.026652174070477486, "learning_rate": 1e-05, "loss": 0.0419, "num_tokens": 2268145.0, "reward": 0.875, "reward_std": 0.2925041913986206, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 1.3046342134475708, "sampling/importance_sampling_ratio/mean": 0.9999422430992126, "sampling/importance_sampling_ratio/min": 0.7187533378601074, "sampling/sampling_logp_difference/max": 0.33023715019226074, "sampling/sampling_logp_difference/mean": 0.011149080470204353, "step": 99 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.53125, "completions/max_length": 768.0, "completions/max_terminated_length": 703.0, "completions/mean_length": 679.34375, "completions/mean_terminated_length": 578.86669921875, "completions/min_length": 343.0, "completions/min_terminated_length": 343.0, "entropy": 0.5958646275103092, "epoch": 0.05353319057815846, "frac_reward_zero_std": 0.5, "grad_norm": 0.018127331510186195, "learning_rate": 1e-05, "loss": 0.0501, "num_tokens": 2294052.0, "reward": 0.375, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.375, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.6077208518981934, "sampling/importance_sampling_ratio/mean": 1.0001519918441772, "sampling/importance_sampling_ratio/min": 0.6073815226554871, "sampling/sampling_logp_difference/max": 0.4985980987548828, "sampling/sampling_logp_difference/mean": 0.014581737108528614, "step": 100 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.415401053847745e-05, "clip_ratio/low_min": 4.415401053847745e-05, "clip_ratio/region_mean": 4.415401053847745e-05, "completions/clipped_ratio": 0.5, "completions/max_length": 768.0, "completions/max_terminated_length": 725.0, "completions/mean_length": 654.96875, "completions/mean_terminated_length": 541.9375, "completions/min_length": 313.0, "completions/min_terminated_length": 313.0, "entropy": 0.7058289349079132, "epoch": 0.05406852248394004, "frac_reward_zero_std": 0.5, "grad_norm": 0.01279167179018259, "learning_rate": 1e-05, "loss": 0.0115, "num_tokens": 2319051.0, "reward": 0.5, "reward_std": 0.2314550280570984, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5080004930496216, "sampling/importance_sampling_ratio/max": 1.3844224214553833, "sampling/importance_sampling_ratio/mean": 0.9999025464057922, "sampling/importance_sampling_ratio/min": 0.6449640989303589, "sampling/sampling_logp_difference/max": 0.43856072425842285, "sampling/sampling_logp_difference/mean": 0.015389555133879185, "step": 101 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 722.0, "completions/mean_length": 528.40625, "completions/mean_terminated_length": 503.6206970214844, "completions/min_length": 292.0, "completions/min_terminated_length": 292.0, "entropy": 0.5093299299478531, "epoch": 0.05460385438972163, "frac_reward_zero_std": 0.0, "grad_norm": 0.01063461508601904, "learning_rate": 1e-05, "loss": 0.0487, "num_tokens": 2339472.0, "reward": 0.78125, "reward_std": 0.3987956643104553, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.4027093648910522, "sampling/importance_sampling_ratio/mean": 0.999841570854187, "sampling/importance_sampling_ratio/min": 0.4838002026081085, "sampling/sampling_logp_difference/max": 0.7260832786560059, "sampling/sampling_logp_difference/mean": 0.012134391814470291, "step": 102 }, { "clip_ratio/high_max": 6.55479816487059e-05, "clip_ratio/high_mean": 6.55479816487059e-05, "clip_ratio/low_mean": 0.00010203138299402781, "clip_ratio/low_min": 0.00010203138299402781, "clip_ratio/region_mean": 0.0001675793646427337, "completions/clipped_ratio": 0.34375, "completions/max_length": 768.0, "completions/max_terminated_length": 766.0, "completions/mean_length": 639.84375, "completions/mean_terminated_length": 572.7142944335938, "completions/min_length": 253.0, "completions/min_terminated_length": 253.0, "entropy": 0.3798535577952862, "epoch": 0.05513918629550321, "frac_reward_zero_std": 0.0, "grad_norm": 0.01762148179113865, "learning_rate": 1e-05, "loss": 0.1169, "num_tokens": 2363603.0, "reward": 0.59375, "reward_std": 0.4534739851951599, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.3716785907745361, "sampling/importance_sampling_ratio/mean": 0.9998103380203247, "sampling/importance_sampling_ratio/min": 0.35052594542503357, "sampling/sampling_logp_difference/max": 1.0483205318450928, "sampling/sampling_logp_difference/mean": 0.010568685829639435, "step": 103 }, { "clip_ratio/high_max": 7.208765600807965e-05, "clip_ratio/high_mean": 7.208765600807965e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 7.208765600807965e-05, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 745.0, "completions/mean_length": 533.4375, "completions/mean_terminated_length": 509.17242431640625, "completions/min_length": 234.0, "completions/min_terminated_length": 234.0, "entropy": 0.3971620574593544, "epoch": 0.055674518201284794, "frac_reward_zero_std": 0.5, "grad_norm": 0.004956406075507402, "learning_rate": 1e-05, "loss": 0.0906, "num_tokens": 2384033.0, "reward": 0.90625, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.3015110492706299, "sampling/importance_sampling_ratio/mean": 0.9997437000274658, "sampling/importance_sampling_ratio/min": 0.6486752033233643, "sampling/sampling_logp_difference/max": 0.43282318115234375, "sampling/sampling_logp_difference/mean": 0.011015697382390499, "step": 104 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 758.0, "completions/mean_length": 550.4375, "completions/mean_terminated_length": 535.933349609375, "completions/min_length": 294.0, "completions/min_terminated_length": 294.0, "entropy": 0.4596591927111149, "epoch": 0.05620985010706638, "frac_reward_zero_std": 0.5, "grad_norm": 0.007774466648697853, "learning_rate": 1e-05, "loss": -0.0272, "num_tokens": 2405175.0, "reward": 0.75, "reward_std": 0.2587745785713196, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.4565011262893677, "sampling/importance_sampling_ratio/mean": 0.9999442100524902, "sampling/importance_sampling_ratio/min": 0.718629777431488, "sampling/sampling_logp_difference/max": 0.3760371208190918, "sampling/sampling_logp_difference/mean": 0.011729179881513119, "step": 105 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 753.0, "completions/mean_length": 598.25, "completions/mean_terminated_length": 559.0769653320312, "completions/min_length": 308.0, "completions/min_terminated_length": 308.0, "entropy": 0.37072158232331276, "epoch": 0.056745182012847964, "frac_reward_zero_std": 0.5, "grad_norm": 0.014647644944489002, "learning_rate": 1e-05, "loss": 0.0362, "num_tokens": 2427967.0, "reward": 0.78125, "reward_std": 0.2630178928375244, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.2708412408828735, "sampling/importance_sampling_ratio/mean": 0.99993896484375, "sampling/importance_sampling_ratio/min": 0.7111266851425171, "sampling/sampling_logp_difference/max": 0.34090471267700195, "sampling/sampling_logp_difference/mean": 0.010582929477095604, "step": 106 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00010191806359216571, "clip_ratio/low_min": 0.00010191806359216571, "clip_ratio/region_mean": 0.00010191806359216571, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 732.0, "completions/mean_length": 586.09375, "completions/mean_terminated_length": 525.4583740234375, "completions/min_length": 246.0, "completions/min_terminated_length": 246.0, "entropy": 0.42987873405218124, "epoch": 0.05728051391862955, "frac_reward_zero_std": 0.0, "grad_norm": 0.006501646712422371, "learning_rate": 1e-05, "loss": 0.0117, "num_tokens": 2450434.0, "reward": 0.65625, "reward_std": 0.4218915104866028, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.3616158962249756, "sampling/importance_sampling_ratio/mean": 1.0002968311309814, "sampling/importance_sampling_ratio/min": 0.7154538631439209, "sampling/sampling_logp_difference/max": 0.3348381519317627, "sampling/sampling_logp_difference/mean": 0.01188938319683075, "step": 107 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.2272575228707865e-05, "clip_ratio/low_min": 4.2272575228707865e-05, "clip_ratio/region_mean": 4.2272575228707865e-05, "completions/clipped_ratio": 0.46875, "completions/max_length": 768.0, "completions/max_terminated_length": 767.0, "completions/mean_length": 699.96875, "completions/mean_terminated_length": 639.941162109375, "completions/min_length": 488.0, "completions/min_terminated_length": 488.0, "entropy": 0.4379051923751831, "epoch": 0.057815845824411134, "frac_reward_zero_std": 0.25, "grad_norm": 0.005879446864128113, "learning_rate": 1e-05, "loss": 0.0104, "num_tokens": 2476785.0, "reward": 0.34375, "reward_std": 0.3377464711666107, "rewards/accuracy_reward/mean": 0.34375, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.2980092763900757, "sampling/importance_sampling_ratio/mean": 0.9998241066932678, "sampling/importance_sampling_ratio/min": 0.6074737906455994, "sampling/sampling_logp_difference/max": 0.4984462261199951, "sampling/sampling_logp_difference/mean": 0.01169486902654171, "step": 108 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00014120041669229977, "clip_ratio/low_min": 0.00014120041669229977, "clip_ratio/region_mean": 0.00014120041669229977, "completions/clipped_ratio": 0.4375, "completions/max_length": 768.0, "completions/max_terminated_length": 734.0, "completions/mean_length": 685.3125, "completions/mean_terminated_length": 621.0, "completions/min_length": 365.0, "completions/min_terminated_length": 365.0, "entropy": 0.478887639939785, "epoch": 0.05835117773019272, "frac_reward_zero_std": 0.25, "grad_norm": 0.008670409210026264, "learning_rate": 1e-05, "loss": 0.0565, "num_tokens": 2503187.0, "reward": 0.59375, "reward_std": 0.3377464711666107, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.2992057800292969, "sampling/importance_sampling_ratio/mean": 1.000105381011963, "sampling/importance_sampling_ratio/min": 0.7309198975563049, "sampling/sampling_logp_difference/max": 0.3134514093399048, "sampling/sampling_logp_difference/mean": 0.011437108740210533, "step": 109 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00023890918237157166, "clip_ratio/low_min": 0.00023890918237157166, "clip_ratio/region_mean": 0.00023890918237157166, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 754.0, "completions/mean_length": 514.5625, "completions/mean_terminated_length": 488.3448181152344, "completions/min_length": 226.0, "completions/min_terminated_length": 226.0, "entropy": 0.5211943462491035, "epoch": 0.058886509635974305, "frac_reward_zero_std": 0.25, "grad_norm": 0.01529600378125906, "learning_rate": 1e-05, "loss": 0.0501, "num_tokens": 2523613.0, "reward": 0.6875, "reward_std": 0.3924051821231842, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.4445958137512207, "sampling/importance_sampling_ratio/mean": 0.9999364018440247, "sampling/importance_sampling_ratio/min": 0.7228748798370361, "sampling/sampling_logp_difference/max": 0.3678295612335205, "sampling/sampling_logp_difference/mean": 0.013533909805119038, "step": 110 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00021671498325304128, "clip_ratio/low_min": 0.00021671498325304128, "clip_ratio/region_mean": 0.00021671498325304128, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 746.0, "completions/mean_length": 577.59375, "completions/mean_terminated_length": 533.6538696289062, "completions/min_length": 278.0, "completions/min_terminated_length": 278.0, "entropy": 0.4334462657570839, "epoch": 0.059421841541755886, "frac_reward_zero_std": 0.25, "grad_norm": 0.01121017150580883, "learning_rate": 1e-05, "loss": 0.0219, "num_tokens": 2545800.0, "reward": 0.25, "reward_std": 0.3514062464237213, "rewards/accuracy_reward/mean": 0.25, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.5534660816192627, "sampling/importance_sampling_ratio/mean": 0.9999406933784485, "sampling/importance_sampling_ratio/min": 0.6609777808189392, "sampling/sampling_logp_difference/max": 0.4404885768890381, "sampling/sampling_logp_difference/mean": 0.012062346562743187, "step": 111 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.560498175327666e-05, "clip_ratio/low_min": 5.560498175327666e-05, "clip_ratio/region_mean": 5.560498175327666e-05, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 765.0, "completions/mean_length": 618.6875, "completions/mean_terminated_length": 568.9166870117188, "completions/min_length": 234.0, "completions/min_terminated_length": 234.0, "entropy": 0.5460376776754856, "epoch": 0.059957173447537475, "frac_reward_zero_std": 0.0, "grad_norm": 0.010709904134273529, "learning_rate": 1e-05, "loss": 0.0829, "num_tokens": 2569158.0, "reward": 0.65625, "reward_std": 0.4628904461860657, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.3855860233306885, "sampling/importance_sampling_ratio/mean": 0.9998103976249695, "sampling/importance_sampling_ratio/min": 0.6119871139526367, "sampling/sampling_logp_difference/max": 0.4910440444946289, "sampling/sampling_logp_difference/mean": 0.013752535916864872, "step": 112 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.9135218432638794e-05, "clip_ratio/low_min": 4.9135218432638794e-05, "clip_ratio/region_mean": 4.9135218432638794e-05, "completions/clipped_ratio": 0.375, "completions/max_length": 768.0, "completions/max_terminated_length": 691.0, "completions/mean_length": 631.875, "completions/mean_terminated_length": 550.2000122070312, "completions/min_length": 270.0, "completions/min_terminated_length": 270.0, "entropy": 0.4064355418086052, "epoch": 0.06049250535331906, "frac_reward_zero_std": 0.0, "grad_norm": 0.01075573731213808, "learning_rate": 1e-05, "loss": 0.0475, "num_tokens": 2593002.0, "reward": 0.5, "reward_std": 0.4355512857437134, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5080004930496216, "sampling/importance_sampling_ratio/max": 1.610756754875183, "sampling/importance_sampling_ratio/mean": 0.9999549388885498, "sampling/importance_sampling_ratio/min": 0.4840478301048279, "sampling/sampling_logp_difference/max": 0.7255716323852539, "sampling/sampling_logp_difference/mean": 0.011302068829536438, "step": 113 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.758279465022497e-05, "clip_ratio/low_min": 4.758279465022497e-05, "clip_ratio/region_mean": 4.758279465022497e-05, "completions/clipped_ratio": 0.3125, "completions/max_length": 768.0, "completions/max_terminated_length": 761.0, "completions/mean_length": 654.53125, "completions/mean_terminated_length": 602.95458984375, "completions/min_length": 323.0, "completions/min_terminated_length": 323.0, "entropy": 0.3643207363784313, "epoch": 0.061027837259100645, "frac_reward_zero_std": 0.0, "grad_norm": 0.018865488469600677, "learning_rate": 1e-05, "loss": 0.0161, "num_tokens": 2617595.0, "reward": 0.6875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.421895980834961, "sampling/importance_sampling_ratio/mean": 0.9998403787612915, "sampling/importance_sampling_ratio/min": 0.5850104093551636, "sampling/sampling_logp_difference/max": 0.536125659942627, "sampling/sampling_logp_difference/mean": 0.009876951575279236, "step": 114 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 716.0, "completions/mean_length": 544.9375, "completions/mean_terminated_length": 521.862060546875, "completions/min_length": 376.0, "completions/min_terminated_length": 376.0, "entropy": 0.4699717238545418, "epoch": 0.06156316916488223, "frac_reward_zero_std": 0.5, "grad_norm": 0.007969534024596214, "learning_rate": 1e-05, "loss": 0.0742, "num_tokens": 2638785.0, "reward": 0.5625, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.360924243927002, "sampling/importance_sampling_ratio/mean": 1.000280499458313, "sampling/importance_sampling_ratio/min": 0.6759506464004517, "sampling/sampling_logp_difference/max": 0.3916351795196533, "sampling/sampling_logp_difference/mean": 0.011387757956981659, "step": 115 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 9.80437034741044e-05, "clip_ratio/low_min": 9.80437034741044e-05, "clip_ratio/region_mean": 9.80437034741044e-05, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 762.0, "completions/mean_length": 621.0625, "completions/mean_terminated_length": 572.0833740234375, "completions/min_length": 408.0, "completions/min_terminated_length": 408.0, "entropy": 0.35027093812823296, "epoch": 0.06209850107066381, "frac_reward_zero_std": 0.5, "grad_norm": 0.006722511723637581, "learning_rate": 1e-05, "loss": -0.0039, "num_tokens": 2662747.0, "reward": 0.78125, "reward_std": 0.2630178928375244, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.4416756629943848, "sampling/importance_sampling_ratio/mean": 1.0001320838928223, "sampling/importance_sampling_ratio/min": 0.7622732520103455, "sampling/sampling_logp_difference/max": 0.36580610275268555, "sampling/sampling_logp_difference/mean": 0.010345006361603737, "step": 116 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 751.0, "completions/mean_length": 597.65625, "completions/mean_terminated_length": 566.1111450195312, "completions/min_length": 374.0, "completions/min_terminated_length": 374.0, "entropy": 0.3591281324625015, "epoch": 0.0626338329764454, "frac_reward_zero_std": 0.5, "grad_norm": 0.015098914504051208, "learning_rate": 1e-05, "loss": 0.0252, "num_tokens": 2685552.0, "reward": 0.90625, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.4866523742675781, "sampling/importance_sampling_ratio/mean": 1.0001282691955566, "sampling/importance_sampling_ratio/min": 0.6318407654762268, "sampling/sampling_logp_difference/max": 0.4591178894042969, "sampling/sampling_logp_difference/mean": 0.010378815233707428, "step": 117 }, { "clip_ratio/high_max": 0.00013217430387157947, "clip_ratio/high_mean": 0.00013217430387157947, "clip_ratio/low_mean": 6.645401299465448e-05, "clip_ratio/low_min": 6.645401299465448e-05, "clip_ratio/region_mean": 0.00019862831686623394, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 704.0, "completions/mean_length": 562.8125, "completions/mean_terminated_length": 494.41668701171875, "completions/min_length": 260.0, "completions/min_terminated_length": 260.0, "entropy": 0.31836752966046333, "epoch": 0.06316916488222699, "frac_reward_zero_std": 0.25, "grad_norm": 0.010840936563909054, "learning_rate": 1e-05, "loss": 0.0123, "num_tokens": 2706810.0, "reward": 0.59375, "reward_std": 0.3608423173427582, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.2777732610702515, "sampling/importance_sampling_ratio/mean": 0.9999618530273438, "sampling/importance_sampling_ratio/min": 0.607105016708374, "sampling/sampling_logp_difference/max": 0.4990534782409668, "sampling/sampling_logp_difference/mean": 0.00928731169551611, "step": 118 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.456327951629646e-05, "clip_ratio/low_min": 4.456327951629646e-05, "clip_ratio/region_mean": 4.456327951629646e-05, "completions/clipped_ratio": 0.625, "completions/max_length": 768.0, "completions/max_terminated_length": 767.0, "completions/mean_length": 716.5, "completions/mean_terminated_length": 630.6666870117188, "completions/min_length": 320.0, "completions/min_terminated_length": 320.0, "entropy": 0.45794132724404335, "epoch": 0.06370449678800856, "frac_reward_zero_std": 0.25, "grad_norm": 0.015280363149940968, "learning_rate": 1e-05, "loss": 0.0099, "num_tokens": 2734178.0, "reward": 0.125, "reward_std": 0.292504221200943, "rewards/accuracy_reward/mean": 0.125, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 1.3790987730026245, "sampling/importance_sampling_ratio/mean": 0.9996334910392761, "sampling/importance_sampling_ratio/min": 0.4426460266113281, "sampling/sampling_logp_difference/max": 0.814984917640686, "sampling/sampling_logp_difference/mean": 0.012173281982541084, "step": 119 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.367110497900285e-05, "clip_ratio/low_min": 5.367110497900285e-05, "clip_ratio/region_mean": 5.367110497900285e-05, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 737.0, "completions/mean_length": 549.90625, "completions/mean_terminated_length": 535.36669921875, "completions/min_length": 221.0, "completions/min_terminated_length": 221.0, "entropy": 0.42989468201994896, "epoch": 0.06423982869379015, "frac_reward_zero_std": 0.5, "grad_norm": 0.0034308782778680325, "learning_rate": 1e-05, "loss": 0.039, "num_tokens": 2755271.0, "reward": 0.59375, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.3252981901168823, "sampling/importance_sampling_ratio/mean": 1.0002095699310303, "sampling/importance_sampling_ratio/min": 0.7031346559524536, "sampling/sampling_logp_difference/max": 0.3522069454193115, "sampling/sampling_logp_difference/mean": 0.011786284856498241, "step": 120 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0001159017119789496, "clip_ratio/low_min": 0.0001159017119789496, "clip_ratio/region_mean": 0.0001159017119789496, "completions/clipped_ratio": 0.3125, "completions/max_length": 768.0, "completions/max_terminated_length": 747.0, "completions/mean_length": 604.25, "completions/mean_terminated_length": 529.8181762695312, "completions/min_length": 320.0, "completions/min_terminated_length": 320.0, "entropy": 0.39081184566020966, "epoch": 0.06477516059957174, "frac_reward_zero_std": 0.5, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0095, "num_tokens": 2778375.0, "reward": 0.5, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5080004930496216, "sampling/importance_sampling_ratio/max": 1.2685933113098145, "sampling/importance_sampling_ratio/mean": 1.0003443956375122, "sampling/importance_sampling_ratio/min": 0.713601291179657, "sampling/sampling_logp_difference/max": 0.3374309539794922, "sampling/sampling_logp_difference/mean": 0.011368668638169765, "step": 121 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 758.0, "completions/mean_length": 557.34375, "completions/mean_terminated_length": 535.5516967773438, "completions/min_length": 226.0, "completions/min_terminated_length": 226.0, "entropy": 0.4360150769352913, "epoch": 0.06531049250535331, "frac_reward_zero_std": 0.25, "grad_norm": 0.012217814102768898, "learning_rate": 1e-05, "loss": 0.0288, "num_tokens": 2800706.0, "reward": 0.75, "reward_std": 0.3514062762260437, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.3406745195388794, "sampling/importance_sampling_ratio/mean": 0.9999719858169556, "sampling/importance_sampling_ratio/min": 0.490824818611145, "sampling/sampling_logp_difference/max": 0.7116680145263672, "sampling/sampling_logp_difference/mean": 0.012638597749173641, "step": 122 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00013946122999186628, "clip_ratio/low_min": 0.00013946122999186628, "clip_ratio/region_mean": 0.00013946122999186628, "completions/clipped_ratio": 0.5, "completions/max_length": 768.0, "completions/max_terminated_length": 762.0, "completions/mean_length": 671.625, "completions/mean_terminated_length": 575.25, "completions/min_length": 367.0, "completions/min_terminated_length": 367.0, "entropy": 0.34236453101038933, "epoch": 0.0658458244111349, "frac_reward_zero_std": 0.25, "grad_norm": 0.015692198649048805, "learning_rate": 1e-05, "loss": 0.0391, "num_tokens": 2826134.0, "reward": 0.46875, "reward_std": 0.3061639964580536, "rewards/accuracy_reward/mean": 0.46875, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.4008420705795288, "sampling/importance_sampling_ratio/mean": 0.999833881855011, "sampling/importance_sampling_ratio/min": 0.7842716574668884, "sampling/sampling_logp_difference/max": 0.33707356452941895, "sampling/sampling_logp_difference/mean": 0.009405428543686867, "step": 123 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.1845705456798896e-05, "clip_ratio/low_min": 5.1845705456798896e-05, "clip_ratio/region_mean": 5.1845705456798896e-05, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 745.0, "completions/mean_length": 491.8125, "completions/mean_terminated_length": 473.4000244140625, "completions/min_length": 260.0, "completions/min_terminated_length": 260.0, "entropy": 0.4624374695122242, "epoch": 0.06638115631691649, "frac_reward_zero_std": 0.5, "grad_norm": 0.008339719846844673, "learning_rate": 1e-05, "loss": 0.0588, "num_tokens": 2845304.0, "reward": 0.8125, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.3198801279067993, "sampling/importance_sampling_ratio/mean": 1.0002994537353516, "sampling/importance_sampling_ratio/min": 0.5299164056777954, "sampling/sampling_logp_difference/max": 0.6350359916687012, "sampling/sampling_logp_difference/mean": 0.011694015003740788, "step": 124 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.34375, "completions/max_length": 768.0, "completions/max_terminated_length": 704.0, "completions/mean_length": 611.5625, "completions/mean_terminated_length": 529.6190795898438, "completions/min_length": 381.0, "completions/min_terminated_length": 381.0, "entropy": 0.5198213867843151, "epoch": 0.06691648822269808, "frac_reward_zero_std": 0.0, "grad_norm": 0.022294864058494568, "learning_rate": 1e-05, "loss": 0.0512, "num_tokens": 2868842.0, "reward": 0.4375, "reward_std": 0.4492306709289551, "rewards/accuracy_reward/mean": 0.4375, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.4437618255615234, "sampling/importance_sampling_ratio/mean": 1.0000042915344238, "sampling/importance_sampling_ratio/min": 0.6878994703292847, "sampling/sampling_logp_difference/max": 0.374112606048584, "sampling/sampling_logp_difference/mean": 0.01330228615552187, "step": 125 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00014667538198409602, "clip_ratio/low_min": 0.00014667538198409602, "clip_ratio/region_mean": 0.00014667538198409602, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 764.0, "completions/mean_length": 595.6875, "completions/mean_terminated_length": 555.923095703125, "completions/min_length": 321.0, "completions/min_terminated_length": 321.0, "entropy": 0.4224700592458248, "epoch": 0.06745182012847965, "frac_reward_zero_std": 0.0, "grad_norm": 0.01445845514535904, "learning_rate": 1e-05, "loss": -0.0236, "num_tokens": 2891416.0, "reward": 0.46875, "reward_std": 0.521792471408844, "rewards/accuracy_reward/mean": 0.46875, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.3168127536773682, "sampling/importance_sampling_ratio/mean": 0.9996733665466309, "sampling/importance_sampling_ratio/min": 0.6416264772415161, "sampling/sampling_logp_difference/max": 0.44374895095825195, "sampling/sampling_logp_difference/mean": 0.01204050425440073, "step": 126 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.645889905281365e-05, "clip_ratio/low_min": 5.645889905281365e-05, "clip_ratio/region_mean": 5.645889905281365e-05, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 751.0, "completions/mean_length": 511.65625, "completions/mean_terminated_length": 464.1851806640625, "completions/min_length": 76.0, "completions/min_terminated_length": 76.0, "entropy": 0.7288839221000671, "epoch": 0.06798715203426124, "frac_reward_zero_std": 0.25, "grad_norm": 0.026353413239121437, "learning_rate": 1e-05, "loss": -0.0769, "num_tokens": 2912117.0, "reward": 0.34375, "reward_std": 0.3471629321575165, "rewards/accuracy_reward/mean": 0.34375, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.4234583377838135, "sampling/importance_sampling_ratio/mean": 1.0000054836273193, "sampling/importance_sampling_ratio/min": 0.739001452922821, "sampling/sampling_logp_difference/max": 0.3530893325805664, "sampling/sampling_logp_difference/mean": 0.01712021417915821, "step": 127 }, { "clip_ratio/high_max": 4.716981129604392e-05, "clip_ratio/high_mean": 4.716981129604392e-05, "clip_ratio/low_mean": 9.705970660434105e-05, "clip_ratio/low_min": 9.705970660434105e-05, "clip_ratio/region_mean": 0.00014422951790038496, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 756.0, "completions/mean_length": 606.90625, "completions/mean_terminated_length": 553.2083740234375, "completions/min_length": 304.0, "completions/min_terminated_length": 304.0, "entropy": 0.5988429039716721, "epoch": 0.06852248394004283, "frac_reward_zero_std": 0.0, "grad_norm": 0.012898923829197884, "learning_rate": 1e-05, "loss": 0.076, "num_tokens": 2935474.0, "reward": 0.5, "reward_std": 0.44403791427612305, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5080004930496216, "sampling/importance_sampling_ratio/max": 1.4216952323913574, "sampling/importance_sampling_ratio/mean": 1.000023365020752, "sampling/importance_sampling_ratio/min": 0.7354314923286438, "sampling/sampling_logp_difference/max": 0.3518500328063965, "sampling/sampling_logp_difference/mean": 0.015339154750108719, "step": 128 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0001535204501124099, "clip_ratio/low_min": 0.0001535204501124099, "clip_ratio/region_mean": 0.0001535204501124099, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 759.0, "completions/mean_length": 597.375, "completions/mean_terminated_length": 558.0, "completions/min_length": 408.0, "completions/min_terminated_length": 408.0, "entropy": 0.5921585410833359, "epoch": 0.0690578158458244, "frac_reward_zero_std": 0.5, "grad_norm": 0.004030481446534395, "learning_rate": 1e-05, "loss": 0.0072, "num_tokens": 2959342.0, "reward": 0.75, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.4255763292312622, "sampling/importance_sampling_ratio/mean": 0.999934196472168, "sampling/importance_sampling_ratio/min": 0.7412431836128235, "sampling/sampling_logp_difference/max": 0.35457611083984375, "sampling/sampling_logp_difference/mean": 0.013684538193047047, "step": 129 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0001262442019651644, "clip_ratio/low_min": 0.0001262442019651644, "clip_ratio/region_mean": 0.0001262442019651644, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 753.0, "completions/mean_length": 524.875, "completions/mean_terminated_length": 490.14288330078125, "completions/min_length": 267.0, "completions/min_terminated_length": 267.0, "entropy": 0.38221365958452225, "epoch": 0.069593147751606, "frac_reward_zero_std": 0.5, "grad_norm": 0.007477280683815479, "learning_rate": 1e-05, "loss": 0.0205, "num_tokens": 2980474.0, "reward": 0.65625, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.3646857738494873, "sampling/importance_sampling_ratio/mean": 1.0000596046447754, "sampling/importance_sampling_ratio/min": 0.6582945585250854, "sampling/sampling_logp_difference/max": 0.4181027412414551, "sampling/sampling_logp_difference/mean": 0.011347775347530842, "step": 130 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 9.609833432477899e-05, "clip_ratio/low_min": 9.609833432477899e-05, "clip_ratio/region_mean": 9.609833432477899e-05, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 759.0, "completions/mean_length": 605.46875, "completions/mean_terminated_length": 567.9615478515625, "completions/min_length": 353.0, "completions/min_terminated_length": 353.0, "entropy": 0.4094332605600357, "epoch": 0.07012847965738758, "frac_reward_zero_std": 0.0, "grad_norm": 0.011829032562673092, "learning_rate": 1e-05, "loss": 0.0585, "num_tokens": 3003297.0, "reward": 0.65625, "reward_std": 0.47137710452079773, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.2982062101364136, "sampling/importance_sampling_ratio/mean": 1.0001816749572754, "sampling/importance_sampling_ratio/min": 0.6821297407150269, "sampling/sampling_logp_difference/max": 0.382535457611084, "sampling/sampling_logp_difference/mean": 0.01232686173170805, "step": 131 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.6766577472444624e-05, "clip_ratio/low_min": 5.6766577472444624e-05, "clip_ratio/region_mean": 5.6766577472444624e-05, "completions/clipped_ratio": 0.28125, "completions/max_length": 768.0, "completions/max_terminated_length": 744.0, "completions/mean_length": 590.65625, "completions/mean_terminated_length": 521.2608642578125, "completions/min_length": 211.0, "completions/min_terminated_length": 211.0, "entropy": 0.3864295594394207, "epoch": 0.07066381156316917, "frac_reward_zero_std": 0.25, "grad_norm": 0.016091959550976753, "learning_rate": 1e-05, "loss": 0.048, "num_tokens": 3026230.0, "reward": 0.71875, "reward_std": 0.378745436668396, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.4359915256500244, "sampling/importance_sampling_ratio/mean": 0.9996742010116577, "sampling/importance_sampling_ratio/min": 0.7111673355102539, "sampling/sampling_logp_difference/max": 0.36185550689697266, "sampling/sampling_logp_difference/mean": 0.011034548282623291, "step": 132 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.292125206324272e-05, "clip_ratio/low_min": 5.292125206324272e-05, "clip_ratio/region_mean": 5.292125206324272e-05, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 760.0, "completions/mean_length": 542.40625, "completions/mean_terminated_length": 527.36669921875, "completions/min_length": 257.0, "completions/min_terminated_length": 257.0, "entropy": 0.3996155969798565, "epoch": 0.07119914346895075, "frac_reward_zero_std": 0.25, "grad_norm": 0.015324263833463192, "learning_rate": 1e-05, "loss": 0.0475, "num_tokens": 3047355.0, "reward": 0.75, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.3368474245071411, "sampling/importance_sampling_ratio/mean": 1.000064730644226, "sampling/importance_sampling_ratio/min": 0.6329601407051086, "sampling/sampling_logp_difference/max": 0.4573478698730469, "sampling/sampling_logp_difference/mean": 0.011135010048747063, "step": 133 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 762.0, "completions/mean_length": 624.1875, "completions/mean_terminated_length": 576.25, "completions/min_length": 276.0, "completions/min_terminated_length": 276.0, "entropy": 0.3773885704576969, "epoch": 0.07173447537473233, "frac_reward_zero_std": 0.5, "grad_norm": 0.002062029903754592, "learning_rate": 1e-05, "loss": 0.0356, "num_tokens": 3071337.0, "reward": 0.8125, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.3878531455993652, "sampling/importance_sampling_ratio/mean": 0.9996427297592163, "sampling/importance_sampling_ratio/min": 0.6468920111656189, "sampling/sampling_logp_difference/max": 0.4355759620666504, "sampling/sampling_logp_difference/mean": 0.01128737535327673, "step": 134 }, { "clip_ratio/high_max": 6.262525130296126e-05, "clip_ratio/high_mean": 6.262525130296126e-05, "clip_ratio/low_mean": 5.910165418754332e-05, "clip_ratio/low_min": 5.910165418754332e-05, "clip_ratio/region_mean": 0.00012172690549050458, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 724.0, "completions/mean_length": 509.5625, "completions/mean_terminated_length": 472.64288330078125, "completions/min_length": 222.0, "completions/min_terminated_length": 222.0, "entropy": 0.4508117511868477, "epoch": 0.07226980728051392, "frac_reward_zero_std": 0.5, "grad_norm": 0.007427507545799017, "learning_rate": 1e-05, "loss": 0.0781, "num_tokens": 3091315.0, "reward": 0.71875, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.4224412441253662, "sampling/importance_sampling_ratio/mean": 0.9999260902404785, "sampling/importance_sampling_ratio/min": 0.23657158017158508, "sampling/sampling_logp_difference/max": 1.4415044784545898, "sampling/sampling_logp_difference/mean": 0.01295729074627161, "step": 135 }, { "clip_ratio/high_max": 4.328254726715386e-05, "clip_ratio/high_mean": 4.328254726715386e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 4.328254726715386e-05, "completions/clipped_ratio": 0.28125, "completions/max_length": 768.0, "completions/max_terminated_length": 727.0, "completions/mean_length": 612.5, "completions/mean_terminated_length": 551.6521606445312, "completions/min_length": 305.0, "completions/min_terminated_length": 305.0, "entropy": 0.3710810951888561, "epoch": 0.0728051391862955, "frac_reward_zero_std": 0.25, "grad_norm": 0.0049619609490036964, "learning_rate": 1e-05, "loss": 0.0789, "num_tokens": 3114939.0, "reward": 0.8125, "reward_std": 0.3104073107242584, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.3776192665100098, "sampling/importance_sampling_ratio/mean": 0.9997196793556213, "sampling/importance_sampling_ratio/min": 0.6667206883430481, "sampling/sampling_logp_difference/max": 0.4053840637207031, "sampling/sampling_logp_difference/mean": 0.00999332033097744, "step": 136 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 666.0, "completions/mean_length": 539.71875, "completions/mean_terminated_length": 497.4444580078125, "completions/min_length": 275.0, "completions/min_terminated_length": 275.0, "entropy": 0.37066930159926414, "epoch": 0.07334047109207709, "frac_reward_zero_std": 0.25, "grad_norm": 0.0033103583846241236, "learning_rate": 1e-05, "loss": 0.0609, "num_tokens": 3135642.0, "reward": 0.78125, "reward_std": 0.3061639666557312, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.3367377519607544, "sampling/importance_sampling_ratio/mean": 0.9998750686645508, "sampling/importance_sampling_ratio/min": 0.46006274223327637, "sampling/sampling_logp_difference/max": 0.7763924598693848, "sampling/sampling_logp_difference/mean": 0.010519558563828468, "step": 137 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00025644271227065474, "clip_ratio/low_min": 0.00025644271227065474, "clip_ratio/region_mean": 0.00025644271227065474, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 751.0, "completions/mean_length": 602.09375, "completions/mean_terminated_length": 571.370361328125, "completions/min_length": 159.0, "completions/min_terminated_length": 159.0, "entropy": 0.4131225012242794, "epoch": 0.07387580299785867, "frac_reward_zero_std": 0.0, "grad_norm": 0.015307110734283924, "learning_rate": 1e-05, "loss": 0.0993, "num_tokens": 3158629.0, "reward": 0.59375, "reward_std": 0.5123760104179382, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.4257789850234985, "sampling/importance_sampling_ratio/mean": 0.9999570250511169, "sampling/importance_sampling_ratio/min": 0.7596944570541382, "sampling/sampling_logp_difference/max": 0.3547182083129883, "sampling/sampling_logp_difference/mean": 0.011196279898285866, "step": 138 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.036261063651182e-05, "clip_ratio/low_min": 5.036261063651182e-05, "clip_ratio/region_mean": 5.036261063651182e-05, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 762.0, "completions/mean_length": 525.875, "completions/mean_terminated_length": 491.2857360839844, "completions/min_length": 173.0, "completions/min_terminated_length": 173.0, "entropy": 0.3909878507256508, "epoch": 0.07441113490364026, "frac_reward_zero_std": 0.25, "grad_norm": 0.017996681854128838, "learning_rate": 1e-05, "loss": 0.0453, "num_tokens": 3178657.0, "reward": 0.65625, "reward_std": 0.3608423173427582, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.3117836713790894, "sampling/importance_sampling_ratio/mean": 1.0001606941223145, "sampling/importance_sampling_ratio/min": 0.6962023973464966, "sampling/sampling_logp_difference/max": 0.36211490631103516, "sampling/sampling_logp_difference/mean": 0.011155658401548862, "step": 139 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.1597337258281186e-05, "clip_ratio/low_min": 4.1597337258281186e-05, "clip_ratio/region_mean": 4.1597337258281186e-05, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 751.0, "completions/mean_length": 605.21875, "completions/mean_terminated_length": 550.9583740234375, "completions/min_length": 278.0, "completions/min_terminated_length": 278.0, "entropy": 0.4558464288711548, "epoch": 0.07494646680942184, "frac_reward_zero_std": 0.0, "grad_norm": 0.011749648489058018, "learning_rate": 1e-05, "loss": 0.0399, "num_tokens": 3201344.0, "reward": 0.65625, "reward_std": 0.3808925747871399, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.336584210395813, "sampling/importance_sampling_ratio/mean": 1.0002506971359253, "sampling/importance_sampling_ratio/min": 0.61990886926651, "sampling/sampling_logp_difference/max": 0.4781827926635742, "sampling/sampling_logp_difference/mean": 0.01124854851514101, "step": 140 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.34375, "completions/max_length": 768.0, "completions/max_terminated_length": 764.0, "completions/mean_length": 675.0, "completions/mean_terminated_length": 626.2857055664062, "completions/min_length": 483.0, "completions/min_terminated_length": 483.0, "entropy": 0.409841600805521, "epoch": 0.07548179871520343, "frac_reward_zero_std": 0.25, "grad_norm": 0.009168663993477821, "learning_rate": 1e-05, "loss": 0.018, "num_tokens": 3226832.0, "reward": 0.59375, "reward_std": 0.3377465009689331, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.588563323020935, "sampling/importance_sampling_ratio/mean": 1.000203251838684, "sampling/importance_sampling_ratio/min": 0.686457633972168, "sampling/sampling_logp_difference/max": 0.4628300666809082, "sampling/sampling_logp_difference/mean": 0.01134434062987566, "step": 141 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 682.0, "completions/mean_length": 478.53125, "completions/mean_terminated_length": 469.19354248046875, "completions/min_length": 227.0, "completions/min_terminated_length": 227.0, "entropy": 0.36848144605755806, "epoch": 0.07601713062098502, "frac_reward_zero_std": 0.75, "grad_norm": 0.007296881638467312, "learning_rate": 1e-05, "loss": -0.023, "num_tokens": 3245497.0, "reward": 0.84375, "reward_std": 0.1293872892856598, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.2988835573196411, "sampling/importance_sampling_ratio/mean": 0.9998937249183655, "sampling/importance_sampling_ratio/min": 0.5236707925796509, "sampling/sampling_logp_difference/max": 0.6468920707702637, "sampling/sampling_logp_difference/mean": 0.010770690627396107, "step": 142 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0001506620683358051, "clip_ratio/low_min": 0.0001506620683358051, "clip_ratio/region_mean": 0.0001506620683358051, "completions/clipped_ratio": 0.34375, "completions/max_length": 768.0, "completions/max_terminated_length": 757.0, "completions/mean_length": 659.5, "completions/mean_terminated_length": 602.6666870117188, "completions/min_length": 351.0, "completions/min_terminated_length": 351.0, "entropy": 0.6066175773739815, "epoch": 0.07655246252676659, "frac_reward_zero_std": 0.0, "grad_norm": 0.009867552667856216, "learning_rate": 1e-05, "loss": 0.0742, "num_tokens": 3270265.0, "reward": 0.59375, "reward_std": 0.4534739851951599, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.5600217580795288, "sampling/importance_sampling_ratio/mean": 1.0000202655792236, "sampling/importance_sampling_ratio/min": 0.5449547171592712, "sampling/sampling_logp_difference/max": 0.6070525646209717, "sampling/sampling_logp_difference/mean": 0.013590005226433277, "step": 143 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 724.0, "completions/mean_length": 515.34375, "completions/mean_terminated_length": 468.5555725097656, "completions/min_length": 277.0, "completions/min_terminated_length": 277.0, "entropy": 0.39527034014463425, "epoch": 0.07708779443254818, "frac_reward_zero_std": 0.75, "grad_norm": 0.00386427016928792, "learning_rate": 1e-05, "loss": 0.0279, "num_tokens": 3290372.0, "reward": 0.8125, "reward_std": 0.1157275140285492, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.3550678491592407, "sampling/importance_sampling_ratio/mean": 0.9998969435691833, "sampling/importance_sampling_ratio/min": 0.7136199474334717, "sampling/sampling_logp_difference/max": 0.337404727935791, "sampling/sampling_logp_difference/mean": 0.011627251282334328, "step": 144 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 768.0, "completions/mean_length": 556.0625, "completions/mean_terminated_length": 534.137939453125, "completions/min_length": 76.0, "completions/min_terminated_length": 76.0, "entropy": 0.4369659088551998, "epoch": 0.07762312633832977, "frac_reward_zero_std": 0.75, "grad_norm": 0.012809324078261852, "learning_rate": 1e-05, "loss": -0.0447, "num_tokens": 3311822.0, "reward": 0.90625, "reward_std": 0.1293872892856598, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.4632304906845093, "sampling/importance_sampling_ratio/mean": 1.0001205205917358, "sampling/importance_sampling_ratio/min": 0.7598057389259338, "sampling/sampling_logp_difference/max": 0.3806467056274414, "sampling/sampling_logp_difference/mean": 0.011080690659582615, "step": 145 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.567928747041151e-05, "clip_ratio/low_min": 5.567928747041151e-05, "clip_ratio/region_mean": 5.567928747041151e-05, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 746.0, "completions/mean_length": 537.28125, "completions/mean_terminated_length": 460.375, "completions/min_length": 202.0, "completions/min_terminated_length": 202.0, "entropy": 0.4744526818394661, "epoch": 0.07815845824411134, "frac_reward_zero_std": 0.5, "grad_norm": 0.011317940428853035, "learning_rate": 1e-05, "loss": 0.0471, "num_tokens": 3332503.0, "reward": 0.8125, "reward_std": 0.249358132481575, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.2816593647003174, "sampling/importance_sampling_ratio/mean": 0.999502420425415, "sampling/importance_sampling_ratio/min": 0.6133771538734436, "sampling/sampling_logp_difference/max": 0.48877525329589844, "sampling/sampling_logp_difference/mean": 0.012451513670384884, "step": 146 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00017116693561547436, "clip_ratio/low_min": 0.00017116693561547436, "clip_ratio/region_mean": 0.00017116693561547436, "completions/clipped_ratio": 0.34375, "completions/max_length": 768.0, "completions/max_terminated_length": 729.0, "completions/mean_length": 593.5, "completions/mean_terminated_length": 502.0952453613281, "completions/min_length": 245.0, "completions/min_terminated_length": 245.0, "entropy": 0.5474798791110516, "epoch": 0.07869379014989293, "frac_reward_zero_std": 0.0, "grad_norm": 0.008388844318687916, "learning_rate": 1e-05, "loss": 0.0919, "num_tokens": 3355151.0, "reward": 0.5, "reward_std": 0.44403791427612305, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5080004930496216, "sampling/importance_sampling_ratio/max": 1.2929469347000122, "sampling/importance_sampling_ratio/mean": 0.9996303915977478, "sampling/importance_sampling_ratio/min": 0.5192931294441223, "sampling/sampling_logp_difference/max": 0.6552867889404297, "sampling/sampling_logp_difference/mean": 0.013475755229592323, "step": 147 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.58534411538858e-05, "clip_ratio/low_min": 5.58534411538858e-05, "clip_ratio/region_mean": 5.58534411538858e-05, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 768.0, "completions/mean_length": 539.625, "completions/mean_terminated_length": 497.3333435058594, "completions/min_length": 251.0, "completions/min_terminated_length": 251.0, "entropy": 0.3991433121263981, "epoch": 0.07922912205567452, "frac_reward_zero_std": 0.25, "grad_norm": 0.005428406875580549, "learning_rate": 1e-05, "loss": 0.0806, "num_tokens": 3376067.0, "reward": 0.625, "reward_std": 0.3104073107242584, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.2868996858596802, "sampling/importance_sampling_ratio/mean": 0.9997435808181763, "sampling/importance_sampling_ratio/min": 0.5954288244247437, "sampling/sampling_logp_difference/max": 0.5184733867645264, "sampling/sampling_logp_difference/mean": 0.011121694929897785, "step": 148 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 765.0, "completions/mean_length": 573.21875, "completions/mean_terminated_length": 537.1481323242188, "completions/min_length": 291.0, "completions/min_terminated_length": 291.0, "entropy": 0.36447297781705856, "epoch": 0.07976445396145611, "frac_reward_zero_std": 0.0, "grad_norm": 0.009165140800178051, "learning_rate": 1e-05, "loss": 0.0768, "num_tokens": 3397874.0, "reward": 0.71875, "reward_std": 0.4628904461860657, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.3582088947296143, "sampling/importance_sampling_ratio/mean": 0.9998113512992859, "sampling/importance_sampling_ratio/min": 0.623322069644928, "sampling/sampling_logp_difference/max": 0.4726918935775757, "sampling/sampling_logp_difference/mean": 0.010827361606061459, "step": 149 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 9.735202183946967e-05, "clip_ratio/low_min": 9.735202183946967e-05, "clip_ratio/region_mean": 9.735202183946967e-05, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 757.0, "completions/mean_length": 515.3125, "completions/mean_terminated_length": 507.1612854003906, "completions/min_length": 263.0, "completions/min_terminated_length": 263.0, "entropy": 0.3126232773065567, "epoch": 0.08029978586723768, "frac_reward_zero_std": 0.5, "grad_norm": 0.0048127430491149426, "learning_rate": 1e-05, "loss": -0.0217, "num_tokens": 3417988.0, "reward": 0.78125, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.3360776901245117, "sampling/importance_sampling_ratio/mean": 0.9999316930770874, "sampling/importance_sampling_ratio/min": 0.7416387796401978, "sampling/sampling_logp_difference/max": 0.2988929748535156, "sampling/sampling_logp_difference/mean": 0.009871783666312695, "step": 150 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00017024551198119298, "clip_ratio/low_min": 0.00017024551198119298, "clip_ratio/region_mean": 0.00017024551198119298, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 761.0, "completions/mean_length": 608.75, "completions/mean_terminated_length": 586.0, "completions/min_length": 254.0, "completions/min_terminated_length": 254.0, "entropy": 0.3984130211174488, "epoch": 0.08083511777301927, "frac_reward_zero_std": 0.25, "grad_norm": 0.003835368202999234, "learning_rate": 1e-05, "loss": 0.0299, "num_tokens": 3441460.0, "reward": 0.5, "reward_std": 0.3745020925998688, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5080004930496216, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0003395080566406, "sampling/importance_sampling_ratio/min": 0.7250773906707764, "sampling/sampling_logp_difference/max": 0.8758184909820557, "sampling/sampling_logp_difference/mean": 0.011509026400744915, "step": 151 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.724044942529872e-05, "clip_ratio/low_min": 6.724044942529872e-05, "clip_ratio/region_mean": 6.724044942529872e-05, "completions/clipped_ratio": 0.34375, "completions/max_length": 768.0, "completions/max_terminated_length": 765.0, "completions/mean_length": 606.125, "completions/mean_terminated_length": 521.3333129882812, "completions/min_length": 175.0, "completions/min_terminated_length": 175.0, "entropy": 0.4458345100283623, "epoch": 0.08137044967880086, "frac_reward_zero_std": 0.25, "grad_norm": 0.008848967961966991, "learning_rate": 1e-05, "loss": 0.0694, "num_tokens": 3464696.0, "reward": 0.625, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.4253044128417969, "sampling/importance_sampling_ratio/mean": 0.9997562766075134, "sampling/importance_sampling_ratio/min": 0.4247634708881378, "sampling/sampling_logp_difference/max": 0.8562228679656982, "sampling/sampling_logp_difference/mean": 0.012223445810377598, "step": 152 }, { "clip_ratio/high_max": 5.5704098485875875e-05, "clip_ratio/high_mean": 5.5704098485875875e-05, "clip_ratio/low_mean": 9.4307182735065e-05, "clip_ratio/low_min": 9.4307182735065e-05, "clip_ratio/region_mean": 0.00015001128122094087, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 757.0, "completions/mean_length": 619.46875, "completions/mean_terminated_length": 569.9583740234375, "completions/min_length": 256.0, "completions/min_terminated_length": 256.0, "entropy": 0.4194677323102951, "epoch": 0.08190578158458243, "frac_reward_zero_std": 0.25, "grad_norm": 0.007004813756793737, "learning_rate": 1e-05, "loss": 0.0624, "num_tokens": 3488143.0, "reward": 0.6875, "reward_std": 0.3514062464237213, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.4491379261016846, "sampling/importance_sampling_ratio/mean": 1.000097393989563, "sampling/importance_sampling_ratio/min": 0.3960610032081604, "sampling/sampling_logp_difference/max": 0.9261870384216309, "sampling/sampling_logp_difference/mean": 0.012135032564401627, "step": 153 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00010301547808921896, "clip_ratio/low_min": 0.00010301547808921896, "clip_ratio/region_mean": 0.00010301547808921896, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 768.0, "completions/mean_length": 577.875, "completions/mean_terminated_length": 542.6666870117188, "completions/min_length": 340.0, "completions/min_terminated_length": 340.0, "entropy": 0.5109613388776779, "epoch": 0.08244111349036402, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0164, "num_tokens": 3510731.0, "reward": 0.78125, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.3617922067642212, "sampling/importance_sampling_ratio/mean": 0.9996497631072998, "sampling/importance_sampling_ratio/min": 0.635783314704895, "sampling/sampling_logp_difference/max": 0.4528975486755371, "sampling/sampling_logp_difference/mean": 0.013292813673615456, "step": 154 }, { "clip_ratio/high_max": 4.8885412979871035e-05, "clip_ratio/high_mean": 4.8885412979871035e-05, "clip_ratio/low_mean": 4.6921923058107495e-05, "clip_ratio/low_min": 4.6921923058107495e-05, "clip_ratio/region_mean": 9.580733603797853e-05, "completions/clipped_ratio": 0.3125, "completions/max_length": 768.0, "completions/max_terminated_length": 717.0, "completions/mean_length": 582.4375, "completions/mean_terminated_length": 498.0909118652344, "completions/min_length": 239.0, "completions/min_terminated_length": 239.0, "entropy": 0.3889825753867626, "epoch": 0.08297644539614561, "frac_reward_zero_std": 0.25, "grad_norm": 0.022144809365272522, "learning_rate": 1e-05, "loss": 0.0725, "num_tokens": 3533225.0, "reward": 0.65625, "reward_std": 0.3061639666557312, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.335777997970581, "sampling/importance_sampling_ratio/mean": 1.000206470489502, "sampling/importance_sampling_ratio/min": 0.5145502090454102, "sampling/sampling_logp_difference/max": 0.6644620895385742, "sampling/sampling_logp_difference/mean": 0.010203310288488865, "step": 155 }, { "clip_ratio/high_max": 0.00015236240142257884, "clip_ratio/high_mean": 0.00015236240142257884, "clip_ratio/low_mean": 4.633061689673923e-05, "clip_ratio/low_min": 4.633061689673923e-05, "clip_ratio/region_mean": 0.00019869301831931807, "completions/clipped_ratio": 0.28125, "completions/max_length": 768.0, "completions/max_terminated_length": 626.0, "completions/mean_length": 554.9375, "completions/mean_terminated_length": 471.5652160644531, "completions/min_length": 236.0, "completions/min_terminated_length": 236.0, "entropy": 0.3928482383489609, "epoch": 0.0835117773019272, "frac_reward_zero_std": 0.25, "grad_norm": 0.01015439536422491, "learning_rate": 1e-05, "loss": 0.0289, "num_tokens": 3554879.0, "reward": 0.65625, "reward_std": 0.3061639964580536, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.4893429279327393, "sampling/importance_sampling_ratio/mean": 1.0001931190490723, "sampling/importance_sampling_ratio/min": 0.6360498070716858, "sampling/sampling_logp_difference/max": 0.45247840881347656, "sampling/sampling_logp_difference/mean": 0.011713199317455292, "step": 156 }, { "clip_ratio/high_max": 0.00012404915469232947, "clip_ratio/high_mean": 0.00012404915469232947, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00012404915469232947, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 629.0, "completions/mean_length": 481.15625, "completions/mean_terminated_length": 451.4827575683594, "completions/min_length": 202.0, "completions/min_terminated_length": 202.0, "entropy": 0.5006781779229641, "epoch": 0.08404710920770878, "frac_reward_zero_std": 0.5, "grad_norm": 0.01573370024561882, "learning_rate": 1e-05, "loss": 0.0707, "num_tokens": 3573804.0, "reward": 0.84375, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.4681521654129028, "sampling/importance_sampling_ratio/mean": 0.9999622106552124, "sampling/importance_sampling_ratio/min": 0.5331178307533264, "sampling/sampling_logp_difference/max": 0.6290128231048584, "sampling/sampling_logp_difference/mean": 0.01235072873532772, "step": 157 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.28125, "completions/max_length": 768.0, "completions/max_terminated_length": 700.0, "completions/mean_length": 535.75, "completions/mean_terminated_length": 444.86956787109375, "completions/min_length": 61.0, "completions/min_terminated_length": 61.0, "entropy": 0.4573538564145565, "epoch": 0.08458244111349036, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": -0.0254, "num_tokens": 3594844.0, "reward": 0.6875, "reward_std": 0.1157275140285492, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.3168195486068726, "sampling/importance_sampling_ratio/mean": 1.000217080116272, "sampling/importance_sampling_ratio/min": 0.672410786151886, "sampling/sampling_logp_difference/max": 0.39688587188720703, "sampling/sampling_logp_difference/mean": 0.012935231439769268, "step": 158 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.543802424450405e-05, "clip_ratio/low_min": 4.543802424450405e-05, "clip_ratio/region_mean": 4.543802424450405e-05, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 753.0, "completions/mean_length": 594.0, "completions/mean_terminated_length": 553.84619140625, "completions/min_length": 371.0, "completions/min_terminated_length": 371.0, "entropy": 0.43147435039281845, "epoch": 0.08511777301927195, "frac_reward_zero_std": 0.5, "grad_norm": 0.014614534564316273, "learning_rate": 1e-05, "loss": 0.0512, "num_tokens": 3617548.0, "reward": 0.71875, "reward_std": 0.2630178928375244, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.6154052019119263, "sampling/importance_sampling_ratio/mean": 0.9999131560325623, "sampling/importance_sampling_ratio/min": 0.665272057056427, "sampling/sampling_logp_difference/max": 0.4795858860015869, "sampling/sampling_logp_difference/mean": 0.01227882131934166, "step": 159 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 9.910868539009243e-05, "clip_ratio/low_min": 9.910868539009243e-05, "clip_ratio/region_mean": 9.910868539009243e-05, "completions/clipped_ratio": 0.28125, "completions/max_length": 768.0, "completions/max_terminated_length": 718.0, "completions/mean_length": 587.875, "completions/mean_terminated_length": 517.3912963867188, "completions/min_length": 264.0, "completions/min_terminated_length": 264.0, "entropy": 0.6415975391864777, "epoch": 0.08565310492505353, "frac_reward_zero_std": 0.0, "grad_norm": 0.014024967327713966, "learning_rate": 1e-05, "loss": 0.0666, "num_tokens": 3639960.0, "reward": 0.71875, "reward_std": 0.4628904461860657, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.328179121017456, "sampling/importance_sampling_ratio/mean": 0.9996159076690674, "sampling/importance_sampling_ratio/min": 0.4720625877380371, "sampling/sampling_logp_difference/max": 0.7506437301635742, "sampling/sampling_logp_difference/mean": 0.01451694406569004, "step": 160 }, { "clip_ratio/high_max": 0.00011413964602979831, "clip_ratio/high_mean": 0.00011413964602979831, "clip_ratio/low_mean": 0.0001784025298547931, "clip_ratio/low_min": 0.0001784025298547931, "clip_ratio/region_mean": 0.0002925421758845914, "completions/clipped_ratio": 0.40625, "completions/max_length": 768.0, "completions/max_terminated_length": 733.0, "completions/mean_length": 624.96875, "completions/mean_terminated_length": 527.1052856445312, "completions/min_length": 208.0, "completions/min_terminated_length": 208.0, "entropy": 0.610698401927948, "epoch": 0.08618843683083512, "frac_reward_zero_std": 0.0, "grad_norm": 0.011172808706760406, "learning_rate": 1e-05, "loss": 0.1149, "num_tokens": 3663719.0, "reward": 0.46875, "reward_std": 0.47137707471847534, "rewards/accuracy_reward/mean": 0.46875, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.3557729721069336, "sampling/importance_sampling_ratio/mean": 1.0003107786178589, "sampling/importance_sampling_ratio/min": 0.7412463426589966, "sampling/sampling_logp_difference/max": 0.30437183380126953, "sampling/sampling_logp_difference/mean": 0.01587352715432644, "step": 161 }, { "clip_ratio/high_max": 5.6382497859885916e-05, "clip_ratio/high_mean": 5.6382497859885916e-05, "clip_ratio/low_mean": 0.00017631493028602563, "clip_ratio/low_min": 0.00017631493028602563, "clip_ratio/region_mean": 0.00023269742814591154, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 741.0, "completions/mean_length": 575.0, "completions/mean_terminated_length": 530.4615478515625, "completions/min_length": 329.0, "completions/min_terminated_length": 329.0, "entropy": 0.4971841536462307, "epoch": 0.0867237687366167, "frac_reward_zero_std": 0.5, "grad_norm": 0.004410617519170046, "learning_rate": 1e-05, "loss": 0.0459, "num_tokens": 3685967.0, "reward": 0.71875, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.4432697296142578, "sampling/importance_sampling_ratio/mean": 1.0001212358474731, "sampling/importance_sampling_ratio/min": 0.7172502875328064, "sampling/sampling_logp_difference/max": 0.3669111728668213, "sampling/sampling_logp_difference/mean": 0.013134903274476528, "step": 162 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.59375, "completions/max_length": 768.0, "completions/max_terminated_length": 764.0, "completions/mean_length": 644.84375, "completions/mean_terminated_length": 464.8461608886719, "completions/min_length": 206.0, "completions/min_terminated_length": 206.0, "entropy": 0.662838339805603, "epoch": 0.0872591006423983, "frac_reward_zero_std": 0.75, "grad_norm": 0.004100402817130089, "learning_rate": 1e-05, "loss": -0.0033, "num_tokens": 3710938.0, "reward": 0.28125, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.28125, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.4349920749664307, "sampling/importance_sampling_ratio/mean": 1.0004535913467407, "sampling/importance_sampling_ratio/min": 0.5462856292724609, "sampling/sampling_logp_difference/max": 0.6046133041381836, "sampling/sampling_logp_difference/mean": 0.017001347616314888, "step": 163 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00013185653369873762, "clip_ratio/low_min": 0.00013185653369873762, "clip_ratio/region_mean": 0.00013185653369873762, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 751.0, "completions/mean_length": 583.84375, "completions/mean_terminated_length": 522.4583740234375, "completions/min_length": 181.0, "completions/min_terminated_length": 181.0, "entropy": 0.514828659594059, "epoch": 0.08779443254817987, "frac_reward_zero_std": 0.25, "grad_norm": 0.010722745209932327, "learning_rate": 1e-05, "loss": 0.0237, "num_tokens": 3733253.0, "reward": 0.53125, "reward_std": 0.3377464711666107, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.4557102918624878, "sampling/importance_sampling_ratio/mean": 1.0003795623779297, "sampling/importance_sampling_ratio/min": 0.7550866603851318, "sampling/sampling_logp_difference/max": 0.37549400329589844, "sampling/sampling_logp_difference/mean": 0.0130677605047822, "step": 164 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00010053625737782568, "clip_ratio/low_min": 0.00010053625737782568, "clip_ratio/region_mean": 0.00010053625737782568, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 761.0, "completions/mean_length": 592.4375, "completions/mean_terminated_length": 543.2799682617188, "completions/min_length": 32.0, "completions/min_terminated_length": 32.0, "entropy": 0.39805489405989647, "epoch": 0.08832976445396146, "frac_reward_zero_std": 0.25, "grad_norm": 0.010290173813700676, "learning_rate": 1e-05, "loss": -0.0248, "num_tokens": 3755411.0, "reward": 0.40625, "reward_std": 0.3966485261917114, "rewards/accuracy_reward/mean": 0.40625, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.2697464227676392, "sampling/importance_sampling_ratio/mean": 1.0000461339950562, "sampling/importance_sampling_ratio/min": 0.5553558468818665, "sampling/sampling_logp_difference/max": 0.5881462097167969, "sampling/sampling_logp_difference/mean": 0.010934079997241497, "step": 165 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 753.0, "completions/mean_length": 574.5625, "completions/mean_terminated_length": 538.74072265625, "completions/min_length": 286.0, "completions/min_terminated_length": 286.0, "entropy": 0.3548167943954468, "epoch": 0.08886509635974305, "frac_reward_zero_std": 0.0, "grad_norm": 0.020712891593575478, "learning_rate": 1e-05, "loss": 0.0697, "num_tokens": 3777453.0, "reward": 0.6875, "reward_std": 0.4492306709289551, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.4247281551361084, "sampling/importance_sampling_ratio/mean": 0.9998741149902344, "sampling/importance_sampling_ratio/min": 0.7023288607597351, "sampling/sampling_logp_difference/max": 0.35398101806640625, "sampling/sampling_logp_difference/mean": 0.010358977131545544, "step": 166 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 768.0, "completions/mean_length": 573.28125, "completions/mean_terminated_length": 553.137939453125, "completions/min_length": 352.0, "completions/min_terminated_length": 352.0, "entropy": 0.39695925265550613, "epoch": 0.08940042826552462, "frac_reward_zero_std": 0.25, "grad_norm": 0.009602807462215424, "learning_rate": 1e-05, "loss": 0.0377, "num_tokens": 3799270.0, "reward": 0.78125, "reward_std": 0.3377464711666107, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.2755472660064697, "sampling/importance_sampling_ratio/mean": 0.999658465385437, "sampling/importance_sampling_ratio/min": 0.5595287680625916, "sampling/sampling_logp_difference/max": 0.580660343170166, "sampling/sampling_logp_difference/mean": 0.011128809303045273, "step": 167 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.047411807230674e-05, "clip_ratio/low_min": 6.047411807230674e-05, "clip_ratio/region_mean": 6.047411807230674e-05, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 715.0, "completions/mean_length": 504.21875, "completions/mean_terminated_length": 466.5357360839844, "completions/min_length": 162.0, "completions/min_terminated_length": 162.0, "entropy": 0.45196371898055077, "epoch": 0.08993576017130621, "frac_reward_zero_std": 0.25, "grad_norm": 0.004683290142565966, "learning_rate": 1e-05, "loss": 0.0917, "num_tokens": 3818909.0, "reward": 0.875, "reward_std": 0.292504221200943, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 1.46566903591156, "sampling/importance_sampling_ratio/mean": 0.999906063079834, "sampling/importance_sampling_ratio/min": 0.6109153628349304, "sampling/sampling_logp_difference/max": 0.4927968978881836, "sampling/sampling_logp_difference/mean": 0.011523962952196598, "step": 168 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.34375, "completions/max_length": 768.0, "completions/max_terminated_length": 732.0, "completions/mean_length": 552.4375, "completions/mean_terminated_length": 439.5238037109375, "completions/min_length": 167.0, "completions/min_terminated_length": 167.0, "entropy": 0.5149292200803757, "epoch": 0.0904710920770878, "frac_reward_zero_std": 0.5, "grad_norm": 0.0017130245687440038, "learning_rate": 1e-05, "loss": 0.0647, "num_tokens": 3840803.0, "reward": 0.5625, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.572034478187561, "sampling/importance_sampling_ratio/mean": 1.0001252889633179, "sampling/importance_sampling_ratio/min": 0.7152678370475769, "sampling/sampling_logp_difference/max": 0.45237064361572266, "sampling/sampling_logp_difference/mean": 0.013675541616976261, "step": 169 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.792944673681632e-05, "clip_ratio/low_min": 4.792944673681632e-05, "clip_ratio/region_mean": 4.792944673681632e-05, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 593.0, "completions/mean_length": 511.6875, "completions/mean_terminated_length": 475.0714416503906, "completions/min_length": 273.0, "completions/min_terminated_length": 273.0, "entropy": 0.47603290528059006, "epoch": 0.09100642398286937, "frac_reward_zero_std": 0.5, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0046, "num_tokens": 3861041.0, "reward": 0.75, "reward_std": 0.2314550280570984, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.43668794631958, "sampling/importance_sampling_ratio/mean": 1.000077247619629, "sampling/importance_sampling_ratio/min": 0.6418737769126892, "sampling/sampling_logp_difference/max": 0.44336366653442383, "sampling/sampling_logp_difference/mean": 0.013669274747371674, "step": 170 }, { "clip_ratio/high_max": 4.765535777551122e-05, "clip_ratio/high_mean": 4.765535777551122e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 4.765535777551122e-05, "completions/clipped_ratio": 0.34375, "completions/max_length": 768.0, "completions/max_terminated_length": 695.0, "completions/mean_length": 594.71875, "completions/mean_terminated_length": 503.952392578125, "completions/min_length": 47.0, "completions/min_terminated_length": 47.0, "entropy": 0.35366255417466164, "epoch": 0.09154175588865096, "frac_reward_zero_std": 0.5, "grad_norm": 0.0049919020384550095, "learning_rate": 1e-05, "loss": -0.0555, "num_tokens": 3884000.0, "reward": 0.59375, "reward_std": 0.22201895713806152, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.2704286575317383, "sampling/importance_sampling_ratio/mean": 1.000166654586792, "sampling/importance_sampling_ratio/min": 0.6700748801231384, "sampling/sampling_logp_difference/max": 0.40036582946777344, "sampling/sampling_logp_difference/mean": 0.010553712956607342, "step": 171 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 753.0, "completions/mean_length": 537.9375, "completions/mean_terminated_length": 522.6000366210938, "completions/min_length": 335.0, "completions/min_terminated_length": 335.0, "entropy": 0.4217514730989933, "epoch": 0.09207708779443255, "frac_reward_zero_std": 0.75, "grad_norm": 0.004006472881883383, "learning_rate": 1e-05, "loss": 0.0315, "num_tokens": 3904670.0, "reward": 0.96875, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.96875, "rewards/accuracy_reward/std": 0.1767766922712326, "sampling/importance_sampling_ratio/max": 1.4209245443344116, "sampling/importance_sampling_ratio/mean": 1.0002920627593994, "sampling/importance_sampling_ratio/min": 0.5328462719917297, "sampling/sampling_logp_difference/max": 0.6295223236083984, "sampling/sampling_logp_difference/mean": 0.01134710293263197, "step": 172 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 760.0, "completions/mean_length": 570.5625, "completions/mean_terminated_length": 550.137939453125, "completions/min_length": 325.0, "completions/min_terminated_length": 325.0, "entropy": 0.4218416325747967, "epoch": 0.09261241970021414, "frac_reward_zero_std": 0.75, "grad_norm": 0.002963853068649769, "learning_rate": 1e-05, "loss": -0.0135, "num_tokens": 3927008.0, "reward": 0.4375, "reward_std": 0.1157275140285492, "rewards/accuracy_reward/mean": 0.4375, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.4399760961532593, "sampling/importance_sampling_ratio/mean": 1.0001033544540405, "sampling/importance_sampling_ratio/min": 0.590807318687439, "sampling/sampling_logp_difference/max": 0.5262653827667236, "sampling/sampling_logp_difference/mean": 0.012068185023963451, "step": 173 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 638.0, "completions/mean_length": 457.46875, "completions/mean_terminated_length": 436.7666931152344, "completions/min_length": 233.0, "completions/min_terminated_length": 233.0, "entropy": 0.36681827902793884, "epoch": 0.09314775160599571, "frac_reward_zero_std": 0.25, "grad_norm": 0.009849708527326584, "learning_rate": 1e-05, "loss": 0.0329, "num_tokens": 3945191.0, "reward": 0.75, "reward_std": 0.3514062464237213, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.5823709964752197, "sampling/importance_sampling_ratio/mean": 1.0002021789550781, "sampling/importance_sampling_ratio/min": 0.7909921407699585, "sampling/sampling_logp_difference/max": 0.4589242935180664, "sampling/sampling_logp_difference/mean": 0.010868269018828869, "step": 174 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.520795482676476e-05, "clip_ratio/low_min": 4.520795482676476e-05, "clip_ratio/region_mean": 4.520795482676476e-05, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 761.0, "completions/mean_length": 608.03125, "completions/mean_terminated_length": 571.1154174804688, "completions/min_length": 336.0, "completions/min_terminated_length": 336.0, "entropy": 0.41545870900154114, "epoch": 0.0936830835117773, "frac_reward_zero_std": 0.25, "grad_norm": 0.006024455651640892, "learning_rate": 1e-05, "loss": 0.026, "num_tokens": 3968568.0, "reward": 0.5625, "reward_std": 0.292504221200943, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.771843433380127, "sampling/importance_sampling_ratio/mean": 1.0000227689743042, "sampling/importance_sampling_ratio/min": 0.6450721621513367, "sampling/sampling_logp_difference/max": 0.5720205307006836, "sampling/sampling_logp_difference/mean": 0.011757463216781616, "step": 175 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 759.0, "completions/mean_length": 555.34375, "completions/mean_terminated_length": 484.4583435058594, "completions/min_length": 347.0, "completions/min_terminated_length": 347.0, "entropy": 0.4451611191034317, "epoch": 0.09421841541755889, "frac_reward_zero_std": 0.25, "grad_norm": 0.007888618856668472, "learning_rate": 1e-05, "loss": 0.0464, "num_tokens": 3990555.0, "reward": 0.375, "reward_std": 0.3104073107242584, "rewards/accuracy_reward/mean": 0.375, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0000252723693848, "sampling/importance_sampling_ratio/min": 0.6822840571403503, "sampling/sampling_logp_difference/max": 1.145115852355957, "sampling/sampling_logp_difference/mean": 0.01246458850800991, "step": 176 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.46875, "completions/max_length": 768.0, "completions/max_terminated_length": 768.0, "completions/mean_length": 648.59375, "completions/mean_terminated_length": 543.2352905273438, "completions/min_length": 357.0, "completions/min_terminated_length": 357.0, "entropy": 0.5301221944391727, "epoch": 0.09475374732334046, "frac_reward_zero_std": 0.5, "grad_norm": 0.006469637155532837, "learning_rate": 1e-05, "loss": 0.0262, "num_tokens": 4015790.0, "reward": 0.25, "reward_std": 0.26726123690605164, "rewards/accuracy_reward/mean": 0.25, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.4269806146621704, "sampling/importance_sampling_ratio/mean": 1.0002111196517944, "sampling/importance_sampling_ratio/min": 0.7875787615776062, "sampling/sampling_logp_difference/max": 0.3555607795715332, "sampling/sampling_logp_difference/mean": 0.012512173503637314, "step": 177 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 585.0, "completions/mean_length": 489.28125, "completions/mean_terminated_length": 396.375, "completions/min_length": 198.0, "completions/min_terminated_length": 198.0, "entropy": 0.5177386812865734, "epoch": 0.09528907922912205, "frac_reward_zero_std": 0.25, "grad_norm": 0.006932341493666172, "learning_rate": 1e-05, "loss": -0.0428, "num_tokens": 4034903.0, "reward": 0.53125, "reward_std": 0.3061639666557312, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.3761708736419678, "sampling/importance_sampling_ratio/mean": 1.0001049041748047, "sampling/importance_sampling_ratio/min": 0.6697771549224854, "sampling/sampling_logp_difference/max": 0.40081024169921875, "sampling/sampling_logp_difference/mean": 0.013819603249430656, "step": 178 }, { "clip_ratio/high_max": 7.982119859661907e-05, "clip_ratio/high_mean": 7.982119859661907e-05, "clip_ratio/low_mean": 4.5306271204026416e-05, "clip_ratio/low_min": 4.5306271204026416e-05, "clip_ratio/region_mean": 0.00012512746980064549, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 712.0, "completions/mean_length": 535.03125, "completions/mean_terminated_length": 501.7500305175781, "completions/min_length": 286.0, "completions/min_terminated_length": 286.0, "entropy": 0.5154776051640511, "epoch": 0.09582441113490364, "frac_reward_zero_std": 0.25, "grad_norm": 0.009936594404280186, "learning_rate": 1e-05, "loss": 0.0506, "num_tokens": 4055600.0, "reward": 0.4375, "reward_std": 0.3514062464237213, "rewards/accuracy_reward/mean": 0.4375, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.3805251121520996, "sampling/importance_sampling_ratio/mean": 1.0003159046173096, "sampling/importance_sampling_ratio/min": 0.5657485127449036, "sampling/sampling_logp_difference/max": 0.5696055889129639, "sampling/sampling_logp_difference/mean": 0.013102860189974308, "step": 179 }, { "clip_ratio/high_max": 5.9495479945326224e-05, "clip_ratio/high_mean": 5.9495479945326224e-05, "clip_ratio/low_mean": 4.829984391108155e-05, "clip_ratio/low_min": 4.829984391108155e-05, "clip_ratio/region_mean": 0.00010779532385640778, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 754.0, "completions/mean_length": 586.6875, "completions/mean_terminated_length": 544.84619140625, "completions/min_length": 319.0, "completions/min_terminated_length": 319.0, "entropy": 0.3384445421397686, "epoch": 0.09635974304068523, "frac_reward_zero_std": 0.25, "grad_norm": 0.0049512009136378765, "learning_rate": 1e-05, "loss": 0.0639, "num_tokens": 4078358.0, "reward": 0.71875, "reward_std": 0.3608423173427582, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.2987746000289917, "sampling/importance_sampling_ratio/mean": 0.999905526638031, "sampling/importance_sampling_ratio/min": 0.3770841360092163, "sampling/sampling_logp_difference/max": 0.9752869606018066, "sampling/sampling_logp_difference/mean": 0.009852582588791847, "step": 180 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0001063377276295796, "clip_ratio/low_min": 0.0001063377276295796, "clip_ratio/region_mean": 0.0001063377276295796, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 695.0, "completions/mean_length": 500.96875, "completions/mean_terminated_length": 462.8214416503906, "completions/min_length": 268.0, "completions/min_terminated_length": 268.0, "entropy": 0.5007325038313866, "epoch": 0.0968950749464668, "frac_reward_zero_std": 0.0, "grad_norm": 0.013404779136180878, "learning_rate": 1e-05, "loss": 0.1112, "num_tokens": 4097981.0, "reward": 0.5625, "reward_std": 0.3945523500442505, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.3490186929702759, "sampling/importance_sampling_ratio/mean": 1.0000747442245483, "sampling/importance_sampling_ratio/min": 0.6561200618743896, "sampling/sampling_logp_difference/max": 0.42141151428222656, "sampling/sampling_logp_difference/mean": 0.013804241083562374, "step": 181 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.427913518156856e-05, "clip_ratio/low_min": 4.427913518156856e-05, "clip_ratio/region_mean": 4.427913518156856e-05, "completions/clipped_ratio": 0.28125, "completions/max_length": 768.0, "completions/max_terminated_length": 725.0, "completions/mean_length": 629.25, "completions/mean_terminated_length": 574.95654296875, "completions/min_length": 411.0, "completions/min_terminated_length": 411.0, "entropy": 0.3962312899529934, "epoch": 0.0974304068522484, "frac_reward_zero_std": 0.5, "grad_norm": 0.0059975991025567055, "learning_rate": 1e-05, "loss": 0.0009, "num_tokens": 4121725.0, "reward": 0.75, "reward_std": 0.2314550280570984, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.3976343870162964, "sampling/importance_sampling_ratio/mean": 1.0000478029251099, "sampling/importance_sampling_ratio/min": 0.7750535607337952, "sampling/sampling_logp_difference/max": 0.3347811698913574, "sampling/sampling_logp_difference/mean": 0.011098440736532211, "step": 182 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.3125, "completions/max_length": 768.0, "completions/max_terminated_length": 757.0, "completions/mean_length": 650.84375, "completions/mean_terminated_length": 597.5909423828125, "completions/min_length": 353.0, "completions/min_terminated_length": 353.0, "entropy": 0.4029821865260601, "epoch": 0.09796573875802998, "frac_reward_zero_std": 0.5, "grad_norm": 0.004008599556982517, "learning_rate": 1e-05, "loss": 0.0272, "num_tokens": 4146952.0, "reward": 0.65625, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.2983002662658691, "sampling/importance_sampling_ratio/mean": 1.000109314918518, "sampling/importance_sampling_ratio/min": 0.6508774757385254, "sampling/sampling_logp_difference/max": 0.42943382263183594, "sampling/sampling_logp_difference/mean": 0.011147662065923214, "step": 183 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00011048150190617889, "clip_ratio/low_min": 0.00011048150190617889, "clip_ratio/region_mean": 0.00011048150190617889, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 699.0, "completions/mean_length": 556.96875, "completions/mean_terminated_length": 486.625, "completions/min_length": 321.0, "completions/min_terminated_length": 321.0, "entropy": 0.4544137492775917, "epoch": 0.09850107066381156, "frac_reward_zero_std": 0.5, "grad_norm": 0.011129366233944893, "learning_rate": 1e-05, "loss": 0.0533, "num_tokens": 4169095.0, "reward": 0.78125, "reward_std": 0.2630178928375244, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.3407338857650757, "sampling/importance_sampling_ratio/mean": 0.999880313873291, "sampling/importance_sampling_ratio/min": 0.4061211049556732, "sampling/sampling_logp_difference/max": 0.9011039137840271, "sampling/sampling_logp_difference/mean": 0.011544800363481045, "step": 184 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 740.0, "completions/mean_length": 546.28125, "completions/mean_terminated_length": 531.5, "completions/min_length": 280.0, "completions/min_terminated_length": 280.0, "entropy": 0.35225995630025864, "epoch": 0.09903640256959315, "frac_reward_zero_std": 0.5, "grad_norm": 0.01049327664077282, "learning_rate": 1e-05, "loss": 0.0165, "num_tokens": 4190336.0, "reward": 0.90625, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.3624271154403687, "sampling/importance_sampling_ratio/mean": 0.9999856948852539, "sampling/importance_sampling_ratio/min": 0.6186681389808655, "sampling/sampling_logp_difference/max": 0.48018622398376465, "sampling/sampling_logp_difference/mean": 0.010624032467603683, "step": 185 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 9.007704647956416e-05, "clip_ratio/low_min": 9.007704647956416e-05, "clip_ratio/region_mean": 9.007704647956416e-05, "completions/clipped_ratio": 0.3125, "completions/max_length": 768.0, "completions/max_terminated_length": 759.0, "completions/mean_length": 653.5625, "completions/mean_terminated_length": 601.5454711914062, "completions/min_length": 409.0, "completions/min_terminated_length": 409.0, "entropy": 0.4254562631249428, "epoch": 0.09957173447537473, "frac_reward_zero_std": 0.0, "grad_norm": 0.010685713030397892, "learning_rate": 1e-05, "loss": 0.0386, "num_tokens": 4215210.0, "reward": 0.59375, "reward_std": 0.4534739851951599, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.816921591758728, "sampling/importance_sampling_ratio/mean": 0.999805212020874, "sampling/importance_sampling_ratio/min": 0.012927633710205555, "sampling/sampling_logp_difference/max": 4.348388195037842, "sampling/sampling_logp_difference/mean": 0.012169590219855309, "step": 186 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00019159849398420192, "clip_ratio/low_min": 0.00019159849398420192, "clip_ratio/region_mean": 0.00019159849398420192, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 639.0, "completions/mean_length": 476.03125, "completions/mean_terminated_length": 434.3214416503906, "completions/min_length": 137.0, "completions/min_terminated_length": 137.0, "entropy": 0.4416114613413811, "epoch": 0.10010706638115632, "frac_reward_zero_std": 0.5, "grad_norm": 0.005831408780068159, "learning_rate": 1e-05, "loss": 0.0415, "num_tokens": 4233715.0, "reward": 0.09375, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.09375, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.2900828123092651, "sampling/importance_sampling_ratio/mean": 1.0000747442245483, "sampling/importance_sampling_ratio/min": 0.7412513494491577, "sampling/sampling_logp_difference/max": 0.29941558837890625, "sampling/sampling_logp_difference/mean": 0.012264476157724857, "step": 187 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.5553937525255606e-05, "clip_ratio/low_min": 4.5553937525255606e-05, "clip_ratio/region_mean": 4.5553937525255606e-05, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 757.0, "completions/mean_length": 588.5, "completions/mean_terminated_length": 555.25927734375, "completions/min_length": 355.0, "completions/min_terminated_length": 355.0, "entropy": 0.3835572823882103, "epoch": 0.1006423982869379, "frac_reward_zero_std": 0.25, "grad_norm": 0.013091735541820526, "learning_rate": 1e-05, "loss": 0.079, "num_tokens": 4256451.0, "reward": 0.625, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.5063872337341309, "sampling/importance_sampling_ratio/mean": 0.9999738335609436, "sampling/importance_sampling_ratio/min": 0.37463682889938354, "sampling/sampling_logp_difference/max": 0.9817981719970703, "sampling/sampling_logp_difference/mean": 0.0106464559212327, "step": 188 }, { "clip_ratio/high_max": 5.089576370664872e-05, "clip_ratio/high_mean": 5.089576370664872e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 5.089576370664872e-05, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 762.0, "completions/mean_length": 588.03125, "completions/mean_terminated_length": 554.7037353515625, "completions/min_length": 315.0, "completions/min_terminated_length": 315.0, "entropy": 0.4354232996702194, "epoch": 0.10117773019271949, "frac_reward_zero_std": 0.25, "grad_norm": 0.012282553128898144, "learning_rate": 1e-05, "loss": 0.0078, "num_tokens": 4279044.0, "reward": 0.71875, "reward_std": 0.3608423173427582, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.3277099132537842, "sampling/importance_sampling_ratio/mean": 1.0001565217971802, "sampling/importance_sampling_ratio/min": 0.6818786859512329, "sampling/sampling_logp_difference/max": 0.3829035758972168, "sampling/sampling_logp_difference/mean": 0.012432866729795933, "step": 189 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00018578031085780822, "clip_ratio/low_min": 0.00018578031085780822, "clip_ratio/region_mean": 0.00018578031085780822, "completions/clipped_ratio": 0.625, "completions/max_length": 768.0, "completions/max_terminated_length": 742.0, "completions/mean_length": 684.96875, "completions/mean_terminated_length": 546.5833740234375, "completions/min_length": 432.0, "completions/min_terminated_length": 432.0, "entropy": 0.5379181317985058, "epoch": 0.10171306209850108, "frac_reward_zero_std": 0.75, "grad_norm": 0.0023843999952077866, "learning_rate": 1e-05, "loss": 0.0123, "num_tokens": 4305499.0, "reward": 0.03125, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.03125, "rewards/accuracy_reward/std": 0.1767766922712326, "sampling/importance_sampling_ratio/max": 1.3420454263687134, "sampling/importance_sampling_ratio/mean": 1.0002247095108032, "sampling/importance_sampling_ratio/min": 0.7700905799865723, "sampling/sampling_logp_difference/max": 0.29419493675231934, "sampling/sampling_logp_difference/mean": 0.01415625587105751, "step": 190 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.1736228013178334e-05, "clip_ratio/low_min": 4.1736228013178334e-05, "clip_ratio/region_mean": 4.1736228013178334e-05, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 763.0, "completions/mean_length": 645.375, "completions/mean_terminated_length": 604.5, "completions/min_length": 318.0, "completions/min_terminated_length": 318.0, "entropy": 0.3155921474099159, "epoch": 0.10224839400428265, "frac_reward_zero_std": 0.25, "grad_norm": 0.006586519535630941, "learning_rate": 1e-05, "loss": 0.0194, "num_tokens": 4330479.0, "reward": 0.59375, "reward_std": 0.3377464711666107, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.3585346937179565, "sampling/importance_sampling_ratio/mean": 1.0000219345092773, "sampling/importance_sampling_ratio/min": 0.6594923138618469, "sampling/sampling_logp_difference/max": 0.41628503799438477, "sampling/sampling_logp_difference/mean": 0.009408274665474892, "step": 191 }, { "clip_ratio/high_max": 0.00010500150165171362, "clip_ratio/high_mean": 0.00010500150165171362, "clip_ratio/low_mean": 4.41852243966423e-05, "clip_ratio/low_min": 4.41852243966423e-05, "clip_ratio/region_mean": 0.00014918672604835592, "completions/clipped_ratio": 0.4375, "completions/max_length": 768.0, "completions/max_terminated_length": 732.0, "completions/mean_length": 671.40625, "completions/mean_terminated_length": 596.2777709960938, "completions/min_length": 446.0, "completions/min_terminated_length": 446.0, "entropy": 0.4409182704985142, "epoch": 0.10278372591006424, "frac_reward_zero_std": 0.0, "grad_norm": 0.013342789374291897, "learning_rate": 1e-05, "loss": 0.0426, "num_tokens": 4355868.0, "reward": 0.34375, "reward_std": 0.4218915104866028, "rewards/accuracy_reward/mean": 0.34375, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.4797009229660034, "sampling/importance_sampling_ratio/mean": 1.000070333480835, "sampling/importance_sampling_ratio/min": 0.7045159935951233, "sampling/sampling_logp_difference/max": 0.39183998107910156, "sampling/sampling_logp_difference/mean": 0.012391703203320503, "step": 192 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00014794354865443893, "clip_ratio/low_min": 0.00014794354865443893, "clip_ratio/region_mean": 0.00014794354865443893, "completions/clipped_ratio": 0.40625, "completions/max_length": 768.0, "completions/max_terminated_length": 679.0, "completions/mean_length": 622.5, "completions/mean_terminated_length": 522.9473876953125, "completions/min_length": 383.0, "completions/min_terminated_length": 383.0, "entropy": 0.49924905225634575, "epoch": 0.10331905781584583, "frac_reward_zero_std": 0.25, "grad_norm": 0.008667836897075176, "learning_rate": 1e-05, "loss": 0.0616, "num_tokens": 4379716.0, "reward": 0.59375, "reward_std": 0.3377464711666107, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.4387460947036743, "sampling/importance_sampling_ratio/mean": 0.9998687505722046, "sampling/importance_sampling_ratio/min": 0.7021949887275696, "sampling/sampling_logp_difference/max": 0.363771915435791, "sampling/sampling_logp_difference/mean": 0.013727799989283085, "step": 193 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.475474270293489e-05, "clip_ratio/low_min": 4.475474270293489e-05, "clip_ratio/region_mean": 4.475474270293489e-05, "completions/clipped_ratio": 0.625, "completions/max_length": 768.0, "completions/max_terminated_length": 761.0, "completions/mean_length": 702.53125, "completions/mean_terminated_length": 593.4166870117188, "completions/min_length": 434.0, "completions/min_terminated_length": 434.0, "entropy": 0.55833300948143, "epoch": 0.10385438972162742, "frac_reward_zero_std": 0.25, "grad_norm": 0.01016563642770052, "learning_rate": 1e-05, "loss": 0.0434, "num_tokens": 4406357.0, "reward": 0.25, "reward_std": 0.3514062464237213, "rewards/accuracy_reward/mean": 0.25, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.7262457609176636, "sampling/importance_sampling_ratio/mean": 0.9997605681419373, "sampling/importance_sampling_ratio/min": 0.7002395391464233, "sampling/sampling_logp_difference/max": 0.5459489822387695, "sampling/sampling_logp_difference/mean": 0.014348062686622143, "step": 194 }, { "clip_ratio/high_max": 0.00011837556303362362, "clip_ratio/high_mean": 0.00011837556303362362, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00011837556303362362, "completions/clipped_ratio": 0.0, "completions/max_length": 761.0, "completions/max_terminated_length": 761.0, "completions/mean_length": 560.84375, "completions/mean_terminated_length": 560.84375, "completions/min_length": 263.0, "completions/min_terminated_length": 263.0, "entropy": 0.3299122229218483, "epoch": 0.10438972162740899, "frac_reward_zero_std": 0.25, "grad_norm": 0.01046715397387743, "learning_rate": 1e-05, "loss": 0.0722, "num_tokens": 4427864.0, "reward": 0.875, "reward_std": 0.292504221200943, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 1.4348326921463013, "sampling/importance_sampling_ratio/mean": 0.9998531937599182, "sampling/importance_sampling_ratio/min": 0.7042420506477356, "sampling/sampling_logp_difference/max": 0.36104822158813477, "sampling/sampling_logp_difference/mean": 0.00976661778986454, "step": 195 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.271951860981062e-05, "clip_ratio/low_min": 6.271951860981062e-05, "clip_ratio/region_mean": 6.271951860981062e-05, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 748.0, "completions/mean_length": 525.90625, "completions/mean_terminated_length": 518.0967407226562, "completions/min_length": 323.0, "completions/min_terminated_length": 323.0, "entropy": 0.30321745201945305, "epoch": 0.10492505353319058, "frac_reward_zero_std": 0.0, "grad_norm": 0.011381514370441437, "learning_rate": 1e-05, "loss": 0.0069, "num_tokens": 4448381.0, "reward": 0.6875, "reward_std": 0.4082317352294922, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.3842442035675049, "sampling/importance_sampling_ratio/mean": 1.0000147819519043, "sampling/importance_sampling_ratio/min": 0.47558218240737915, "sampling/sampling_logp_difference/max": 0.7432155609130859, "sampling/sampling_logp_difference/mean": 0.009583337232470512, "step": 196 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.40625, "completions/max_length": 768.0, "completions/max_terminated_length": 753.0, "completions/mean_length": 612.84375, "completions/mean_terminated_length": 506.6842041015625, "completions/min_length": 267.0, "completions/min_terminated_length": 267.0, "entropy": 0.5010893531143665, "epoch": 0.10546038543897217, "frac_reward_zero_std": 0.25, "grad_norm": 0.015702135860919952, "learning_rate": 1e-05, "loss": 0.0208, "num_tokens": 4471688.0, "reward": 0.625, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.4714659452438354, "sampling/importance_sampling_ratio/mean": 1.0001922845840454, "sampling/importance_sampling_ratio/min": 0.6426209807395935, "sampling/sampling_logp_difference/max": 0.4422001838684082, "sampling/sampling_logp_difference/mean": 0.013482176698744297, "step": 197 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00010368593939347193, "clip_ratio/low_min": 0.00010368593939347193, "clip_ratio/region_mean": 0.00010368593939347193, "completions/clipped_ratio": 0.28125, "completions/max_length": 768.0, "completions/max_terminated_length": 763.0, "completions/mean_length": 586.375, "completions/mean_terminated_length": 515.3043823242188, "completions/min_length": 256.0, "completions/min_terminated_length": 256.0, "entropy": 0.37887583673000336, "epoch": 0.10599571734475374, "frac_reward_zero_std": 0.0, "grad_norm": 0.019367409870028496, "learning_rate": 1e-05, "loss": 0.0901, "num_tokens": 4494548.0, "reward": 0.5, "reward_std": 0.44403791427612305, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5080004930496216, "sampling/importance_sampling_ratio/max": 1.6417173147201538, "sampling/importance_sampling_ratio/mean": 0.9996522665023804, "sampling/importance_sampling_ratio/min": 0.5451266169548035, "sampling/sampling_logp_difference/max": 0.6067371368408203, "sampling/sampling_logp_difference/mean": 0.010913314297795296, "step": 198 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.2244002543156967e-05, "clip_ratio/low_min": 4.2244002543156967e-05, "clip_ratio/region_mean": 4.2244002543156967e-05, "completions/clipped_ratio": 0.34375, "completions/max_length": 768.0, "completions/max_terminated_length": 748.0, "completions/mean_length": 574.96875, "completions/mean_terminated_length": 473.8571472167969, "completions/min_length": 327.0, "completions/min_terminated_length": 327.0, "entropy": 0.3855096586048603, "epoch": 0.10653104925053533, "frac_reward_zero_std": 0.25, "grad_norm": 0.008093578740954399, "learning_rate": 1e-05, "loss": 0.1202, "num_tokens": 4516411.0, "reward": 0.625, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.2982853651046753, "sampling/importance_sampling_ratio/mean": 0.9998065829277039, "sampling/importance_sampling_ratio/min": 0.6771559119224548, "sampling/sampling_logp_difference/max": 0.38985371589660645, "sampling/sampling_logp_difference/mean": 0.011096972040832043, "step": 199 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 691.0, "completions/mean_length": 471.25, "completions/mean_terminated_length": 451.4667053222656, "completions/min_length": 137.0, "completions/min_terminated_length": 137.0, "entropy": 0.43591704219579697, "epoch": 0.10706638115631692, "frac_reward_zero_std": 0.25, "grad_norm": 0.0033567268401384354, "learning_rate": 1e-05, "loss": 0.0702, "num_tokens": 4535019.0, "reward": 0.84375, "reward_std": 0.3061639964580536, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.3899041414260864, "sampling/importance_sampling_ratio/mean": 1.0000931024551392, "sampling/importance_sampling_ratio/min": 0.6976861953735352, "sampling/sampling_logp_difference/max": 0.3599858283996582, "sampling/sampling_logp_difference/mean": 0.013180355541408062, "step": 200 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 757.0, "completions/mean_length": 519.90625, "completions/mean_terminated_length": 494.2413635253906, "completions/min_length": 236.0, "completions/min_terminated_length": 236.0, "entropy": 0.3362443335354328, "epoch": 0.1076017130620985, "frac_reward_zero_std": 0.5, "grad_norm": 0.0033777381759136915, "learning_rate": 1e-05, "loss": -0.0141, "num_tokens": 4554952.0, "reward": 0.9375, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.9375, "rewards/accuracy_reward/std": 0.24593468010425568, "sampling/importance_sampling_ratio/max": 1.2886570692062378, "sampling/importance_sampling_ratio/mean": 1.0002273321151733, "sampling/importance_sampling_ratio/min": 0.6547046303749084, "sampling/sampling_logp_difference/max": 0.4235711097717285, "sampling/sampling_logp_difference/mean": 0.010121806524693966, "step": 201 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 745.0, "completions/mean_length": 598.71875, "completions/mean_terminated_length": 551.3200073242188, "completions/min_length": 356.0, "completions/min_terminated_length": 356.0, "entropy": 0.31316785886883736, "epoch": 0.10813704496788008, "frac_reward_zero_std": 0.5, "grad_norm": 0.0072125643491744995, "learning_rate": 1e-05, "loss": 0.0626, "num_tokens": 4577591.0, "reward": 0.84375, "reward_std": 0.22201895713806152, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.4741240739822388, "sampling/importance_sampling_ratio/mean": 1.0000097751617432, "sampling/importance_sampling_ratio/min": 0.739193856716156, "sampling/sampling_logp_difference/max": 0.388063907623291, "sampling/sampling_logp_difference/mean": 0.009544799104332924, "step": 202 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 711.0, "completions/mean_length": 551.375, "completions/mean_terminated_length": 528.9655151367188, "completions/min_length": 277.0, "completions/min_terminated_length": 277.0, "entropy": 0.3635198548436165, "epoch": 0.10867237687366167, "frac_reward_zero_std": 0.25, "grad_norm": 0.007378813344985247, "learning_rate": 1e-05, "loss": 0.0863, "num_tokens": 4598667.0, "reward": 0.78125, "reward_std": 0.3061639666557312, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.4234294891357422, "sampling/importance_sampling_ratio/mean": 0.9999359250068665, "sampling/importance_sampling_ratio/min": 0.713753342628479, "sampling/sampling_logp_difference/max": 0.3530690670013428, "sampling/sampling_logp_difference/mean": 0.010574338026344776, "step": 203 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00015298444486688823, "clip_ratio/low_min": 0.00015298444486688823, "clip_ratio/region_mean": 0.00015298444486688823, "completions/clipped_ratio": 0.375, "completions/max_length": 768.0, "completions/max_terminated_length": 725.0, "completions/mean_length": 660.9375, "completions/mean_terminated_length": 596.7000122070312, "completions/min_length": 395.0, "completions/min_terminated_length": 395.0, "entropy": 0.4041886292397976, "epoch": 0.10920770877944326, "frac_reward_zero_std": 0.0, "grad_norm": 0.013808957301080227, "learning_rate": 1e-05, "loss": 0.0327, "num_tokens": 4623601.0, "reward": 0.4375, "reward_std": 0.4492306709289551, "rewards/accuracy_reward/mean": 0.4375, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.3408790826797485, "sampling/importance_sampling_ratio/mean": 1.0000574588775635, "sampling/importance_sampling_ratio/min": 0.7060325741767883, "sampling/sampling_logp_difference/max": 0.34809398651123047, "sampling/sampling_logp_difference/mean": 0.011319300159811974, "step": 204 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 765.0, "completions/mean_length": 515.15625, "completions/mean_terminated_length": 444.3599853515625, "completions/min_length": 259.0, "completions/min_terminated_length": 259.0, "entropy": 0.6250455193221569, "epoch": 0.10974304068522484, "frac_reward_zero_std": 0.5, "grad_norm": 0.007350800558924675, "learning_rate": 1e-05, "loss": -0.0154, "num_tokens": 4644134.0, "reward": 0.71875, "reward_std": 0.2630178928375244, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.457682728767395, "sampling/importance_sampling_ratio/mean": 1.0000706911087036, "sampling/importance_sampling_ratio/min": 0.7095704078674316, "sampling/sampling_logp_difference/max": 0.3768479824066162, "sampling/sampling_logp_difference/mean": 0.015043867751955986, "step": 205 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 711.0, "completions/mean_length": 549.1875, "completions/mean_terminated_length": 508.6666564941406, "completions/min_length": 305.0, "completions/min_terminated_length": 305.0, "entropy": 0.39418603852391243, "epoch": 0.11027837259100642, "frac_reward_zero_std": 0.5, "grad_norm": 0.0042163603939116, "learning_rate": 1e-05, "loss": 0.0581, "num_tokens": 4665684.0, "reward": 0.90625, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.5864545106887817, "sampling/importance_sampling_ratio/mean": 0.9994666576385498, "sampling/importance_sampling_ratio/min": 0.5278363823890686, "sampling/sampling_logp_difference/max": 0.6389689445495605, "sampling/sampling_logp_difference/mean": 0.011330260895192623, "step": 206 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.976114723831415e-05, "clip_ratio/low_min": 4.976114723831415e-05, "clip_ratio/region_mean": 4.976114723831415e-05, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 750.0, "completions/mean_length": 571.78125, "completions/mean_terminated_length": 543.75, "completions/min_length": 304.0, "completions/min_terminated_length": 304.0, "entropy": 0.4335530735552311, "epoch": 0.11081370449678801, "frac_reward_zero_std": 0.25, "grad_norm": 0.002859686966985464, "learning_rate": 1e-05, "loss": 0.0697, "num_tokens": 4687573.0, "reward": 0.6875, "reward_std": 0.292504221200943, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.550080418586731, "sampling/importance_sampling_ratio/mean": 0.9997480511665344, "sampling/importance_sampling_ratio/min": 0.7509428858757019, "sampling/sampling_logp_difference/max": 0.4383068084716797, "sampling/sampling_logp_difference/mean": 0.011670460924506187, "step": 207 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00022257391174207442, "clip_ratio/low_min": 0.00022257391174207442, "clip_ratio/region_mean": 0.00022257391174207442, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 757.0, "completions/mean_length": 597.96875, "completions/mean_terminated_length": 541.2916870117188, "completions/min_length": 370.0, "completions/min_terminated_length": 370.0, "entropy": 0.4528747610747814, "epoch": 0.11134903640256959, "frac_reward_zero_std": 0.5, "grad_norm": 0.012036731466650963, "learning_rate": 1e-05, "loss": 0.0409, "num_tokens": 4710412.0, "reward": 0.65625, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0001055002212524, "sampling/importance_sampling_ratio/min": 0.7277035713195801, "sampling/sampling_logp_difference/max": 0.757115364074707, "sampling/sampling_logp_difference/mean": 0.012383118271827698, "step": 208 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.002000762033276e-05, "clip_ratio/low_min": 5.002000762033276e-05, "clip_ratio/region_mean": 5.002000762033276e-05, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 721.0, "completions/mean_length": 553.34375, "completions/mean_terminated_length": 539.0333862304688, "completions/min_length": 323.0, "completions/min_terminated_length": 323.0, "entropy": 0.40291527658700943, "epoch": 0.11188436830835118, "frac_reward_zero_std": 0.0, "grad_norm": 0.004115377552807331, "learning_rate": 1e-05, "loss": 0.0368, "num_tokens": 4731567.0, "reward": 0.59375, "reward_std": 0.4218915104866028, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.3464263677597046, "sampling/importance_sampling_ratio/mean": 0.9997650384902954, "sampling/importance_sampling_ratio/min": 0.7798070907592773, "sampling/sampling_logp_difference/max": 0.2974538803100586, "sampling/sampling_logp_difference/mean": 0.011762114241719246, "step": 209 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 666.0, "completions/mean_length": 526.6875, "completions/mean_terminated_length": 446.25, "completions/min_length": 248.0, "completions/min_terminated_length": 248.0, "entropy": 0.44322478026151657, "epoch": 0.11241970021413276, "frac_reward_zero_std": 0.25, "grad_norm": 0.009065919555723667, "learning_rate": 1e-05, "loss": 0.0643, "num_tokens": 4751861.0, "reward": 0.625, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.4833462238311768, "sampling/importance_sampling_ratio/mean": 0.9995315670967102, "sampling/importance_sampling_ratio/min": 0.765781581401825, "sampling/sampling_logp_difference/max": 0.3943004608154297, "sampling/sampling_logp_difference/mean": 0.012334013357758522, "step": 210 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 759.0, "completions/mean_length": 584.90625, "completions/mean_terminated_length": 523.875, "completions/min_length": 309.0, "completions/min_terminated_length": 309.0, "entropy": 0.3720376640558243, "epoch": 0.11295503211991435, "frac_reward_zero_std": 0.75, "grad_norm": 0.009011111222207546, "learning_rate": 1e-05, "loss": 0.0261, "num_tokens": 4774082.0, "reward": 0.65625, "reward_std": 0.1293872892856598, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.502943992614746, "sampling/importance_sampling_ratio/mean": 1.0001670122146606, "sampling/importance_sampling_ratio/min": 0.6831524968147278, "sampling/sampling_logp_difference/max": 0.4074258804321289, "sampling/sampling_logp_difference/mean": 0.010632229968905449, "step": 211 }, { "clip_ratio/high_max": 5.6637971283635125e-05, "clip_ratio/high_mean": 5.6637971283635125e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 5.6637971283635125e-05, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 732.0, "completions/mean_length": 536.0, "completions/mean_terminated_length": 520.5333862304688, "completions/min_length": 336.0, "completions/min_terminated_length": 336.0, "entropy": 0.3791799619793892, "epoch": 0.11349036402569593, "frac_reward_zero_std": 0.25, "grad_norm": 0.009205985814332962, "learning_rate": 1e-05, "loss": 0.0078, "num_tokens": 4794986.0, "reward": 0.625, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.4290452003479004, "sampling/importance_sampling_ratio/mean": 0.9997783899307251, "sampling/importance_sampling_ratio/min": 0.7603805065155029, "sampling/sampling_logp_difference/max": 0.3570065498352051, "sampling/sampling_logp_difference/mean": 0.010945064947009087, "step": 212 }, { "clip_ratio/high_max": 4.2866940930252895e-05, "clip_ratio/high_mean": 4.2866940930252895e-05, "clip_ratio/low_mean": 0.00022432948389905505, "clip_ratio/low_min": 0.00022432948389905505, "clip_ratio/region_mean": 0.00026719642482930794, "completions/clipped_ratio": 0.4375, "completions/max_length": 768.0, "completions/max_terminated_length": 763.0, "completions/mean_length": 679.5, "completions/mean_terminated_length": 610.6666870117188, "completions/min_length": 401.0, "completions/min_terminated_length": 401.0, "entropy": 0.4458453692495823, "epoch": 0.11402569593147752, "frac_reward_zero_std": 0.0, "grad_norm": 0.0072832051664590836, "learning_rate": 1e-05, "loss": 0.0266, "num_tokens": 4820618.0, "reward": 0.4375, "reward_std": 0.4492306709289551, "rewards/accuracy_reward/mean": 0.4375, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.4859117269515991, "sampling/importance_sampling_ratio/mean": 0.9998586177825928, "sampling/importance_sampling_ratio/min": 0.6292915940284729, "sampling/sampling_logp_difference/max": 0.46316051483154297, "sampling/sampling_logp_difference/mean": 0.01230671163648367, "step": 213 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.283178325043991e-05, "clip_ratio/low_min": 5.283178325043991e-05, "clip_ratio/region_mean": 5.283178325043991e-05, "completions/clipped_ratio": 0.375, "completions/max_length": 768.0, "completions/max_terminated_length": 685.0, "completions/mean_length": 630.25, "completions/mean_terminated_length": 547.6000366210938, "completions/min_length": 339.0, "completions/min_terminated_length": 339.0, "entropy": 0.4581412896513939, "epoch": 0.1145610278372591, "frac_reward_zero_std": 0.5, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0157, "num_tokens": 4844802.0, "reward": 0.46875, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.46875, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.635581612586975, "sampling/importance_sampling_ratio/mean": 0.9998361468315125, "sampling/importance_sampling_ratio/min": 0.7378471493721008, "sampling/sampling_logp_difference/max": 0.49199843406677246, "sampling/sampling_logp_difference/mean": 0.01316896453499794, "step": 214 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 720.0, "completions/mean_length": 536.5625, "completions/mean_terminated_length": 521.1333618164062, "completions/min_length": 307.0, "completions/min_terminated_length": 307.0, "entropy": 0.35761559009552, "epoch": 0.11509635974304068, "frac_reward_zero_std": 0.25, "grad_norm": 0.019765743985772133, "learning_rate": 1e-05, "loss": -0.0135, "num_tokens": 4865948.0, "reward": 0.875, "reward_std": 0.2925041913986206, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 1.4213097095489502, "sampling/importance_sampling_ratio/mean": 0.9999829530715942, "sampling/importance_sampling_ratio/min": 0.5343505144119263, "sampling/sampling_logp_difference/max": 0.6267032623291016, "sampling/sampling_logp_difference/mean": 0.010485834442079067, "step": 215 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 693.0, "completions/mean_length": 560.0625, "completions/mean_terminated_length": 490.75, "completions/min_length": 344.0, "completions/min_terminated_length": 344.0, "entropy": 0.4157865084707737, "epoch": 0.11563169164882227, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0412, "num_tokens": 4887566.0, "reward": 0.71875, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.556046724319458, "sampling/importance_sampling_ratio/mean": 1.0002392530441284, "sampling/importance_sampling_ratio/min": 0.7258667945861816, "sampling/sampling_logp_difference/max": 0.44214844703674316, "sampling/sampling_logp_difference/mean": 0.011593695729970932, "step": 216 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 691.0, "completions/mean_length": 549.65625, "completions/mean_terminated_length": 518.4642944335938, "completions/min_length": 312.0, "completions/min_terminated_length": 312.0, "entropy": 0.34925256110727787, "epoch": 0.11616702355460386, "frac_reward_zero_std": 0.5, "grad_norm": 0.01225263997912407, "learning_rate": 1e-05, "loss": 0.0054, "num_tokens": 4908755.0, "reward": 0.71875, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.4094576835632324, "sampling/importance_sampling_ratio/mean": 1.0001062154769897, "sampling/importance_sampling_ratio/min": 0.6197565793991089, "sampling/sampling_logp_difference/max": 0.47842860221862793, "sampling/sampling_logp_difference/mean": 0.011083544231951237, "step": 217 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00013465177471516654, "clip_ratio/low_min": 0.00013465177471516654, "clip_ratio/region_mean": 0.00013465177471516654, "completions/clipped_ratio": 0.59375, "completions/max_length": 768.0, "completions/max_terminated_length": 712.0, "completions/mean_length": 694.5625, "completions/mean_terminated_length": 587.2307739257812, "completions/min_length": 426.0, "completions/min_terminated_length": 426.0, "entropy": 0.5617510974407196, "epoch": 0.11670235546038545, "frac_reward_zero_std": 0.5, "grad_norm": 0.008800877258181572, "learning_rate": 1e-05, "loss": 0.0277, "num_tokens": 4935429.0, "reward": 0.25, "reward_std": 0.2587745785713196, "rewards/accuracy_reward/mean": 0.25, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.3903852701187134, "sampling/importance_sampling_ratio/mean": 0.9998555779457092, "sampling/importance_sampling_ratio/min": 0.4617181420326233, "sampling/sampling_logp_difference/max": 0.7728006839752197, "sampling/sampling_logp_difference/mean": 0.014596041291952133, "step": 218 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 8.682675252202898e-05, "clip_ratio/low_min": 8.682675252202898e-05, "clip_ratio/region_mean": 8.682675252202898e-05, "completions/clipped_ratio": 0.4375, "completions/max_length": 768.0, "completions/max_terminated_length": 765.0, "completions/mean_length": 663.375, "completions/mean_terminated_length": 582.0, "completions/min_length": 283.0, "completions/min_terminated_length": 283.0, "entropy": 0.48347389698028564, "epoch": 0.11723768736616702, "frac_reward_zero_std": 0.0, "grad_norm": 0.010706407949328423, "learning_rate": 1e-05, "loss": 0.0167, "num_tokens": 4960833.0, "reward": 0.40625, "reward_std": 0.4218915104866028, "rewards/accuracy_reward/mean": 0.40625, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.5666464567184448, "sampling/importance_sampling_ratio/mean": 0.9998987913131714, "sampling/importance_sampling_ratio/min": 0.5342758893966675, "sampling/sampling_logp_difference/max": 0.6268428564071655, "sampling/sampling_logp_difference/mean": 0.013395486399531364, "step": 219 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00015082780009834096, "clip_ratio/low_min": 0.00015082780009834096, "clip_ratio/region_mean": 0.00015082780009834096, "completions/clipped_ratio": 0.28125, "completions/max_length": 768.0, "completions/max_terminated_length": 700.0, "completions/mean_length": 566.78125, "completions/mean_terminated_length": 488.0434875488281, "completions/min_length": 290.0, "completions/min_terminated_length": 290.0, "entropy": 0.5746861696243286, "epoch": 0.11777301927194861, "frac_reward_zero_std": 0.25, "grad_norm": 0.021935919299721718, "learning_rate": 1e-05, "loss": 0.1448, "num_tokens": 4982834.0, "reward": 0.6875, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.405480146408081, "sampling/importance_sampling_ratio/mean": 1.0001972913742065, "sampling/importance_sampling_ratio/min": 0.6488431692123413, "sampling/sampling_logp_difference/max": 0.43256425857543945, "sampling/sampling_logp_difference/mean": 0.013339702971279621, "step": 220 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 752.0, "completions/mean_length": 574.125, "completions/mean_terminated_length": 538.2222290039062, "completions/min_length": 243.0, "completions/min_terminated_length": 243.0, "entropy": 0.3626738339662552, "epoch": 0.1183083511777302, "frac_reward_zero_std": 0.25, "grad_norm": 0.011936082504689693, "learning_rate": 1e-05, "loss": 0.0502, "num_tokens": 5004766.0, "reward": 0.84375, "reward_std": 0.3061639964580536, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.3696194887161255, "sampling/importance_sampling_ratio/mean": 1.0002532005310059, "sampling/importance_sampling_ratio/min": 0.6588507294654846, "sampling/sampling_logp_difference/max": 0.41725826263427734, "sampling/sampling_logp_difference/mean": 0.01117282547056675, "step": 221 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0001569602572999429, "clip_ratio/low_min": 0.0001569602572999429, "clip_ratio/region_mean": 0.0001569602572999429, "completions/clipped_ratio": 0.28125, "completions/max_length": 768.0, "completions/max_terminated_length": 750.0, "completions/mean_length": 584.3125, "completions/mean_terminated_length": 512.434814453125, "completions/min_length": 225.0, "completions/min_terminated_length": 225.0, "entropy": 0.42847940325737, "epoch": 0.11884368308351177, "frac_reward_zero_std": 0.0, "grad_norm": 0.010854063555598259, "learning_rate": 1e-05, "loss": -0.0077, "num_tokens": 5026904.0, "reward": 0.40625, "reward_std": 0.4534739851951599, "rewards/accuracy_reward/mean": 0.40625, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.36565363407135, "sampling/importance_sampling_ratio/mean": 0.9997027516365051, "sampling/importance_sampling_ratio/min": 0.6231113076210022, "sampling/sampling_logp_difference/max": 0.47303009033203125, "sampling/sampling_logp_difference/mean": 0.012134634889662266, "step": 222 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.999999873689376e-05, "clip_ratio/low_min": 4.999999873689376e-05, "clip_ratio/region_mean": 4.999999873689376e-05, "completions/clipped_ratio": 0.34375, "completions/max_length": 768.0, "completions/max_terminated_length": 748.0, "completions/mean_length": 606.3125, "completions/mean_terminated_length": 521.6190795898438, "completions/min_length": 228.0, "completions/min_terminated_length": 228.0, "entropy": 0.44984953850507736, "epoch": 0.11937901498929336, "frac_reward_zero_std": 0.25, "grad_norm": 0.009545247070491314, "learning_rate": 1e-05, "loss": 0.0274, "num_tokens": 5050130.0, "reward": 0.5625, "reward_std": 0.3745020925998688, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.385037899017334, "sampling/importance_sampling_ratio/mean": 1.0000183582305908, "sampling/importance_sampling_ratio/min": 0.7200703024864197, "sampling/sampling_logp_difference/max": 0.3284064531326294, "sampling/sampling_logp_difference/mean": 0.012084768153727055, "step": 223 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 752.0, "completions/mean_length": 602.3125, "completions/mean_terminated_length": 564.0769653320312, "completions/min_length": 361.0, "completions/min_terminated_length": 361.0, "entropy": 0.3796752355992794, "epoch": 0.11991434689507495, "frac_reward_zero_std": 0.0, "grad_norm": 0.017084071412682533, "learning_rate": 1e-05, "loss": 0.0475, "num_tokens": 5072548.0, "reward": 0.8125, "reward_std": 0.4082317352294922, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.297909140586853, "sampling/importance_sampling_ratio/mean": 0.9999545812606812, "sampling/importance_sampling_ratio/min": 0.7055865526199341, "sampling/sampling_logp_difference/max": 0.3487257957458496, "sampling/sampling_logp_difference/mean": 0.010714119300246239, "step": 224 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.5, "completions/max_length": 768.0, "completions/max_terminated_length": 712.0, "completions/mean_length": 662.53125, "completions/mean_terminated_length": 557.0625, "completions/min_length": 404.0, "completions/min_terminated_length": 404.0, "entropy": 0.48830728977918625, "epoch": 0.12044967880085652, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.02, "num_tokens": 5097821.0, "reward": 0.46875, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.46875, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.4616483449935913, "sampling/importance_sampling_ratio/mean": 1.0001503229141235, "sampling/importance_sampling_ratio/min": 0.564860999584198, "sampling/sampling_logp_difference/max": 0.5711755752563477, "sampling/sampling_logp_difference/mean": 0.013598200865089893, "step": 225 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 9.84872312983498e-05, "clip_ratio/low_min": 9.84872312983498e-05, "clip_ratio/region_mean": 9.84872312983498e-05, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 726.0, "completions/mean_length": 578.59375, "completions/mean_terminated_length": 515.4583740234375, "completions/min_length": 172.0, "completions/min_terminated_length": 172.0, "entropy": 0.49579084664583206, "epoch": 0.12098501070663811, "frac_reward_zero_std": 0.25, "grad_norm": 0.01154306624084711, "learning_rate": 1e-05, "loss": 0.0863, "num_tokens": 5120392.0, "reward": 0.53125, "reward_std": 0.378745436668396, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.6366358995437622, "sampling/importance_sampling_ratio/mean": 0.9999333620071411, "sampling/importance_sampling_ratio/min": 0.5065307021141052, "sampling/sampling_logp_difference/max": 0.6801702976226807, "sampling/sampling_logp_difference/mean": 0.01242026686668396, "step": 226 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0002624518565426115, "clip_ratio/low_min": 0.0002624518565426115, "clip_ratio/region_mean": 0.0002624518565426115, "completions/clipped_ratio": 0.625, "completions/max_length": 768.0, "completions/max_terminated_length": 761.0, "completions/mean_length": 711.53125, "completions/mean_terminated_length": 617.4166870117188, "completions/min_length": 494.0, "completions/min_terminated_length": 494.0, "entropy": 0.6983655318617821, "epoch": 0.1215203426124197, "frac_reward_zero_std": 0.0, "grad_norm": 0.020195450633764267, "learning_rate": 1e-05, "loss": 0.0289, "num_tokens": 5147929.0, "reward": 0.1875, "reward_std": 0.3945523500442505, "rewards/accuracy_reward/mean": 0.1875, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.6050238609313965, "sampling/importance_sampling_ratio/mean": 0.9996456503868103, "sampling/importance_sampling_ratio/min": 0.625224769115448, "sampling/sampling_logp_difference/max": 0.47313857078552246, "sampling/sampling_logp_difference/mean": 0.017196593806147575, "step": 227 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.796623034053482e-05, "clip_ratio/low_min": 4.796623034053482e-05, "clip_ratio/region_mean": 4.796623034053482e-05, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 712.0, "completions/mean_length": 587.59375, "completions/mean_terminated_length": 554.1851806640625, "completions/min_length": 354.0, "completions/min_terminated_length": 354.0, "entropy": 0.32752997800707817, "epoch": 0.12205567451820129, "frac_reward_zero_std": 0.25, "grad_norm": 0.003851040732115507, "learning_rate": 1e-05, "loss": 0.0239, "num_tokens": 5170444.0, "reward": 0.71875, "reward_std": 0.2651650309562683, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.4324769973754883, "sampling/importance_sampling_ratio/mean": 1.0000749826431274, "sampling/importance_sampling_ratio/min": 0.7732476592063904, "sampling/sampling_logp_difference/max": 0.3594050407409668, "sampling/sampling_logp_difference/mean": 0.009868286550045013, "step": 228 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00017453929103794508, "clip_ratio/low_min": 0.00017453929103794508, "clip_ratio/region_mean": 0.00017453929103794508, "completions/clipped_ratio": 0.3125, "completions/max_length": 768.0, "completions/max_terminated_length": 753.0, "completions/mean_length": 646.53125, "completions/mean_terminated_length": 591.3181762695312, "completions/min_length": 368.0, "completions/min_terminated_length": 368.0, "entropy": 0.3754481337964535, "epoch": 0.12259100642398287, "frac_reward_zero_std": 0.0, "grad_norm": 0.00590737909078598, "learning_rate": 1e-05, "loss": 0.0197, "num_tokens": 5194869.0, "reward": 0.5625, "reward_std": 0.47655022144317627, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.3619716167449951, "sampling/importance_sampling_ratio/mean": 1.0001739263534546, "sampling/importance_sampling_ratio/min": 0.6845538020133972, "sampling/sampling_logp_difference/max": 0.37898802757263184, "sampling/sampling_logp_difference/mean": 0.010325351729989052, "step": 229 }, { "clip_ratio/high_max": 7.318500865949318e-05, "clip_ratio/high_mean": 7.318500865949318e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 7.318500865949318e-05, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 681.0, "completions/mean_length": 504.125, "completions/mean_terminated_length": 486.5333557128906, "completions/min_length": 270.0, "completions/min_terminated_length": 270.0, "entropy": 0.3812452107667923, "epoch": 0.12312633832976445, "frac_reward_zero_std": 0.25, "grad_norm": 0.0037694680504500866, "learning_rate": 1e-05, "loss": -0.0002, "num_tokens": 5214745.0, "reward": 0.90625, "reward_std": 0.2651650309562683, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.5416697263717651, "sampling/importance_sampling_ratio/mean": 0.9997934699058533, "sampling/importance_sampling_ratio/min": 0.6923429369926453, "sampling/sampling_logp_difference/max": 0.43286609649658203, "sampling/sampling_logp_difference/mean": 0.011613366194069386, "step": 230 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.278716344037093e-05, "clip_ratio/low_min": 5.278716344037093e-05, "clip_ratio/region_mean": 5.278716344037093e-05, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 767.0, "completions/mean_length": 534.96875, "completions/mean_terminated_length": 491.8148193359375, "completions/min_length": 297.0, "completions/min_terminated_length": 297.0, "entropy": 0.49364667758345604, "epoch": 0.12366167023554604, "frac_reward_zero_std": 0.5, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": -0.0083, "num_tokens": 5235528.0, "reward": 0.6875, "reward_std": 0.249358132481575, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.5320346355438232, "sampling/importance_sampling_ratio/mean": 1.0000654458999634, "sampling/importance_sampling_ratio/min": 0.6256884932518005, "sampling/sampling_logp_difference/max": 0.46890270709991455, "sampling/sampling_logp_difference/mean": 0.012900633737444878, "step": 231 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.555555617320351e-05, "clip_ratio/low_min": 5.555555617320351e-05, "clip_ratio/region_mean": 5.555555617320351e-05, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 738.0, "completions/mean_length": 541.6875, "completions/mean_terminated_length": 478.3199768066406, "completions/min_length": 231.0, "completions/min_terminated_length": 231.0, "entropy": 0.3573584407567978, "epoch": 0.12419700214132762, "frac_reward_zero_std": 0.5, "grad_norm": 0.002308719092980027, "learning_rate": 1e-05, "loss": 0.0239, "num_tokens": 5256486.0, "reward": 0.78125, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.526574730873108, "sampling/importance_sampling_ratio/mean": 1.0002806186676025, "sampling/importance_sampling_ratio/min": 0.7009093165397644, "sampling/sampling_logp_difference/max": 0.423026442527771, "sampling/sampling_logp_difference/mean": 0.011031332425773144, "step": 232 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00018253981397720054, "clip_ratio/low_min": 0.00018253981397720054, "clip_ratio/region_mean": 0.00018253981397720054, "completions/clipped_ratio": 0.53125, "completions/max_length": 768.0, "completions/max_terminated_length": 757.0, "completions/mean_length": 677.65625, "completions/mean_terminated_length": 575.2667236328125, "completions/min_length": 438.0, "completions/min_terminated_length": 438.0, "entropy": 0.433471143245697, "epoch": 0.1247323340471092, "frac_reward_zero_std": 0.25, "grad_norm": 0.005887295119464397, "learning_rate": 1e-05, "loss": 0.0168, "num_tokens": 5282179.0, "reward": 0.34375, "reward_std": 0.3787454068660736, "rewards/accuracy_reward/mean": 0.34375, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.9891812801361084, "sampling/importance_sampling_ratio/mean": 1.0000905990600586, "sampling/importance_sampling_ratio/min": 0.6790370941162109, "sampling/sampling_logp_difference/max": 0.6877231597900391, "sampling/sampling_logp_difference/mean": 0.011292919516563416, "step": 233 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00015975481073837727, "clip_ratio/low_min": 0.00015975481073837727, "clip_ratio/region_mean": 0.00015975481073837727, "completions/clipped_ratio": 0.28125, "completions/max_length": 768.0, "completions/max_terminated_length": 742.0, "completions/mean_length": 599.15625, "completions/mean_terminated_length": 533.0869750976562, "completions/min_length": 366.0, "completions/min_terminated_length": 366.0, "entropy": 0.4720841348171234, "epoch": 0.1252676659528908, "frac_reward_zero_std": 0.5, "grad_norm": 0.0028473229613155127, "learning_rate": 1e-05, "loss": 0.0933, "num_tokens": 5304928.0, "reward": 0.625, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.5715802907943726, "sampling/importance_sampling_ratio/mean": 1.0001165866851807, "sampling/importance_sampling_ratio/min": 0.670651376247406, "sampling/sampling_logp_difference/max": 0.45208168029785156, "sampling/sampling_logp_difference/mean": 0.013936175964772701, "step": 234 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 699.0, "completions/mean_length": 517.65625, "completions/mean_terminated_length": 491.75860595703125, "completions/min_length": 217.0, "completions/min_terminated_length": 217.0, "entropy": 0.45977578312158585, "epoch": 0.12580299785867238, "frac_reward_zero_std": 0.5, "grad_norm": 0.004914202727377415, "learning_rate": 1e-05, "loss": 0.0165, "num_tokens": 5325693.0, "reward": 0.75, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.3612115383148193, "sampling/importance_sampling_ratio/mean": 0.9997780919075012, "sampling/importance_sampling_ratio/min": 0.7317842841148376, "sampling/sampling_logp_difference/max": 0.3122694492340088, "sampling/sampling_logp_difference/mean": 0.012379621155560017, "step": 235 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 729.0, "completions/mean_length": 559.6875, "completions/mean_terminated_length": 501.3599853515625, "completions/min_length": 299.0, "completions/min_terminated_length": 299.0, "entropy": 0.48104002699255943, "epoch": 0.12633832976445397, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0393, "num_tokens": 5348611.0, "reward": 0.6875, "reward_std": 0.1157275140285492, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.3752455711364746, "sampling/importance_sampling_ratio/mean": 0.9995938539505005, "sampling/importance_sampling_ratio/min": 0.6394771337509155, "sampling/sampling_logp_difference/max": 0.44710445404052734, "sampling/sampling_logp_difference/mean": 0.013101525604724884, "step": 236 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0001430235934094526, "clip_ratio/low_min": 0.0001430235934094526, "clip_ratio/region_mean": 0.0001430235934094526, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 742.0, "completions/mean_length": 540.59375, "completions/mean_terminated_length": 488.11541748046875, "completions/min_length": 249.0, "completions/min_terminated_length": 249.0, "entropy": 0.3864423558115959, "epoch": 0.12687366167023553, "frac_reward_zero_std": 0.5, "grad_norm": 0.0048812697641551495, "learning_rate": 1e-05, "loss": -0.0265, "num_tokens": 5370014.0, "reward": 0.75, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.299454689025879, "sampling/importance_sampling_ratio/mean": 1.0000145435333252, "sampling/importance_sampling_ratio/min": 0.7125881910324097, "sampling/sampling_logp_difference/max": 0.3388516902923584, "sampling/sampling_logp_difference/mean": 0.01179390400648117, "step": 237 }, { "clip_ratio/high_max": 4.8449612222611904e-05, "clip_ratio/high_mean": 4.8449612222611904e-05, "clip_ratio/low_mean": 9.447983029531315e-05, "clip_ratio/low_min": 9.447983029531315e-05, "clip_ratio/region_mean": 0.00014292944251792505, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 756.0, "completions/mean_length": 603.9375, "completions/mean_terminated_length": 558.0, "completions/min_length": 269.0, "completions/min_terminated_length": 269.0, "entropy": 0.3614657334983349, "epoch": 0.12740899357601712, "frac_reward_zero_std": 0.0, "grad_norm": 0.01957891508936882, "learning_rate": 1e-05, "loss": 0.0009, "num_tokens": 5392988.0, "reward": 0.71875, "reward_std": 0.4218915104866028, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.6607438325881958, "sampling/importance_sampling_ratio/mean": 1.0000667572021484, "sampling/importance_sampling_ratio/min": 0.7405574321746826, "sampling/sampling_logp_difference/max": 0.507265567779541, "sampling/sampling_logp_difference/mean": 0.01022407691925764, "step": 238 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.3125, "completions/max_length": 768.0, "completions/max_terminated_length": 729.0, "completions/mean_length": 594.71875, "completions/mean_terminated_length": 515.95458984375, "completions/min_length": 249.0, "completions/min_terminated_length": 249.0, "entropy": 0.44415390864014626, "epoch": 0.1279443254817987, "frac_reward_zero_std": 0.5, "grad_norm": 0.0031295267399400473, "learning_rate": 1e-05, "loss": 0.0267, "num_tokens": 5416299.0, "reward": 0.65625, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.4592589139938354, "sampling/importance_sampling_ratio/mean": 1.0006099939346313, "sampling/importance_sampling_ratio/min": 0.6672098636627197, "sampling/sampling_logp_difference/max": 0.4046506881713867, "sampling/sampling_logp_difference/mean": 0.012880331836640835, "step": 239 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.7313158322358504e-05, "clip_ratio/low_min": 5.7313158322358504e-05, "clip_ratio/region_mean": 5.7313158322358504e-05, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 662.0, "completions/mean_length": 535.25, "completions/mean_terminated_length": 481.5384826660156, "completions/min_length": 302.0, "completions/min_terminated_length": 302.0, "entropy": 0.5112757124006748, "epoch": 0.1284796573875803, "frac_reward_zero_std": 0.25, "grad_norm": 0.015150925144553185, "learning_rate": 1e-05, "loss": 0.0354, "num_tokens": 5437403.0, "reward": 0.5625, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.4094347953796387, "sampling/importance_sampling_ratio/mean": 1.0003074407577515, "sampling/importance_sampling_ratio/min": 0.7242313027381897, "sampling/sampling_logp_difference/max": 0.3431887626647949, "sampling/sampling_logp_difference/mean": 0.014134557917714119, "step": 240 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00011728248864528723, "clip_ratio/low_min": 0.00011728248864528723, "clip_ratio/region_mean": 0.00011728248864528723, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 763.0, "completions/mean_length": 583.1875, "completions/mean_terminated_length": 521.5833740234375, "completions/min_length": 10.0, "completions/min_terminated_length": 10.0, "entropy": 0.4333116486668587, "epoch": 0.1290149892933619, "frac_reward_zero_std": 0.25, "grad_norm": 0.011063863523304462, "learning_rate": 1e-05, "loss": 0.0412, "num_tokens": 5459801.0, "reward": 0.625, "reward_std": 0.3924051821231842, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.333984613418579, "sampling/importance_sampling_ratio/mean": 0.9999924302101135, "sampling/importance_sampling_ratio/min": 0.698438823223114, "sampling/sampling_logp_difference/max": 0.35890769958496094, "sampling/sampling_logp_difference/mean": 0.012637640349566936, "step": 241 }, { "clip_ratio/high_max": 4.197447924525477e-05, "clip_ratio/high_mean": 4.197447924525477e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 4.197447924525477e-05, "completions/clipped_ratio": 0.3125, "completions/max_length": 768.0, "completions/max_terminated_length": 753.0, "completions/mean_length": 615.28125, "completions/mean_terminated_length": 545.8636474609375, "completions/min_length": 314.0, "completions/min_terminated_length": 314.0, "entropy": 0.40572981908917427, "epoch": 0.12955032119914348, "frac_reward_zero_std": 0.25, "grad_norm": 0.0034186160191893578, "learning_rate": 1e-05, "loss": 0.0382, "num_tokens": 5483442.0, "reward": 0.65625, "reward_std": 0.2651650309562683, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.2892894744873047, "sampling/importance_sampling_ratio/mean": 1.0002504587173462, "sampling/importance_sampling_ratio/min": 0.5395433902740479, "sampling/sampling_logp_difference/max": 0.6170320510864258, "sampling/sampling_logp_difference/mean": 0.011737329885363579, "step": 242 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.49688154226169e-05, "clip_ratio/low_min": 6.49688154226169e-05, "clip_ratio/region_mean": 6.49688154226169e-05, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 768.0, "completions/mean_length": 581.15625, "completions/mean_terminated_length": 561.8275756835938, "completions/min_length": 244.0, "completions/min_terminated_length": 244.0, "entropy": 0.34565189108252525, "epoch": 0.13008565310492506, "frac_reward_zero_std": 0.0, "grad_norm": 0.006013086065649986, "learning_rate": 1e-05, "loss": 0.0372, "num_tokens": 5505615.0, "reward": 0.65625, "reward_std": 0.4355708956718445, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.3997101783752441, "sampling/importance_sampling_ratio/mean": 0.999925971031189, "sampling/importance_sampling_ratio/min": 0.6597829461097717, "sampling/sampling_logp_difference/max": 0.4158444404602051, "sampling/sampling_logp_difference/mean": 0.010271354578435421, "step": 243 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 754.0, "completions/mean_length": 543.125, "completions/mean_terminated_length": 468.16668701171875, "completions/min_length": 257.0, "completions/min_terminated_length": 257.0, "entropy": 0.40996165573596954, "epoch": 0.13062098501070663, "frac_reward_zero_std": 0.5, "grad_norm": 0.013304386287927628, "learning_rate": 1e-05, "loss": 0.0606, "num_tokens": 5526739.0, "reward": 0.75, "reward_std": 0.26726123690605164, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.2847847938537598, "sampling/importance_sampling_ratio/mean": 0.9998285174369812, "sampling/importance_sampling_ratio/min": 0.6456636786460876, "sampling/sampling_logp_difference/max": 0.43747663497924805, "sampling/sampling_logp_difference/mean": 0.011566098779439926, "step": 244 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00013087704064673744, "clip_ratio/low_min": 0.00013087704064673744, "clip_ratio/region_mean": 0.00013087704064673744, "completions/clipped_ratio": 0.40625, "completions/max_length": 768.0, "completions/max_terminated_length": 757.0, "completions/mean_length": 675.0625, "completions/mean_terminated_length": 611.4736938476562, "completions/min_length": 350.0, "completions/min_terminated_length": 350.0, "entropy": 0.4671286754310131, "epoch": 0.1311563169164882, "frac_reward_zero_std": 0.25, "grad_norm": 0.007557358127087355, "learning_rate": 1e-05, "loss": 0.0517, "num_tokens": 5552573.0, "reward": 0.46875, "reward_std": 0.35564959049224854, "rewards/accuracy_reward/mean": 0.46875, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.6242812871932983, "sampling/importance_sampling_ratio/mean": 0.9999816417694092, "sampling/importance_sampling_ratio/min": 0.7106167078018188, "sampling/sampling_logp_difference/max": 0.4850654602050781, "sampling/sampling_logp_difference/mean": 0.013589474372565746, "step": 245 }, { "clip_ratio/high_max": 0.0001058401758200489, "clip_ratio/high_mean": 0.0001058401758200489, "clip_ratio/low_mean": 6.003842281643301e-05, "clip_ratio/low_min": 6.003842281643301e-05, "clip_ratio/region_mean": 0.0001658785986364819, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 722.0, "completions/mean_length": 547.65625, "completions/mean_terminated_length": 532.9666748046875, "completions/min_length": 291.0, "completions/min_terminated_length": 291.0, "entropy": 0.40443161129951477, "epoch": 0.1316916488222698, "frac_reward_zero_std": 0.0, "grad_norm": 0.011984323151409626, "learning_rate": 1e-05, "loss": 0.0347, "num_tokens": 5573706.0, "reward": 0.53125, "reward_std": 0.3808925747871399, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.5090829133987427, "sampling/importance_sampling_ratio/mean": 0.9999943375587463, "sampling/importance_sampling_ratio/min": 0.6009371280670166, "sampling/sampling_logp_difference/max": 0.5092649459838867, "sampling/sampling_logp_difference/mean": 0.011626939289271832, "step": 246 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00018206869208370335, "clip_ratio/low_min": 0.00018206869208370335, "clip_ratio/region_mean": 0.00018206869208370335, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 738.0, "completions/mean_length": 538.15625, "completions/mean_terminated_length": 495.59259033203125, "completions/min_length": 279.0, "completions/min_terminated_length": 279.0, "entropy": 0.37508508935570717, "epoch": 0.1322269807280514, "frac_reward_zero_std": 0.0, "grad_norm": 0.016642436385154724, "learning_rate": 1e-05, "loss": 0.0751, "num_tokens": 5594767.0, "reward": 0.65625, "reward_std": 0.4628904461860657, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.4443391561508179, "sampling/importance_sampling_ratio/mean": 0.9999256730079651, "sampling/importance_sampling_ratio/min": 0.7800746560096741, "sampling/sampling_logp_difference/max": 0.36765193939208984, "sampling/sampling_logp_difference/mean": 0.011149761267006397, "step": 247 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.3995681810192764e-05, "clip_ratio/low_min": 5.3995681810192764e-05, "clip_ratio/region_mean": 5.3995681810192764e-05, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 743.0, "completions/mean_length": 525.21875, "completions/mean_terminated_length": 517.3870849609375, "completions/min_length": 330.0, "completions/min_terminated_length": 330.0, "entropy": 0.3842773735523224, "epoch": 0.13276231263383298, "frac_reward_zero_std": 0.25, "grad_norm": 0.013428544625639915, "learning_rate": 1e-05, "loss": 0.0636, "num_tokens": 5614830.0, "reward": 0.84375, "reward_std": 0.3198433816432953, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.4436107873916626, "sampling/importance_sampling_ratio/mean": 0.999886691570282, "sampling/importance_sampling_ratio/min": 0.7112228274345398, "sampling/sampling_logp_difference/max": 0.36714744567871094, "sampling/sampling_logp_difference/mean": 0.011435474269092083, "step": 248 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00011107330647064373, "clip_ratio/low_min": 0.00011107330647064373, "clip_ratio/region_mean": 0.00011107330647064373, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 743.0, "completions/mean_length": 566.65625, "completions/mean_terminated_length": 520.1923217773438, "completions/min_length": 312.0, "completions/min_terminated_length": 312.0, "entropy": 0.4407459795475006, "epoch": 0.13329764453961457, "frac_reward_zero_std": 0.25, "grad_norm": 0.008654451929032803, "learning_rate": 1e-05, "loss": 0.06, "num_tokens": 5636955.0, "reward": 0.625, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.3829621076583862, "sampling/importance_sampling_ratio/mean": 1.0000816583633423, "sampling/importance_sampling_ratio/min": 0.7467939257621765, "sampling/sampling_logp_difference/max": 0.32422757148742676, "sampling/sampling_logp_difference/mean": 0.011981659568846226, "step": 249 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.693954178947024e-05, "clip_ratio/low_min": 4.693954178947024e-05, "clip_ratio/region_mean": 4.693954178947024e-05, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 719.0, "completions/mean_length": 530.25, "completions/mean_terminated_length": 463.67999267578125, "completions/min_length": 203.0, "completions/min_terminated_length": 203.0, "entropy": 0.5331684872508049, "epoch": 0.13383297644539616, "frac_reward_zero_std": 0.5, "grad_norm": 0.0049696266651153564, "learning_rate": 1e-05, "loss": 0.0073, "num_tokens": 5658067.0, "reward": 0.53125, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.44119393825531, "sampling/importance_sampling_ratio/mean": 1.0000510215759277, "sampling/importance_sampling_ratio/min": 0.5510550737380981, "sampling/sampling_logp_difference/max": 0.5959205627441406, "sampling/sampling_logp_difference/mean": 0.013961504213511944, "step": 250 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.08543525938876e-05, "clip_ratio/low_min": 5.08543525938876e-05, "clip_ratio/region_mean": 5.08543525938876e-05, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 751.0, "completions/mean_length": 586.03125, "completions/mean_terminated_length": 560.0357666015625, "completions/min_length": 362.0, "completions/min_terminated_length": 362.0, "entropy": 0.3641898073256016, "epoch": 0.13436830835117772, "frac_reward_zero_std": 0.25, "grad_norm": 0.01703214831650257, "learning_rate": 1e-05, "loss": 0.0301, "num_tokens": 5680404.0, "reward": 0.65625, "reward_std": 0.378745436668396, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.4004987478256226, "sampling/importance_sampling_ratio/mean": 1.0002009868621826, "sampling/importance_sampling_ratio/min": 0.7036284804344177, "sampling/sampling_logp_difference/max": 0.3515048027038574, "sampling/sampling_logp_difference/mean": 0.01081143319606781, "step": 251 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00016276042151730508, "clip_ratio/low_min": 0.00016276042151730508, "clip_ratio/region_mean": 0.00016276042151730508, "completions/clipped_ratio": 0.71875, "completions/max_length": 768.0, "completions/max_terminated_length": 765.0, "completions/mean_length": 706.90625, "completions/mean_terminated_length": 550.7777709960938, "completions/min_length": 369.0, "completions/min_terminated_length": 369.0, "entropy": 0.6074881665408611, "epoch": 0.1349036402569593, "frac_reward_zero_std": 0.0, "grad_norm": 0.009656330570578575, "learning_rate": 1e-05, "loss": 0.076, "num_tokens": 5707537.0, "reward": 0.34375, "reward_std": 0.4397946000099182, "rewards/accuracy_reward/mean": 0.34375, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.4766384363174438, "sampling/importance_sampling_ratio/mean": 1.000126838684082, "sampling/importance_sampling_ratio/min": 0.6533249616622925, "sampling/sampling_logp_difference/max": 0.42568063735961914, "sampling/sampling_logp_difference/mean": 0.01621743105351925, "step": 252 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 696.0, "completions/mean_length": 504.90625, "completions/mean_terminated_length": 496.4193420410156, "completions/min_length": 302.0, "completions/min_terminated_length": 302.0, "entropy": 0.28246027417480946, "epoch": 0.1354389721627409, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0446, "num_tokens": 5727046.0, "reward": 0.96875, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.96875, "rewards/accuracy_reward/std": 0.1767766922712326, "sampling/importance_sampling_ratio/max": 1.2752832174301147, "sampling/importance_sampling_ratio/mean": 1.0000879764556885, "sampling/importance_sampling_ratio/min": 0.7149717807769775, "sampling/sampling_logp_difference/max": 0.3355121612548828, "sampling/sampling_logp_difference/mean": 0.008951162919402122, "step": 253 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.770992381963879e-05, "clip_ratio/low_min": 4.770992381963879e-05, "clip_ratio/region_mean": 4.770992381963879e-05, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 744.0, "completions/mean_length": 587.34375, "completions/mean_terminated_length": 536.760009765625, "completions/min_length": 295.0, "completions/min_terminated_length": 295.0, "entropy": 0.46078526228666306, "epoch": 0.13597430406852248, "frac_reward_zero_std": 0.25, "grad_norm": 0.008114302530884743, "learning_rate": 1e-05, "loss": 0.0234, "num_tokens": 5749641.0, "reward": 0.6875, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.4352549314498901, "sampling/importance_sampling_ratio/mean": 1.0001859664916992, "sampling/importance_sampling_ratio/min": 0.7173982262611389, "sampling/sampling_logp_difference/max": 0.3613424301147461, "sampling/sampling_logp_difference/mean": 0.013140284456312656, "step": 254 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00012068824435118586, "clip_ratio/low_min": 0.00012068824435118586, "clip_ratio/region_mean": 0.00012068824435118586, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 766.0, "completions/mean_length": 495.90625, "completions/mean_terminated_length": 487.1290283203125, "completions/min_length": 293.0, "completions/min_terminated_length": 293.0, "entropy": 0.32981937006115913, "epoch": 0.13650963597430407, "frac_reward_zero_std": 0.5, "grad_norm": 0.008741078898310661, "learning_rate": 1e-05, "loss": 0.0194, "num_tokens": 5768766.0, "reward": 0.8125, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.2965061664581299, "sampling/importance_sampling_ratio/mean": 1.0001507997512817, "sampling/importance_sampling_ratio/min": 0.6830660104751587, "sampling/sampling_logp_difference/max": 0.3811638355255127, "sampling/sampling_logp_difference/mean": 0.010534428060054779, "step": 255 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 761.0, "completions/mean_length": 559.0625, "completions/mean_terminated_length": 510.8461608886719, "completions/min_length": 281.0, "completions/min_terminated_length": 281.0, "entropy": 0.47280626744031906, "epoch": 0.13704496788008566, "frac_reward_zero_std": 0.5, "grad_norm": 0.008221897296607494, "learning_rate": 1e-05, "loss": 0.0166, "num_tokens": 5790224.0, "reward": 0.5625, "reward_std": 0.249358132481575, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.355219841003418, "sampling/importance_sampling_ratio/mean": 1.0001143217086792, "sampling/importance_sampling_ratio/min": 0.782821774482727, "sampling/sampling_logp_difference/max": 0.30396366119384766, "sampling/sampling_logp_difference/mean": 0.013127387501299381, "step": 256 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 759.0, "completions/mean_length": 552.1875, "completions/mean_terminated_length": 537.800048828125, "completions/min_length": 329.0, "completions/min_terminated_length": 329.0, "entropy": 0.39103712514042854, "epoch": 0.13758029978586725, "frac_reward_zero_std": 0.5, "grad_norm": 0.0028837770223617554, "learning_rate": 1e-05, "loss": -0.005, "num_tokens": 5811094.0, "reward": 0.75, "reward_std": 0.2314550280570984, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.2821139097213745, "sampling/importance_sampling_ratio/mean": 1.0000741481781006, "sampling/importance_sampling_ratio/min": 0.6260427832603455, "sampling/sampling_logp_difference/max": 0.4683365821838379, "sampling/sampling_logp_difference/mean": 0.011365697719156742, "step": 257 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 721.0, "completions/mean_length": 533.90625, "completions/mean_terminated_length": 518.300048828125, "completions/min_length": 311.0, "completions/min_terminated_length": 311.0, "entropy": 0.3572545275092125, "epoch": 0.1381156316916488, "frac_reward_zero_std": 0.5, "grad_norm": 0.0038978864904493093, "learning_rate": 1e-05, "loss": -0.0356, "num_tokens": 5831635.0, "reward": 0.75, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.5498930215835571, "sampling/importance_sampling_ratio/mean": 0.9995436072349548, "sampling/importance_sampling_ratio/min": 0.5019902586936951, "sampling/sampling_logp_difference/max": 0.6891746520996094, "sampling/sampling_logp_difference/mean": 0.011141622439026833, "step": 258 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0001326857745880261, "clip_ratio/low_min": 0.0001326857745880261, "clip_ratio/region_mean": 0.0001326857745880261, "completions/clipped_ratio": 0.4375, "completions/max_length": 768.0, "completions/max_terminated_length": 758.0, "completions/mean_length": 669.0625, "completions/mean_terminated_length": 592.1111450195312, "completions/min_length": 341.0, "completions/min_terminated_length": 341.0, "entropy": 0.444234162569046, "epoch": 0.1386509635974304, "frac_reward_zero_std": 0.5, "grad_norm": 0.003991715610027313, "learning_rate": 1e-05, "loss": -0.0024, "num_tokens": 5856781.0, "reward": 0.1875, "reward_std": 0.249358132481575, "rewards/accuracy_reward/mean": 0.1875, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.308322787284851, "sampling/importance_sampling_ratio/mean": 0.9999985098838806, "sampling/importance_sampling_ratio/min": 0.6211650967597961, "sampling/sampling_logp_difference/max": 0.47615838050842285, "sampling/sampling_logp_difference/mean": 0.011888940818607807, "step": 259 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 745.0, "completions/mean_length": 556.0, "completions/mean_terminated_length": 534.0689697265625, "completions/min_length": 319.0, "completions/min_terminated_length": 319.0, "entropy": 0.3420996032655239, "epoch": 0.139186295503212, "frac_reward_zero_std": 0.0, "grad_norm": 0.014006612822413445, "learning_rate": 1e-05, "loss": 0.0753, "num_tokens": 5877957.0, "reward": 0.53125, "reward_std": 0.5038893818855286, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.2864056825637817, "sampling/importance_sampling_ratio/mean": 0.9997958540916443, "sampling/importance_sampling_ratio/min": 0.6887884736061096, "sampling/sampling_logp_difference/max": 0.3728210926055908, "sampling/sampling_logp_difference/mean": 0.010547701269388199, "step": 260 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.000139106712595094, "clip_ratio/low_min": 0.000139106712595094, "clip_ratio/region_mean": 0.000139106712595094, "completions/clipped_ratio": 0.375, "completions/max_length": 768.0, "completions/max_terminated_length": 750.0, "completions/mean_length": 629.28125, "completions/mean_terminated_length": 546.0499877929688, "completions/min_length": 363.0, "completions/min_terminated_length": 363.0, "entropy": 0.5140561163425446, "epoch": 0.13972162740899358, "frac_reward_zero_std": 0.5, "grad_norm": 0.008071399293839931, "learning_rate": 1e-05, "loss": -0.0269, "num_tokens": 5902118.0, "reward": 0.40625, "reward_std": 0.22201895713806152, "rewards/accuracy_reward/mean": 0.40625, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.4044896364212036, "sampling/importance_sampling_ratio/mean": 1.0003046989440918, "sampling/importance_sampling_ratio/min": 0.7481744289398193, "sampling/sampling_logp_difference/max": 0.3396739959716797, "sampling/sampling_logp_difference/mean": 0.01442759856581688, "step": 261 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 9.36329597607255e-05, "clip_ratio/low_min": 9.36329597607255e-05, "clip_ratio/region_mean": 9.36329597607255e-05, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 678.0, "completions/mean_length": 509.875, "completions/mean_terminated_length": 437.5999755859375, "completions/min_length": 222.0, "completions/min_terminated_length": 222.0, "entropy": 0.4288115091621876, "epoch": 0.14025695931477516, "frac_reward_zero_std": 0.0, "grad_norm": 0.018146095797419548, "learning_rate": 1e-05, "loss": 0.0464, "num_tokens": 5921810.0, "reward": 0.53125, "reward_std": 0.4628904461860657, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.3087602853775024, "sampling/importance_sampling_ratio/mean": 0.9999146461486816, "sampling/importance_sampling_ratio/min": 0.732586145401001, "sampling/sampling_logp_difference/max": 0.3111743927001953, "sampling/sampling_logp_difference/mean": 0.01181797869503498, "step": 262 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00010246776582789607, "clip_ratio/low_min": 0.00010246776582789607, "clip_ratio/region_mean": 0.00010246776582789607, "completions/clipped_ratio": 0.3125, "completions/max_length": 768.0, "completions/max_terminated_length": 736.0, "completions/mean_length": 600.59375, "completions/mean_terminated_length": 524.5, "completions/min_length": 340.0, "completions/min_terminated_length": 340.0, "entropy": 0.43451959267258644, "epoch": 0.14079229122055675, "frac_reward_zero_std": 0.0, "grad_norm": 0.011997496709227562, "learning_rate": 1e-05, "loss": 0.0171, "num_tokens": 5944581.0, "reward": 0.40625, "reward_std": 0.48928019404411316, "rewards/accuracy_reward/mean": 0.40625, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.4284863471984863, "sampling/importance_sampling_ratio/mean": 0.9999670386314392, "sampling/importance_sampling_ratio/min": 0.5742226839065552, "sampling/sampling_logp_difference/max": 0.5547380447387695, "sampling/sampling_logp_difference/mean": 0.012722854502499104, "step": 263 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 735.0, "completions/mean_length": 522.625, "completions/mean_terminated_length": 466.0000305175781, "completions/min_length": 248.0, "completions/min_terminated_length": 248.0, "entropy": 0.4376721791923046, "epoch": 0.14132762312633834, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.009, "num_tokens": 5965257.0, "reward": 0.1875, "reward_std": 0.1157275140285492, "rewards/accuracy_reward/mean": 0.1875, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.312913179397583, "sampling/importance_sampling_ratio/mean": 1.0000436305999756, "sampling/importance_sampling_ratio/min": 0.7160667181015015, "sampling/sampling_logp_difference/max": 0.333981990814209, "sampling/sampling_logp_difference/mean": 0.013143196702003479, "step": 264 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.3125, "completions/max_length": 768.0, "completions/max_terminated_length": 734.0, "completions/mean_length": 632.34375, "completions/mean_terminated_length": 570.6818237304688, "completions/min_length": 385.0, "completions/min_terminated_length": 385.0, "entropy": 0.4544507935643196, "epoch": 0.1418629550321199, "frac_reward_zero_std": 0.5, "grad_norm": 0.013826858252286911, "learning_rate": 1e-05, "loss": 0.0514, "num_tokens": 5989164.0, "reward": 0.5625, "reward_std": 0.249358132481575, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.4265916347503662, "sampling/importance_sampling_ratio/mean": 1.0002870559692383, "sampling/importance_sampling_ratio/min": 0.6972841024398804, "sampling/sampling_logp_difference/max": 0.3605623245239258, "sampling/sampling_logp_difference/mean": 0.012940780259668827, "step": 265 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.469979416579008e-05, "clip_ratio/low_min": 6.469979416579008e-05, "clip_ratio/region_mean": 6.469979416579008e-05, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 760.0, "completions/mean_length": 592.375, "completions/mean_terminated_length": 543.2000122070312, "completions/min_length": 199.0, "completions/min_terminated_length": 199.0, "entropy": 0.5141049586236477, "epoch": 0.1423982869379015, "frac_reward_zero_std": 0.0, "grad_norm": 0.015743238851428032, "learning_rate": 1e-05, "loss": 0.0378, "num_tokens": 6012272.0, "reward": 0.65625, "reward_std": 0.47137710452079773, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.759840965270996, "sampling/importance_sampling_ratio/mean": 0.9995988011360168, "sampling/importance_sampling_ratio/min": 0.7500654458999634, "sampling/sampling_logp_difference/max": 0.5652234554290771, "sampling/sampling_logp_difference/mean": 0.014157882891595364, "step": 266 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 744.0, "completions/max_terminated_length": 744.0, "completions/mean_length": 504.5625, "completions/mean_terminated_length": 504.5625, "completions/min_length": 303.0, "completions/min_terminated_length": 303.0, "entropy": 0.38440496288239956, "epoch": 0.14293361884368308, "frac_reward_zero_std": 0.25, "grad_norm": 0.017426220700144768, "learning_rate": 1e-05, "loss": -0.0089, "num_tokens": 6031738.0, "reward": 0.90625, "reward_std": 0.2651650309562683, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.4726731777191162, "sampling/importance_sampling_ratio/mean": 0.9999735951423645, "sampling/importance_sampling_ratio/min": 0.6526588201522827, "sampling/sampling_logp_difference/max": 0.4267008304595947, "sampling/sampling_logp_difference/mean": 0.011597482487559319, "step": 267 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.858142347075045e-05, "clip_ratio/low_min": 4.858142347075045e-05, "clip_ratio/region_mean": 4.858142347075045e-05, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 699.0, "completions/mean_length": 560.78125, "completions/mean_terminated_length": 502.7599792480469, "completions/min_length": 257.0, "completions/min_terminated_length": 257.0, "entropy": 0.35142942145466805, "epoch": 0.14346895074946467, "frac_reward_zero_std": 0.25, "grad_norm": 0.007161566521972418, "learning_rate": 1e-05, "loss": -0.0032, "num_tokens": 6053307.0, "reward": 0.78125, "reward_std": 0.3377465009689331, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.400130271911621, "sampling/importance_sampling_ratio/mean": 1.000180721282959, "sampling/importance_sampling_ratio/min": 0.7326385378837585, "sampling/sampling_logp_difference/max": 0.3365652561187744, "sampling/sampling_logp_difference/mean": 0.009951233863830566, "step": 268 }, { "clip_ratio/high_max": 6.887052586534992e-05, "clip_ratio/high_mean": 6.887052586534992e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 6.887052586534992e-05, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 760.0, "completions/mean_length": 484.5625, "completions/mean_terminated_length": 455.2413635253906, "completions/min_length": 234.0, "completions/min_terminated_length": 234.0, "entropy": 0.41471974551677704, "epoch": 0.14400428265524626, "frac_reward_zero_std": 0.0, "grad_norm": 0.013710001483559608, "learning_rate": 1e-05, "loss": -0.0051, "num_tokens": 6072165.0, "reward": 0.5, "reward_std": 0.4082317352294922, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5080004930496216, "sampling/importance_sampling_ratio/max": 1.4713863134384155, "sampling/importance_sampling_ratio/mean": 0.9999328255653381, "sampling/importance_sampling_ratio/min": 0.6127001643180847, "sampling/sampling_logp_difference/max": 0.4898796081542969, "sampling/sampling_logp_difference/mean": 0.012783356010913849, "step": 269 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 755.0, "completions/mean_length": 495.59375, "completions/mean_terminated_length": 456.6785888671875, "completions/min_length": 258.0, "completions/min_terminated_length": 258.0, "entropy": 0.4830066040158272, "epoch": 0.14453961456102785, "frac_reward_zero_std": 0.5, "grad_norm": 0.0023124227300286293, "learning_rate": 1e-05, "loss": 0.0654, "num_tokens": 6091312.0, "reward": 0.875, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 1.2866265773773193, "sampling/importance_sampling_ratio/mean": 1.0001729726791382, "sampling/importance_sampling_ratio/min": 0.6610352993011475, "sampling/sampling_logp_difference/max": 0.41394805908203125, "sampling/sampling_logp_difference/mean": 0.012617088854312897, "step": 270 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 9.790906915441155e-05, "clip_ratio/low_min": 9.790906915441155e-05, "clip_ratio/region_mean": 9.790906915441155e-05, "completions/clipped_ratio": 0.3125, "completions/max_length": 768.0, "completions/max_terminated_length": 733.0, "completions/mean_length": 613.28125, "completions/mean_terminated_length": 542.95458984375, "completions/min_length": 323.0, "completions/min_terminated_length": 323.0, "entropy": 0.413370955735445, "epoch": 0.14507494646680943, "frac_reward_zero_std": 0.25, "grad_norm": 0.009229264222085476, "learning_rate": 1e-05, "loss": 0.0575, "num_tokens": 6114313.0, "reward": 0.65625, "reward_std": 0.3966485261917114, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.373965859413147, "sampling/importance_sampling_ratio/mean": 1.0002484321594238, "sampling/importance_sampling_ratio/min": 0.7165717482566833, "sampling/sampling_logp_difference/max": 0.3332768678665161, "sampling/sampling_logp_difference/mean": 0.012102281674742699, "step": 271 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 651.0, "completions/mean_length": 550.3125, "completions/mean_terminated_length": 519.2142944335938, "completions/min_length": 253.0, "completions/min_terminated_length": 253.0, "entropy": 0.41443566605448723, "epoch": 0.145610278372591, "frac_reward_zero_std": 0.25, "grad_norm": 0.00584491016343236, "learning_rate": 1e-05, "loss": -0.0001, "num_tokens": 6135643.0, "reward": 0.78125, "reward_std": 0.3377465009689331, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.6588126420974731, "sampling/importance_sampling_ratio/mean": 1.000101923942566, "sampling/importance_sampling_ratio/min": 0.39590126276016235, "sampling/sampling_logp_difference/max": 0.9265904426574707, "sampling/sampling_logp_difference/mean": 0.012218380346894264, "step": 272 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.796623034053482e-05, "clip_ratio/low_min": 4.796623034053482e-05, "clip_ratio/region_mean": 4.796623034053482e-05, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 767.0, "completions/mean_length": 479.875, "completions/mean_terminated_length": 426.5185241699219, "completions/min_length": 176.0, "completions/min_terminated_length": 176.0, "entropy": 0.43369118496775627, "epoch": 0.14614561027837258, "frac_reward_zero_std": 0.0, "grad_norm": 0.020795246586203575, "learning_rate": 1e-05, "loss": 0.016, "num_tokens": 6154295.0, "reward": 0.46875, "reward_std": 0.4397946000099182, "rewards/accuracy_reward/mean": 0.46875, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.3095465898513794, "sampling/importance_sampling_ratio/mean": 0.9997921586036682, "sampling/importance_sampling_ratio/min": 0.6530885696411133, "sampling/sampling_logp_difference/max": 0.4260425567626953, "sampling/sampling_logp_difference/mean": 0.013379311189055443, "step": 273 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.2105042414041236e-05, "clip_ratio/low_min": 5.2105042414041236e-05, "clip_ratio/region_mean": 5.2105042414041236e-05, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 743.0, "completions/mean_length": 578.4375, "completions/mean_terminated_length": 525.3599853515625, "completions/min_length": 292.0, "completions/min_terminated_length": 292.0, "entropy": 0.3660472631454468, "epoch": 0.14668094218415417, "frac_reward_zero_std": 0.5, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0219, "num_tokens": 6176293.0, "reward": 0.40625, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.40625, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.4787331819534302, "sampling/importance_sampling_ratio/mean": 1.0000271797180176, "sampling/importance_sampling_ratio/min": 0.6978476047515869, "sampling/sampling_logp_difference/max": 0.3911857604980469, "sampling/sampling_logp_difference/mean": 0.010998491197824478, "step": 274 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 734.0, "completions/mean_length": 542.4375, "completions/mean_terminated_length": 490.3846435546875, "completions/min_length": 267.0, "completions/min_terminated_length": 267.0, "entropy": 0.5499711148440838, "epoch": 0.14721627408993576, "frac_reward_zero_std": 0.25, "grad_norm": 0.015120045281946659, "learning_rate": 1e-05, "loss": 0.1106, "num_tokens": 6197379.0, "reward": 0.71875, "reward_std": 0.3787454068660736, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.3861445188522339, "sampling/importance_sampling_ratio/mean": 0.9997429847717285, "sampling/importance_sampling_ratio/min": 0.6772403717041016, "sampling/sampling_logp_difference/max": 0.38972902297973633, "sampling/sampling_logp_difference/mean": 0.015370858833193779, "step": 275 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00010428234600112773, "clip_ratio/low_min": 0.00010428234600112773, "clip_ratio/region_mean": 0.00010428234600112773, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 753.0, "completions/mean_length": 542.15625, "completions/mean_terminated_length": 490.0384826660156, "completions/min_length": 313.0, "completions/min_terminated_length": 313.0, "entropy": 0.4484858773648739, "epoch": 0.14775160599571735, "frac_reward_zero_std": 0.5, "grad_norm": 0.002846313640475273, "learning_rate": 1e-05, "loss": 0.0359, "num_tokens": 6218568.0, "reward": 0.84375, "reward_std": 0.22201895713806152, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.3625099658966064, "sampling/importance_sampling_ratio/mean": 1.00043523311615, "sampling/importance_sampling_ratio/min": 0.6556280851364136, "sampling/sampling_logp_difference/max": 0.4221615791320801, "sampling/sampling_logp_difference/mean": 0.012717328034341335, "step": 276 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00015897070625214837, "clip_ratio/low_min": 0.00015897070625214837, "clip_ratio/region_mean": 0.00015897070625214837, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 756.0, "completions/mean_length": 588.28125, "completions/mean_terminated_length": 546.8077392578125, "completions/min_length": 268.0, "completions/min_terminated_length": 268.0, "entropy": 0.3977391868829727, "epoch": 0.14828693790149894, "frac_reward_zero_std": 0.25, "grad_norm": 0.008605397306382656, "learning_rate": 1e-05, "loss": 0.0472, "num_tokens": 6241449.0, "reward": 0.65625, "reward_std": 0.378745436668396, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.5247626304626465, "sampling/importance_sampling_ratio/mean": 0.9999570250511169, "sampling/importance_sampling_ratio/min": 0.7116842269897461, "sampling/sampling_logp_difference/max": 0.42183876037597656, "sampling/sampling_logp_difference/mean": 0.011903348378837109, "step": 277 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 734.0, "completions/mean_length": 569.8125, "completions/mean_terminated_length": 541.5, "completions/min_length": 322.0, "completions/min_terminated_length": 322.0, "entropy": 0.33303431794047356, "epoch": 0.14882226980728053, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 6263451.0, "reward": 0.5, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5080004930496216, "sampling/importance_sampling_ratio/max": 1.506096363067627, "sampling/importance_sampling_ratio/mean": 1.0001150369644165, "sampling/importance_sampling_ratio/min": 0.7046422958374023, "sampling/sampling_logp_difference/max": 0.40952110290527344, "sampling/sampling_logp_difference/mean": 0.010882987640798092, "step": 278 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00025589248616597615, "clip_ratio/low_min": 0.00025589248616597615, "clip_ratio/region_mean": 0.00025589248616597615, "completions/clipped_ratio": 0.34375, "completions/max_length": 768.0, "completions/max_terminated_length": 735.0, "completions/mean_length": 597.09375, "completions/mean_terminated_length": 507.5714416503906, "completions/min_length": 320.0, "completions/min_terminated_length": 320.0, "entropy": 0.7360854744911194, "epoch": 0.1493576017130621, "frac_reward_zero_std": 0.5, "grad_norm": 0.0050915502943098545, "learning_rate": 1e-05, "loss": 0.0607, "num_tokens": 6287118.0, "reward": 0.40625, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.40625, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0002696514129639, "sampling/importance_sampling_ratio/min": 0.6256908178329468, "sampling/sampling_logp_difference/max": 0.928502082824707, "sampling/sampling_logp_difference/mean": 0.01567136123776436, "step": 279 }, { "clip_ratio/high_max": 6.834335363237187e-05, "clip_ratio/high_mean": 6.834335363237187e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 6.834335363237187e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 642.0, "completions/max_terminated_length": 642.0, "completions/mean_length": 431.71875, "completions/mean_terminated_length": 431.71875, "completions/min_length": 234.0, "completions/min_terminated_length": 234.0, "entropy": 0.2850475553423166, "epoch": 0.14989293361884368, "frac_reward_zero_std": 0.25, "grad_norm": 0.011554716154932976, "learning_rate": 1e-05, "loss": 0.006, "num_tokens": 6304021.0, "reward": 0.84375, "reward_std": 0.3198433816432953, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.3164055347442627, "sampling/importance_sampling_ratio/mean": 1.0000574588775635, "sampling/importance_sampling_ratio/min": 0.5672076940536499, "sampling/sampling_logp_difference/max": 0.5670297145843506, "sampling/sampling_logp_difference/mean": 0.00954667292535305, "step": 280 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 715.0, "completions/mean_length": 574.96875, "completions/mean_terminated_length": 562.1000366210938, "completions/min_length": 351.0, "completions/min_terminated_length": 351.0, "entropy": 0.4140516445040703, "epoch": 0.15042826552462527, "frac_reward_zero_std": 0.5, "grad_norm": 0.002030004281550646, "learning_rate": 1e-05, "loss": -0.0161, "num_tokens": 6326292.0, "reward": 0.8125, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.403630018234253, "sampling/importance_sampling_ratio/mean": 0.9998246431350708, "sampling/importance_sampling_ratio/min": 0.6674271821975708, "sampling/sampling_logp_difference/max": 0.404325008392334, "sampling/sampling_logp_difference/mean": 0.012476157397031784, "step": 281 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00010199918324360624, "clip_ratio/low_min": 0.00010199918324360624, "clip_ratio/region_mean": 0.00010199918324360624, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 764.0, "completions/mean_length": 551.28125, "completions/mean_terminated_length": 544.290283203125, "completions/min_length": 346.0, "completions/min_terminated_length": 346.0, "entropy": 0.3446253836154938, "epoch": 0.15096359743040685, "frac_reward_zero_std": 0.5, "grad_norm": 0.007887735031545162, "learning_rate": 1e-05, "loss": -0.0259, "num_tokens": 6347365.0, "reward": 0.8125, "reward_std": 0.249358132481575, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.43865966796875, "sampling/importance_sampling_ratio/mean": 1.000063180923462, "sampling/importance_sampling_ratio/min": 0.5418812036514282, "sampling/sampling_logp_difference/max": 0.612708568572998, "sampling/sampling_logp_difference/mean": 0.01101731602102518, "step": 282 }, { "clip_ratio/high_max": 5.5580258049303666e-05, "clip_ratio/high_mean": 5.5580258049303666e-05, "clip_ratio/low_mean": 4.4326239731162786e-05, "clip_ratio/low_min": 4.4326239731162786e-05, "clip_ratio/region_mean": 9.990649778046645e-05, "completions/clipped_ratio": 0.65625, "completions/max_length": 768.0, "completions/max_terminated_length": 764.0, "completions/mean_length": 690.375, "completions/mean_terminated_length": 542.1818237304688, "completions/min_length": 321.0, "completions/min_terminated_length": 321.0, "entropy": 0.4889049455523491, "epoch": 0.15149892933618844, "frac_reward_zero_std": 0.25, "grad_norm": 0.014394853264093399, "learning_rate": 1e-05, "loss": 0.0643, "num_tokens": 6373537.0, "reward": 0.375, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.375, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.585191249847412, "sampling/importance_sampling_ratio/mean": 1.0000473260879517, "sampling/importance_sampling_ratio/min": 0.6346446871757507, "sampling/sampling_logp_difference/max": 0.460705041885376, "sampling/sampling_logp_difference/mean": 0.012962386943399906, "step": 283 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00012900454385089688, "clip_ratio/low_min": 0.00012900454385089688, "clip_ratio/region_mean": 0.00012900454385089688, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 723.0, "completions/mean_length": 562.46875, "completions/mean_terminated_length": 515.0384521484375, "completions/min_length": 318.0, "completions/min_terminated_length": 318.0, "entropy": 0.4238313138484955, "epoch": 0.15203426124197003, "frac_reward_zero_std": 0.5, "grad_norm": 0.015825411304831505, "learning_rate": 1e-05, "loss": 0.0603, "num_tokens": 6395344.0, "reward": 0.78125, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.6557939052581787, "sampling/importance_sampling_ratio/mean": 1.0001264810562134, "sampling/importance_sampling_ratio/min": 0.42597201466560364, "sampling/sampling_logp_difference/max": 0.8533816337585449, "sampling/sampling_logp_difference/mean": 0.011711214669048786, "step": 284 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00012601517300936393, "clip_ratio/low_min": 0.00012601517300936393, "clip_ratio/region_mean": 0.00012601517300936393, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 735.0, "completions/mean_length": 535.90625, "completions/mean_terminated_length": 470.91998291015625, "completions/min_length": 250.0, "completions/min_terminated_length": 250.0, "entropy": 0.4247622564435005, "epoch": 0.15256959314775162, "frac_reward_zero_std": 0.5, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0074, "num_tokens": 6416453.0, "reward": 0.625, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.3460848331451416, "sampling/importance_sampling_ratio/mean": 0.9999608397483826, "sampling/importance_sampling_ratio/min": 0.6614370942115784, "sampling/sampling_logp_difference/max": 0.4133404493331909, "sampling/sampling_logp_difference/mean": 0.012951512821018696, "step": 285 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.879586024093442e-05, "clip_ratio/low_min": 5.879586024093442e-05, "clip_ratio/region_mean": 5.879586024093442e-05, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 720.0, "completions/mean_length": 520.21875, "completions/mean_terminated_length": 503.70001220703125, "completions/min_length": 297.0, "completions/min_terminated_length": 297.0, "entropy": 0.39539410546422005, "epoch": 0.15310492505353318, "frac_reward_zero_std": 0.25, "grad_norm": 0.0048318887129426, "learning_rate": 1e-05, "loss": 0.0713, "num_tokens": 6436668.0, "reward": 0.90625, "reward_std": 0.2651650309562683, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.354333519935608, "sampling/importance_sampling_ratio/mean": 0.9997705221176147, "sampling/importance_sampling_ratio/min": 0.6209386587142944, "sampling/sampling_logp_difference/max": 0.47652292251586914, "sampling/sampling_logp_difference/mean": 0.011598610319197178, "step": 286 }, { "clip_ratio/high_max": 6.319514795904979e-05, "clip_ratio/high_mean": 6.319514795904979e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 6.319514795904979e-05, "completions/clipped_ratio": 0.28125, "completions/max_length": 768.0, "completions/max_terminated_length": 716.0, "completions/mean_length": 584.9375, "completions/mean_terminated_length": 513.3043823242188, "completions/min_length": 254.0, "completions/min_terminated_length": 254.0, "entropy": 0.5881256833672523, "epoch": 0.15364025695931477, "frac_reward_zero_std": 0.0, "grad_norm": 0.00954087171703577, "learning_rate": 1e-05, "loss": 0.0026, "num_tokens": 6459010.0, "reward": 0.46875, "reward_std": 0.4218915104866028, "rewards/accuracy_reward/mean": 0.46875, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.3682405948638916, "sampling/importance_sampling_ratio/mean": 0.999691367149353, "sampling/importance_sampling_ratio/min": 0.16219860315322876, "sampling/sampling_logp_difference/max": 1.8189337253570557, "sampling/sampling_logp_difference/mean": 0.01596952974796295, "step": 287 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.627915768651292e-05, "clip_ratio/low_min": 4.627915768651292e-05, "clip_ratio/region_mean": 4.627915768651292e-05, "completions/clipped_ratio": 0.28125, "completions/max_length": 768.0, "completions/max_terminated_length": 758.0, "completions/mean_length": 632.25, "completions/mean_terminated_length": 579.1304321289062, "completions/min_length": 415.0, "completions/min_terminated_length": 415.0, "entropy": 0.3997486066073179, "epoch": 0.15417558886509636, "frac_reward_zero_std": 0.25, "grad_norm": 0.004742585588246584, "learning_rate": 1e-05, "loss": 0.0729, "num_tokens": 6483682.0, "reward": 0.625, "reward_std": 0.3650856614112854, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.3171533346176147, "sampling/importance_sampling_ratio/mean": 0.9998674988746643, "sampling/importance_sampling_ratio/min": 0.6995875835418701, "sampling/sampling_logp_difference/max": 0.35726428031921387, "sampling/sampling_logp_difference/mean": 0.011724493466317654, "step": 288 }, { "clip_ratio/high_max": 4.958349745720625e-05, "clip_ratio/high_mean": 4.958349745720625e-05, "clip_ratio/low_mean": 0.00020616228721337393, "clip_ratio/low_min": 0.00020616228721337393, "clip_ratio/region_mean": 0.0002557457846705802, "completions/clipped_ratio": 0.28125, "completions/max_length": 768.0, "completions/max_terminated_length": 691.0, "completions/mean_length": 605.5625, "completions/mean_terminated_length": 542.0, "completions/min_length": 348.0, "completions/min_terminated_length": 348.0, "entropy": 0.41044608503580093, "epoch": 0.15471092077087795, "frac_reward_zero_std": 0.0, "grad_norm": 0.012034800834953785, "learning_rate": 1e-05, "loss": -0.031, "num_tokens": 6506732.0, "reward": 0.59375, "reward_std": 0.4218915104866028, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.4353028535842896, "sampling/importance_sampling_ratio/mean": 1.0002548694610596, "sampling/importance_sampling_ratio/min": 0.5381591320037842, "sampling/sampling_logp_difference/max": 0.6196010112762451, "sampling/sampling_logp_difference/mean": 0.012869988568127155, "step": 289 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.4216485548531637e-05, "clip_ratio/low_min": 4.4216485548531637e-05, "clip_ratio/region_mean": 4.4216485548531637e-05, "completions/clipped_ratio": 0.34375, "completions/max_length": 768.0, "completions/max_terminated_length": 761.0, "completions/mean_length": 630.03125, "completions/mean_terminated_length": 557.7619018554688, "completions/min_length": 235.0, "completions/min_terminated_length": 235.0, "entropy": 0.3749997690320015, "epoch": 0.15524625267665954, "frac_reward_zero_std": 0.25, "grad_norm": 0.01595156081020832, "learning_rate": 1e-05, "loss": 0.0419, "num_tokens": 6530765.0, "reward": 0.46875, "reward_std": 0.3608423173427582, "rewards/accuracy_reward/mean": 0.46875, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.4583345651626587, "sampling/importance_sampling_ratio/mean": 1.0001922845840454, "sampling/importance_sampling_ratio/min": 0.6439215540885925, "sampling/sampling_logp_difference/max": 0.44017839431762695, "sampling/sampling_logp_difference/mean": 0.011338340118527412, "step": 290 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 737.0, "completions/mean_length": 576.5625, "completions/mean_terminated_length": 532.3846435546875, "completions/min_length": 222.0, "completions/min_terminated_length": 222.0, "entropy": 0.30455345287919044, "epoch": 0.15578158458244112, "frac_reward_zero_std": 0.5, "grad_norm": 0.0046155196614563465, "learning_rate": 1e-05, "loss": 0.0296, "num_tokens": 6552679.0, "reward": 0.875, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 1.3491917848587036, "sampling/importance_sampling_ratio/mean": 0.9999334216117859, "sampling/importance_sampling_ratio/min": 0.6191225051879883, "sampling/sampling_logp_difference/max": 0.47945213317871094, "sampling/sampling_logp_difference/mean": 0.009973663836717606, "step": 291 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.7339449995197356e-05, "clip_ratio/low_min": 5.7339449995197356e-05, "clip_ratio/region_mean": 5.7339449995197356e-05, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 764.0, "completions/mean_length": 566.09375, "completions/mean_terminated_length": 528.7037353515625, "completions/min_length": 309.0, "completions/min_terminated_length": 309.0, "entropy": 0.3746066950261593, "epoch": 0.15631691648822268, "frac_reward_zero_std": 0.25, "grad_norm": 0.010570799931883812, "learning_rate": 1e-05, "loss": 0.0421, "num_tokens": 6574674.0, "reward": 0.78125, "reward_std": 0.3377465009689331, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.6160459518432617, "sampling/importance_sampling_ratio/mean": 1.000115990638733, "sampling/importance_sampling_ratio/min": 0.663177490234375, "sampling/sampling_logp_difference/max": 0.4799823760986328, "sampling/sampling_logp_difference/mean": 0.012090645730495453, "step": 292 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00015084210826898925, "clip_ratio/low_min": 0.00015084210826898925, "clip_ratio/region_mean": 0.00015084210826898925, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 763.0, "completions/mean_length": 642.15625, "completions/mean_terminated_length": 600.2083740234375, "completions/min_length": 377.0, "completions/min_terminated_length": 377.0, "entropy": 0.46960141137242317, "epoch": 0.15685224839400427, "frac_reward_zero_std": 0.0, "grad_norm": 0.006962858606129885, "learning_rate": 1e-05, "loss": 0.0248, "num_tokens": 6599495.0, "reward": 0.5, "reward_std": 0.4492306709289551, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5080004930496216, "sampling/importance_sampling_ratio/max": 1.350098729133606, "sampling/importance_sampling_ratio/mean": 1.0000478029251099, "sampling/importance_sampling_ratio/min": 0.7264196872711182, "sampling/sampling_logp_difference/max": 0.3196272850036621, "sampling/sampling_logp_difference/mean": 0.013042032718658447, "step": 293 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 722.0, "completions/mean_length": 535.21875, "completions/mean_terminated_length": 457.625, "completions/min_length": 309.0, "completions/min_terminated_length": 309.0, "entropy": 0.47649625316262245, "epoch": 0.15738758029978586, "frac_reward_zero_std": 0.25, "grad_norm": 0.005913050379604101, "learning_rate": 1e-05, "loss": 0.1513, "num_tokens": 6620262.0, "reward": 0.6875, "reward_std": 0.2925041913986206, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.8259203433990479, "sampling/importance_sampling_ratio/mean": 0.9998753666877747, "sampling/importance_sampling_ratio/min": 0.7434327602386475, "sampling/sampling_logp_difference/max": 0.6020841598510742, "sampling/sampling_logp_difference/mean": 0.014077939093112946, "step": 294 }, { "clip_ratio/high_max": 6.351625779643655e-05, "clip_ratio/high_mean": 6.351625779643655e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 6.351625779643655e-05, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 647.0, "completions/mean_length": 494.46875, "completions/mean_terminated_length": 455.39288330078125, "completions/min_length": 292.0, "completions/min_terminated_length": 292.0, "entropy": 0.42062101513147354, "epoch": 0.15792291220556745, "frac_reward_zero_std": 0.5, "grad_norm": 0.010757889598608017, "learning_rate": 1e-05, "loss": 0.0413, "num_tokens": 6639845.0, "reward": 0.8125, "reward_std": 0.249358132481575, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.4439904689788818, "sampling/importance_sampling_ratio/mean": 1.0000611543655396, "sampling/importance_sampling_ratio/min": 0.6982812881469727, "sampling/sampling_logp_difference/max": 0.36741042137145996, "sampling/sampling_logp_difference/mean": 0.012994526885449886, "step": 295 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 755.0, "completions/mean_length": 561.5625, "completions/mean_terminated_length": 532.0714721679688, "completions/min_length": 338.0, "completions/min_terminated_length": 338.0, "entropy": 0.45113179460167885, "epoch": 0.15845824411134904, "frac_reward_zero_std": 0.5, "grad_norm": 0.014833190478384495, "learning_rate": 1e-05, "loss": 0.0159, "num_tokens": 6661119.0, "reward": 0.78125, "reward_std": 0.2630178928375244, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.3035452365875244, "sampling/importance_sampling_ratio/mean": 0.999748170375824, "sampling/importance_sampling_ratio/min": 0.7063039541244507, "sampling/sampling_logp_difference/max": 0.34770965576171875, "sampling/sampling_logp_difference/mean": 0.012418637052178383, "step": 296 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00017841537919593975, "clip_ratio/low_min": 0.00017841537919593975, "clip_ratio/region_mean": 0.00017841537919593975, "completions/clipped_ratio": 0.28125, "completions/max_length": 768.0, "completions/max_terminated_length": 732.0, "completions/mean_length": 562.3125, "completions/mean_terminated_length": 481.82611083984375, "completions/min_length": 220.0, "completions/min_terminated_length": 220.0, "entropy": 0.4274291656911373, "epoch": 0.15899357601713063, "frac_reward_zero_std": 0.25, "grad_norm": 0.007896609604358673, "learning_rate": 1e-05, "loss": 0.0588, "num_tokens": 6682857.0, "reward": 0.53125, "reward_std": 0.35564959049224854, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.6551094055175781, "sampling/importance_sampling_ratio/mean": 0.9993698000907898, "sampling/importance_sampling_ratio/min": 0.6031726598739624, "sampling/sampling_logp_difference/max": 0.505551815032959, "sampling/sampling_logp_difference/mean": 0.012979673221707344, "step": 297 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 647.0, "completions/mean_length": 447.40625, "completions/mean_terminated_length": 437.06451416015625, "completions/min_length": 266.0, "completions/min_terminated_length": 266.0, "entropy": 0.4299395754933357, "epoch": 0.15952890792291222, "frac_reward_zero_std": 0.75, "grad_norm": 0.004124860744923353, "learning_rate": 1e-05, "loss": 0.0293, "num_tokens": 6700470.0, "reward": 0.9375, "reward_std": 0.1157275140285492, "rewards/accuracy_reward/mean": 0.9375, "rewards/accuracy_reward/std": 0.24593468010425568, "sampling/importance_sampling_ratio/max": 1.3150358200073242, "sampling/importance_sampling_ratio/mean": 0.9992318749427795, "sampling/importance_sampling_ratio/min": 0.5699240565299988, "sampling/sampling_logp_difference/max": 0.5622521638870239, "sampling/sampling_logp_difference/mean": 0.01316915825009346, "step": 298 }, { "clip_ratio/high_max": 5.173841054784134e-05, "clip_ratio/high_mean": 5.173841054784134e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 5.173841054784134e-05, "completions/clipped_ratio": 0.4375, "completions/max_length": 768.0, "completions/max_terminated_length": 682.0, "completions/mean_length": 612.28125, "completions/mean_terminated_length": 491.1666564941406, "completions/min_length": 254.0, "completions/min_terminated_length": 254.0, "entropy": 0.4086042698472738, "epoch": 0.16006423982869378, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": -0.0149, "num_tokens": 6724039.0, "reward": 0.46875, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.46875, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.500494360923767, "sampling/importance_sampling_ratio/mean": 1.0000535249710083, "sampling/importance_sampling_ratio/min": 0.6633061766624451, "sampling/sampling_logp_difference/max": 0.4105186462402344, "sampling/sampling_logp_difference/mean": 0.011918322183191776, "step": 299 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 736.0, "completions/mean_length": 589.71875, "completions/mean_terminated_length": 548.5769653320312, "completions/min_length": 289.0, "completions/min_terminated_length": 289.0, "entropy": 0.3584157060831785, "epoch": 0.16059957173447537, "frac_reward_zero_std": 0.25, "grad_norm": 0.004774761851876974, "learning_rate": 1e-05, "loss": 0.0465, "num_tokens": 6746910.0, "reward": 0.6875, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.545341968536377, "sampling/importance_sampling_ratio/mean": 0.9997391104698181, "sampling/importance_sampling_ratio/min": 0.7122886776924133, "sampling/sampling_logp_difference/max": 0.4352452754974365, "sampling/sampling_logp_difference/mean": 0.011193247511982918, "step": 300 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.413545634131879e-05, "clip_ratio/low_min": 6.413545634131879e-05, "clip_ratio/region_mean": 6.413545634131879e-05, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 762.0, "completions/mean_length": 490.46875, "completions/mean_terminated_length": 439.0740661621094, "completions/min_length": 217.0, "completions/min_terminated_length": 217.0, "entropy": 0.4165109172463417, "epoch": 0.16113490364025695, "frac_reward_zero_std": 0.25, "grad_norm": 0.009167240932583809, "learning_rate": 1e-05, "loss": 0.0491, "num_tokens": 6766317.0, "reward": 0.625, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.4356251955032349, "sampling/importance_sampling_ratio/mean": 1.000047206878662, "sampling/importance_sampling_ratio/min": 0.6076681017875671, "sampling/sampling_logp_difference/max": 0.4981265068054199, "sampling/sampling_logp_difference/mean": 0.013111790642142296, "step": 301 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 767.0, "completions/mean_length": 549.5, "completions/mean_terminated_length": 518.2857666015625, "completions/min_length": 182.0, "completions/min_terminated_length": 182.0, "entropy": 0.40456661209464073, "epoch": 0.16167023554603854, "frac_reward_zero_std": 0.25, "grad_norm": 0.01440720446407795, "learning_rate": 1e-05, "loss": 0.0694, "num_tokens": 6787725.0, "reward": 0.75, "reward_std": 0.3514062464237213, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.3568141460418701, "sampling/importance_sampling_ratio/mean": 1.0003018379211426, "sampling/importance_sampling_ratio/min": 0.6405941247940063, "sampling/sampling_logp_difference/max": 0.4453592300415039, "sampling/sampling_logp_difference/mean": 0.012022528797388077, "step": 302 }, { "clip_ratio/high_max": 4.461099160835147e-05, "clip_ratio/high_mean": 4.461099160835147e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 4.461099160835147e-05, "completions/clipped_ratio": 0.5, "completions/max_length": 768.0, "completions/max_terminated_length": 746.0, "completions/mean_length": 657.0, "completions/mean_terminated_length": 546.0, "completions/min_length": 354.0, "completions/min_terminated_length": 354.0, "entropy": 0.5198293589055538, "epoch": 0.16220556745182013, "frac_reward_zero_std": 0.5, "grad_norm": 0.012838945724070072, "learning_rate": 1e-05, "loss": 0.0428, "num_tokens": 6812285.0, "reward": 0.5625, "reward_std": 0.2587745785713196, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.3716440200805664, "sampling/importance_sampling_ratio/mean": 1.000239372253418, "sampling/importance_sampling_ratio/min": 0.7117252349853516, "sampling/sampling_logp_difference/max": 0.3400633931159973, "sampling/sampling_logp_difference/mean": 0.014528676867485046, "step": 303 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.3125, "completions/max_length": 768.0, "completions/max_terminated_length": 685.0, "completions/mean_length": 561.84375, "completions/mean_terminated_length": 468.1363830566406, "completions/min_length": 297.0, "completions/min_terminated_length": 297.0, "entropy": 0.607463214546442, "epoch": 0.16274089935760172, "frac_reward_zero_std": 0.25, "grad_norm": 0.005798875819891691, "learning_rate": 1e-05, "loss": 0.068, "num_tokens": 6834048.0, "reward": 0.5625, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9998785853385925, "sampling/importance_sampling_ratio/min": 0.6077396273612976, "sampling/sampling_logp_difference/max": 0.7001485824584961, "sampling/sampling_logp_difference/mean": 0.015073039568960667, "step": 304 }, { "clip_ratio/high_max": 5.195345147512853e-05, "clip_ratio/high_mean": 5.195345147512853e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 5.195345147512853e-05, "completions/clipped_ratio": 0.34375, "completions/max_length": 768.0, "completions/max_terminated_length": 764.0, "completions/mean_length": 576.84375, "completions/mean_terminated_length": 476.71429443359375, "completions/min_length": 298.0, "completions/min_terminated_length": 298.0, "entropy": 0.6039410643279552, "epoch": 0.1632762312633833, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0079, "num_tokens": 6856163.0, "reward": 0.4375, "reward_std": 0.1157275140285492, "rewards/accuracy_reward/mean": 0.4375, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.5125415325164795, "sampling/importance_sampling_ratio/mean": 0.9999698996543884, "sampling/importance_sampling_ratio/min": 0.45265570282936096, "sampling/sampling_logp_difference/max": 0.7926235198974609, "sampling/sampling_logp_difference/mean": 0.017051981762051582, "step": 305 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00010205216676695272, "clip_ratio/low_min": 0.00010205216676695272, "clip_ratio/region_mean": 0.00010205216676695272, "completions/clipped_ratio": 0.375, "completions/max_length": 768.0, "completions/max_terminated_length": 696.0, "completions/mean_length": 575.625, "completions/mean_terminated_length": 460.20001220703125, "completions/min_length": 270.0, "completions/min_terminated_length": 270.0, "entropy": 0.4296734184026718, "epoch": 0.16381156316916487, "frac_reward_zero_std": 0.0, "grad_norm": 0.014403407461941242, "learning_rate": 1e-05, "loss": 0.1056, "num_tokens": 6878343.0, "reward": 0.59375, "reward_std": 0.4807935357093811, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9999287128448486, "sampling/importance_sampling_ratio/min": 0.7375364899635315, "sampling/sampling_logp_difference/max": 0.6974811553955078, "sampling/sampling_logp_difference/mean": 0.01293996162712574, "step": 306 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.992012691218406e-05, "clip_ratio/low_min": 4.992012691218406e-05, "clip_ratio/region_mean": 4.992012691218406e-05, "completions/clipped_ratio": 0.28125, "completions/max_length": 768.0, "completions/max_terminated_length": 735.0, "completions/mean_length": 607.78125, "completions/mean_terminated_length": 545.0869750976562, "completions/min_length": 292.0, "completions/min_terminated_length": 292.0, "entropy": 0.3507284037768841, "epoch": 0.16434689507494646, "frac_reward_zero_std": 0.25, "grad_norm": 0.012407874688506126, "learning_rate": 1e-05, "loss": 0.0623, "num_tokens": 6902232.0, "reward": 0.6875, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.5367474555969238, "sampling/importance_sampling_ratio/mean": 0.9999766945838928, "sampling/importance_sampling_ratio/min": 0.6973090171813965, "sampling/sampling_logp_difference/max": 0.4296681880950928, "sampling/sampling_logp_difference/mean": 0.010848877020180225, "step": 307 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.625, "completions/max_length": 768.0, "completions/max_terminated_length": 768.0, "completions/mean_length": 701.8125, "completions/mean_terminated_length": 591.5, "completions/min_length": 429.0, "completions/min_terminated_length": 429.0, "entropy": 0.5943136811256409, "epoch": 0.16488222698072805, "frac_reward_zero_std": 0.5, "grad_norm": 0.007644558325409889, "learning_rate": 1e-05, "loss": 0.0287, "num_tokens": 6929282.0, "reward": 0.1875, "reward_std": 0.249358132481575, "rewards/accuracy_reward/mean": 0.1875, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.4155625104904175, "sampling/importance_sampling_ratio/mean": 1.0000989437103271, "sampling/importance_sampling_ratio/min": 0.630094051361084, "sampling/sampling_logp_difference/max": 0.4618861675262451, "sampling/sampling_logp_difference/mean": 0.014622432179749012, "step": 308 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 9.484563270234503e-05, "clip_ratio/low_min": 9.484563270234503e-05, "clip_ratio/region_mean": 9.484563270234503e-05, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 758.0, "completions/mean_length": 571.03125, "completions/mean_terminated_length": 525.5769653320312, "completions/min_length": 309.0, "completions/min_terminated_length": 309.0, "entropy": 0.35181776620447636, "epoch": 0.16541755888650964, "frac_reward_zero_std": 0.5, "grad_norm": 0.005052079446613789, "learning_rate": 1e-05, "loss": 0.0434, "num_tokens": 6950915.0, "reward": 0.875, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 1.5886921882629395, "sampling/importance_sampling_ratio/mean": 1.0000771284103394, "sampling/importance_sampling_ratio/min": 0.700106680393219, "sampling/sampling_logp_difference/max": 0.46291112899780273, "sampling/sampling_logp_difference/mean": 0.010631774552166462, "step": 309 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.8809059080667794e-05, "clip_ratio/low_min": 4.8809059080667794e-05, "clip_ratio/region_mean": 4.8809059080667794e-05, "completions/clipped_ratio": 0.46875, "completions/max_length": 768.0, "completions/max_terminated_length": 740.0, "completions/mean_length": 636.96875, "completions/mean_terminated_length": 521.3529663085938, "completions/min_length": 306.0, "completions/min_terminated_length": 306.0, "entropy": 0.37192361056804657, "epoch": 0.16595289079229122, "frac_reward_zero_std": 0.0, "grad_norm": 0.019923802465200424, "learning_rate": 1e-05, "loss": 0.0388, "num_tokens": 6975178.0, "reward": 0.53125, "reward_std": 0.4218915104866028, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.371717929840088, "sampling/importance_sampling_ratio/mean": 0.9998781085014343, "sampling/importance_sampling_ratio/min": 0.7024648785591125, "sampling/sampling_logp_difference/max": 0.35315990447998047, "sampling/sampling_logp_difference/mean": 0.010792315006256104, "step": 310 }, { "clip_ratio/high_max": 5.715592124033719e-05, "clip_ratio/high_mean": 5.715592124033719e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 5.715592124033719e-05, "completions/clipped_ratio": 0.34375, "completions/max_length": 768.0, "completions/max_terminated_length": 695.0, "completions/mean_length": 561.84375, "completions/mean_terminated_length": 453.8571472167969, "completions/min_length": 250.0, "completions/min_terminated_length": 250.0, "entropy": 0.466085322201252, "epoch": 0.1664882226980728, "frac_reward_zero_std": 0.5, "grad_norm": 0.009683496318757534, "learning_rate": 1e-05, "loss": 0.0455, "num_tokens": 6997693.0, "reward": 0.5, "reward_std": 0.26726123690605164, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5080004930496216, "sampling/importance_sampling_ratio/max": 1.596548318862915, "sampling/importance_sampling_ratio/mean": 1.0000803470611572, "sampling/importance_sampling_ratio/min": 0.5866580605506897, "sampling/sampling_logp_difference/max": 0.5333130955696106, "sampling/sampling_logp_difference/mean": 0.013436157256364822, "step": 311 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 7.88146280683577e-05, "clip_ratio/low_min": 7.88146280683577e-05, "clip_ratio/region_mean": 7.88146280683577e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 679.0, "completions/max_terminated_length": 679.0, "completions/mean_length": 415.3125, "completions/mean_terminated_length": 415.3125, "completions/min_length": 200.0, "completions/min_terminated_length": 200.0, "entropy": 0.41137731820344925, "epoch": 0.1670235546038544, "frac_reward_zero_std": 0.25, "grad_norm": 0.014312022365629673, "learning_rate": 1e-05, "loss": -0.0326, "num_tokens": 7013967.0, "reward": 0.875, "reward_std": 0.2925041913986206, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 1.7482023239135742, "sampling/importance_sampling_ratio/mean": 1.0004349946975708, "sampling/importance_sampling_ratio/min": 0.7703766822814941, "sampling/sampling_logp_difference/max": 0.5585880279541016, "sampling/sampling_logp_difference/mean": 0.012294778600335121, "step": 312 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 732.0, "completions/mean_length": 513.59375, "completions/mean_terminated_length": 477.2500305175781, "completions/min_length": 193.0, "completions/min_terminated_length": 193.0, "entropy": 0.34391969814896584, "epoch": 0.16755888650963596, "frac_reward_zero_std": 0.5, "grad_norm": 0.003923412878066301, "learning_rate": 1e-05, "loss": 0.0187, "num_tokens": 7034434.0, "reward": 0.6875, "reward_std": 0.249358132481575, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.322226881980896, "sampling/importance_sampling_ratio/mean": 1.0004199743270874, "sampling/importance_sampling_ratio/min": 0.65497225522995, "sampling/sampling_logp_difference/max": 0.42316246032714844, "sampling/sampling_logp_difference/mean": 0.010589324869215488, "step": 313 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 634.0, "completions/mean_length": 432.9375, "completions/mean_terminated_length": 422.1290283203125, "completions/min_length": 287.0, "completions/min_terminated_length": 287.0, "entropy": 0.35445887967944145, "epoch": 0.16809421841541755, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0608, "num_tokens": 7051712.0, "reward": 0.96875, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.96875, "rewards/accuracy_reward/std": 0.1767766922712326, "sampling/importance_sampling_ratio/max": 1.3308967351913452, "sampling/importance_sampling_ratio/mean": 0.9997856616973877, "sampling/importance_sampling_ratio/min": 0.7060892581939697, "sampling/sampling_logp_difference/max": 0.34801363945007324, "sampling/sampling_logp_difference/mean": 0.011282389983534813, "step": 314 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 756.0, "completions/mean_length": 540.09375, "completions/mean_terminated_length": 524.9000244140625, "completions/min_length": 280.0, "completions/min_terminated_length": 280.0, "entropy": 0.4350382909178734, "epoch": 0.16862955032119914, "frac_reward_zero_std": 0.5, "grad_norm": 0.006640596780925989, "learning_rate": 1e-05, "loss": 0.0339, "num_tokens": 7072467.0, "reward": 0.875, "reward_std": 0.2314550280570984, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 1.2996902465820312, "sampling/importance_sampling_ratio/mean": 1.0000028610229492, "sampling/importance_sampling_ratio/min": 0.7655131816864014, "sampling/sampling_logp_difference/max": 0.2672088146209717, "sampling/sampling_logp_difference/mean": 0.012654009275138378, "step": 315 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.4375, "completions/max_length": 768.0, "completions/max_terminated_length": 726.0, "completions/mean_length": 614.0, "completions/mean_terminated_length": 494.22222900390625, "completions/min_length": 295.0, "completions/min_terminated_length": 295.0, "entropy": 0.6355114802718163, "epoch": 0.16916488222698073, "frac_reward_zero_std": 0.75, "grad_norm": 0.007677148096263409, "learning_rate": 1e-05, "loss": 0.0147, "num_tokens": 7095763.0, "reward": 0.40625, "reward_std": 0.1293872892856598, "rewards/accuracy_reward/mean": 0.40625, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.4618357419967651, "sampling/importance_sampling_ratio/mean": 0.9997237920761108, "sampling/importance_sampling_ratio/min": 0.6855196952819824, "sampling/sampling_logp_difference/max": 0.37969303131103516, "sampling/sampling_logp_difference/mean": 0.01625332050025463, "step": 316 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 8.626638737041503e-05, "clip_ratio/low_min": 8.626638737041503e-05, "clip_ratio/region_mean": 8.626638737041503e-05, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 723.0, "completions/mean_length": 558.28125, "completions/mean_terminated_length": 499.55999755859375, "completions/min_length": 260.0, "completions/min_terminated_length": 260.0, "entropy": 0.3198474645614624, "epoch": 0.16970021413276232, "frac_reward_zero_std": 0.25, "grad_norm": 0.0033788979053497314, "learning_rate": 1e-05, "loss": 0.0412, "num_tokens": 7117252.0, "reward": 0.75, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.270493745803833, "sampling/importance_sampling_ratio/mean": 0.9998461008071899, "sampling/importance_sampling_ratio/min": 0.6651964783668518, "sampling/sampling_logp_difference/max": 0.4076728820800781, "sampling/sampling_logp_difference/mean": 0.01024382933974266, "step": 317 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 737.0, "completions/mean_length": 499.53125, "completions/mean_terminated_length": 490.8709411621094, "completions/min_length": 332.0, "completions/min_terminated_length": 332.0, "entropy": 0.3893941566348076, "epoch": 0.1702355460385439, "frac_reward_zero_std": 0.25, "grad_norm": 0.015175970271229744, "learning_rate": 1e-05, "loss": 0.0093, "num_tokens": 7136957.0, "reward": 0.6875, "reward_std": 0.3514062464237213, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.3951607942581177, "sampling/importance_sampling_ratio/mean": 1.0000625848770142, "sampling/importance_sampling_ratio/min": 0.6843892931938171, "sampling/sampling_logp_difference/max": 0.3792283535003662, "sampling/sampling_logp_difference/mean": 0.011749176308512688, "step": 318 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.59375, "completions/max_length": 768.0, "completions/max_terminated_length": 743.0, "completions/mean_length": 702.3125, "completions/mean_terminated_length": 606.3077392578125, "completions/min_length": 412.0, "completions/min_terminated_length": 412.0, "entropy": 0.6505813598632812, "epoch": 0.1707708779443255, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 7163991.0, "reward": 0.25, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.25, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.3685336112976074, "sampling/importance_sampling_ratio/mean": 0.999991774559021, "sampling/importance_sampling_ratio/min": 0.5428902506828308, "sampling/sampling_logp_difference/max": 0.6108481884002686, "sampling/sampling_logp_difference/mean": 0.016265524551272392, "step": 319 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.48028658865951e-05, "clip_ratio/low_min": 4.48028658865951e-05, "clip_ratio/region_mean": 4.48028658865951e-05, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 758.0, "completions/mean_length": 602.53125, "completions/mean_terminated_length": 571.888916015625, "completions/min_length": 352.0, "completions/min_terminated_length": 352.0, "entropy": 0.31928472593426704, "epoch": 0.17130620985010706, "frac_reward_zero_std": 0.75, "grad_norm": 0.0050224303267896175, "learning_rate": 1e-05, "loss": 0.0228, "num_tokens": 7187520.0, "reward": 0.875, "reward_std": 0.13363061845302582, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 1.4198600053787231, "sampling/importance_sampling_ratio/mean": 0.9996873736381531, "sampling/importance_sampling_ratio/min": 0.4986302852630615, "sampling/sampling_logp_difference/max": 0.6958904266357422, "sampling/sampling_logp_difference/mean": 0.010240714997053146, "step": 320 }, { "clip_ratio/high_max": 5.560498175327666e-05, "clip_ratio/high_mean": 5.560498175327666e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 5.560498175327666e-05, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 745.0, "completions/mean_length": 581.75, "completions/mean_terminated_length": 529.5999755859375, "completions/min_length": 198.0, "completions/min_terminated_length": 198.0, "entropy": 0.3794487714767456, "epoch": 0.17184154175588864, "frac_reward_zero_std": 0.0, "grad_norm": 0.00835055485367775, "learning_rate": 1e-05, "loss": 0.0692, "num_tokens": 7209624.0, "reward": 0.59375, "reward_std": 0.4807935357093811, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.3938608169555664, "sampling/importance_sampling_ratio/mean": 1.0001498460769653, "sampling/importance_sampling_ratio/min": 0.744274914264679, "sampling/sampling_logp_difference/max": 0.3320775032043457, "sampling/sampling_logp_difference/mean": 0.011007760651409626, "step": 321 }, { "clip_ratio/high_max": 5.026135841035284e-05, "clip_ratio/high_mean": 5.026135841035284e-05, "clip_ratio/low_mean": 9.896593837765977e-05, "clip_ratio/low_min": 9.896593837765977e-05, "clip_ratio/region_mean": 0.0001492272967880126, "completions/clipped_ratio": 0.28125, "completions/max_length": 768.0, "completions/max_terminated_length": 735.0, "completions/mean_length": 581.34375, "completions/mean_terminated_length": 508.3043518066406, "completions/min_length": 305.0, "completions/min_terminated_length": 305.0, "entropy": 0.4647734649479389, "epoch": 0.17237687366167023, "frac_reward_zero_std": 0.0, "grad_norm": 0.016229046508669853, "learning_rate": 1e-05, "loss": 0.1117, "num_tokens": 7232283.0, "reward": 0.625, "reward_std": 0.49022960662841797, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.799541711807251, "sampling/importance_sampling_ratio/mean": 1.0000022649765015, "sampling/importance_sampling_ratio/min": 0.6365535259246826, "sampling/sampling_logp_difference/max": 0.5875320434570312, "sampling/sampling_logp_difference/mean": 0.013627829030156136, "step": 322 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.191029777051881e-05, "clip_ratio/low_min": 5.191029777051881e-05, "clip_ratio/region_mean": 5.191029777051881e-05, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 750.0, "completions/mean_length": 577.5625, "completions/mean_terminated_length": 514.0833740234375, "completions/min_length": 310.0, "completions/min_terminated_length": 310.0, "entropy": 0.4767385721206665, "epoch": 0.17291220556745182, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0296, "num_tokens": 7254709.0, "reward": 0.65625, "reward_std": 0.1293872892856598, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.4214529991149902, "sampling/importance_sampling_ratio/mean": 1.0003039836883545, "sampling/importance_sampling_ratio/min": 0.7111378908157349, "sampling/sampling_logp_difference/max": 0.35167956352233887, "sampling/sampling_logp_difference/mean": 0.013442853465676308, "step": 323 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 751.0, "completions/mean_length": 623.46875, "completions/mean_terminated_length": 583.0, "completions/min_length": 333.0, "completions/min_terminated_length": 333.0, "entropy": 0.49009233340620995, "epoch": 0.1734475374732334, "frac_reward_zero_std": 0.25, "grad_norm": 0.016870608553290367, "learning_rate": 1e-05, "loss": 0.0539, "num_tokens": 7278428.0, "reward": 0.65625, "reward_std": 0.2651650309562683, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.7307504415512085, "sampling/importance_sampling_ratio/mean": 1.0006048679351807, "sampling/importance_sampling_ratio/min": 0.7399675846099854, "sampling/sampling_logp_difference/max": 0.5485551357269287, "sampling/sampling_logp_difference/mean": 0.013142632320523262, "step": 324 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.1440329116303474e-05, "clip_ratio/low_min": 5.1440329116303474e-05, "clip_ratio/region_mean": 5.1440329116303474e-05, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 740.0, "completions/mean_length": 607.15625, "completions/mean_terminated_length": 570.0385131835938, "completions/min_length": 370.0, "completions/min_terminated_length": 370.0, "entropy": 0.36767593026161194, "epoch": 0.173982869379015, "frac_reward_zero_std": 0.25, "grad_norm": 0.0196528360247612, "learning_rate": 1e-05, "loss": 0.07, "num_tokens": 7301817.0, "reward": 0.75, "reward_std": 0.3514062464237213, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.3494623899459839, "sampling/importance_sampling_ratio/mean": 0.9995817542076111, "sampling/importance_sampling_ratio/min": 0.6885505318641663, "sampling/sampling_logp_difference/max": 0.373166561126709, "sampling/sampling_logp_difference/mean": 0.011270838789641857, "step": 325 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.83746116515249e-05, "clip_ratio/low_min": 4.83746116515249e-05, "clip_ratio/region_mean": 4.83746116515249e-05, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 763.0, "completions/mean_length": 574.25, "completions/mean_terminated_length": 509.66668701171875, "completions/min_length": 277.0, "completions/min_terminated_length": 277.0, "entropy": 0.4425664134323597, "epoch": 0.1745182012847966, "frac_reward_zero_std": 0.5, "grad_norm": 0.01783912628889084, "learning_rate": 1e-05, "loss": 0.0823, "num_tokens": 7324025.0, "reward": 0.625, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.3928987979888916, "sampling/importance_sampling_ratio/mean": 0.9998793601989746, "sampling/importance_sampling_ratio/min": 0.7173065543174744, "sampling/sampling_logp_difference/max": 0.33225202560424805, "sampling/sampling_logp_difference/mean": 0.012686881236732006, "step": 326 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00012377842358546332, "clip_ratio/low_min": 0.00012377842358546332, "clip_ratio/region_mean": 0.00012377842358546332, "completions/clipped_ratio": 0.28125, "completions/max_length": 768.0, "completions/max_terminated_length": 603.0, "completions/mean_length": 547.28125, "completions/mean_terminated_length": 460.9130554199219, "completions/min_length": 300.0, "completions/min_terminated_length": 300.0, "entropy": 0.43768591433763504, "epoch": 0.17505353319057815, "frac_reward_zero_std": 0.25, "grad_norm": 0.01271110214293003, "learning_rate": 1e-05, "loss": 0.1081, "num_tokens": 7345146.0, "reward": 0.6875, "reward_std": 0.292504221200943, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.404424786567688, "sampling/importance_sampling_ratio/mean": 1.0002367496490479, "sampling/importance_sampling_ratio/min": 0.775164008140564, "sampling/sampling_logp_difference/max": 0.339627742767334, "sampling/sampling_logp_difference/mean": 0.013005612418055534, "step": 327 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 765.0, "completions/mean_length": 572.34375, "completions/mean_terminated_length": 527.1923217773438, "completions/min_length": 285.0, "completions/min_terminated_length": 285.0, "entropy": 0.3581581190228462, "epoch": 0.17558886509635974, "frac_reward_zero_std": 0.0, "grad_norm": 0.008894316852092743, "learning_rate": 1e-05, "loss": 0.0195, "num_tokens": 7367581.0, "reward": 0.6875, "reward_std": 0.4492306709289551, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.4476288557052612, "sampling/importance_sampling_ratio/mean": 1.000088095664978, "sampling/importance_sampling_ratio/min": 0.7586356401443481, "sampling/sampling_logp_difference/max": 0.36992692947387695, "sampling/sampling_logp_difference/mean": 0.011610949411988258, "step": 328 }, { "clip_ratio/high_max": 7.697044202359393e-05, "clip_ratio/high_mean": 7.697044202359393e-05, "clip_ratio/low_mean": 6.047411807230674e-05, "clip_ratio/low_min": 6.047411807230674e-05, "clip_ratio/region_mean": 0.00013744456009590067, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 741.0, "completions/mean_length": 533.0625, "completions/mean_terminated_length": 525.4838256835938, "completions/min_length": 323.0, "completions/min_terminated_length": 323.0, "entropy": 0.38617009297013283, "epoch": 0.17612419700214133, "frac_reward_zero_std": 0.25, "grad_norm": 0.008866837248206139, "learning_rate": 1e-05, "loss": -0.036, "num_tokens": 7388487.0, "reward": 0.75, "reward_std": 0.3514062464237213, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.3624176979064941, "sampling/importance_sampling_ratio/mean": 0.9998299479484558, "sampling/importance_sampling_ratio/min": 0.6679694056510925, "sampling/sampling_logp_difference/max": 0.40351295471191406, "sampling/sampling_logp_difference/mean": 0.01204304676502943, "step": 329 }, { "clip_ratio/high_max": 4.8281188355758786e-05, "clip_ratio/high_mean": 4.8281188355758786e-05, "clip_ratio/low_mean": 0.00010299221685272641, "clip_ratio/low_min": 0.00010299221685272641, "clip_ratio/region_mean": 0.0001512734052084852, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 758.0, "completions/mean_length": 588.75, "completions/mean_terminated_length": 555.5555419921875, "completions/min_length": 221.0, "completions/min_terminated_length": 221.0, "entropy": 0.5742395892739296, "epoch": 0.1766595289079229, "frac_reward_zero_std": 0.25, "grad_norm": 0.011959652416408062, "learning_rate": 1e-05, "loss": -0.0274, "num_tokens": 7411607.0, "reward": 0.71875, "reward_std": 0.3608423173427582, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.3552334308624268, "sampling/importance_sampling_ratio/mean": 1.0000379085540771, "sampling/importance_sampling_ratio/min": 0.733167827129364, "sampling/sampling_logp_difference/max": 0.3103806972503662, "sampling/sampling_logp_difference/mean": 0.01542259193956852, "step": 330 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 709.0, "completions/mean_length": 466.03125, "completions/mean_terminated_length": 456.2903137207031, "completions/min_length": 296.0, "completions/min_terminated_length": 296.0, "entropy": 0.4616716504096985, "epoch": 0.1771948608137045, "frac_reward_zero_std": 0.5, "grad_norm": 0.01152541022747755, "learning_rate": 1e-05, "loss": 0.007, "num_tokens": 7429936.0, "reward": 0.8125, "reward_std": 0.249358132481575, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.3555200099945068, "sampling/importance_sampling_ratio/mean": 0.9999625086784363, "sampling/importance_sampling_ratio/min": 0.6500133872032166, "sampling/sampling_logp_difference/max": 0.43076229095458984, "sampling/sampling_logp_difference/mean": 0.013224722817540169, "step": 331 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 714.0, "completions/mean_length": 552.71875, "completions/mean_terminated_length": 492.44000244140625, "completions/min_length": 64.0, "completions/min_terminated_length": 64.0, "entropy": 0.4250369407236576, "epoch": 0.1777301927194861, "frac_reward_zero_std": 0.0, "grad_norm": 0.01536703109741211, "learning_rate": 1e-05, "loss": 0.0528, "num_tokens": 7451031.0, "reward": 0.75, "reward_std": 0.4355512857437134, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.3783841133117676, "sampling/importance_sampling_ratio/mean": 0.9997681379318237, "sampling/importance_sampling_ratio/min": 0.6973655819892883, "sampling/sampling_logp_difference/max": 0.360445499420166, "sampling/sampling_logp_difference/mean": 0.012299950234591961, "step": 332 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.375, "completions/max_length": 768.0, "completions/max_terminated_length": 696.0, "completions/mean_length": 608.8125, "completions/mean_terminated_length": 513.2999877929688, "completions/min_length": 309.0, "completions/min_terminated_length": 309.0, "entropy": 0.44707604870200157, "epoch": 0.17826552462526768, "frac_reward_zero_std": 0.25, "grad_norm": 0.008990665897727013, "learning_rate": 1e-05, "loss": 0.0209, "num_tokens": 7474097.0, "reward": 0.53125, "reward_std": 0.378745436668396, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.3583874702453613, "sampling/importance_sampling_ratio/mean": 0.9996063113212585, "sampling/importance_sampling_ratio/min": 0.7477813363075256, "sampling/sampling_logp_difference/max": 0.30629831552505493, "sampling/sampling_logp_difference/mean": 0.011986833065748215, "step": 333 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00011857796562253498, "clip_ratio/low_min": 0.00011857796562253498, "clip_ratio/region_mean": 0.00011857796562253498, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 764.0, "completions/mean_length": 540.34375, "completions/mean_terminated_length": 476.5999755859375, "completions/min_length": 131.0, "completions/min_terminated_length": 131.0, "entropy": 0.6022821851074696, "epoch": 0.17880085653104924, "frac_reward_zero_std": 0.0, "grad_norm": 0.013709074817597866, "learning_rate": 1e-05, "loss": 0.0002, "num_tokens": 7495308.0, "reward": 0.59375, "reward_std": 0.4807935357093811, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.5634510517120361, "sampling/importance_sampling_ratio/mean": 1.0003116130828857, "sampling/importance_sampling_ratio/min": 0.563000500202179, "sampling/sampling_logp_difference/max": 0.5744748115539551, "sampling/sampling_logp_difference/mean": 0.015677832067012787, "step": 334 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.3125, "completions/max_length": 768.0, "completions/max_terminated_length": 759.0, "completions/mean_length": 590.90625, "completions/mean_terminated_length": 510.40911865234375, "completions/min_length": 316.0, "completions/min_terminated_length": 316.0, "entropy": 0.4364216811954975, "epoch": 0.17933618843683083, "frac_reward_zero_std": 0.0, "grad_norm": 0.011650744825601578, "learning_rate": 1e-05, "loss": 0.0991, "num_tokens": 7518073.0, "reward": 0.6875, "reward_std": 0.44403791427612305, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.4041688442230225, "sampling/importance_sampling_ratio/mean": 1.000081181526184, "sampling/importance_sampling_ratio/min": 0.7033185362815857, "sampling/sampling_logp_difference/max": 0.3519454002380371, "sampling/sampling_logp_difference/mean": 0.012732066214084625, "step": 335 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 762.0, "completions/mean_length": 607.53125, "completions/mean_terminated_length": 570.5, "completions/min_length": 376.0, "completions/min_terminated_length": 376.0, "entropy": 0.31602367386221886, "epoch": 0.17987152034261242, "frac_reward_zero_std": 0.0, "grad_norm": 0.005973147228360176, "learning_rate": 1e-05, "loss": 0.097, "num_tokens": 7541450.0, "reward": 0.75, "reward_std": 0.4492306709289551, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.4350091218948364, "sampling/importance_sampling_ratio/mean": 0.9999140501022339, "sampling/importance_sampling_ratio/min": 0.7572264671325684, "sampling/sampling_logp_difference/max": 0.36117124557495117, "sampling/sampling_logp_difference/mean": 0.01013131719082594, "step": 336 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.1888750022044405e-05, "clip_ratio/low_min": 5.1888750022044405e-05, "clip_ratio/region_mean": 5.1888750022044405e-05, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 768.0, "completions/mean_length": 615.0, "completions/mean_terminated_length": 564.0, "completions/min_length": 237.0, "completions/min_terminated_length": 237.0, "entropy": 0.3818574286997318, "epoch": 0.180406852248394, "frac_reward_zero_std": 0.25, "grad_norm": 0.007271267473697662, "learning_rate": 1e-05, "loss": 0.0051, "num_tokens": 7565898.0, "reward": 0.40625, "reward_std": 0.3608423173427582, "rewards/accuracy_reward/mean": 0.40625, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.476676344871521, "sampling/importance_sampling_ratio/mean": 0.9995765089988708, "sampling/importance_sampling_ratio/min": 0.6412937641143799, "sampling/sampling_logp_difference/max": 0.4442676603794098, "sampling/sampling_logp_difference/mean": 0.011861482635140419, "step": 337 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 744.0, "completions/mean_length": 517.53125, "completions/mean_terminated_length": 500.8333740234375, "completions/min_length": 305.0, "completions/min_terminated_length": 305.0, "entropy": 0.35317888110876083, "epoch": 0.1809421841541756, "frac_reward_zero_std": 0.25, "grad_norm": 0.008780697360634804, "learning_rate": 1e-05, "loss": -0.0228, "num_tokens": 7586003.0, "reward": 0.8125, "reward_std": 0.3104073107242584, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.449013590812683, "sampling/importance_sampling_ratio/mean": 1.000103235244751, "sampling/importance_sampling_ratio/min": 0.7200444936752319, "sampling/sampling_logp_difference/max": 0.37088310718536377, "sampling/sampling_logp_difference/mean": 0.01070118136703968, "step": 338 }, { "clip_ratio/high_max": 4.9662296078167856e-05, "clip_ratio/high_mean": 4.9662296078167856e-05, "clip_ratio/low_mean": 4.9662296078167856e-05, "clip_ratio/low_min": 4.9662296078167856e-05, "clip_ratio/region_mean": 9.932459215633571e-05, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 704.0, "completions/mean_length": 502.5, "completions/mean_terminated_length": 475.03448486328125, "completions/min_length": 212.0, "completions/min_terminated_length": 212.0, "entropy": 0.3665863908827305, "epoch": 0.18147751605995718, "frac_reward_zero_std": 0.25, "grad_norm": 0.00488157058134675, "learning_rate": 1e-05, "loss": 0.0782, "num_tokens": 7605659.0, "reward": 0.875, "reward_std": 0.292504221200943, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 1.3858939409255981, "sampling/importance_sampling_ratio/mean": 0.9999417662620544, "sampling/importance_sampling_ratio/min": 0.68963623046875, "sampling/sampling_logp_difference/max": 0.37159109115600586, "sampling/sampling_logp_difference/mean": 0.011676940135657787, "step": 339 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 753.0, "completions/mean_length": 567.03125, "completions/mean_terminated_length": 529.8148193359375, "completions/min_length": 264.0, "completions/min_terminated_length": 264.0, "entropy": 0.3395811803638935, "epoch": 0.18201284796573874, "frac_reward_zero_std": 0.0, "grad_norm": 0.010610250756144524, "learning_rate": 1e-05, "loss": 0.1187, "num_tokens": 7627124.0, "reward": 0.8125, "reward_std": 0.3945523500442505, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.4310650825500488, "sampling/importance_sampling_ratio/mean": 1.0000576972961426, "sampling/importance_sampling_ratio/min": 0.6813254952430725, "sampling/sampling_logp_difference/max": 0.3837151527404785, "sampling/sampling_logp_difference/mean": 0.010930436663329601, "step": 340 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00011768460171879269, "clip_ratio/low_min": 0.00011768460171879269, "clip_ratio/region_mean": 0.00011768460171879269, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 764.0, "completions/mean_length": 529.84375, "completions/mean_terminated_length": 463.1600036621094, "completions/min_length": 283.0, "completions/min_terminated_length": 283.0, "entropy": 0.4085201546549797, "epoch": 0.18254817987152033, "frac_reward_zero_std": 0.25, "grad_norm": 0.008732098154723644, "learning_rate": 1e-05, "loss": 0.0371, "num_tokens": 7648263.0, "reward": 0.59375, "reward_std": 0.3198433816432953, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.4668234586715698, "sampling/importance_sampling_ratio/mean": 1.0001531839370728, "sampling/importance_sampling_ratio/min": 0.6989305019378662, "sampling/sampling_logp_difference/max": 0.3830990791320801, "sampling/sampling_logp_difference/mean": 0.012125181034207344, "step": 341 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 691.0, "completions/mean_length": 511.15625, "completions/mean_terminated_length": 484.5862121582031, "completions/min_length": 235.0, "completions/min_terminated_length": 235.0, "entropy": 0.37089499831199646, "epoch": 0.18308351177730192, "frac_reward_zero_std": 0.5, "grad_norm": 0.006230442319065332, "learning_rate": 1e-05, "loss": 0.0112, "num_tokens": 7668484.0, "reward": 0.875, "reward_std": 0.2314550280570984, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 1.4687354564666748, "sampling/importance_sampling_ratio/mean": 1.0003365278244019, "sampling/importance_sampling_ratio/min": 0.729246199131012, "sampling/sampling_logp_difference/max": 0.384401798248291, "sampling/sampling_logp_difference/mean": 0.01226845383644104, "step": 342 }, { "clip_ratio/high_max": 6.541077891597524e-05, "clip_ratio/high_mean": 6.541077891597524e-05, "clip_ratio/low_mean": 6.620762724196538e-05, "clip_ratio/low_min": 6.620762724196538e-05, "clip_ratio/region_mean": 0.00013161840615794063, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 627.0, "completions/mean_length": 469.40625, "completions/mean_terminated_length": 459.774169921875, "completions/min_length": 318.0, "completions/min_terminated_length": 318.0, "entropy": 0.3941527418792248, "epoch": 0.1836188436830835, "frac_reward_zero_std": 0.25, "grad_norm": 0.00416077533736825, "learning_rate": 1e-05, "loss": 0.0712, "num_tokens": 7686961.0, "reward": 0.90625, "reward_std": 0.2651650309562683, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.4136728048324585, "sampling/importance_sampling_ratio/mean": 0.9997018575668335, "sampling/importance_sampling_ratio/min": 0.7000539302825928, "sampling/sampling_logp_difference/max": 0.356597900390625, "sampling/sampling_logp_difference/mean": 0.01219549123197794, "step": 343 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 761.0, "completions/mean_length": 597.0, "completions/mean_terminated_length": 549.1199951171875, "completions/min_length": 306.0, "completions/min_terminated_length": 306.0, "entropy": 0.43536716513335705, "epoch": 0.1841541755888651, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0173, "num_tokens": 7710473.0, "reward": 0.46875, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.46875, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.511583685874939, "sampling/importance_sampling_ratio/mean": 1.0004948377609253, "sampling/importance_sampling_ratio/min": 0.4704502820968628, "sampling/sampling_logp_difference/max": 0.7540650367736816, "sampling/sampling_logp_difference/mean": 0.012780013494193554, "step": 344 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 736.0, "completions/mean_length": 560.875, "completions/mean_terminated_length": 531.2857666015625, "completions/min_length": 312.0, "completions/min_terminated_length": 312.0, "entropy": 0.37558479234576225, "epoch": 0.1846895074946467, "frac_reward_zero_std": 0.5, "grad_norm": 0.029174676164984703, "learning_rate": 1e-05, "loss": 0.0137, "num_tokens": 7732181.0, "reward": 0.84375, "reward_std": 0.22201895713806152, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.4214953184127808, "sampling/importance_sampling_ratio/mean": 1.000383734703064, "sampling/importance_sampling_ratio/min": 0.7225403785705566, "sampling/sampling_logp_difference/max": 0.35170936584472656, "sampling/sampling_logp_difference/mean": 0.011788973584771156, "step": 345 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 765.0, "completions/mean_length": 527.15625, "completions/mean_terminated_length": 502.2413635253906, "completions/min_length": 192.0, "completions/min_terminated_length": 192.0, "entropy": 0.4374905489385128, "epoch": 0.18522483940042828, "frac_reward_zero_std": 0.5, "grad_norm": 0.0033343981485813856, "learning_rate": 1e-05, "loss": 0.0643, "num_tokens": 7752810.0, "reward": 0.75, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.954554796218872, "sampling/importance_sampling_ratio/mean": 1.0001956224441528, "sampling/importance_sampling_ratio/min": 0.6426865458488464, "sampling/sampling_logp_difference/max": 0.6701624393463135, "sampling/sampling_logp_difference/mean": 0.012586500495672226, "step": 346 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.587155854096636e-05, "clip_ratio/low_min": 4.587155854096636e-05, "clip_ratio/region_mean": 4.587155854096636e-05, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 759.0, "completions/mean_length": 584.15625, "completions/mean_terminated_length": 550.1111450195312, "completions/min_length": 232.0, "completions/min_terminated_length": 232.0, "entropy": 0.3952989913523197, "epoch": 0.18576017130620984, "frac_reward_zero_std": 0.25, "grad_norm": 0.008386149071156979, "learning_rate": 1e-05, "loss": -0.009, "num_tokens": 7775431.0, "reward": 0.53125, "reward_std": 0.3471629321575165, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.317029356956482, "sampling/importance_sampling_ratio/mean": 0.9999770522117615, "sampling/importance_sampling_ratio/min": 0.677424430847168, "sampling/sampling_logp_difference/max": 0.38945722579956055, "sampling/sampling_logp_difference/mean": 0.011926273815333843, "step": 347 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 723.0, "completions/mean_length": 465.84375, "completions/mean_terminated_length": 445.70001220703125, "completions/min_length": 319.0, "completions/min_terminated_length": 319.0, "entropy": 0.3603689447045326, "epoch": 0.18629550321199143, "frac_reward_zero_std": 0.25, "grad_norm": 0.008089978247880936, "learning_rate": 1e-05, "loss": -0.0137, "num_tokens": 7793946.0, "reward": 0.78125, "reward_std": 0.3061639964580536, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.3611254692077637, "sampling/importance_sampling_ratio/mean": 0.9999449253082275, "sampling/importance_sampling_ratio/min": 0.7004105448722839, "sampling/sampling_logp_difference/max": 0.35608863830566406, "sampling/sampling_logp_difference/mean": 0.011440886184573174, "step": 348 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.131362922838889e-05, "clip_ratio/low_min": 5.131362922838889e-05, "clip_ratio/region_mean": 5.131362922838889e-05, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 708.0, "completions/mean_length": 495.28125, "completions/mean_terminated_length": 404.375, "completions/min_length": 186.0, "completions/min_terminated_length": 186.0, "entropy": 0.45432041212916374, "epoch": 0.18683083511777301, "frac_reward_zero_std": 0.25, "grad_norm": 0.018666157498955727, "learning_rate": 1e-05, "loss": 0.0106, "num_tokens": 7813187.0, "reward": 0.46875, "reward_std": 0.3198433816432953, "rewards/accuracy_reward/mean": 0.46875, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.3087438344955444, "sampling/importance_sampling_ratio/mean": 0.9999655485153198, "sampling/importance_sampling_ratio/min": 0.6450303196907043, "sampling/sampling_logp_difference/max": 0.4384579658508301, "sampling/sampling_logp_difference/mean": 0.013790863566100597, "step": 349 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 760.0, "completions/mean_length": 535.8125, "completions/mean_terminated_length": 511.7930908203125, "completions/min_length": 340.0, "completions/min_terminated_length": 340.0, "entropy": 0.3715438283979893, "epoch": 0.1873661670235546, "frac_reward_zero_std": 0.75, "grad_norm": 0.0037117162719368935, "learning_rate": 1e-05, "loss": 0.0082, "num_tokens": 7834165.0, "reward": 0.78125, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.4204705953598022, "sampling/importance_sampling_ratio/mean": 1.0002492666244507, "sampling/importance_sampling_ratio/min": 0.6965051293373108, "sampling/sampling_logp_difference/max": 0.3616800308227539, "sampling/sampling_logp_difference/mean": 0.011881536804139614, "step": 350 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.0797665355494246e-05, "clip_ratio/low_min": 6.0797665355494246e-05, "clip_ratio/region_mean": 6.0797665355494246e-05, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 701.0, "completions/mean_length": 502.1875, "completions/mean_terminated_length": 427.7599792480469, "completions/min_length": 229.0, "completions/min_terminated_length": 229.0, "entropy": 0.43836427107453346, "epoch": 0.1879014989293362, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": -0.0015, "num_tokens": 7854003.0, "reward": 0.53125, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.2850538492202759, "sampling/importance_sampling_ratio/mean": 0.9995579123497009, "sampling/importance_sampling_ratio/min": 0.59810471534729, "sampling/sampling_logp_difference/max": 0.5139894485473633, "sampling/sampling_logp_difference/mean": 0.012770895846188068, "step": 351 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00010048757030745037, "clip_ratio/low_min": 0.00010048757030745037, "clip_ratio/region_mean": 0.00010048757030745037, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 743.0, "completions/mean_length": 584.96875, "completions/mean_terminated_length": 533.719970703125, "completions/min_length": 308.0, "completions/min_terminated_length": 308.0, "entropy": 0.41844386607408524, "epoch": 0.18843683083511778, "frac_reward_zero_std": 0.0, "grad_norm": 0.015738243237137794, "learning_rate": 1e-05, "loss": 0.0174, "num_tokens": 7876474.0, "reward": 0.6875, "reward_std": 0.4355512857437134, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.760148525238037, "sampling/importance_sampling_ratio/mean": 1.0003736019134521, "sampling/importance_sampling_ratio/min": 0.7047797441482544, "sampling/sampling_logp_difference/max": 0.5653982162475586, "sampling/sampling_logp_difference/mean": 0.012314577586948872, "step": 352 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.28125, "completions/max_length": 768.0, "completions/max_terminated_length": 646.0, "completions/mean_length": 553.9375, "completions/mean_terminated_length": 470.1739196777344, "completions/min_length": 250.0, "completions/min_terminated_length": 250.0, "entropy": 0.5313730835914612, "epoch": 0.18897216274089937, "frac_reward_zero_std": 0.5, "grad_norm": 0.0038939369842410088, "learning_rate": 1e-05, "loss": 0.0235, "num_tokens": 7897976.0, "reward": 0.625, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.5618528127670288, "sampling/importance_sampling_ratio/mean": 1.0003228187561035, "sampling/importance_sampling_ratio/min": 0.6975950598716736, "sampling/sampling_logp_difference/max": 0.44587278366088867, "sampling/sampling_logp_difference/mean": 0.015048760920763016, "step": 353 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 714.0, "completions/mean_length": 583.375, "completions/mean_terminated_length": 564.27587890625, "completions/min_length": 234.0, "completions/min_terminated_length": 234.0, "entropy": 0.3671898692846298, "epoch": 0.18950749464668093, "frac_reward_zero_std": 0.5, "grad_norm": 0.007814828306436539, "learning_rate": 1e-05, "loss": -0.0143, "num_tokens": 7920100.0, "reward": 0.84375, "reward_std": 0.22201895713806152, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.4202724695205688, "sampling/importance_sampling_ratio/mean": 0.9999342560768127, "sampling/importance_sampling_ratio/min": 0.6841111779212952, "sampling/sampling_logp_difference/max": 0.3796348571777344, "sampling/sampling_logp_difference/mean": 0.011666848324239254, "step": 354 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.367804598994553e-05, "clip_ratio/low_min": 6.367804598994553e-05, "clip_ratio/region_mean": 6.367804598994553e-05, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 761.0, "completions/mean_length": 622.34375, "completions/mean_terminated_length": 588.7307739257812, "completions/min_length": 352.0, "completions/min_terminated_length": 352.0, "entropy": 0.3561982773244381, "epoch": 0.19004282655246252, "frac_reward_zero_std": 0.0, "grad_norm": 0.010745654813945293, "learning_rate": 1e-05, "loss": 0.0504, "num_tokens": 7943879.0, "reward": 0.4375, "reward_std": 0.5081326961517334, "rewards/accuracy_reward/mean": 0.4375, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.6341825723648071, "sampling/importance_sampling_ratio/mean": 0.9998425841331482, "sampling/importance_sampling_ratio/min": 0.7280601859092712, "sampling/sampling_logp_difference/max": 0.49114274978637695, "sampling/sampling_logp_difference/mean": 0.01099233515560627, "step": 355 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.5625, "completions/max_length": 768.0, "completions/max_terminated_length": 672.0, "completions/mean_length": 679.3125, "completions/mean_terminated_length": 565.2857666015625, "completions/min_length": 416.0, "completions/min_terminated_length": 416.0, "entropy": 0.5292008481919765, "epoch": 0.1905781584582441, "frac_reward_zero_std": 0.5, "grad_norm": 0.003873254870995879, "learning_rate": 1e-05, "loss": 0.0581, "num_tokens": 7970537.0, "reward": 0.4375, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.4375, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.4053484201431274, "sampling/importance_sampling_ratio/mean": 0.9999608993530273, "sampling/importance_sampling_ratio/min": 0.6079300045967102, "sampling/sampling_logp_difference/max": 0.4976954460144043, "sampling/sampling_logp_difference/mean": 0.014417771250009537, "step": 356 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.172839493956417e-05, "clip_ratio/low_min": 6.172839493956417e-05, "clip_ratio/region_mean": 6.172839493956417e-05, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 680.0, "completions/mean_length": 497.34375, "completions/mean_terminated_length": 458.6785888671875, "completions/min_length": 99.0, "completions/min_terminated_length": 99.0, "entropy": 0.33522066473960876, "epoch": 0.1911134903640257, "frac_reward_zero_std": 0.5, "grad_norm": 0.005561012774705887, "learning_rate": 1e-05, "loss": 0.0407, "num_tokens": 7989564.0, "reward": 0.71875, "reward_std": 0.2630178928375244, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.3128960132598877, "sampling/importance_sampling_ratio/mean": 0.9998707175254822, "sampling/importance_sampling_ratio/min": 0.5142340660095215, "sampling/sampling_logp_difference/max": 0.665076732635498, "sampling/sampling_logp_difference/mean": 0.010443082079291344, "step": 357 }, { "clip_ratio/high_max": 0.00010860121983569115, "clip_ratio/high_mean": 0.00010860121983569115, "clip_ratio/low_mean": 5.1355793402763084e-05, "clip_ratio/low_min": 5.1355793402763084e-05, "clip_ratio/region_mean": 0.00015995701323845424, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 765.0, "completions/mean_length": 607.53125, "completions/mean_terminated_length": 554.0416870117188, "completions/min_length": 301.0, "completions/min_terminated_length": 301.0, "entropy": 0.40050872415304184, "epoch": 0.19164882226980728, "frac_reward_zero_std": 0.25, "grad_norm": 0.013117474503815174, "learning_rate": 1e-05, "loss": -0.0281, "num_tokens": 8012533.0, "reward": 0.5625, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.3459420204162598, "sampling/importance_sampling_ratio/mean": 1.0003983974456787, "sampling/importance_sampling_ratio/min": 0.5469598174095154, "sampling/sampling_logp_difference/max": 0.6033799648284912, "sampling/sampling_logp_difference/mean": 0.011664224788546562, "step": 358 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.289885666570626e-05, "clip_ratio/low_min": 5.289885666570626e-05, "clip_ratio/region_mean": 5.289885666570626e-05, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 649.0, "completions/mean_length": 501.4375, "completions/mean_terminated_length": 439.923095703125, "completions/min_length": 195.0, "completions/min_terminated_length": 195.0, "entropy": 0.5156456530094147, "epoch": 0.19218415417558887, "frac_reward_zero_std": 0.5, "grad_norm": 0.018113363534212112, "learning_rate": 1e-05, "loss": 0.0591, "num_tokens": 8032603.0, "reward": 0.78125, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.4613863229751587, "sampling/importance_sampling_ratio/mean": 0.9998681545257568, "sampling/importance_sampling_ratio/min": 0.700824499130249, "sampling/sampling_logp_difference/max": 0.37938547134399414, "sampling/sampling_logp_difference/mean": 0.015054013580083847, "step": 359 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00010434056457597762, "clip_ratio/low_min": 0.00010434056457597762, "clip_ratio/region_mean": 0.00010434056457597762, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 669.0, "completions/mean_length": 528.1875, "completions/mean_terminated_length": 461.03997802734375, "completions/min_length": 241.0, "completions/min_terminated_length": 241.0, "entropy": 0.3673683628439903, "epoch": 0.19271948608137046, "frac_reward_zero_std": 0.25, "grad_norm": 0.006402954924851656, "learning_rate": 1e-05, "loss": 0.0682, "num_tokens": 8053081.0, "reward": 0.59375, "reward_std": 0.3061639964580536, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.633599042892456, "sampling/importance_sampling_ratio/mean": 0.9999885559082031, "sampling/importance_sampling_ratio/min": 0.7214100360870361, "sampling/sampling_logp_difference/max": 0.4907855987548828, "sampling/sampling_logp_difference/mean": 0.012097165919840336, "step": 360 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.439979188144207e-05, "clip_ratio/low_min": 6.439979188144207e-05, "clip_ratio/region_mean": 6.439979188144207e-05, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 750.0, "completions/mean_length": 549.0, "completions/mean_terminated_length": 517.7142944335938, "completions/min_length": 288.0, "completions/min_terminated_length": 288.0, "entropy": 0.3545941151678562, "epoch": 0.19325481798715202, "frac_reward_zero_std": 0.25, "grad_norm": 0.0052088904194533825, "learning_rate": 1e-05, "loss": -0.006, "num_tokens": 8074649.0, "reward": 0.5625, "reward_std": 0.2925041913986206, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.2932671308517456, "sampling/importance_sampling_ratio/mean": 0.999854564666748, "sampling/importance_sampling_ratio/min": 0.6198790669441223, "sampling/sampling_logp_difference/max": 0.47823095321655273, "sampling/sampling_logp_difference/mean": 0.01117474865168333, "step": 361 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 747.0, "completions/mean_length": 508.90625, "completions/mean_terminated_length": 449.1153869628906, "completions/min_length": 257.0, "completions/min_terminated_length": 257.0, "entropy": 0.37205641344189644, "epoch": 0.1937901498929336, "frac_reward_zero_std": 0.5, "grad_norm": 0.01416859868913889, "learning_rate": 1e-05, "loss": 0.0557, "num_tokens": 8095174.0, "reward": 0.78125, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.4567490816116333, "sampling/importance_sampling_ratio/mean": 1.0000224113464355, "sampling/importance_sampling_ratio/min": 0.6502732038497925, "sampling/sampling_logp_difference/max": 0.4303627014160156, "sampling/sampling_logp_difference/mean": 0.012502220459282398, "step": 362 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.3125, "completions/max_length": 768.0, "completions/max_terminated_length": 767.0, "completions/mean_length": 625.03125, "completions/mean_terminated_length": 560.0454711914062, "completions/min_length": 349.0, "completions/min_terminated_length": 349.0, "entropy": 0.3963019587099552, "epoch": 0.1943254817987152, "frac_reward_zero_std": 0.0, "grad_norm": 0.013624940067529678, "learning_rate": 1e-05, "loss": 0.0821, "num_tokens": 8119263.0, "reward": 0.46875, "reward_std": 0.47137707471847534, "rewards/accuracy_reward/mean": 0.46875, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.8045307397842407, "sampling/importance_sampling_ratio/mean": 0.9999462962150574, "sampling/importance_sampling_ratio/min": 0.6579004526138306, "sampling/sampling_logp_difference/max": 0.5903005599975586, "sampling/sampling_logp_difference/mean": 0.012125816196203232, "step": 363 }, { "clip_ratio/high_max": 4.8904537834459916e-05, "clip_ratio/high_mean": 4.8904537834459916e-05, "clip_ratio/low_mean": 5.8220772189088166e-05, "clip_ratio/low_min": 5.8220772189088166e-05, "clip_ratio/region_mean": 0.00010712531002354808, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 763.0, "completions/mean_length": 557.875, "completions/mean_terminated_length": 487.8333435058594, "completions/min_length": 211.0, "completions/min_terminated_length": 211.0, "entropy": 0.386314794421196, "epoch": 0.1948608137044968, "frac_reward_zero_std": 0.25, "grad_norm": 0.014079393818974495, "learning_rate": 1e-05, "loss": -0.0407, "num_tokens": 8141307.0, "reward": 0.53125, "reward_std": 0.3471629321575165, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.2817705869674683, "sampling/importance_sampling_ratio/mean": 1.0008752346038818, "sampling/importance_sampling_ratio/min": 0.6971936821937561, "sampling/sampling_logp_difference/max": 0.36069202423095703, "sampling/sampling_logp_difference/mean": 0.012224792502820492, "step": 364 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 751.0, "completions/mean_length": 579.59375, "completions/mean_terminated_length": 573.51611328125, "completions/min_length": 393.0, "completions/min_terminated_length": 393.0, "entropy": 0.31728189811110497, "epoch": 0.19539614561027838, "frac_reward_zero_std": 0.25, "grad_norm": 0.01950717344880104, "learning_rate": 1e-05, "loss": -0.0291, "num_tokens": 8163582.0, "reward": 0.5625, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.4351892471313477, "sampling/importance_sampling_ratio/mean": 0.9998736381530762, "sampling/importance_sampling_ratio/min": 0.697848916053772, "sampling/sampling_logp_difference/max": 0.3612966537475586, "sampling/sampling_logp_difference/mean": 0.009971254505217075, "step": 365 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 753.0, "completions/mean_length": 488.0625, "completions/mean_terminated_length": 459.10345458984375, "completions/min_length": 247.0, "completions/min_terminated_length": 247.0, "entropy": 0.39941294491291046, "epoch": 0.19593147751605997, "frac_reward_zero_std": 0.5, "grad_norm": 0.006344099994748831, "learning_rate": 1e-05, "loss": 0.0189, "num_tokens": 8182824.0, "reward": 0.84375, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.3463642597198486, "sampling/importance_sampling_ratio/mean": 0.9999861717224121, "sampling/importance_sampling_ratio/min": 0.6290184855461121, "sampling/sampling_logp_difference/max": 0.4635946750640869, "sampling/sampling_logp_difference/mean": 0.011870177462697029, "step": 366 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.427702853921801e-05, "clip_ratio/low_min": 5.427702853921801e-05, "clip_ratio/region_mean": 5.427702853921801e-05, "completions/clipped_ratio": 0.34375, "completions/max_length": 768.0, "completions/max_terminated_length": 623.0, "completions/mean_length": 555.65625, "completions/mean_terminated_length": 444.4285888671875, "completions/min_length": 292.0, "completions/min_terminated_length": 292.0, "entropy": 0.4908212870359421, "epoch": 0.19646680942184155, "frac_reward_zero_std": 0.0, "grad_norm": 0.02309938333928585, "learning_rate": 1e-05, "loss": 0.1003, "num_tokens": 8204773.0, "reward": 0.53125, "reward_std": 0.4397946000099182, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.4570693969726562, "sampling/importance_sampling_ratio/mean": 0.9994863271713257, "sampling/importance_sampling_ratio/min": 0.4955500364303589, "sampling/sampling_logp_difference/max": 0.7020869255065918, "sampling/sampling_logp_difference/mean": 0.014599011279642582, "step": 367 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 714.0, "completions/max_terminated_length": 714.0, "completions/mean_length": 476.03125, "completions/mean_terminated_length": 476.03125, "completions/min_length": 245.0, "completions/min_terminated_length": 245.0, "entropy": 0.4153149016201496, "epoch": 0.19700214132762311, "frac_reward_zero_std": 0.75, "grad_norm": 0.002889768686145544, "learning_rate": 1e-05, "loss": -0.0296, "num_tokens": 8223782.0, "reward": 0.71875, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.7374821901321411, "sampling/importance_sampling_ratio/mean": 1.0005125999450684, "sampling/importance_sampling_ratio/min": 0.6276167631149292, "sampling/sampling_logp_difference/max": 0.5524370670318604, "sampling/sampling_logp_difference/mean": 0.012754712253808975, "step": 368 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0001386214207741432, "clip_ratio/low_min": 0.0001386214207741432, "clip_ratio/region_mean": 0.0001386214207741432, "completions/clipped_ratio": 0.0, "completions/max_length": 737.0, "completions/max_terminated_length": 737.0, "completions/mean_length": 468.25, "completions/mean_terminated_length": 468.25, "completions/min_length": 210.0, "completions/min_terminated_length": 210.0, "entropy": 0.3890383690595627, "epoch": 0.1975374732334047, "frac_reward_zero_std": 0.5, "grad_norm": 0.022693229839205742, "learning_rate": 1e-05, "loss": -0.0174, "num_tokens": 8242150.0, "reward": 0.8125, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.439625859260559, "sampling/importance_sampling_ratio/mean": 1.000091791152954, "sampling/importance_sampling_ratio/min": 0.6898723244667053, "sampling/sampling_logp_difference/max": 0.371248722076416, "sampling/sampling_logp_difference/mean": 0.011646188795566559, "step": 369 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.345177826005965e-05, "clip_ratio/low_min": 6.345177826005965e-05, "clip_ratio/region_mean": 6.345177826005965e-05, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 744.0, "completions/mean_length": 495.125, "completions/mean_terminated_length": 466.89654541015625, "completions/min_length": 274.0, "completions/min_terminated_length": 274.0, "entropy": 0.42222118377685547, "epoch": 0.1980728051391863, "frac_reward_zero_std": 0.25, "grad_norm": 0.011758032254874706, "learning_rate": 1e-05, "loss": 0.0572, "num_tokens": 8261770.0, "reward": 0.53125, "reward_std": 0.3471629321575165, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.3863540887832642, "sampling/importance_sampling_ratio/mean": 0.9994974136352539, "sampling/importance_sampling_ratio/min": 0.7322672605514526, "sampling/sampling_logp_difference/max": 0.3266773223876953, "sampling/sampling_logp_difference/mean": 0.01309940405189991, "step": 370 }, { "clip_ratio/high_max": 0.00011445986456237733, "clip_ratio/high_mean": 0.00011445986456237733, "clip_ratio/low_mean": 0.00013022391067352146, "clip_ratio/low_min": 0.00013022391067352146, "clip_ratio/region_mean": 0.0002446837752358988, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 731.0, "completions/mean_length": 537.46875, "completions/mean_terminated_length": 494.77777099609375, "completions/min_length": 342.0, "completions/min_terminated_length": 342.0, "entropy": 0.4241586662828922, "epoch": 0.19860813704496788, "frac_reward_zero_std": 0.0, "grad_norm": 0.005418703425675631, "learning_rate": 1e-05, "loss": 0.0116, "num_tokens": 8283161.0, "reward": 0.78125, "reward_std": 0.4218915104866028, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.5823909044265747, "sampling/importance_sampling_ratio/mean": 1.0000547170639038, "sampling/importance_sampling_ratio/min": 0.579826831817627, "sampling/sampling_logp_difference/max": 0.5450257658958435, "sampling/sampling_logp_difference/mean": 0.012620266526937485, "step": 371 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.798464578925632e-05, "clip_ratio/low_min": 4.798464578925632e-05, "clip_ratio/region_mean": 4.798464578925632e-05, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 765.0, "completions/mean_length": 558.625, "completions/mean_terminated_length": 544.6666870117188, "completions/min_length": 288.0, "completions/min_terminated_length": 288.0, "entropy": 0.3315769210457802, "epoch": 0.19914346895074947, "frac_reward_zero_std": 0.0, "grad_norm": 0.011637786403298378, "learning_rate": 1e-05, "loss": 0.0771, "num_tokens": 8304453.0, "reward": 0.65625, "reward_std": 0.4944729208946228, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.36954927444458, "sampling/importance_sampling_ratio/mean": 1.000084638595581, "sampling/importance_sampling_ratio/min": 0.6938039660453796, "sampling/sampling_logp_difference/max": 0.36556577682495117, "sampling/sampling_logp_difference/mean": 0.010713249444961548, "step": 372 }, { "clip_ratio/high_max": 5.236698780208826e-05, "clip_ratio/high_mean": 5.236698780208826e-05, "clip_ratio/low_mean": 4.74743646918796e-05, "clip_ratio/low_min": 4.74743646918796e-05, "clip_ratio/region_mean": 9.984135249396786e-05, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 749.0, "completions/mean_length": 603.15625, "completions/mean_terminated_length": 548.2083740234375, "completions/min_length": 396.0, "completions/min_terminated_length": 396.0, "entropy": 0.4163140803575516, "epoch": 0.19967880085653106, "frac_reward_zero_std": 0.0, "grad_norm": 0.01129836030304432, "learning_rate": 1e-05, "loss": 0.0385, "num_tokens": 8328066.0, "reward": 0.53125, "reward_std": 0.4807935357093811, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.4257516860961914, "sampling/importance_sampling_ratio/mean": 1.0003459453582764, "sampling/importance_sampling_ratio/min": 0.6285473108291626, "sampling/sampling_logp_difference/max": 0.4643440246582031, "sampling/sampling_logp_difference/mean": 0.012463225051760674, "step": 373 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.58043250546325e-05, "clip_ratio/low_min": 4.58043250546325e-05, "clip_ratio/region_mean": 4.58043250546325e-05, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 767.0, "completions/mean_length": 551.28125, "completions/mean_terminated_length": 520.3214721679688, "completions/min_length": 288.0, "completions/min_terminated_length": 288.0, "entropy": 0.3488573506474495, "epoch": 0.20021413276231265, "frac_reward_zero_std": 0.0, "grad_norm": 0.014986801892518997, "learning_rate": 1e-05, "loss": 0.0967, "num_tokens": 8349131.0, "reward": 0.78125, "reward_std": 0.4218915104866028, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.3101072311401367, "sampling/importance_sampling_ratio/mean": 0.9999088644981384, "sampling/importance_sampling_ratio/min": 0.6880736351013184, "sampling/sampling_logp_difference/max": 0.3738594055175781, "sampling/sampling_logp_difference/mean": 0.010670335032045841, "step": 374 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.3125, "completions/max_length": 768.0, "completions/max_terminated_length": 600.0, "completions/mean_length": 545.75, "completions/mean_terminated_length": 444.727294921875, "completions/min_length": 277.0, "completions/min_terminated_length": 277.0, "entropy": 0.4944803100079298, "epoch": 0.2007494646680942, "frac_reward_zero_std": 0.5, "grad_norm": 0.005338540766388178, "learning_rate": 1e-05, "loss": 0.0431, "num_tokens": 8370531.0, "reward": 0.625, "reward_std": 0.2314550280570984, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.5178382396697998, "sampling/importance_sampling_ratio/mean": 0.999999463558197, "sampling/importance_sampling_ratio/min": 0.7099547386169434, "sampling/sampling_logp_difference/max": 0.41728711128234863, "sampling/sampling_logp_difference/mean": 0.014006479643285275, "step": 375 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.638218888430856e-05, "clip_ratio/low_min": 4.638218888430856e-05, "clip_ratio/region_mean": 4.638218888430856e-05, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 759.0, "completions/mean_length": 569.46875, "completions/mean_terminated_length": 532.7037353515625, "completions/min_length": 279.0, "completions/min_terminated_length": 279.0, "entropy": 0.3828435130417347, "epoch": 0.2012847965738758, "frac_reward_zero_std": 0.25, "grad_norm": 0.020134570077061653, "learning_rate": 1e-05, "loss": 0.0551, "num_tokens": 8392642.0, "reward": 0.8125, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.4648139476776123, "sampling/importance_sampling_ratio/mean": 1.000060796737671, "sampling/importance_sampling_ratio/min": 0.7272341847419739, "sampling/sampling_logp_difference/max": 0.3817281723022461, "sampling/sampling_logp_difference/mean": 0.011920085176825523, "step": 376 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00011117736357846297, "clip_ratio/low_min": 0.00011117736357846297, "clip_ratio/region_mean": 0.00011117736357846297, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 764.0, "completions/mean_length": 619.90625, "completions/mean_terminated_length": 570.5416870117188, "completions/min_length": 312.0, "completions/min_terminated_length": 312.0, "entropy": 0.39548028632998466, "epoch": 0.20182012847965738, "frac_reward_zero_std": 0.25, "grad_norm": 0.006651423405855894, "learning_rate": 1e-05, "loss": 0.0123, "num_tokens": 8416855.0, "reward": 0.6875, "reward_std": 0.3514062464237213, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.2932239770889282, "sampling/importance_sampling_ratio/mean": 0.9998322129249573, "sampling/importance_sampling_ratio/min": 0.7805500030517578, "sampling/sampling_logp_difference/max": 0.2571382522583008, "sampling/sampling_logp_difference/mean": 0.011906609870493412, "step": 377 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.669406189350411e-05, "clip_ratio/low_min": 4.669406189350411e-05, "clip_ratio/region_mean": 4.669406189350411e-05, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 766.0, "completions/mean_length": 580.0625, "completions/mean_terminated_length": 527.4400024414062, "completions/min_length": 315.0, "completions/min_terminated_length": 315.0, "entropy": 0.4088635481894016, "epoch": 0.20235546038543897, "frac_reward_zero_std": 0.25, "grad_norm": 0.011806381866335869, "learning_rate": 1e-05, "loss": 0.0068, "num_tokens": 8438897.0, "reward": 0.6875, "reward_std": 0.2925041913986206, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.3545053005218506, "sampling/importance_sampling_ratio/mean": 0.9998217225074768, "sampling/importance_sampling_ratio/min": 0.7168411612510681, "sampling/sampling_logp_difference/max": 0.3329010009765625, "sampling/sampling_logp_difference/mean": 0.012179611250758171, "step": 378 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 677.0, "completions/mean_length": 560.8125, "completions/mean_terminated_length": 491.75, "completions/min_length": 266.0, "completions/min_terminated_length": 266.0, "entropy": 0.38923313096165657, "epoch": 0.20289079229122056, "frac_reward_zero_std": 0.25, "grad_norm": 0.010750584304332733, "learning_rate": 1e-05, "loss": 0.0021, "num_tokens": 8460651.0, "reward": 0.6875, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.387181282043457, "sampling/importance_sampling_ratio/mean": 0.9999170899391174, "sampling/importance_sampling_ratio/min": 0.7010316252708435, "sampling/sampling_logp_difference/max": 0.355202317237854, "sampling/sampling_logp_difference/mean": 0.01274655107408762, "step": 379 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 9.636845788918436e-05, "clip_ratio/low_min": 9.636845788918436e-05, "clip_ratio/region_mean": 9.636845788918436e-05, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 700.0, "completions/mean_length": 537.59375, "completions/mean_terminated_length": 484.423095703125, "completions/min_length": 274.0, "completions/min_terminated_length": 274.0, "entropy": 0.43413735553622246, "epoch": 0.20342612419700215, "frac_reward_zero_std": 0.25, "grad_norm": 0.011824137531220913, "learning_rate": 1e-05, "loss": 0.0937, "num_tokens": 8481398.0, "reward": 0.65625, "reward_std": 0.3966485261917114, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.3749160766601562, "sampling/importance_sampling_ratio/mean": 1.000190258026123, "sampling/importance_sampling_ratio/min": 0.699332594871521, "sampling/sampling_logp_difference/max": 0.35762882232666016, "sampling/sampling_logp_difference/mean": 0.013350420631468296, "step": 380 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 759.0, "completions/mean_length": 536.71875, "completions/mean_terminated_length": 521.300048828125, "completions/min_length": 290.0, "completions/min_terminated_length": 290.0, "entropy": 0.3783344514667988, "epoch": 0.20396145610278374, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0184, "num_tokens": 8502261.0, "reward": 0.9375, "reward_std": 0.1157275140285492, "rewards/accuracy_reward/mean": 0.9375, "rewards/accuracy_reward/std": 0.24593468010425568, "sampling/importance_sampling_ratio/max": 1.347518801689148, "sampling/importance_sampling_ratio/mean": 0.9999199509620667, "sampling/importance_sampling_ratio/min": 0.17710205912590027, "sampling/sampling_logp_difference/max": 1.7310290336608887, "sampling/sampling_logp_difference/mean": 0.011571547947824001, "step": 381 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 764.0, "completions/mean_length": 490.9375, "completions/mean_terminated_length": 451.357177734375, "completions/min_length": 205.0, "completions/min_terminated_length": 205.0, "entropy": 0.3456154465675354, "epoch": 0.2044967880085653, "frac_reward_zero_std": 0.5, "grad_norm": 0.013170991092920303, "learning_rate": 1e-05, "loss": 0.0837, "num_tokens": 8521483.0, "reward": 0.90625, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.3907924890518188, "sampling/importance_sampling_ratio/mean": 1.0000041723251343, "sampling/importance_sampling_ratio/min": 0.605514645576477, "sampling/sampling_logp_difference/max": 0.5016765594482422, "sampling/sampling_logp_difference/mean": 0.011147287674248219, "step": 382 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 7.594167982460931e-05, "clip_ratio/low_min": 7.594167982460931e-05, "clip_ratio/region_mean": 7.594167982460931e-05, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 725.0, "completions/mean_length": 536.40625, "completions/mean_terminated_length": 503.3214416503906, "completions/min_length": 239.0, "completions/min_terminated_length": 239.0, "entropy": 0.38958995789289474, "epoch": 0.2050321199143469, "frac_reward_zero_std": 0.25, "grad_norm": 0.01350939180701971, "learning_rate": 1e-05, "loss": 0.0677, "num_tokens": 8542160.0, "reward": 0.8125, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.4676110744476318, "sampling/importance_sampling_ratio/mean": 1.000357747077942, "sampling/importance_sampling_ratio/min": 0.7181212902069092, "sampling/sampling_logp_difference/max": 0.3836359977722168, "sampling/sampling_logp_difference/mean": 0.012127463705837727, "step": 383 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 736.0, "completions/mean_length": 583.9375, "completions/mean_terminated_length": 522.5833740234375, "completions/min_length": 343.0, "completions/min_terminated_length": 343.0, "entropy": 0.46484004333615303, "epoch": 0.20556745182012848, "frac_reward_zero_std": 0.5, "grad_norm": 0.009122509509325027, "learning_rate": 1e-05, "loss": 0.035, "num_tokens": 8564878.0, "reward": 0.5625, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.6090890169143677, "sampling/importance_sampling_ratio/mean": 0.9999677538871765, "sampling/importance_sampling_ratio/min": 0.7001003623008728, "sampling/sampling_logp_difference/max": 0.47566819190979004, "sampling/sampling_logp_difference/mean": 0.013996811583638191, "step": 384 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 9.758871601661667e-05, "clip_ratio/low_min": 9.758871601661667e-05, "clip_ratio/region_mean": 9.758871601661667e-05, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 750.0, "completions/mean_length": 615.625, "completions/mean_terminated_length": 572.9599609375, "completions/min_length": 387.0, "completions/min_terminated_length": 387.0, "entropy": 0.39049726352095604, "epoch": 0.20610278372591007, "frac_reward_zero_std": 0.0, "grad_norm": 0.025379296392202377, "learning_rate": 1e-05, "loss": 0.0438, "num_tokens": 8588426.0, "reward": 0.53125, "reward_std": 0.4628904461860657, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.4643515348434448, "sampling/importance_sampling_ratio/mean": 1.000016689300537, "sampling/importance_sampling_ratio/min": 0.607111930847168, "sampling/sampling_logp_difference/max": 0.4990421533584595, "sampling/sampling_logp_difference/mean": 0.01193297654390335, "step": 385 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00011785374226747081, "clip_ratio/low_min": 0.00011785374226747081, "clip_ratio/region_mean": 0.00011785374226747081, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 752.0, "completions/mean_length": 566.40625, "completions/mean_terminated_length": 537.607177734375, "completions/min_length": 303.0, "completions/min_terminated_length": 303.0, "entropy": 0.4207429699599743, "epoch": 0.20663811563169165, "frac_reward_zero_std": 0.25, "grad_norm": 0.010531638748943806, "learning_rate": 1e-05, "loss": 0.0627, "num_tokens": 8610159.0, "reward": 0.5625, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.4233890771865845, "sampling/importance_sampling_ratio/mean": 1.000326156616211, "sampling/importance_sampling_ratio/min": 0.7141737937927246, "sampling/sampling_logp_difference/max": 0.3530406951904297, "sampling/sampling_logp_difference/mean": 0.012914708815515041, "step": 386 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00011987982361461036, "clip_ratio/low_min": 0.00011987982361461036, "clip_ratio/region_mean": 0.00011987982361461036, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 752.0, "completions/mean_length": 560.9375, "completions/mean_terminated_length": 491.91668701171875, "completions/min_length": 244.0, "completions/min_terminated_length": 244.0, "entropy": 0.29077574610710144, "epoch": 0.20717344753747324, "frac_reward_zero_std": 0.5, "grad_norm": 0.012365398928523064, "learning_rate": 1e-05, "loss": 0.0157, "num_tokens": 8631797.0, "reward": 0.46875, "reward_std": 0.2630178928375244, "rewards/accuracy_reward/mean": 0.46875, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.422987937927246, "sampling/importance_sampling_ratio/mean": 0.9994892477989197, "sampling/importance_sampling_ratio/min": 0.6514736413955688, "sampling/sampling_logp_difference/max": 0.42851829528808594, "sampling/sampling_logp_difference/mean": 0.01023923885077238, "step": 387 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 708.0, "completions/mean_length": 541.78125, "completions/mean_terminated_length": 509.46429443359375, "completions/min_length": 194.0, "completions/min_terminated_length": 194.0, "entropy": 0.3683685436844826, "epoch": 0.20770877944325483, "frac_reward_zero_std": 0.75, "grad_norm": 0.0029659071005880833, "learning_rate": 1e-05, "loss": -0.0355, "num_tokens": 8652998.0, "reward": 0.71875, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.2972773313522339, "sampling/importance_sampling_ratio/mean": 1.000082015991211, "sampling/importance_sampling_ratio/min": 0.6745245456695557, "sampling/sampling_logp_difference/max": 0.39374732971191406, "sampling/sampling_logp_difference/mean": 0.011581224389374256, "step": 388 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00012244734898558818, "clip_ratio/low_min": 0.00012244734898558818, "clip_ratio/region_mean": 0.00012244734898558818, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 764.0, "completions/mean_length": 541.875, "completions/mean_terminated_length": 500.0, "completions/min_length": 308.0, "completions/min_terminated_length": 308.0, "entropy": 0.3554263301193714, "epoch": 0.2082441113490364, "frac_reward_zero_std": 0.5, "grad_norm": 0.007479190360754728, "learning_rate": 1e-05, "loss": 0.0059, "num_tokens": 8673658.0, "reward": 0.71875, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.6044725179672241, "sampling/importance_sampling_ratio/mean": 0.9996060132980347, "sampling/importance_sampling_ratio/min": 0.7043182849884033, "sampling/sampling_logp_difference/max": 0.4727950096130371, "sampling/sampling_logp_difference/mean": 0.011445174925029278, "step": 389 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 738.0, "completions/mean_length": 540.65625, "completions/mean_terminated_length": 464.875, "completions/min_length": 223.0, "completions/min_terminated_length": 223.0, "entropy": 0.42104338482022285, "epoch": 0.20877944325481798, "frac_reward_zero_std": 0.5, "grad_norm": 0.0034305029548704624, "learning_rate": 1e-05, "loss": -0.0099, "num_tokens": 8694759.0, "reward": 0.53125, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.5624512434005737, "sampling/importance_sampling_ratio/mean": 0.9999898672103882, "sampling/importance_sampling_ratio/min": 0.7069579362869263, "sampling/sampling_logp_difference/max": 0.4462559223175049, "sampling/sampling_logp_difference/mean": 0.013074529357254505, "step": 390 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 711.0, "completions/mean_length": 472.09375, "completions/mean_terminated_length": 441.4827575683594, "completions/min_length": 197.0, "completions/min_terminated_length": 197.0, "entropy": 0.339879784733057, "epoch": 0.20931477516059957, "frac_reward_zero_std": 0.5, "grad_norm": 0.009654682129621506, "learning_rate": 1e-05, "loss": 0.0432, "num_tokens": 8713314.0, "reward": 0.53125, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.6273754835128784, "sampling/importance_sampling_ratio/mean": 1.0000404119491577, "sampling/importance_sampling_ratio/min": 0.7747105956077576, "sampling/sampling_logp_difference/max": 0.4869685173034668, "sampling/sampling_logp_difference/mean": 0.011338535696268082, "step": 391 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 721.0, "completions/mean_length": 449.28125, "completions/mean_terminated_length": 439.0, "completions/min_length": 156.0, "completions/min_terminated_length": 156.0, "entropy": 0.39829106256365776, "epoch": 0.20985010706638116, "frac_reward_zero_std": 0.5, "grad_norm": 0.015813199803233147, "learning_rate": 1e-05, "loss": -0.0001, "num_tokens": 8731259.0, "reward": 0.8125, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.3294726610183716, "sampling/importance_sampling_ratio/mean": 0.9998857975006104, "sampling/importance_sampling_ratio/min": 0.7047446370124817, "sampling/sampling_logp_difference/max": 0.34991979598999023, "sampling/sampling_logp_difference/mean": 0.012459760531783104, "step": 392 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.607445589499548e-05, "clip_ratio/low_min": 4.607445589499548e-05, "clip_ratio/region_mean": 4.607445589499548e-05, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 764.0, "completions/mean_length": 579.40625, "completions/mean_terminated_length": 516.5416870117188, "completions/min_length": 165.0, "completions/min_terminated_length": 165.0, "entropy": 0.4902245067059994, "epoch": 0.21038543897216275, "frac_reward_zero_std": 0.5, "grad_norm": 0.0068397908471524715, "learning_rate": 1e-05, "loss": 0.0321, "num_tokens": 8753632.0, "reward": 0.625, "reward_std": 0.2314550280570984, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.2944300174713135, "sampling/importance_sampling_ratio/mean": 0.9996551275253296, "sampling/importance_sampling_ratio/min": 0.6621697545051575, "sampling/sampling_logp_difference/max": 0.4122333526611328, "sampling/sampling_logp_difference/mean": 0.01450817659497261, "step": 393 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00011655086200335063, "clip_ratio/low_min": 0.00011655086200335063, "clip_ratio/region_mean": 0.00011655086200335063, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 754.0, "completions/mean_length": 546.84375, "completions/mean_terminated_length": 484.91998291015625, "completions/min_length": 306.0, "completions/min_terminated_length": 306.0, "entropy": 0.5028059035539627, "epoch": 0.21092077087794434, "frac_reward_zero_std": 0.0, "grad_norm": 0.020201072096824646, "learning_rate": 1e-05, "loss": 0.0305, "num_tokens": 8775035.0, "reward": 0.59375, "reward_std": 0.4628904461860657, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.3976861238479614, "sampling/importance_sampling_ratio/mean": 1.0001035928726196, "sampling/importance_sampling_ratio/min": 0.5971508622169495, "sampling/sampling_logp_difference/max": 0.5155854821205139, "sampling/sampling_logp_difference/mean": 0.014604815281927586, "step": 394 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 737.0, "completions/mean_length": 561.9375, "completions/mean_terminated_length": 548.2000122070312, "completions/min_length": 262.0, "completions/min_terminated_length": 262.0, "entropy": 0.31315233558416367, "epoch": 0.2114561027837259, "frac_reward_zero_std": 0.5, "grad_norm": 0.010948101989924908, "learning_rate": 1e-05, "loss": 0.0285, "num_tokens": 8797289.0, "reward": 0.84375, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.6328351497650146, "sampling/importance_sampling_ratio/mean": 0.9999956488609314, "sampling/importance_sampling_ratio/min": 0.7200225591659546, "sampling/sampling_logp_difference/max": 0.49031782150268555, "sampling/sampling_logp_difference/mean": 0.010241299867630005, "step": 395 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 765.0, "completions/mean_length": 565.84375, "completions/mean_terminated_length": 536.9642944335938, "completions/min_length": 321.0, "completions/min_terminated_length": 321.0, "entropy": 0.28643761575222015, "epoch": 0.21199143468950749, "frac_reward_zero_std": 0.25, "grad_norm": 0.006162581965327263, "learning_rate": 1e-05, "loss": -0.0147, "num_tokens": 8819196.0, "reward": 0.6875, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.4234240055084229, "sampling/importance_sampling_ratio/mean": 0.9998257160186768, "sampling/importance_sampling_ratio/min": 0.5781893134117126, "sampling/sampling_logp_difference/max": 0.547853946685791, "sampling/sampling_logp_difference/mean": 0.009983896277844906, "step": 396 }, { "clip_ratio/high_max": 5.835667616338469e-05, "clip_ratio/high_mean": 5.835667616338469e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 5.835667616338469e-05, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 701.0, "completions/mean_length": 506.25, "completions/mean_terminated_length": 468.857177734375, "completions/min_length": 282.0, "completions/min_terminated_length": 282.0, "entropy": 0.36724481359124184, "epoch": 0.21252676659528907, "frac_reward_zero_std": 0.5, "grad_norm": 0.004155013710260391, "learning_rate": 1e-05, "loss": 0.0533, "num_tokens": 8839356.0, "reward": 0.6875, "reward_std": 0.249358132481575, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.431915521621704, "sampling/importance_sampling_ratio/mean": 0.9999504685401917, "sampling/importance_sampling_ratio/min": 0.6292864680290222, "sampling/sampling_logp_difference/max": 0.463168740272522, "sampling/sampling_logp_difference/mean": 0.01224219985306263, "step": 397 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 631.0, "completions/mean_length": 464.53125, "completions/mean_terminated_length": 444.3000183105469, "completions/min_length": 231.0, "completions/min_terminated_length": 231.0, "entropy": 0.456684835255146, "epoch": 0.21306209850107066, "frac_reward_zero_std": 0.0, "grad_norm": 0.013800724409520626, "learning_rate": 1e-05, "loss": -0.014, "num_tokens": 8858837.0, "reward": 0.5625, "reward_std": 0.47655022144317627, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.399673342704773, "sampling/importance_sampling_ratio/mean": 1.0001552104949951, "sampling/importance_sampling_ratio/min": 0.7700387835502625, "sampling/sampling_logp_difference/max": 0.3362388610839844, "sampling/sampling_logp_difference/mean": 0.014229970052838326, "step": 398 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.33275563409552e-05, "clip_ratio/low_min": 4.33275563409552e-05, "clip_ratio/region_mean": 4.33275563409552e-05, "completions/clipped_ratio": 0.28125, "completions/max_length": 768.0, "completions/max_terminated_length": 766.0, "completions/mean_length": 630.5, "completions/mean_terminated_length": 576.6956787109375, "completions/min_length": 286.0, "completions/min_terminated_length": 286.0, "entropy": 0.43897998705506325, "epoch": 0.21359743040685225, "frac_reward_zero_std": 0.25, "grad_norm": 0.005376460030674934, "learning_rate": 1e-05, "loss": 0.0637, "num_tokens": 8882869.0, "reward": 0.59375, "reward_std": 0.3061639964580536, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.4658955335617065, "sampling/importance_sampling_ratio/mean": 1.000138759613037, "sampling/importance_sampling_ratio/min": 0.7277007102966309, "sampling/sampling_logp_difference/max": 0.38246631622314453, "sampling/sampling_logp_difference/mean": 0.012856343761086464, "step": 399 }, { "clip_ratio/high_max": 5.468066592584364e-05, "clip_ratio/high_mean": 5.468066592584364e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 5.468066592584364e-05, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 725.0, "completions/mean_length": 526.0, "completions/mean_terminated_length": 509.86669921875, "completions/min_length": 295.0, "completions/min_terminated_length": 295.0, "entropy": 0.30568647384643555, "epoch": 0.21413276231263384, "frac_reward_zero_std": 0.0, "grad_norm": 0.020914584398269653, "learning_rate": 1e-05, "loss": 0.0765, "num_tokens": 8903293.0, "reward": 0.59375, "reward_std": 0.4534739851951599, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.4682308435440063, "sampling/importance_sampling_ratio/mean": 1.000411868095398, "sampling/importance_sampling_ratio/min": 0.6125555634498596, "sampling/sampling_logp_difference/max": 0.4901156425476074, "sampling/sampling_logp_difference/mean": 0.01073786523193121, "step": 400 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.2105042414041236e-05, "clip_ratio/low_min": 5.2105042414041236e-05, "clip_ratio/region_mean": 5.2105042414041236e-05, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 731.0, "completions/mean_length": 582.59375, "completions/mean_terminated_length": 548.25927734375, "completions/min_length": 253.0, "completions/min_terminated_length": 253.0, "entropy": 0.2904437258839607, "epoch": 0.21466809421841543, "frac_reward_zero_std": 0.25, "grad_norm": 0.0036875430960208178, "learning_rate": 1e-05, "loss": 0.0899, "num_tokens": 8925400.0, "reward": 0.84375, "reward_std": 0.3061639666557312, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.4000310897827148, "sampling/importance_sampling_ratio/mean": 1.0001499652862549, "sampling/importance_sampling_ratio/min": 0.605435848236084, "sampling/sampling_logp_difference/max": 0.5018067359924316, "sampling/sampling_logp_difference/mean": 0.009563757106661797, "step": 401 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 754.0, "completions/mean_length": 448.03125, "completions/mean_terminated_length": 437.70965576171875, "completions/min_length": 240.0, "completions/min_terminated_length": 240.0, "entropy": 0.3770170584321022, "epoch": 0.215203426124197, "frac_reward_zero_std": 0.25, "grad_norm": 0.011369570158421993, "learning_rate": 1e-05, "loss": -0.0049, "num_tokens": 8942953.0, "reward": 0.59375, "reward_std": 0.3061639964580536, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.4061670303344727, "sampling/importance_sampling_ratio/mean": 1.000148892402649, "sampling/importance_sampling_ratio/min": 0.7544295787811279, "sampling/sampling_logp_difference/max": 0.3408675193786621, "sampling/sampling_logp_difference/mean": 0.0119005823507905, "step": 402 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 686.0, "completions/mean_length": 570.15625, "completions/mean_terminated_length": 514.760009765625, "completions/min_length": 243.0, "completions/min_terminated_length": 243.0, "entropy": 0.3198831807821989, "epoch": 0.21573875802997858, "frac_reward_zero_std": 0.25, "grad_norm": 0.005257037468254566, "learning_rate": 1e-05, "loss": 0.0479, "num_tokens": 8964846.0, "reward": 0.65625, "reward_std": 0.3198433816432953, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.4231312274932861, "sampling/importance_sampling_ratio/mean": 0.9996529817581177, "sampling/importance_sampling_ratio/min": 0.6967759132385254, "sampling/sampling_logp_difference/max": 0.36129140853881836, "sampling/sampling_logp_difference/mean": 0.010347031056880951, "step": 403 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00011215791892027482, "clip_ratio/low_min": 0.00011215791892027482, "clip_ratio/region_mean": 0.00011215791892027482, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 740.0, "completions/mean_length": 474.5625, "completions/mean_terminated_length": 465.0967712402344, "completions/min_length": 235.0, "completions/min_terminated_length": 235.0, "entropy": 0.38425299152731895, "epoch": 0.21627408993576017, "frac_reward_zero_std": 0.25, "grad_norm": 0.010644332505762577, "learning_rate": 1e-05, "loss": -0.0006, "num_tokens": 8983448.0, "reward": 0.8125, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.445139765739441, "sampling/importance_sampling_ratio/mean": 0.9999208450317383, "sampling/importance_sampling_ratio/min": 0.6692987084388733, "sampling/sampling_logp_difference/max": 0.40152478218078613, "sampling/sampling_logp_difference/mean": 0.012317796237766743, "step": 404 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.32821832166519e-05, "clip_ratio/low_min": 5.32821832166519e-05, "clip_ratio/region_mean": 5.32821832166519e-05, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 637.0, "completions/mean_length": 479.5, "completions/mean_terminated_length": 470.19354248046875, "completions/min_length": 281.0, "completions/min_terminated_length": 281.0, "entropy": 0.33991294354200363, "epoch": 0.21680942184154176, "frac_reward_zero_std": 0.5, "grad_norm": 0.00593644380569458, "learning_rate": 1e-05, "loss": 0.0168, "num_tokens": 9002560.0, "reward": 0.875, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 1.4745886325836182, "sampling/importance_sampling_ratio/mean": 1.0000718832015991, "sampling/importance_sampling_ratio/min": 0.700054943561554, "sampling/sampling_logp_difference/max": 0.3883790969848633, "sampling/sampling_logp_difference/mean": 0.011626236140727997, "step": 405 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 692.0, "completions/mean_length": 511.21875, "completions/mean_terminated_length": 494.10003662109375, "completions/min_length": 321.0, "completions/min_terminated_length": 321.0, "entropy": 0.3885650038719177, "epoch": 0.21734475374732334, "frac_reward_zero_std": 0.25, "grad_norm": 0.011330134235322475, "learning_rate": 1e-05, "loss": 0.0533, "num_tokens": 9022535.0, "reward": 0.5625, "reward_std": 0.3745020925998688, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.5095738172531128, "sampling/importance_sampling_ratio/mean": 1.0001286268234253, "sampling/importance_sampling_ratio/min": 0.6953168511390686, "sampling/sampling_logp_difference/max": 0.41182732582092285, "sampling/sampling_logp_difference/mean": 0.012502442114055157, "step": 406 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.3533189202426e-05, "clip_ratio/low_min": 5.3533189202426e-05, "clip_ratio/region_mean": 5.3533189202426e-05, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 762.0, "completions/mean_length": 537.75, "completions/mean_terminated_length": 522.4000244140625, "completions/min_length": 191.0, "completions/min_terminated_length": 191.0, "entropy": 0.3448605425655842, "epoch": 0.21788008565310493, "frac_reward_zero_std": 0.25, "grad_norm": 0.01386034395545721, "learning_rate": 1e-05, "loss": -0.058, "num_tokens": 9043839.0, "reward": 0.59375, "reward_std": 0.3061639964580536, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.3660922050476074, "sampling/importance_sampling_ratio/mean": 1.000069499015808, "sampling/importance_sampling_ratio/min": 0.6906020045280457, "sampling/sampling_logp_difference/max": 0.3701915740966797, "sampling/sampling_logp_difference/mean": 0.011480391025543213, "step": 407 }, { "clip_ratio/high_max": 5.5285272537730634e-05, "clip_ratio/high_mean": 5.5285272537730634e-05, "clip_ratio/low_mean": 5.912961205467582e-05, "clip_ratio/low_min": 5.912961205467582e-05, "clip_ratio/region_mean": 0.00011441488459240645, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 752.0, "completions/mean_length": 569.53125, "completions/mean_terminated_length": 532.7777709960938, "completions/min_length": 325.0, "completions/min_terminated_length": 325.0, "entropy": 0.2892577201128006, "epoch": 0.21841541755888652, "frac_reward_zero_std": 0.25, "grad_norm": 0.0128361526876688, "learning_rate": 1e-05, "loss": 0.0604, "num_tokens": 9065408.0, "reward": 0.71875, "reward_std": 0.3198433816432953, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.5042604207992554, "sampling/importance_sampling_ratio/mean": 0.9997498989105225, "sampling/importance_sampling_ratio/min": 0.7064053416252136, "sampling/sampling_logp_difference/max": 0.40830135345458984, "sampling/sampling_logp_difference/mean": 0.009226069785654545, "step": 408 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.069010537932627e-05, "clip_ratio/low_min": 4.069010537932627e-05, "clip_ratio/region_mean": 4.069010537932627e-05, "completions/clipped_ratio": 0.40625, "completions/max_length": 768.0, "completions/max_terminated_length": 764.0, "completions/mean_length": 615.875, "completions/mean_terminated_length": 511.78948974609375, "completions/min_length": 159.0, "completions/min_terminated_length": 159.0, "entropy": 0.6065098270773888, "epoch": 0.21895074946466808, "frac_reward_zero_std": 0.25, "grad_norm": 0.0186065174639225, "learning_rate": 1e-05, "loss": 0.0543, "num_tokens": 9088772.0, "reward": 0.53125, "reward_std": 0.3471629321575165, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.3956139087677002, "sampling/importance_sampling_ratio/mean": 1.0001424551010132, "sampling/importance_sampling_ratio/min": 0.6796916723251343, "sampling/sampling_logp_difference/max": 0.38611602783203125, "sampling/sampling_logp_difference/mean": 0.016564738005399704, "step": 409 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 9.758587111718953e-05, "clip_ratio/low_min": 9.758587111718953e-05, "clip_ratio/region_mean": 9.758587111718953e-05, "completions/clipped_ratio": 0.3125, "completions/max_length": 768.0, "completions/max_terminated_length": 761.0, "completions/mean_length": 578.53125, "completions/mean_terminated_length": 492.40911865234375, "completions/min_length": 314.0, "completions/min_terminated_length": 314.0, "entropy": 0.5033793821930885, "epoch": 0.21948608137044967, "frac_reward_zero_std": 0.25, "grad_norm": 0.0057432218454778194, "learning_rate": 1e-05, "loss": 0.1145, "num_tokens": 9111373.0, "reward": 0.53125, "reward_std": 0.3471629321575165, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.4896433353424072, "sampling/importance_sampling_ratio/mean": 1.0002572536468506, "sampling/importance_sampling_ratio/min": 0.7555832266807556, "sampling/sampling_logp_difference/max": 0.39853668212890625, "sampling/sampling_logp_difference/mean": 0.013849738985300064, "step": 410 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.34375, "completions/max_length": 768.0, "completions/max_terminated_length": 768.0, "completions/mean_length": 597.09375, "completions/mean_terminated_length": 507.5714416503906, "completions/min_length": 262.0, "completions/min_terminated_length": 262.0, "entropy": 0.6196390092372894, "epoch": 0.22002141327623126, "frac_reward_zero_std": 0.5, "grad_norm": 0.00944578181952238, "learning_rate": 1e-05, "loss": 0.0325, "num_tokens": 9134904.0, "reward": 0.5, "reward_std": 0.26726123690605164, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5080004930496216, "sampling/importance_sampling_ratio/max": 1.372896671295166, "sampling/importance_sampling_ratio/mean": 0.9999515414237976, "sampling/importance_sampling_ratio/min": 0.7232800126075745, "sampling/sampling_logp_difference/max": 0.3239588737487793, "sampling/sampling_logp_difference/mean": 0.016830801963806152, "step": 411 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 744.0, "completions/mean_length": 591.0, "completions/mean_terminated_length": 541.4400024414062, "completions/min_length": 322.0, "completions/min_terminated_length": 322.0, "entropy": 0.41069863736629486, "epoch": 0.22055674518201285, "frac_reward_zero_std": 0.5, "grad_norm": 0.011540129780769348, "learning_rate": 1e-05, "loss": 0.0398, "num_tokens": 9157360.0, "reward": 0.5625, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.3424558639526367, "sampling/importance_sampling_ratio/mean": 0.999802827835083, "sampling/importance_sampling_ratio/min": 0.6996087431907654, "sampling/sampling_logp_difference/max": 0.35723400115966797, "sampling/sampling_logp_difference/mean": 0.012154701165854931, "step": 412 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.80769231216982e-05, "clip_ratio/low_min": 4.80769231216982e-05, "clip_ratio/region_mean": 4.80769231216982e-05, "completions/clipped_ratio": 0.3125, "completions/max_length": 768.0, "completions/max_terminated_length": 732.0, "completions/mean_length": 604.125, "completions/mean_terminated_length": 529.6363525390625, "completions/min_length": 310.0, "completions/min_terminated_length": 310.0, "entropy": 0.4063192680478096, "epoch": 0.22109207708779444, "frac_reward_zero_std": 0.25, "grad_norm": 0.013522643595933914, "learning_rate": 1e-05, "loss": 0.0541, "num_tokens": 9181020.0, "reward": 0.59375, "reward_std": 0.3061639964580536, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.8388729095458984, "sampling/importance_sampling_ratio/mean": 0.9999910593032837, "sampling/importance_sampling_ratio/min": 0.6064245104789734, "sampling/sampling_logp_difference/max": 0.6091527938842773, "sampling/sampling_logp_difference/mean": 0.012002038769423962, "step": 413 }, { "clip_ratio/high_max": 7.237985118990764e-05, "clip_ratio/high_mean": 7.237985118990764e-05, "clip_ratio/low_mean": 0.00012242898810654879, "clip_ratio/low_min": 0.00012242898810654879, "clip_ratio/region_mean": 0.00019480883929645643, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 698.0, "completions/mean_length": 521.4375, "completions/mean_terminated_length": 439.25, "completions/min_length": 274.0, "completions/min_terminated_length": 274.0, "entropy": 0.4026436246931553, "epoch": 0.22162740899357602, "frac_reward_zero_std": 0.25, "grad_norm": 0.004440732765942812, "learning_rate": 1e-05, "loss": 0.0381, "num_tokens": 9201306.0, "reward": 0.5, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5080004930496216, "sampling/importance_sampling_ratio/max": 1.3815584182739258, "sampling/importance_sampling_ratio/mean": 1.0002615451812744, "sampling/importance_sampling_ratio/min": 0.40709826350212097, "sampling/sampling_logp_difference/max": 0.8987007141113281, "sampling/sampling_logp_difference/mean": 0.012605084106326103, "step": 414 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 9.0698194981087e-05, "clip_ratio/low_min": 9.0698194981087e-05, "clip_ratio/region_mean": 9.0698194981087e-05, "completions/clipped_ratio": 0.40625, "completions/max_length": 768.0, "completions/max_terminated_length": 704.0, "completions/mean_length": 655.09375, "completions/mean_terminated_length": 577.8421020507812, "completions/min_length": 399.0, "completions/min_terminated_length": 399.0, "entropy": 0.44373518601059914, "epoch": 0.2221627408993576, "frac_reward_zero_std": 0.25, "grad_norm": 0.012777360156178474, "learning_rate": 1e-05, "loss": -0.0018, "num_tokens": 9226389.0, "reward": 0.40625, "reward_std": 0.3471629321575165, "rewards/accuracy_reward/mean": 0.40625, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.6369410753250122, "sampling/importance_sampling_ratio/mean": 1.0006061792373657, "sampling/importance_sampling_ratio/min": 0.6410353779792786, "sampling/sampling_logp_difference/max": 0.4928293228149414, "sampling/sampling_logp_difference/mean": 0.012791387736797333, "step": 415 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.34375, "completions/max_length": 768.0, "completions/max_terminated_length": 747.0, "completions/mean_length": 622.15625, "completions/mean_terminated_length": 545.7619018554688, "completions/min_length": 277.0, "completions/min_terminated_length": 277.0, "entropy": 0.4507865346968174, "epoch": 0.22269807280513917, "frac_reward_zero_std": 0.5, "grad_norm": 0.015010213479399681, "learning_rate": 1e-05, "loss": 0.0346, "num_tokens": 9249730.0, "reward": 0.65625, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.2977815866470337, "sampling/importance_sampling_ratio/mean": 1.0001089572906494, "sampling/importance_sampling_ratio/min": 0.6073641777038574, "sampling/sampling_logp_difference/max": 0.498626708984375, "sampling/sampling_logp_difference/mean": 0.01323573850095272, "step": 416 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00019604443150456063, "clip_ratio/low_min": 0.00019604443150456063, "clip_ratio/region_mean": 0.00019604443150456063, "completions/clipped_ratio": 0.34375, "completions/max_length": 768.0, "completions/max_terminated_length": 714.0, "completions/mean_length": 587.65625, "completions/mean_terminated_length": 493.19049072265625, "completions/min_length": 180.0, "completions/min_terminated_length": 180.0, "entropy": 0.4715174324810505, "epoch": 0.22323340471092076, "frac_reward_zero_std": 0.5, "grad_norm": 0.013832558877766132, "learning_rate": 1e-05, "loss": 0.0106, "num_tokens": 9272495.0, "reward": 0.59375, "reward_std": 0.22201895713806152, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.2830474376678467, "sampling/importance_sampling_ratio/mean": 0.999737024307251, "sampling/importance_sampling_ratio/min": 0.5005362629890442, "sampling/sampling_logp_difference/max": 0.692075252532959, "sampling/sampling_logp_difference/mean": 0.013608349487185478, "step": 417 }, { "clip_ratio/high_max": 5.6947606935864314e-05, "clip_ratio/high_mean": 5.6947606935864314e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 5.6947606935864314e-05, "completions/clipped_ratio": 0.3125, "completions/max_length": 768.0, "completions/max_terminated_length": 732.0, "completions/mean_length": 587.0, "completions/mean_terminated_length": 504.727294921875, "completions/min_length": 329.0, "completions/min_terminated_length": 329.0, "entropy": 0.6519546769559383, "epoch": 0.22376873661670235, "frac_reward_zero_std": 0.5, "grad_norm": 0.021211620420217514, "learning_rate": 1e-05, "loss": 0.0725, "num_tokens": 9295959.0, "reward": 0.5625, "reward_std": 0.249358132481575, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.4707138538360596, "sampling/importance_sampling_ratio/mean": 1.0002268552780151, "sampling/importance_sampling_ratio/min": 0.5587266683578491, "sampling/sampling_logp_difference/max": 0.5820949077606201, "sampling/sampling_logp_difference/mean": 0.017742017284035683, "step": 418 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.4324205848388374e-05, "clip_ratio/low_min": 5.4324205848388374e-05, "clip_ratio/region_mean": 5.4324205848388374e-05, "completions/clipped_ratio": 0.34375, "completions/max_length": 768.0, "completions/max_terminated_length": 726.0, "completions/mean_length": 615.46875, "completions/mean_terminated_length": 535.5714111328125, "completions/min_length": 362.0, "completions/min_terminated_length": 362.0, "entropy": 0.3769516982138157, "epoch": 0.22430406852248394, "frac_reward_zero_std": 0.25, "grad_norm": 0.0035179273691028357, "learning_rate": 1e-05, "loss": 0.0939, "num_tokens": 9319446.0, "reward": 0.625, "reward_std": 0.3104073107242584, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.3247960805892944, "sampling/importance_sampling_ratio/mean": 0.9999805688858032, "sampling/importance_sampling_ratio/min": 0.6951610445976257, "sampling/sampling_logp_difference/max": 0.36361169815063477, "sampling/sampling_logp_difference/mean": 0.010986430570483208, "step": 419 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 742.0, "completions/mean_length": 630.78125, "completions/mean_terminated_length": 585.0416870117188, "completions/min_length": 413.0, "completions/min_terminated_length": 413.0, "entropy": 0.31577062606811523, "epoch": 0.22483940042826553, "frac_reward_zero_std": 0.5, "grad_norm": 0.0051013389602303505, "learning_rate": 1e-05, "loss": 0.0481, "num_tokens": 9343263.0, "reward": 0.53125, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.4925427436828613, "sampling/importance_sampling_ratio/mean": 1.0003926753997803, "sampling/importance_sampling_ratio/min": 0.7386350035667419, "sampling/sampling_logp_difference/max": 0.4004812240600586, "sampling/sampling_logp_difference/mean": 0.010470103472471237, "step": 420 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 761.0, "completions/mean_length": 505.5625, "completions/mean_terminated_length": 497.0967712402344, "completions/min_length": 294.0, "completions/min_terminated_length": 294.0, "entropy": 0.32444397918879986, "epoch": 0.22537473233404712, "frac_reward_zero_std": 0.0, "grad_norm": 0.006141118239611387, "learning_rate": 1e-05, "loss": 0.0871, "num_tokens": 9363265.0, "reward": 0.5, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5080004930496216, "sampling/importance_sampling_ratio/max": 1.416572093963623, "sampling/importance_sampling_ratio/mean": 0.9998906254768372, "sampling/importance_sampling_ratio/min": 0.7291104197502136, "sampling/sampling_logp_difference/max": 0.3482398986816406, "sampling/sampling_logp_difference/mean": 0.011126390658318996, "step": 421 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00023380395214189775, "clip_ratio/low_min": 0.00023380395214189775, "clip_ratio/region_mean": 0.00023380395214189775, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 735.0, "completions/mean_length": 563.3125, "completions/mean_terminated_length": 516.0769653320312, "completions/min_length": 348.0, "completions/min_terminated_length": 348.0, "entropy": 0.33970316872000694, "epoch": 0.2259100642398287, "frac_reward_zero_std": 0.0, "grad_norm": 0.01292883139103651, "learning_rate": 1e-05, "loss": 0.0555, "num_tokens": 9385555.0, "reward": 0.59375, "reward_std": 0.4218915104866028, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.5158705711364746, "sampling/importance_sampling_ratio/mean": 0.9997826814651489, "sampling/importance_sampling_ratio/min": 0.6873456239700317, "sampling/sampling_logp_difference/max": 0.41598987579345703, "sampling/sampling_logp_difference/mean": 0.011197828687727451, "step": 422 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 7.094211468938738e-05, "clip_ratio/low_min": 7.094211468938738e-05, "clip_ratio/region_mean": 7.094211468938738e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 706.0, "completions/max_terminated_length": 706.0, "completions/mean_length": 460.3125, "completions/mean_terminated_length": 460.3125, "completions/min_length": 241.0, "completions/min_terminated_length": 241.0, "entropy": 0.3652568534016609, "epoch": 0.22644539614561027, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": -0.04, "num_tokens": 9403701.0, "reward": 0.8125, "reward_std": 0.1157275140285492, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.310981035232544, "sampling/importance_sampling_ratio/mean": 0.9997658729553223, "sampling/importance_sampling_ratio/min": 0.6754453182220459, "sampling/sampling_logp_difference/max": 0.3923830986022949, "sampling/sampling_logp_difference/mean": 0.011771125718951225, "step": 423 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.493170490604825e-05, "clip_ratio/low_min": 4.493170490604825e-05, "clip_ratio/region_mean": 4.493170490604825e-05, "completions/clipped_ratio": 0.28125, "completions/max_length": 768.0, "completions/max_terminated_length": 759.0, "completions/mean_length": 558.9375, "completions/mean_terminated_length": 477.13043212890625, "completions/min_length": 195.0, "completions/min_terminated_length": 195.0, "entropy": 0.44727589562535286, "epoch": 0.22698072805139186, "frac_reward_zero_std": 0.25, "grad_norm": 0.024435359984636307, "learning_rate": 1e-05, "loss": 0.0439, "num_tokens": 9425763.0, "reward": 0.6875, "reward_std": 0.3471825420856476, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.9711543321609497, "sampling/importance_sampling_ratio/mean": 1.0003437995910645, "sampling/importance_sampling_ratio/min": 0.6911482214927673, "sampling/sampling_logp_difference/max": 0.678619384765625, "sampling/sampling_logp_difference/mean": 0.013055948540568352, "step": 424 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 729.0, "completions/mean_length": 533.40625, "completions/mean_terminated_length": 479.2692565917969, "completions/min_length": 263.0, "completions/min_terminated_length": 263.0, "entropy": 0.36951202899217606, "epoch": 0.22751605995717344, "frac_reward_zero_std": 0.25, "grad_norm": 0.027764614671468735, "learning_rate": 1e-05, "loss": 0.1327, "num_tokens": 9446264.0, "reward": 0.625, "reward_std": 0.2925041913986206, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.2995736598968506, "sampling/importance_sampling_ratio/mean": 0.9997979402542114, "sampling/importance_sampling_ratio/min": 0.6881005167961121, "sampling/sampling_logp_difference/max": 0.37382030487060547, "sampling/sampling_logp_difference/mean": 0.011245404370129108, "step": 425 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.946717283222824e-05, "clip_ratio/low_min": 5.946717283222824e-05, "clip_ratio/region_mean": 5.946717283222824e-05, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 763.0, "completions/mean_length": 584.40625, "completions/mean_terminated_length": 533.0, "completions/min_length": 339.0, "completions/min_terminated_length": 339.0, "entropy": 0.31000653095543385, "epoch": 0.22805139186295503, "frac_reward_zero_std": 0.25, "grad_norm": 0.011753677390515804, "learning_rate": 1e-05, "loss": 0.0523, "num_tokens": 9468717.0, "reward": 0.6875, "reward_std": 0.3514062464237213, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.3142720460891724, "sampling/importance_sampling_ratio/mean": 1.0002434253692627, "sampling/importance_sampling_ratio/min": 0.6745606660842896, "sampling/sampling_logp_difference/max": 0.3936936855316162, "sampling/sampling_logp_difference/mean": 0.0103093720972538, "step": 426 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.3125, "completions/max_length": 768.0, "completions/max_terminated_length": 756.0, "completions/mean_length": 606.65625, "completions/mean_terminated_length": 533.3181762695312, "completions/min_length": 309.0, "completions/min_terminated_length": 309.0, "entropy": 0.34468335285782814, "epoch": 0.22858672376873662, "frac_reward_zero_std": 0.5, "grad_norm": 0.014401816762983799, "learning_rate": 1e-05, "loss": 0.0063, "num_tokens": 9492026.0, "reward": 0.5625, "reward_std": 0.2587745785713196, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.000317931175232, "sampling/importance_sampling_ratio/min": 0.5973264575004578, "sampling/sampling_logp_difference/max": 0.7384324073791504, "sampling/sampling_logp_difference/mean": 0.011003500781953335, "step": 427 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 762.0, "completions/max_terminated_length": 762.0, "completions/mean_length": 460.96875, "completions/mean_terminated_length": 460.96875, "completions/min_length": 262.0, "completions/min_terminated_length": 262.0, "entropy": 0.3451710008084774, "epoch": 0.2291220556745182, "frac_reward_zero_std": 0.25, "grad_norm": 0.019438588991761208, "learning_rate": 1e-05, "loss": 0.0016, "num_tokens": 9510321.0, "reward": 0.65625, "reward_std": 0.3966485261917114, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.6240769624710083, "sampling/importance_sampling_ratio/mean": 0.9998361468315125, "sampling/importance_sampling_ratio/min": 0.7779816389083862, "sampling/sampling_logp_difference/max": 0.4849395751953125, "sampling/sampling_logp_difference/mean": 0.01068849116563797, "step": 428 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 753.0, "completions/mean_length": 565.96875, "completions/mean_terminated_length": 537.107177734375, "completions/min_length": 382.0, "completions/min_terminated_length": 382.0, "entropy": 0.31969908252358437, "epoch": 0.2296573875802998, "frac_reward_zero_std": 0.5, "grad_norm": 0.011303309351205826, "learning_rate": 1e-05, "loss": 0.0354, "num_tokens": 9532320.0, "reward": 0.59375, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.4453920125961304, "sampling/importance_sampling_ratio/mean": 1.000016450881958, "sampling/importance_sampling_ratio/min": 0.697854220867157, "sampling/sampling_logp_difference/max": 0.3683805465698242, "sampling/sampling_logp_difference/mean": 0.010260508395731449, "step": 429 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 715.0, "completions/mean_length": 445.9375, "completions/mean_terminated_length": 424.4666748046875, "completions/min_length": 230.0, "completions/min_terminated_length": 230.0, "entropy": 0.3787587955594063, "epoch": 0.23019271948608136, "frac_reward_zero_std": 0.5, "grad_norm": 0.018236199393868446, "learning_rate": 1e-05, "loss": 0.1, "num_tokens": 9550118.0, "reward": 0.90625, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.40244722366333, "sampling/importance_sampling_ratio/mean": 0.9997003078460693, "sampling/importance_sampling_ratio/min": 0.6140540242195129, "sampling/sampling_logp_difference/max": 0.4876723289489746, "sampling/sampling_logp_difference/mean": 0.011893510818481445, "step": 430 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00011145786993438378, "clip_ratio/low_min": 0.00011145786993438378, "clip_ratio/region_mean": 0.00011145786993438378, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 759.0, "completions/mean_length": 478.375, "completions/mean_terminated_length": 424.7407531738281, "completions/min_length": 180.0, "completions/min_terminated_length": 180.0, "entropy": 0.373266514390707, "epoch": 0.23072805139186295, "frac_reward_zero_std": 0.0, "grad_norm": 0.014188689179718494, "learning_rate": 1e-05, "loss": 0.0714, "num_tokens": 9569058.0, "reward": 0.75, "reward_std": 0.4355512857437134, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0001626014709473, "sampling/importance_sampling_ratio/min": 0.7322058081626892, "sampling/sampling_logp_difference/max": 1.0407233238220215, "sampling/sampling_logp_difference/mean": 0.011204676702618599, "step": 431 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.506608067662455e-05, "clip_ratio/low_min": 5.506608067662455e-05, "clip_ratio/region_mean": 5.506608067662455e-05, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 734.0, "completions/mean_length": 544.84375, "completions/mean_terminated_length": 512.9642944335938, "completions/min_length": 314.0, "completions/min_terminated_length": 314.0, "entropy": 0.35175613686442375, "epoch": 0.23126338329764454, "frac_reward_zero_std": 0.25, "grad_norm": 0.015107778832316399, "learning_rate": 1e-05, "loss": -0.0242, "num_tokens": 9590421.0, "reward": 0.65625, "reward_std": 0.3608423173427582, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.393925666809082, "sampling/importance_sampling_ratio/mean": 1.0000779628753662, "sampling/importance_sampling_ratio/min": 0.722366988658905, "sampling/sampling_logp_difference/max": 0.3321239948272705, "sampling/sampling_logp_difference/mean": 0.011217285878956318, "step": 432 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00013936093091615476, "clip_ratio/low_min": 0.00013936093091615476, "clip_ratio/region_mean": 0.00013936093091615476, "completions/clipped_ratio": 0.46875, "completions/max_length": 768.0, "completions/max_terminated_length": 754.0, "completions/mean_length": 671.75, "completions/mean_terminated_length": 586.8235473632812, "completions/min_length": 380.0, "completions/min_terminated_length": 380.0, "entropy": 0.3009270764887333, "epoch": 0.23179871520342613, "frac_reward_zero_std": 0.0, "grad_norm": 0.009971894323825836, "learning_rate": 1e-05, "loss": -0.0092, "num_tokens": 9615749.0, "reward": 0.40625, "reward_std": 0.4944729208946228, "rewards/accuracy_reward/mean": 0.40625, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.557236909866333, "sampling/importance_sampling_ratio/mean": 0.9998192191123962, "sampling/importance_sampling_ratio/min": 0.6994993090629578, "sampling/sampling_logp_difference/max": 0.4429130554199219, "sampling/sampling_logp_difference/mean": 0.009629164822399616, "step": 433 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.527345299720764e-05, "clip_ratio/low_min": 4.527345299720764e-05, "clip_ratio/region_mean": 4.527345299720764e-05, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 759.0, "completions/mean_length": 645.1875, "completions/mean_terminated_length": 610.7999877929688, "completions/min_length": 361.0, "completions/min_terminated_length": 361.0, "entropy": 0.4690527394413948, "epoch": 0.23233404710920771, "frac_reward_zero_std": 0.75, "grad_norm": 0.0030221513006836176, "learning_rate": 1e-05, "loss": 0.0371, "num_tokens": 9640035.0, "reward": 0.40625, "reward_std": 0.1293872892856598, "rewards/accuracy_reward/mean": 0.40625, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.4208089113235474, "sampling/importance_sampling_ratio/mean": 1.0002892017364502, "sampling/importance_sampling_ratio/min": 0.6786139011383057, "sampling/sampling_logp_difference/max": 0.38770294189453125, "sampling/sampling_logp_difference/mean": 0.013197680935263634, "step": 434 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.898454739712179e-05, "clip_ratio/low_min": 6.898454739712179e-05, "clip_ratio/region_mean": 6.898454739712179e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 742.0, "completions/max_terminated_length": 742.0, "completions/mean_length": 469.875, "completions/mean_terminated_length": 469.875, "completions/min_length": 226.0, "completions/min_terminated_length": 226.0, "entropy": 0.38235819712281227, "epoch": 0.2328693790149893, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0011, "num_tokens": 9658831.0, "reward": 0.96875, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.96875, "rewards/accuracy_reward/std": 0.1767766922712326, "sampling/importance_sampling_ratio/max": 1.4322915077209473, "sampling/importance_sampling_ratio/mean": 1.0000020265579224, "sampling/importance_sampling_ratio/min": 0.7452959418296814, "sampling/sampling_logp_difference/max": 0.35927557945251465, "sampling/sampling_logp_difference/mean": 0.012142562307417393, "step": 435 }, { "clip_ratio/high_max": 9.854770905803889e-05, "clip_ratio/high_mean": 9.854770905803889e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 9.854770905803889e-05, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 759.0, "completions/mean_length": 597.34375, "completions/mean_terminated_length": 540.4583740234375, "completions/min_length": 302.0, "completions/min_terminated_length": 302.0, "entropy": 0.34139036759734154, "epoch": 0.2334047109207709, "frac_reward_zero_std": 0.0, "grad_norm": 0.012364211492240429, "learning_rate": 1e-05, "loss": 0.0444, "num_tokens": 9681674.0, "reward": 0.71875, "reward_std": 0.4218915104866028, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.4381734132766724, "sampling/importance_sampling_ratio/mean": 1.0000934600830078, "sampling/importance_sampling_ratio/min": 0.6522260308265686, "sampling/sampling_logp_difference/max": 0.4273641109466553, "sampling/sampling_logp_difference/mean": 0.010575716383755207, "step": 436 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.3146257414482534e-05, "clip_ratio/low_min": 5.3146257414482534e-05, "clip_ratio/region_mean": 5.3146257414482534e-05, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 762.0, "completions/mean_length": 528.375, "completions/mean_terminated_length": 503.5862121582031, "completions/min_length": 285.0, "completions/min_terminated_length": 285.0, "entropy": 0.3023227211087942, "epoch": 0.23394004282655245, "frac_reward_zero_std": 0.0, "grad_norm": 0.017348093912005424, "learning_rate": 1e-05, "loss": 0.062, "num_tokens": 9701814.0, "reward": 0.75, "reward_std": 0.4261348247528076, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9999216794967651, "sampling/importance_sampling_ratio/min": 0.2950405180454254, "sampling/sampling_logp_difference/max": 1.2206425666809082, "sampling/sampling_logp_difference/mean": 0.010344784706830978, "step": 437 }, { "clip_ratio/high_max": 6.243756070034578e-05, "clip_ratio/high_mean": 6.243756070034578e-05, "clip_ratio/low_mean": 0.000227421351155499, "clip_ratio/low_min": 0.000227421351155499, "clip_ratio/region_mean": 0.0002898589191318024, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 741.0, "completions/mean_length": 540.125, "completions/mean_terminated_length": 464.16668701171875, "completions/min_length": 297.0, "completions/min_terminated_length": 297.0, "entropy": 0.33811743929982185, "epoch": 0.23447537473233404, "frac_reward_zero_std": 0.0, "grad_norm": 0.004054592922329903, "learning_rate": 1e-05, "loss": 0.0526, "num_tokens": 9723362.0, "reward": 0.6875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.4524321556091309, "sampling/importance_sampling_ratio/mean": 0.9998065233230591, "sampling/importance_sampling_ratio/min": 0.6455051302909851, "sampling/sampling_logp_difference/max": 0.43772220611572266, "sampling/sampling_logp_difference/mean": 0.01056714728474617, "step": 438 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00016832120309118181, "clip_ratio/low_min": 0.00016832120309118181, "clip_ratio/region_mean": 0.00016832120309118181, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 736.0, "completions/mean_length": 528.59375, "completions/mean_terminated_length": 503.82757568359375, "completions/min_length": 75.0, "completions/min_terminated_length": 75.0, "entropy": 0.4121296629309654, "epoch": 0.23501070663811563, "frac_reward_zero_std": 0.5, "grad_norm": 0.014934531413018703, "learning_rate": 1e-05, "loss": -0.0677, "num_tokens": 9743797.0, "reward": 0.78125, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.4214543104171753, "sampling/importance_sampling_ratio/mean": 1.0002490282058716, "sampling/importance_sampling_ratio/min": 0.6828449964523315, "sampling/sampling_logp_difference/max": 0.3814873695373535, "sampling/sampling_logp_difference/mean": 0.012028890661895275, "step": 439 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00010313531674910337, "clip_ratio/low_min": 0.00010313531674910337, "clip_ratio/region_mean": 0.00010313531674910337, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 729.0, "completions/mean_length": 545.46875, "completions/mean_terminated_length": 494.11541748046875, "completions/min_length": 268.0, "completions/min_terminated_length": 268.0, "entropy": 0.45557525381445885, "epoch": 0.23554603854389722, "frac_reward_zero_std": 0.0, "grad_norm": 0.0137118399143219, "learning_rate": 1e-05, "loss": 0.0342, "num_tokens": 9765124.0, "reward": 0.46875, "reward_std": 0.5038893818855286, "rewards/accuracy_reward/mean": 0.46875, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.4215961694717407, "sampling/importance_sampling_ratio/mean": 1.0003676414489746, "sampling/importance_sampling_ratio/min": 0.6576674580574036, "sampling/sampling_logp_difference/max": 0.4190559387207031, "sampling/sampling_logp_difference/mean": 0.013410435989499092, "step": 440 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 746.0, "completions/mean_length": 469.59375, "completions/mean_terminated_length": 449.70001220703125, "completions/min_length": 232.0, "completions/min_terminated_length": 232.0, "entropy": 0.4070568270981312, "epoch": 0.2360813704496788, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 9783551.0, "reward": 0.75, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.4098912477493286, "sampling/importance_sampling_ratio/mean": 0.9998842477798462, "sampling/importance_sampling_ratio/min": 0.720041036605835, "sampling/sampling_logp_difference/max": 0.34351253509521484, "sampling/sampling_logp_difference/mean": 0.012938451953232288, "step": 441 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00017385257524438202, "clip_ratio/low_min": 0.00017385257524438202, "clip_ratio/region_mean": 0.00017385257524438202, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 739.0, "completions/mean_length": 557.40625, "completions/mean_terminated_length": 527.3214721679688, "completions/min_length": 328.0, "completions/min_terminated_length": 328.0, "entropy": 0.40482622012495995, "epoch": 0.2366167023554604, "frac_reward_zero_std": 0.0, "grad_norm": 0.011935973539948463, "learning_rate": 1e-05, "loss": 0.0967, "num_tokens": 9805308.0, "reward": 0.65625, "reward_std": 0.47137710452079773, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.3014415502548218, "sampling/importance_sampling_ratio/mean": 1.0003654956817627, "sampling/importance_sampling_ratio/min": 0.6488705277442932, "sampling/sampling_logp_difference/max": 0.4325220584869385, "sampling/sampling_logp_difference/mean": 0.012709065340459347, "step": 442 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 730.0, "completions/mean_length": 538.21875, "completions/mean_terminated_length": 514.4483032226562, "completions/min_length": 388.0, "completions/min_terminated_length": 388.0, "entropy": 0.324844416230917, "epoch": 0.23715203426124196, "frac_reward_zero_std": 0.5, "grad_norm": 0.008872278966009617, "learning_rate": 1e-05, "loss": 0.048, "num_tokens": 9825891.0, "reward": 0.84375, "reward_std": 0.22201895713806152, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.4155662059783936, "sampling/importance_sampling_ratio/mean": 1.0002615451812744, "sampling/importance_sampling_ratio/min": 0.7099429965019226, "sampling/sampling_logp_difference/max": 0.34752964973449707, "sampling/sampling_logp_difference/mean": 0.010144418105483055, "step": 443 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.351625779643655e-05, "clip_ratio/low_min": 6.351625779643655e-05, "clip_ratio/region_mean": 6.351625779643655e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 719.0, "completions/max_terminated_length": 719.0, "completions/mean_length": 459.34375, "completions/mean_terminated_length": 459.34375, "completions/min_length": 293.0, "completions/min_terminated_length": 293.0, "entropy": 0.31106376461684704, "epoch": 0.23768736616702354, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": -0.0047, "num_tokens": 9844110.0, "reward": 0.8125, "reward_std": 0.1157275140285492, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.3937691450119019, "sampling/importance_sampling_ratio/mean": 0.9998219609260559, "sampling/importance_sampling_ratio/min": 0.7058107256889343, "sampling/sampling_logp_difference/max": 0.34840822219848633, "sampling/sampling_logp_difference/mean": 0.010355335660278797, "step": 444 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 768.0, "completions/mean_length": 569.78125, "completions/mean_terminated_length": 524.0384521484375, "completions/min_length": 293.0, "completions/min_terminated_length": 293.0, "entropy": 0.3032274544239044, "epoch": 0.23822269807280513, "frac_reward_zero_std": 0.5, "grad_norm": 0.006632843986153603, "learning_rate": 1e-05, "loss": 0.0451, "num_tokens": 9865911.0, "reward": 0.8125, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.3431744575500488, "sampling/importance_sampling_ratio/mean": 0.9997745156288147, "sampling/importance_sampling_ratio/min": 0.6964196562767029, "sampling/sampling_logp_difference/max": 0.3618028163909912, "sampling/sampling_logp_difference/mean": 0.00975789874792099, "step": 445 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.367804598994553e-05, "clip_ratio/low_min": 6.367804598994553e-05, "clip_ratio/region_mean": 6.367804598994553e-05, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 724.0, "completions/mean_length": 559.53125, "completions/mean_terminated_length": 545.6333618164062, "completions/min_length": 345.0, "completions/min_terminated_length": 345.0, "entropy": 0.2855161540210247, "epoch": 0.23875802997858672, "frac_reward_zero_std": 0.25, "grad_norm": 0.017041988670825958, "learning_rate": 1e-05, "loss": 0.0306, "num_tokens": 9887672.0, "reward": 0.8125, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.2953450679779053, "sampling/importance_sampling_ratio/mean": 1.0003676414489746, "sampling/importance_sampling_ratio/min": 0.7011907696723938, "sampling/sampling_logp_difference/max": 0.35497522354125977, "sampling/sampling_logp_difference/mean": 0.009561488404870033, "step": 446 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00015837948740227148, "clip_ratio/low_min": 0.00015837948740227148, "clip_ratio/region_mean": 0.00015837948740227148, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 705.0, "completions/mean_length": 575.53125, "completions/mean_terminated_length": 531.1154174804688, "completions/min_length": 306.0, "completions/min_terminated_length": 306.0, "entropy": 0.3524639941751957, "epoch": 0.2392933618843683, "frac_reward_zero_std": 0.25, "grad_norm": 0.002969164866954088, "learning_rate": 1e-05, "loss": 0.0423, "num_tokens": 9909937.0, "reward": 0.65625, "reward_std": 0.3061639964580536, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.363702654838562, "sampling/importance_sampling_ratio/mean": 1.000001072883606, "sampling/importance_sampling_ratio/min": 0.6369519829750061, "sampling/sampling_logp_difference/max": 0.4510610103607178, "sampling/sampling_logp_difference/mean": 0.011127698235213757, "step": 447 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 713.0, "completions/mean_length": 525.375, "completions/mean_terminated_length": 490.71429443359375, "completions/min_length": 265.0, "completions/min_terminated_length": 265.0, "entropy": 0.3839997462928295, "epoch": 0.2398286937901499, "frac_reward_zero_std": 0.5, "grad_norm": 0.004479656461626291, "learning_rate": 1e-05, "loss": 0.0359, "num_tokens": 9930773.0, "reward": 0.78125, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.308794379234314, "sampling/importance_sampling_ratio/mean": 1.0000007152557373, "sampling/importance_sampling_ratio/min": 0.6819316148757935, "sampling/sampling_logp_difference/max": 0.3828258514404297, "sampling/sampling_logp_difference/mean": 0.011892806738615036, "step": 448 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 764.0, "completions/mean_length": 545.84375, "completions/mean_terminated_length": 522.862060546875, "completions/min_length": 354.0, "completions/min_terminated_length": 354.0, "entropy": 0.37527740001678467, "epoch": 0.2403640256959315, "frac_reward_zero_std": 0.25, "grad_norm": 0.018485892564058304, "learning_rate": 1e-05, "loss": 0.0424, "num_tokens": 9951880.0, "reward": 0.78125, "reward_std": 0.3377464711666107, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.6329386234283447, "sampling/importance_sampling_ratio/mean": 0.9999679923057556, "sampling/importance_sampling_ratio/min": 0.660495400428772, "sampling/sampling_logp_difference/max": 0.49038124084472656, "sampling/sampling_logp_difference/mean": 0.012055926024913788, "step": 449 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.0040031055686995e-05, "clip_ratio/low_min": 5.0040031055686995e-05, "clip_ratio/region_mean": 5.0040031055686995e-05, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 754.0, "completions/mean_length": 531.21875, "completions/mean_terminated_length": 506.72412109375, "completions/min_length": 312.0, "completions/min_terminated_length": 312.0, "entropy": 0.35922387056052685, "epoch": 0.24089935760171305, "frac_reward_zero_std": 0.25, "grad_norm": 0.004421040415763855, "learning_rate": 1e-05, "loss": 0.0502, "num_tokens": 9972255.0, "reward": 0.90625, "reward_std": 0.2651650309562683, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.3125956058502197, "sampling/importance_sampling_ratio/mean": 0.9998888969421387, "sampling/importance_sampling_ratio/min": 0.5535402297973633, "sampling/sampling_logp_difference/max": 0.5914208292961121, "sampling/sampling_logp_difference/mean": 0.011364748701453209, "step": 450 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00011941327466047369, "clip_ratio/low_min": 0.00011941327466047369, "clip_ratio/region_mean": 0.00011941327466047369, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 705.0, "completions/mean_length": 552.625, "completions/mean_terminated_length": 502.923095703125, "completions/min_length": 268.0, "completions/min_terminated_length": 268.0, "entropy": 0.33651861920952797, "epoch": 0.24143468950749464, "frac_reward_zero_std": 0.5, "grad_norm": 0.01763063855469227, "learning_rate": 1e-05, "loss": 0.0031, "num_tokens": 9993547.0, "reward": 0.8125, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.3175828456878662, "sampling/importance_sampling_ratio/mean": 0.9998105764389038, "sampling/importance_sampling_ratio/min": 0.598853588104248, "sampling/sampling_logp_difference/max": 0.5127382278442383, "sampling/sampling_logp_difference/mean": 0.010966668836772442, "step": 451 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 750.0, "completions/mean_length": 470.875, "completions/mean_terminated_length": 461.2903137207031, "completions/min_length": 197.0, "completions/min_terminated_length": 197.0, "entropy": 0.32031749933958054, "epoch": 0.24197002141327623, "frac_reward_zero_std": 0.5, "grad_norm": 0.015869755297899246, "learning_rate": 1e-05, "loss": -0.015, "num_tokens": 10011935.0, "reward": 0.78125, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.4097594022750854, "sampling/importance_sampling_ratio/mean": 0.9998683333396912, "sampling/importance_sampling_ratio/min": 0.710909366607666, "sampling/sampling_logp_difference/max": 0.34341907501220703, "sampling/sampling_logp_difference/mean": 0.011071456596255302, "step": 452 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00010638788808137178, "clip_ratio/low_min": 0.00010638788808137178, "clip_ratio/region_mean": 0.00010638788808137178, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 695.0, "completions/mean_length": 532.09375, "completions/mean_terminated_length": 488.40740966796875, "completions/min_length": 274.0, "completions/min_terminated_length": 274.0, "entropy": 0.41738782078027725, "epoch": 0.24250535331905781, "frac_reward_zero_std": 0.5, "grad_norm": 0.002433771500363946, "learning_rate": 1e-05, "loss": 0.0167, "num_tokens": 10032554.0, "reward": 0.78125, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.7308326959609985, "sampling/importance_sampling_ratio/mean": 1.0001813173294067, "sampling/importance_sampling_ratio/min": 0.7279193997383118, "sampling/sampling_logp_difference/max": 0.5486025810241699, "sampling/sampling_logp_difference/mean": 0.012861635535955429, "step": 453 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00015337190779973753, "clip_ratio/low_min": 0.00015337190779973753, "clip_ratio/region_mean": 0.00015337190779973753, "completions/clipped_ratio": 0.375, "completions/max_length": 768.0, "completions/max_terminated_length": 764.0, "completions/mean_length": 663.4375, "completions/mean_terminated_length": 600.7000122070312, "completions/min_length": 290.0, "completions/min_terminated_length": 290.0, "entropy": 0.396393358707428, "epoch": 0.2430406852248394, "frac_reward_zero_std": 0.25, "grad_norm": 0.011197500862181187, "learning_rate": 1e-05, "loss": 0.0347, "num_tokens": 10057896.0, "reward": 0.34375, "reward_std": 0.3061639964580536, "rewards/accuracy_reward/mean": 0.34375, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.810591697692871, "sampling/importance_sampling_ratio/mean": 1.0000510215759277, "sampling/importance_sampling_ratio/min": 0.5973223447799683, "sampling/sampling_logp_difference/max": 0.593653678894043, "sampling/sampling_logp_difference/mean": 0.012164346873760223, "step": 454 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00014273404303821735, "clip_ratio/low_min": 0.00014273404303821735, "clip_ratio/region_mean": 0.00014273404303821735, "completions/clipped_ratio": 0.34375, "completions/max_length": 768.0, "completions/max_terminated_length": 768.0, "completions/mean_length": 644.5, "completions/mean_terminated_length": 579.8095092773438, "completions/min_length": 332.0, "completions/min_terminated_length": 332.0, "entropy": 0.46060651168227196, "epoch": 0.243576017130621, "frac_reward_zero_std": 0.25, "grad_norm": 0.01803814060986042, "learning_rate": 1e-05, "loss": 0.0575, "num_tokens": 10082760.0, "reward": 0.53125, "reward_std": 0.3198433816432953, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.4235470294952393, "sampling/importance_sampling_ratio/mean": 1.0002251863479614, "sampling/importance_sampling_ratio/min": 0.5989532470703125, "sampling/sampling_logp_difference/max": 0.5125718116760254, "sampling/sampling_logp_difference/mean": 0.013906026259064674, "step": 455 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00019422921104705893, "clip_ratio/low_min": 0.00019422921104705893, "clip_ratio/region_mean": 0.00019422921104705893, "completions/clipped_ratio": 0.40625, "completions/max_length": 768.0, "completions/max_terminated_length": 766.0, "completions/mean_length": 629.53125, "completions/mean_terminated_length": 534.7894897460938, "completions/min_length": 331.0, "completions/min_terminated_length": 331.0, "entropy": 0.5165114663541317, "epoch": 0.24411134903640258, "frac_reward_zero_std": 0.25, "grad_norm": 0.014529377222061157, "learning_rate": 1e-05, "loss": 0.041, "num_tokens": 10106761.0, "reward": 0.53125, "reward_std": 0.3471629321575165, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.389373779296875, "sampling/importance_sampling_ratio/mean": 1.0002028942108154, "sampling/importance_sampling_ratio/min": 0.7022377848625183, "sampling/sampling_logp_difference/max": 0.3534832000732422, "sampling/sampling_logp_difference/mean": 0.015013284981250763, "step": 456 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 735.0, "completions/mean_length": 526.875, "completions/mean_terminated_length": 501.9310302734375, "completions/min_length": 371.0, "completions/min_terminated_length": 371.0, "entropy": 0.3432654179632664, "epoch": 0.24464668094218414, "frac_reward_zero_std": 0.0, "grad_norm": 0.016591932624578476, "learning_rate": 1e-05, "loss": 0.1204, "num_tokens": 10127565.0, "reward": 0.75, "reward_std": 0.4261348247528076, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.4345364570617676, "sampling/importance_sampling_ratio/mean": 1.0000282526016235, "sampling/importance_sampling_ratio/min": 0.6134641766548157, "sampling/sampling_logp_difference/max": 0.488633394241333, "sampling/sampling_logp_difference/mean": 0.011462590657174587, "step": 457 }, { "clip_ratio/high_max": 6.641870277235284e-05, "clip_ratio/high_mean": 6.641870277235284e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 6.641870277235284e-05, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 768.0, "completions/mean_length": 522.75, "completions/mean_terminated_length": 466.15386962890625, "completions/min_length": 315.0, "completions/min_terminated_length": 315.0, "entropy": 0.3245689757168293, "epoch": 0.24518201284796573, "frac_reward_zero_std": 0.5, "grad_norm": 0.008053712546825409, "learning_rate": 1e-05, "loss": 0.0316, "num_tokens": 10147765.0, "reward": 0.8125, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.339956283569336, "sampling/importance_sampling_ratio/mean": 1.0000364780426025, "sampling/importance_sampling_ratio/min": 0.6956804990768433, "sampling/sampling_logp_difference/max": 0.36286473274230957, "sampling/sampling_logp_difference/mean": 0.010648135095834732, "step": 458 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 714.0, "completions/max_terminated_length": 714.0, "completions/mean_length": 474.65625, "completions/mean_terminated_length": 474.65625, "completions/min_length": 142.0, "completions/min_terminated_length": 142.0, "entropy": 0.31016377732157707, "epoch": 0.24571734475374732, "frac_reward_zero_std": 0.25, "grad_norm": 0.022604985162615776, "learning_rate": 1e-05, "loss": -0.0694, "num_tokens": 10166602.0, "reward": 0.875, "reward_std": 0.292504221200943, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 1.3568627834320068, "sampling/importance_sampling_ratio/mean": 0.9997751712799072, "sampling/importance_sampling_ratio/min": 0.7631262540817261, "sampling/sampling_logp_difference/max": 0.3051753044128418, "sampling/sampling_logp_difference/mean": 0.009892992675304413, "step": 459 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.808278703829274e-05, "clip_ratio/low_min": 6.808278703829274e-05, "clip_ratio/region_mean": 6.808278703829274e-05, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 619.0, "completions/mean_length": 480.4375, "completions/mean_terminated_length": 471.1612854003906, "completions/min_length": 274.0, "completions/min_terminated_length": 274.0, "entropy": 0.3669208362698555, "epoch": 0.2462526766595289, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": -0.0223, "num_tokens": 10185200.0, "reward": 0.90625, "reward_std": 0.1293872892856598, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.9818788766860962, "sampling/importance_sampling_ratio/mean": 0.9997610449790955, "sampling/importance_sampling_ratio/min": 0.702907145023346, "sampling/sampling_logp_difference/max": 0.6840453147888184, "sampling/sampling_logp_difference/mean": 0.01118472870439291, "step": 460 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 670.0, "completions/mean_length": 545.25, "completions/mean_terminated_length": 522.2069091796875, "completions/min_length": 388.0, "completions/min_terminated_length": 388.0, "entropy": 0.32584521919488907, "epoch": 0.2467880085653105, "frac_reward_zero_std": 0.5, "grad_norm": 0.019112931564450264, "learning_rate": 1e-05, "loss": 0.0704, "num_tokens": 10206328.0, "reward": 0.9375, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.9375, "rewards/accuracy_reward/std": 0.24593468010425568, "sampling/importance_sampling_ratio/max": 1.4550303220748901, "sampling/importance_sampling_ratio/mean": 0.9998010396957397, "sampling/importance_sampling_ratio/min": 0.6699081063270569, "sampling/sampling_logp_difference/max": 0.40061473846435547, "sampling/sampling_logp_difference/mean": 0.010370478965342045, "step": 461 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 732.0, "completions/mean_length": 543.25, "completions/mean_terminated_length": 491.3846435546875, "completions/min_length": 312.0, "completions/min_terminated_length": 312.0, "entropy": 0.3115219175815582, "epoch": 0.24732334047109208, "frac_reward_zero_std": 0.25, "grad_norm": 0.005989633966237307, "learning_rate": 1e-05, "loss": 0.0305, "num_tokens": 10227560.0, "reward": 0.84375, "reward_std": 0.3061639964580536, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.4669171571731567, "sampling/importance_sampling_ratio/mean": 0.9996733069419861, "sampling/importance_sampling_ratio/min": 0.7713218927383423, "sampling/sampling_logp_difference/max": 0.3831629753112793, "sampling/sampling_logp_difference/mean": 0.010220806114375591, "step": 462 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0002042240921582561, "clip_ratio/low_min": 0.0002042240921582561, "clip_ratio/region_mean": 0.0002042240921582561, "completions/clipped_ratio": 0.34375, "completions/max_length": 768.0, "completions/max_terminated_length": 733.0, "completions/mean_length": 607.125, "completions/mean_terminated_length": 522.857177734375, "completions/min_length": 349.0, "completions/min_terminated_length": 349.0, "entropy": 0.4051716774702072, "epoch": 0.24785867237687367, "frac_reward_zero_std": 0.25, "grad_norm": 0.006537931971251965, "learning_rate": 1e-05, "loss": 0.0595, "num_tokens": 10250988.0, "reward": 0.59375, "reward_std": 0.3198433816432953, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.6197837591171265, "sampling/importance_sampling_ratio/mean": 1.0001596212387085, "sampling/importance_sampling_ratio/min": 0.7747717499732971, "sampling/sampling_logp_difference/max": 0.48229265213012695, "sampling/sampling_logp_difference/mean": 0.012219919823110104, "step": 463 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.491555955610238e-05, "clip_ratio/low_min": 4.491555955610238e-05, "clip_ratio/region_mean": 4.491555955610238e-05, "completions/clipped_ratio": 0.46875, "completions/max_length": 768.0, "completions/max_terminated_length": 709.0, "completions/mean_length": 655.90625, "completions/mean_terminated_length": 557.0, "completions/min_length": 320.0, "completions/min_terminated_length": 320.0, "entropy": 0.3715316690504551, "epoch": 0.24839400428265523, "frac_reward_zero_std": 0.5, "grad_norm": 0.007977494969964027, "learning_rate": 1e-05, "loss": 0.0479, "num_tokens": 10276401.0, "reward": 0.5, "reward_std": 0.2587745785713196, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5080004930496216, "sampling/importance_sampling_ratio/max": 1.4375637769699097, "sampling/importance_sampling_ratio/mean": 0.9999771118164062, "sampling/importance_sampling_ratio/min": 0.6834230422973633, "sampling/sampling_logp_difference/max": 0.38064122200012207, "sampling/sampling_logp_difference/mean": 0.011099442839622498, "step": 464 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 740.0, "completions/mean_length": 510.5, "completions/mean_terminated_length": 483.862060546875, "completions/min_length": 208.0, "completions/min_terminated_length": 208.0, "entropy": 0.3095720000565052, "epoch": 0.24892933618843682, "frac_reward_zero_std": 0.5, "grad_norm": 0.018161436542868614, "learning_rate": 1e-05, "loss": 0.0305, "num_tokens": 10296377.0, "reward": 0.53125, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.4600300788879395, "sampling/importance_sampling_ratio/mean": 0.9999551177024841, "sampling/importance_sampling_ratio/min": 0.7093867063522339, "sampling/sampling_logp_difference/max": 0.37845706939697266, "sampling/sampling_logp_difference/mean": 0.011000224389135838, "step": 465 }, { "clip_ratio/high_max": 6.568575918208808e-05, "clip_ratio/high_mean": 6.568575918208808e-05, "clip_ratio/low_mean": 0.000293447490548715, "clip_ratio/low_min": 0.000293447490548715, "clip_ratio/region_mean": 0.00035913324973080307, "completions/clipped_ratio": 0.375, "completions/max_length": 768.0, "completions/max_terminated_length": 751.0, "completions/mean_length": 585.3125, "completions/mean_terminated_length": 475.70001220703125, "completions/min_length": 312.0, "completions/min_terminated_length": 312.0, "entropy": 0.4775240123271942, "epoch": 0.2494646680942184, "frac_reward_zero_std": 0.0, "grad_norm": 0.007165522780269384, "learning_rate": 1e-05, "loss": 0.0388, "num_tokens": 10318947.0, "reward": 0.28125, "reward_std": 0.4628904461860657, "rewards/accuracy_reward/mean": 0.28125, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.4121737480163574, "sampling/importance_sampling_ratio/mean": 1.0000252723693848, "sampling/importance_sampling_ratio/min": 0.5510292649269104, "sampling/sampling_logp_difference/max": 0.5959672927856445, "sampling/sampling_logp_difference/mean": 0.013995653949677944, "step": 466 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00015253391029546037, "clip_ratio/low_min": 0.00015253391029546037, "clip_ratio/region_mean": 0.00015253391029546037, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 758.0, "completions/mean_length": 594.6875, "completions/mean_terminated_length": 554.6923217773438, "completions/min_length": 332.0, "completions/min_terminated_length": 332.0, "entropy": 0.3064855579286814, "epoch": 0.25, "frac_reward_zero_std": 0.25, "grad_norm": 0.007180414628237486, "learning_rate": 1e-05, "loss": 0.04, "num_tokens": 10341449.0, "reward": 0.71875, "reward_std": 0.3471629321575165, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.8535993099212646, "sampling/importance_sampling_ratio/mean": 0.9999573826789856, "sampling/importance_sampling_ratio/min": 0.6286686062812805, "sampling/sampling_logp_difference/max": 0.6171293258666992, "sampling/sampling_logp_difference/mean": 0.009904849343001842, "step": 467 }, { "clip_ratio/high_max": 5.4561325669055805e-05, "clip_ratio/high_mean": 5.4561325669055805e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 5.4561325669055805e-05, "completions/clipped_ratio": 0.28125, "completions/max_length": 768.0, "completions/max_terminated_length": 764.0, "completions/mean_length": 618.09375, "completions/mean_terminated_length": 559.434814453125, "completions/min_length": 303.0, "completions/min_terminated_length": 303.0, "entropy": 0.43297940120100975, "epoch": 0.2505353319057816, "frac_reward_zero_std": 0.25, "grad_norm": 0.003917201422154903, "learning_rate": 1e-05, "loss": 0.0601, "num_tokens": 10365084.0, "reward": 0.4375, "reward_std": 0.292504221200943, "rewards/accuracy_reward/mean": 0.4375, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.3909175395965576, "sampling/importance_sampling_ratio/mean": 0.9995114803314209, "sampling/importance_sampling_ratio/min": 0.7199997305870056, "sampling/sampling_logp_difference/max": 0.32996368408203125, "sampling/sampling_logp_difference/mean": 0.012611831538379192, "step": 468 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00012666822294704616, "clip_ratio/low_min": 0.00012666822294704616, "clip_ratio/region_mean": 0.00012666822294704616, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 647.0, "completions/mean_length": 449.65625, "completions/mean_terminated_length": 439.3870849609375, "completions/min_length": 274.0, "completions/min_terminated_length": 274.0, "entropy": 0.41355736553668976, "epoch": 0.2510706638115632, "frac_reward_zero_std": 0.25, "grad_norm": 0.00916921067982912, "learning_rate": 1e-05, "loss": 0.0521, "num_tokens": 10382809.0, "reward": 0.65625, "reward_std": 0.3061639964580536, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.333982229232788, "sampling/importance_sampling_ratio/mean": 0.9997520446777344, "sampling/importance_sampling_ratio/min": 0.7351874113082886, "sampling/sampling_logp_difference/max": 0.3076298236846924, "sampling/sampling_logp_difference/mean": 0.013193922117352486, "step": 469 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00012437811528798193, "clip_ratio/low_min": 0.00012437811528798193, "clip_ratio/region_mean": 0.00012437811528798193, "completions/clipped_ratio": 0.3125, "completions/max_length": 768.0, "completions/max_terminated_length": 705.0, "completions/mean_length": 591.0625, "completions/mean_terminated_length": 510.6363830566406, "completions/min_length": 258.0, "completions/min_terminated_length": 258.0, "entropy": 0.4005119614303112, "epoch": 0.25160599571734477, "frac_reward_zero_std": 0.5, "grad_norm": 0.010969983413815498, "learning_rate": 1e-05, "loss": 0.0567, "num_tokens": 10405739.0, "reward": 0.5, "reward_std": 0.26726123690605164, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5080004930496216, "sampling/importance_sampling_ratio/max": 1.2928237915039062, "sampling/importance_sampling_ratio/mean": 0.9999876618385315, "sampling/importance_sampling_ratio/min": 0.46985867619514465, "sampling/sampling_logp_difference/max": 0.7553234100341797, "sampling/sampling_logp_difference/mean": 0.012378847226500511, "step": 470 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.969436460873112e-05, "clip_ratio/low_min": 5.969436460873112e-05, "clip_ratio/region_mean": 5.969436460873112e-05, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 658.0, "completions/mean_length": 551.875, "completions/mean_terminated_length": 479.8333435058594, "completions/min_length": 293.0, "completions/min_terminated_length": 293.0, "entropy": 0.36107574589550495, "epoch": 0.25214132762312635, "frac_reward_zero_std": 0.0, "grad_norm": 0.013910401612520218, "learning_rate": 1e-05, "loss": 0.1243, "num_tokens": 10426871.0, "reward": 0.65625, "reward_std": 0.4355708956718445, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.309815764427185, "sampling/importance_sampling_ratio/mean": 1.000069499015808, "sampling/importance_sampling_ratio/min": 0.6900006532669067, "sampling/sampling_logp_difference/max": 0.3710627555847168, "sampling/sampling_logp_difference/mean": 0.011647488921880722, "step": 471 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.833719867747277e-05, "clip_ratio/low_min": 4.833719867747277e-05, "clip_ratio/region_mean": 4.833719867747277e-05, "completions/clipped_ratio": 0.3125, "completions/max_length": 768.0, "completions/max_terminated_length": 767.0, "completions/mean_length": 625.0625, "completions/mean_terminated_length": 560.0909423828125, "completions/min_length": 376.0, "completions/min_terminated_length": 376.0, "entropy": 0.40370364487171173, "epoch": 0.25267665952890794, "frac_reward_zero_std": 0.25, "grad_norm": 0.004989312961697578, "learning_rate": 1e-05, "loss": 0.0385, "num_tokens": 10450521.0, "reward": 0.53125, "reward_std": 0.3377465009689331, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.394562840461731, "sampling/importance_sampling_ratio/mean": 0.9999212622642517, "sampling/importance_sampling_ratio/min": 0.6457639932632446, "sampling/sampling_logp_difference/max": 0.43732118606567383, "sampling/sampling_logp_difference/mean": 0.011998952366411686, "step": 472 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00019895387231372297, "clip_ratio/low_min": 0.00019895387231372297, "clip_ratio/region_mean": 0.00019895387231372297, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 683.0, "completions/mean_length": 442.3125, "completions/mean_terminated_length": 431.8064270019531, "completions/min_length": 232.0, "completions/min_terminated_length": 232.0, "entropy": 0.37263400107622147, "epoch": 0.25321199143468953, "frac_reward_zero_std": 0.0, "grad_norm": 0.02191942185163498, "learning_rate": 1e-05, "loss": 0.0142, "num_tokens": 10468083.0, "reward": 0.59375, "reward_std": 0.4218915104866028, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.4055616855621338, "sampling/importance_sampling_ratio/mean": 0.9999365210533142, "sampling/importance_sampling_ratio/min": 0.7259685397148132, "sampling/sampling_logp_difference/max": 0.3404369354248047, "sampling/sampling_logp_difference/mean": 0.01224359218031168, "step": 473 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 734.0, "completions/mean_length": 488.46875, "completions/mean_terminated_length": 469.8333435058594, "completions/min_length": 191.0, "completions/min_terminated_length": 191.0, "entropy": 0.34486325457692146, "epoch": 0.25374732334047106, "frac_reward_zero_std": 0.25, "grad_norm": 0.017580291256308556, "learning_rate": 1e-05, "loss": 0.0035, "num_tokens": 10487210.0, "reward": 0.90625, "reward_std": 0.2651650309562683, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.000373363494873, "sampling/importance_sampling_ratio/min": 0.6997325420379639, "sampling/sampling_logp_difference/max": 0.7289609909057617, "sampling/sampling_logp_difference/mean": 0.011850179173052311, "step": 474 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.000192368224816164, "clip_ratio/low_min": 0.000192368224816164, "clip_ratio/region_mean": 0.000192368224816164, "completions/clipped_ratio": 0.34375, "completions/max_length": 768.0, "completions/max_terminated_length": 637.0, "completions/mean_length": 588.28125, "completions/mean_terminated_length": 494.1428527832031, "completions/min_length": 276.0, "completions/min_terminated_length": 276.0, "entropy": 0.4914706163108349, "epoch": 0.25428265524625265, "frac_reward_zero_std": 0.25, "grad_norm": 0.020851751789450645, "learning_rate": 1e-05, "loss": -0.0075, "num_tokens": 10510019.0, "reward": 0.65625, "reward_std": 0.3377464711666107, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.425860047340393, "sampling/importance_sampling_ratio/mean": 1.0002589225769043, "sampling/importance_sampling_ratio/min": 0.47174105048179626, "sampling/sampling_logp_difference/max": 0.7513251304626465, "sampling/sampling_logp_difference/mean": 0.014382414519786835, "step": 475 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00010526915866648778, "clip_ratio/low_min": 0.00010526915866648778, "clip_ratio/region_mean": 0.00010526915866648778, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 742.0, "completions/mean_length": 530.8125, "completions/mean_terminated_length": 506.2758483886719, "completions/min_length": 269.0, "completions/min_terminated_length": 269.0, "entropy": 0.34860437735915184, "epoch": 0.25481798715203424, "frac_reward_zero_std": 0.25, "grad_norm": 0.016587065532803535, "learning_rate": 1e-05, "loss": 0.0155, "num_tokens": 10530765.0, "reward": 0.625, "reward_std": 0.3745020925998688, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.4301687479019165, "sampling/importance_sampling_ratio/mean": 0.9998611211776733, "sampling/importance_sampling_ratio/min": 0.7063822150230408, "sampling/sampling_logp_difference/max": 0.35779237747192383, "sampling/sampling_logp_difference/mean": 0.011494183912873268, "step": 476 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 711.0, "completions/mean_length": 484.84375, "completions/mean_terminated_length": 475.70965576171875, "completions/min_length": 257.0, "completions/min_terminated_length": 257.0, "entropy": 0.3333367947489023, "epoch": 0.25535331905781583, "frac_reward_zero_std": 0.25, "grad_norm": 0.005053653847426176, "learning_rate": 1e-05, "loss": 0.0722, "num_tokens": 10550344.0, "reward": 0.78125, "reward_std": 0.3061639964580536, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.3609040975570679, "sampling/importance_sampling_ratio/mean": 0.9998964667320251, "sampling/importance_sampling_ratio/min": 0.6754359602928162, "sampling/sampling_logp_difference/max": 0.3923969268798828, "sampling/sampling_logp_difference/mean": 0.011056541465222836, "step": 477 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00011185682524228469, "clip_ratio/low_min": 0.00011185682524228469, "clip_ratio/region_mean": 0.00011185682524228469, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 685.0, "completions/mean_length": 488.0625, "completions/mean_terminated_length": 479.0322570800781, "completions/min_length": 277.0, "completions/min_terminated_length": 277.0, "entropy": 0.5512354504317045, "epoch": 0.2558886509635974, "frac_reward_zero_std": 0.25, "grad_norm": 0.01098668109625578, "learning_rate": 1e-05, "loss": 0.0606, "num_tokens": 10570282.0, "reward": 0.75, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.3350861072540283, "sampling/importance_sampling_ratio/mean": 0.9997745752334595, "sampling/importance_sampling_ratio/min": 0.62984699010849, "sampling/sampling_logp_difference/max": 0.4622783660888672, "sampling/sampling_logp_difference/mean": 0.015619696117937565, "step": 478 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.6921675422927365e-05, "clip_ratio/low_min": 5.6921675422927365e-05, "clip_ratio/region_mean": 5.6921675422927365e-05, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 768.0, "completions/mean_length": 532.0, "completions/mean_terminated_length": 465.91998291015625, "completions/min_length": 250.0, "completions/min_terminated_length": 250.0, "entropy": 0.4183240532875061, "epoch": 0.256423982869379, "frac_reward_zero_std": 0.25, "grad_norm": 0.006963755935430527, "learning_rate": 1e-05, "loss": 0.066, "num_tokens": 10591626.0, "reward": 0.59375, "reward_std": 0.3377464711666107, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.4004451036453247, "sampling/importance_sampling_ratio/mean": 0.9999240636825562, "sampling/importance_sampling_ratio/min": 0.7040327191352844, "sampling/sampling_logp_difference/max": 0.35093045234680176, "sampling/sampling_logp_difference/mean": 0.013495264574885368, "step": 479 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.527345299720764e-05, "clip_ratio/low_min": 4.527345299720764e-05, "clip_ratio/region_mean": 4.527345299720764e-05, "completions/clipped_ratio": 0.28125, "completions/max_length": 768.0, "completions/max_terminated_length": 757.0, "completions/mean_length": 628.75, "completions/mean_terminated_length": 574.2608642578125, "completions/min_length": 296.0, "completions/min_terminated_length": 296.0, "entropy": 0.41216954216361046, "epoch": 0.2569593147751606, "frac_reward_zero_std": 0.25, "grad_norm": 0.007541249040514231, "learning_rate": 1e-05, "loss": 0.0468, "num_tokens": 10615450.0, "reward": 0.65625, "reward_std": 0.3608423173427582, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.4162741899490356, "sampling/importance_sampling_ratio/mean": 1.0002094507217407, "sampling/importance_sampling_ratio/min": 0.5041298270225525, "sampling/sampling_logp_difference/max": 0.6849215626716614, "sampling/sampling_logp_difference/mean": 0.01230060774832964, "step": 480 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 760.0, "completions/mean_length": 512.9375, "completions/mean_terminated_length": 495.933349609375, "completions/min_length": 233.0, "completions/min_terminated_length": 233.0, "entropy": 0.3297930024564266, "epoch": 0.2574946466809422, "frac_reward_zero_std": 0.5, "grad_norm": 0.0026471305172890425, "learning_rate": 1e-05, "loss": 0.0516, "num_tokens": 10635208.0, "reward": 0.875, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 1.4261765480041504, "sampling/importance_sampling_ratio/mean": 1.000159740447998, "sampling/importance_sampling_ratio/min": 0.6802457571029663, "sampling/sampling_logp_difference/max": 0.3853011131286621, "sampling/sampling_logp_difference/mean": 0.010556701570749283, "step": 481 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 739.0, "completions/mean_length": 511.46875, "completions/mean_terminated_length": 494.36669921875, "completions/min_length": 311.0, "completions/min_terminated_length": 311.0, "entropy": 0.30446585454046726, "epoch": 0.2580299785867238, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0293, "num_tokens": 10655143.0, "reward": 0.96875, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.96875, "rewards/accuracy_reward/std": 0.1767766922712326, "sampling/importance_sampling_ratio/max": 1.2877490520477295, "sampling/importance_sampling_ratio/mean": 1.0001307725906372, "sampling/importance_sampling_ratio/min": 0.7307831048965454, "sampling/sampling_logp_difference/max": 0.3136385679244995, "sampling/sampling_logp_difference/mean": 0.00997633021324873, "step": 482 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00010371941607445478, "clip_ratio/low_min": 0.00010371941607445478, "clip_ratio/region_mean": 0.00010371941607445478, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 765.0, "completions/mean_length": 545.59375, "completions/mean_terminated_length": 471.4583435058594, "completions/min_length": 254.0, "completions/min_terminated_length": 254.0, "entropy": 0.4034724347293377, "epoch": 0.25856531049250536, "frac_reward_zero_std": 0.25, "grad_norm": 0.013043330982327461, "learning_rate": 1e-05, "loss": 0.1117, "num_tokens": 10676866.0, "reward": 0.6875, "reward_std": 0.3745020925998688, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.4366579055786133, "sampling/importance_sampling_ratio/mean": 0.9998902082443237, "sampling/importance_sampling_ratio/min": 0.7108510732650757, "sampling/sampling_logp_difference/max": 0.3623194694519043, "sampling/sampling_logp_difference/mean": 0.013162062503397465, "step": 483 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 767.0, "completions/mean_length": 546.71875, "completions/mean_terminated_length": 495.65386962890625, "completions/min_length": 249.0, "completions/min_terminated_length": 249.0, "entropy": 0.30012526363134384, "epoch": 0.25910064239828695, "frac_reward_zero_std": 0.25, "grad_norm": 0.008093496784567833, "learning_rate": 1e-05, "loss": 0.0377, "num_tokens": 10698009.0, "reward": 0.84375, "reward_std": 0.3061639666557312, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.4611619710922241, "sampling/importance_sampling_ratio/mean": 1.0001821517944336, "sampling/importance_sampling_ratio/min": 0.6850831508636475, "sampling/sampling_logp_difference/max": 0.37923192977905273, "sampling/sampling_logp_difference/mean": 0.009886610321700573, "step": 484 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 715.0, "completions/mean_length": 538.09375, "completions/mean_terminated_length": 495.5185241699219, "completions/min_length": 214.0, "completions/min_terminated_length": 214.0, "entropy": 0.3680847994983196, "epoch": 0.25963597430406854, "frac_reward_zero_std": 0.25, "grad_norm": 0.010817212052643299, "learning_rate": 1e-05, "loss": 0.0603, "num_tokens": 10719172.0, "reward": 0.78125, "reward_std": 0.3608423173427582, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.5157670974731445, "sampling/importance_sampling_ratio/mean": 0.999823272228241, "sampling/importance_sampling_ratio/min": 0.7624718546867371, "sampling/sampling_logp_difference/max": 0.415921688079834, "sampling/sampling_logp_difference/mean": 0.011580967344343662, "step": 485 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00014238310541259125, "clip_ratio/low_min": 0.00014238310541259125, "clip_ratio/region_mean": 0.00014238310541259125, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 727.0, "completions/mean_length": 474.375, "completions/mean_terminated_length": 454.8000183105469, "completions/min_length": 134.0, "completions/min_terminated_length": 134.0, "entropy": 0.3717295974493027, "epoch": 0.26017130620985013, "frac_reward_zero_std": 0.25, "grad_norm": 0.00501126516610384, "learning_rate": 1e-05, "loss": 0.0412, "num_tokens": 10737856.0, "reward": 0.6875, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.2876932621002197, "sampling/importance_sampling_ratio/mean": 1.000070571899414, "sampling/importance_sampling_ratio/min": 0.6117763519287109, "sampling/sampling_logp_difference/max": 0.49138855934143066, "sampling/sampling_logp_difference/mean": 0.011895447038114071, "step": 486 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 762.0, "completions/mean_length": 581.53125, "completions/mean_terminated_length": 554.8928833007812, "completions/min_length": 285.0, "completions/min_terminated_length": 285.0, "entropy": 0.34515034779906273, "epoch": 0.2607066381156317, "frac_reward_zero_std": 0.25, "grad_norm": 0.024670716375112534, "learning_rate": 1e-05, "loss": 0.0453, "num_tokens": 10760713.0, "reward": 0.59375, "reward_std": 0.3061639964580536, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.4213733673095703, "sampling/importance_sampling_ratio/mean": 1.000146508216858, "sampling/importance_sampling_ratio/min": 0.7005010843276978, "sampling/sampling_logp_difference/max": 0.355959415435791, "sampling/sampling_logp_difference/mean": 0.011290302500128746, "step": 487 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.137690095580183e-05, "clip_ratio/low_min": 5.137690095580183e-05, "clip_ratio/region_mean": 5.137690095580183e-05, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 703.0, "completions/mean_length": 504.875, "completions/mean_terminated_length": 467.2857360839844, "completions/min_length": 272.0, "completions/min_terminated_length": 272.0, "entropy": 0.4183342345058918, "epoch": 0.26124197002141325, "frac_reward_zero_std": 0.5, "grad_norm": 0.011635100468993187, "learning_rate": 1e-05, "loss": 0.0309, "num_tokens": 10780861.0, "reward": 0.65625, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.4351632595062256, "sampling/importance_sampling_ratio/mean": 1.0002400875091553, "sampling/importance_sampling_ratio/min": 0.702856719493866, "sampling/sampling_logp_difference/max": 0.3612785339355469, "sampling/sampling_logp_difference/mean": 0.012863479554653168, "step": 488 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.423212885740213e-05, "clip_ratio/low_min": 4.423212885740213e-05, "clip_ratio/region_mean": 4.423212885740213e-05, "completions/clipped_ratio": 0.3125, "completions/max_length": 768.0, "completions/max_terminated_length": 728.0, "completions/mean_length": 621.4375, "completions/mean_terminated_length": 554.8181762695312, "completions/min_length": 331.0, "completions/min_terminated_length": 331.0, "entropy": 0.4198906682431698, "epoch": 0.26177730192719484, "frac_reward_zero_std": 0.25, "grad_norm": 0.017947958782315254, "learning_rate": 1e-05, "loss": 0.0406, "num_tokens": 10804907.0, "reward": 0.53125, "reward_std": 0.378745436668396, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.442814588546753, "sampling/importance_sampling_ratio/mean": 1.0000669956207275, "sampling/importance_sampling_ratio/min": 0.694574236869812, "sampling/sampling_logp_difference/max": 0.3665957450866699, "sampling/sampling_logp_difference/mean": 0.012978503480553627, "step": 489 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.940588355064392e-05, "clip_ratio/low_min": 6.940588355064392e-05, "clip_ratio/region_mean": 6.940588355064392e-05, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 758.0, "completions/mean_length": 487.125, "completions/mean_terminated_length": 468.4000244140625, "completions/min_length": 228.0, "completions/min_terminated_length": 228.0, "entropy": 0.38713568449020386, "epoch": 0.2623126338329764, "frac_reward_zero_std": 0.5, "grad_norm": 0.017323443666100502, "learning_rate": 1e-05, "loss": -0.0415, "num_tokens": 10824367.0, "reward": 0.78125, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.4198147058486938, "sampling/importance_sampling_ratio/mean": 1.0001780986785889, "sampling/importance_sampling_ratio/min": 0.7587830424308777, "sampling/sampling_logp_difference/max": 0.3505263328552246, "sampling/sampling_logp_difference/mean": 0.012243088334798813, "step": 490 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.28125, "completions/max_length": 768.0, "completions/max_terminated_length": 745.0, "completions/mean_length": 617.375, "completions/mean_terminated_length": 558.434814453125, "completions/min_length": 391.0, "completions/min_terminated_length": 391.0, "entropy": 0.43042561784386635, "epoch": 0.262847965738758, "frac_reward_zero_std": 0.25, "grad_norm": 0.005080129019916058, "learning_rate": 1e-05, "loss": 0.0135, "num_tokens": 10848371.0, "reward": 0.6875, "reward_std": 0.2925041913986206, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.375198483467102, "sampling/importance_sampling_ratio/mean": 0.9998195171356201, "sampling/importance_sampling_ratio/min": 0.7337667346000671, "sampling/sampling_logp_difference/max": 0.31859803199768066, "sampling/sampling_logp_difference/mean": 0.012578554451465607, "step": 491 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.3998592445859686e-05, "clip_ratio/low_min": 4.3998592445859686e-05, "clip_ratio/region_mean": 4.3998592445859686e-05, "completions/clipped_ratio": 0.4375, "completions/max_length": 768.0, "completions/max_terminated_length": 719.0, "completions/mean_length": 652.65625, "completions/mean_terminated_length": 562.9444580078125, "completions/min_length": 380.0, "completions/min_terminated_length": 380.0, "entropy": 0.3964984081685543, "epoch": 0.2633832976445396, "frac_reward_zero_std": 0.5, "grad_norm": 0.0028810864314436913, "learning_rate": 1e-05, "loss": 0.0256, "num_tokens": 10873160.0, "reward": 0.34375, "reward_std": 0.22201895713806152, "rewards/accuracy_reward/mean": 0.34375, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.4145331382751465, "sampling/importance_sampling_ratio/mean": 1.0005985498428345, "sampling/importance_sampling_ratio/min": 0.7429461479187012, "sampling/sampling_logp_difference/max": 0.34679949283599854, "sampling/sampling_logp_difference/mean": 0.011351502500474453, "step": 492 }, { "clip_ratio/high_max": 5.099959162180312e-05, "clip_ratio/high_mean": 5.099959162180312e-05, "clip_ratio/low_mean": 5.0709939387161285e-05, "clip_ratio/low_min": 5.0709939387161285e-05, "clip_ratio/region_mean": 0.0001017095310089644, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 734.0, "completions/mean_length": 588.4375, "completions/mean_terminated_length": 562.7857666015625, "completions/min_length": 396.0, "completions/min_terminated_length": 396.0, "entropy": 0.43003494292497635, "epoch": 0.2639186295503212, "frac_reward_zero_std": 0.25, "grad_norm": 0.01873714290559292, "learning_rate": 1e-05, "loss": 0.0293, "num_tokens": 10895526.0, "reward": 0.65625, "reward_std": 0.3608423173427582, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.961113452911377, "sampling/importance_sampling_ratio/mean": 1.0002150535583496, "sampling/importance_sampling_ratio/min": 0.677449643611908, "sampling/sampling_logp_difference/max": 0.6735124588012695, "sampling/sampling_logp_difference/mean": 0.013516178354620934, "step": 493 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 691.0, "completions/mean_length": 541.6875, "completions/mean_terminated_length": 499.77777099609375, "completions/min_length": 313.0, "completions/min_terminated_length": 313.0, "entropy": 0.4351819381117821, "epoch": 0.2644539614561028, "frac_reward_zero_std": 0.0, "grad_norm": 0.01836119033396244, "learning_rate": 1e-05, "loss": 0.029, "num_tokens": 10916580.0, "reward": 0.625, "reward_std": 0.47655022144317627, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.4574007987976074, "sampling/importance_sampling_ratio/mean": 1.0003912448883057, "sampling/importance_sampling_ratio/min": 0.7238929271697998, "sampling/sampling_logp_difference/max": 0.37665462493896484, "sampling/sampling_logp_difference/mean": 0.013308979570865631, "step": 494 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0001161811851488892, "clip_ratio/low_min": 0.0001161811851488892, "clip_ratio/region_mean": 0.0001161811851488892, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 731.0, "completions/mean_length": 596.5, "completions/mean_terminated_length": 539.3333740234375, "completions/min_length": 341.0, "completions/min_terminated_length": 341.0, "entropy": 0.4781779535114765, "epoch": 0.26498929336188437, "frac_reward_zero_std": 0.25, "grad_norm": 0.020715584978461266, "learning_rate": 1e-05, "loss": 0.0468, "num_tokens": 10939844.0, "reward": 0.59375, "reward_std": 0.3471629321575165, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.4176584482192993, "sampling/importance_sampling_ratio/mean": 1.000109314918518, "sampling/importance_sampling_ratio/min": 0.7280387878417969, "sampling/sampling_logp_difference/max": 0.3490065336227417, "sampling/sampling_logp_difference/mean": 0.013495479710400105, "step": 495 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 7.254788215504959e-05, "clip_ratio/low_min": 7.254788215504959e-05, "clip_ratio/region_mean": 7.254788215504959e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 760.0, "completions/max_terminated_length": 760.0, "completions/mean_length": 419.96875, "completions/mean_terminated_length": 419.96875, "completions/min_length": 58.0, "completions/min_terminated_length": 58.0, "entropy": 0.39975521713495255, "epoch": 0.26552462526766596, "frac_reward_zero_std": 0.25, "grad_norm": 0.005138919688761234, "learning_rate": 1e-05, "loss": -0.1093, "num_tokens": 10956547.0, "reward": 0.71875, "reward_std": 0.2651650309562683, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.4020583629608154, "sampling/importance_sampling_ratio/mean": 0.9998289942741394, "sampling/importance_sampling_ratio/min": 0.5549389123916626, "sampling/sampling_logp_difference/max": 0.5888972282409668, "sampling/sampling_logp_difference/mean": 0.011359790340065956, "step": 496 }, { "clip_ratio/high_max": 4.470672502066009e-05, "clip_ratio/high_mean": 4.470672502066009e-05, "clip_ratio/low_mean": 0.00015187276585493237, "clip_ratio/low_min": 0.00015187276585493237, "clip_ratio/region_mean": 0.00019657949087559246, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 747.0, "completions/mean_length": 618.40625, "completions/mean_terminated_length": 590.7037353515625, "completions/min_length": 352.0, "completions/min_terminated_length": 352.0, "entropy": 0.29839450120925903, "epoch": 0.26605995717344755, "frac_reward_zero_std": 0.25, "grad_norm": 0.005227820016443729, "learning_rate": 1e-05, "loss": -0.0015, "num_tokens": 10980064.0, "reward": 0.8125, "reward_std": 0.3104073107242584, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.3167110681533813, "sampling/importance_sampling_ratio/mean": 0.9995431900024414, "sampling/importance_sampling_ratio/min": 0.6599874496459961, "sampling/sampling_logp_difference/max": 0.41553449630737305, "sampling/sampling_logp_difference/mean": 0.009761122055351734, "step": 497 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 753.0, "completions/mean_length": 545.90625, "completions/mean_terminated_length": 514.1785888671875, "completions/min_length": 309.0, "completions/min_terminated_length": 309.0, "entropy": 0.3104923889040947, "epoch": 0.26659528907922914, "frac_reward_zero_std": 0.5, "grad_norm": 0.006149497348815203, "learning_rate": 1e-05, "loss": 0.0875, "num_tokens": 11001053.0, "reward": 0.84375, "reward_std": 0.22201895713806152, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.3990099430084229, "sampling/importance_sampling_ratio/mean": 0.9997010231018066, "sampling/importance_sampling_ratio/min": 0.7086121439933777, "sampling/sampling_logp_difference/max": 0.34444689750671387, "sampling/sampling_logp_difference/mean": 0.010131840594112873, "step": 498 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 747.0, "completions/mean_length": 518.09375, "completions/mean_terminated_length": 492.2413635253906, "completions/min_length": 357.0, "completions/min_terminated_length": 357.0, "entropy": 0.39401593804359436, "epoch": 0.2671306209850107, "frac_reward_zero_std": 0.5, "grad_norm": 0.013658261857926846, "learning_rate": 1e-05, "loss": 0.0287, "num_tokens": 11021152.0, "reward": 0.75, "reward_std": 0.2587745785713196, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.4280586242675781, "sampling/importance_sampling_ratio/mean": 1.0001189708709717, "sampling/importance_sampling_ratio/min": 0.6740455627441406, "sampling/sampling_logp_difference/max": 0.3944575786590576, "sampling/sampling_logp_difference/mean": 0.012333094142377377, "step": 499 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.4375, "completions/max_length": 768.0, "completions/max_terminated_length": 717.0, "completions/mean_length": 582.5625, "completions/mean_terminated_length": 438.3333435058594, "completions/min_length": 263.0, "completions/min_terminated_length": 263.0, "entropy": 0.522261418402195, "epoch": 0.2676659528907923, "frac_reward_zero_std": 0.5, "grad_norm": 0.010906667448580265, "learning_rate": 1e-05, "loss": 0.012, "num_tokens": 11044378.0, "reward": 0.5, "reward_std": 0.2587745785713196, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5080004930496216, "sampling/importance_sampling_ratio/max": 1.493876576423645, "sampling/importance_sampling_ratio/mean": 0.99949049949646, "sampling/importance_sampling_ratio/min": 0.6382625699043274, "sampling/sampling_logp_difference/max": 0.4490056037902832, "sampling/sampling_logp_difference/mean": 0.015240837819874287, "step": 500 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 7.271669892361388e-05, "clip_ratio/low_min": 7.271669892361388e-05, "clip_ratio/region_mean": 7.271669892361388e-05, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 660.0, "completions/mean_length": 470.40625, "completions/mean_terminated_length": 439.6206970214844, "completions/min_length": 273.0, "completions/min_terminated_length": 273.0, "entropy": 0.45451122894883156, "epoch": 0.2682012847965739, "frac_reward_zero_std": 0.25, "grad_norm": 0.013978679664433002, "learning_rate": 1e-05, "loss": 0.03, "num_tokens": 11063463.0, "reward": 0.71875, "reward_std": 0.3471629321575165, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.4506685733795166, "sampling/importance_sampling_ratio/mean": 0.9997245669364929, "sampling/importance_sampling_ratio/min": 0.7564635872840881, "sampling/sampling_logp_difference/max": 0.3720245361328125, "sampling/sampling_logp_difference/mean": 0.014118613675236702, "step": 501 }, { "clip_ratio/high_max": 6.627784023294225e-05, "clip_ratio/high_mean": 6.627784023294225e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 6.627784023294225e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 752.0, "completions/max_terminated_length": 752.0, "completions/mean_length": 454.90625, "completions/mean_terminated_length": 454.90625, "completions/min_length": 296.0, "completions/min_terminated_length": 296.0, "entropy": 0.34238502010703087, "epoch": 0.26873661670235544, "frac_reward_zero_std": 0.75, "grad_norm": 0.007140727713704109, "learning_rate": 1e-05, "loss": -0.0429, "num_tokens": 11081892.0, "reward": 0.9375, "reward_std": 0.1157275140285492, "rewards/accuracy_reward/mean": 0.9375, "rewards/accuracy_reward/std": 0.24593468010425568, "sampling/importance_sampling_ratio/max": 1.3643701076507568, "sampling/importance_sampling_ratio/mean": 0.9997614622116089, "sampling/importance_sampling_ratio/min": 0.6850076913833618, "sampling/sampling_logp_difference/max": 0.3783252239227295, "sampling/sampling_logp_difference/mean": 0.011558032594621181, "step": 502 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 680.0, "completions/mean_length": 480.15625, "completions/mean_terminated_length": 470.8709411621094, "completions/min_length": 267.0, "completions/min_terminated_length": 267.0, "entropy": 0.36768776178359985, "epoch": 0.269271948608137, "frac_reward_zero_std": 0.5, "grad_norm": 0.013215147890150547, "learning_rate": 1e-05, "loss": 0.0173, "num_tokens": 11101393.0, "reward": 0.875, "reward_std": 0.2314550280570984, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 1.5229955911636353, "sampling/importance_sampling_ratio/mean": 1.0001130104064941, "sampling/importance_sampling_ratio/min": 0.6991862058639526, "sampling/sampling_logp_difference/max": 0.42067909240722656, "sampling/sampling_logp_difference/mean": 0.012127074413001537, "step": 503 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 734.0, "completions/mean_length": 541.75, "completions/mean_terminated_length": 489.5384826660156, "completions/min_length": 221.0, "completions/min_terminated_length": 221.0, "entropy": 0.39395223557949066, "epoch": 0.2698072805139186, "frac_reward_zero_std": 0.0, "grad_norm": 0.013775346800684929, "learning_rate": 1e-05, "loss": 0.0805, "num_tokens": 11122289.0, "reward": 0.71875, "reward_std": 0.4397946000099182, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.5974918603897095, "sampling/importance_sampling_ratio/mean": 0.9998372793197632, "sampling/importance_sampling_ratio/min": 0.7151423096656799, "sampling/sampling_logp_difference/max": 0.46843481063842773, "sampling/sampling_logp_difference/mean": 0.012173264287412167, "step": 504 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0001201498816953972, "clip_ratio/low_min": 0.0001201498816953972, "clip_ratio/region_mean": 0.0001201498816953972, "completions/clipped_ratio": 0.34375, "completions/max_length": 768.0, "completions/max_terminated_length": 754.0, "completions/mean_length": 592.78125, "completions/mean_terminated_length": 501.0, "completions/min_length": 304.0, "completions/min_terminated_length": 304.0, "entropy": 0.43938862904906273, "epoch": 0.2703426124197002, "frac_reward_zero_std": 0.0, "grad_norm": 0.016774991527199745, "learning_rate": 1e-05, "loss": 0.0461, "num_tokens": 11145298.0, "reward": 0.4375, "reward_std": 0.5260357856750488, "rewards/accuracy_reward/mean": 0.4375, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.5081984996795654, "sampling/importance_sampling_ratio/mean": 1.0002576112747192, "sampling/importance_sampling_ratio/min": 0.7404820919036865, "sampling/sampling_logp_difference/max": 0.4109158515930176, "sampling/sampling_logp_difference/mean": 0.013073504902422428, "step": 505 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00012589900143211707, "clip_ratio/low_min": 0.00012589900143211707, "clip_ratio/region_mean": 0.00012589900143211707, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 711.0, "completions/mean_length": 484.125, "completions/mean_terminated_length": 443.5714416503906, "completions/min_length": 250.0, "completions/min_terminated_length": 250.0, "entropy": 0.40190256014466286, "epoch": 0.2708779443254818, "frac_reward_zero_std": 0.5, "grad_norm": 0.01707610860466957, "learning_rate": 1e-05, "loss": 0.032, "num_tokens": 11164822.0, "reward": 0.6875, "reward_std": 0.249358132481575, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.6597273349761963, "sampling/importance_sampling_ratio/mean": 0.9995678067207336, "sampling/importance_sampling_ratio/min": 0.7377364039421082, "sampling/sampling_logp_difference/max": 0.5066533088684082, "sampling/sampling_logp_difference/mean": 0.012648704461753368, "step": 506 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 688.0, "completions/mean_length": 525.5, "completions/mean_terminated_length": 490.857177734375, "completions/min_length": 267.0, "completions/min_terminated_length": 267.0, "entropy": 0.37242839112877846, "epoch": 0.2714132762312634, "frac_reward_zero_std": 0.5, "grad_norm": 0.011503325775265694, "learning_rate": 1e-05, "loss": 0.0556, "num_tokens": 11185726.0, "reward": 0.8125, "reward_std": 0.249358132481575, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.2950011491775513, "sampling/importance_sampling_ratio/mean": 0.9997808337211609, "sampling/importance_sampling_ratio/min": 0.5552176237106323, "sampling/sampling_logp_difference/max": 0.5883951187133789, "sampling/sampling_logp_difference/mean": 0.011658416129648685, "step": 507 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00028234537603566423, "clip_ratio/low_min": 0.00028234537603566423, "clip_ratio/region_mean": 0.00028234537603566423, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 760.0, "completions/mean_length": 565.84375, "completions/mean_terminated_length": 498.4583435058594, "completions/min_length": 261.0, "completions/min_terminated_length": 261.0, "entropy": 0.4553045593202114, "epoch": 0.27194860813704497, "frac_reward_zero_std": 0.25, "grad_norm": 0.015420237556099892, "learning_rate": 1e-05, "loss": 0.0217, "num_tokens": 11207377.0, "reward": 0.46875, "reward_std": 0.3377464711666107, "rewards/accuracy_reward/mean": 0.46875, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.6266615390777588, "sampling/importance_sampling_ratio/mean": 0.9997661709785461, "sampling/importance_sampling_ratio/min": 0.6946967840194702, "sampling/sampling_logp_difference/max": 0.4865298271179199, "sampling/sampling_logp_difference/mean": 0.01380403246730566, "step": 508 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 755.0, "completions/mean_length": 483.0, "completions/mean_terminated_length": 464.0000305175781, "completions/min_length": 309.0, "completions/min_terminated_length": 309.0, "entropy": 0.297705065459013, "epoch": 0.27248394004282656, "frac_reward_zero_std": 0.75, "grad_norm": 0.007576956879347563, "learning_rate": 1e-05, "loss": -0.0033, "num_tokens": 11226609.0, "reward": 0.875, "reward_std": 0.13363061845302582, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 1.4550213813781738, "sampling/importance_sampling_ratio/mean": 1.0000966787338257, "sampling/importance_sampling_ratio/min": 0.7399776577949524, "sampling/sampling_logp_difference/max": 0.3750205636024475, "sampling/sampling_logp_difference/mean": 0.01007781270891428, "step": 509 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 715.0, "completions/mean_length": 504.84375, "completions/mean_terminated_length": 496.3548278808594, "completions/min_length": 234.0, "completions/min_terminated_length": 234.0, "entropy": 0.3161064684391022, "epoch": 0.27301927194860814, "frac_reward_zero_std": 0.5, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0715, "num_tokens": 11246188.0, "reward": 0.875, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 1.3199232816696167, "sampling/importance_sampling_ratio/mean": 0.9998076558113098, "sampling/importance_sampling_ratio/min": 0.6959797739982605, "sampling/sampling_logp_difference/max": 0.36243462562561035, "sampling/sampling_logp_difference/mean": 0.010502434335649014, "step": 510 }, { "clip_ratio/high_max": 5.5506217904621735e-05, "clip_ratio/high_mean": 5.5506217904621735e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 5.5506217904621735e-05, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 754.0, "completions/mean_length": 575.03125, "completions/mean_terminated_length": 562.1666870117188, "completions/min_length": 390.0, "completions/min_terminated_length": 390.0, "entropy": 0.3339088074862957, "epoch": 0.27355460385438973, "frac_reward_zero_std": 0.25, "grad_norm": 0.01582440920174122, "learning_rate": 1e-05, "loss": 0.0028, "num_tokens": 11268429.0, "reward": 0.71875, "reward_std": 0.35564959049224854, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.8644335269927979, "sampling/importance_sampling_ratio/mean": 1.000091552734375, "sampling/importance_sampling_ratio/min": 0.45649251341819763, "sampling/sampling_logp_difference/max": 0.7841830253601074, "sampling/sampling_logp_difference/mean": 0.010884090326726437, "step": 511 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 8.778089977568015e-05, "clip_ratio/low_min": 8.778089977568015e-05, "clip_ratio/region_mean": 8.778089977568015e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 725.0, "completions/max_terminated_length": 725.0, "completions/mean_length": 377.34375, "completions/mean_terminated_length": 377.34375, "completions/min_length": 143.0, "completions/min_terminated_length": 143.0, "entropy": 0.34434812515974045, "epoch": 0.2740899357601713, "frac_reward_zero_std": 0.25, "grad_norm": 0.024170197546482086, "learning_rate": 1e-05, "loss": 0.0183, "num_tokens": 11284296.0, "reward": 0.625, "reward_std": 0.3650856614112854, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.7866148948669434, "sampling/importance_sampling_ratio/mean": 1.0001447200775146, "sampling/importance_sampling_ratio/min": 0.7076085209846497, "sampling/sampling_logp_difference/max": 0.5803227424621582, "sampling/sampling_logp_difference/mean": 0.011088111437857151, "step": 512 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 715.0, "completions/max_terminated_length": 715.0, "completions/mean_length": 476.15625, "completions/mean_terminated_length": 476.15625, "completions/min_length": 252.0, "completions/min_terminated_length": 252.0, "entropy": 0.31146080791950226, "epoch": 0.2746252676659529, "frac_reward_zero_std": 0.5, "grad_norm": 0.006982438266277313, "learning_rate": 1e-05, "loss": -0.0112, "num_tokens": 11303469.0, "reward": 0.75, "reward_std": 0.2587745785713196, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.4836581945419312, "sampling/importance_sampling_ratio/mean": 0.9999073147773743, "sampling/importance_sampling_ratio/min": 0.675791323184967, "sampling/sampling_logp_difference/max": 0.39451074600219727, "sampling/sampling_logp_difference/mean": 0.01101579237729311, "step": 513 }, { "clip_ratio/high_max": 4.740235090139322e-05, "clip_ratio/high_mean": 4.740235090139322e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 4.740235090139322e-05, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 758.0, "completions/mean_length": 583.0, "completions/mean_terminated_length": 540.3077392578125, "completions/min_length": 290.0, "completions/min_terminated_length": 290.0, "entropy": 0.32112976163625717, "epoch": 0.2751605995717345, "frac_reward_zero_std": 0.75, "grad_norm": 0.002383845392614603, "learning_rate": 1e-05, "loss": -0.0029, "num_tokens": 11326253.0, "reward": 0.46875, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.46875, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.461322546005249, "sampling/importance_sampling_ratio/mean": 0.9995347857475281, "sampling/importance_sampling_ratio/min": 0.6077783703804016, "sampling/sampling_logp_difference/max": 0.49794501066207886, "sampling/sampling_logp_difference/mean": 0.010457371361553669, "step": 514 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.28125, "completions/max_length": 768.0, "completions/max_terminated_length": 761.0, "completions/mean_length": 614.3125, "completions/mean_terminated_length": 554.1739501953125, "completions/min_length": 284.0, "completions/min_terminated_length": 284.0, "entropy": 0.2877458203583956, "epoch": 0.2756959314775161, "frac_reward_zero_std": 0.5, "grad_norm": 0.002158162649720907, "learning_rate": 1e-05, "loss": 0.0301, "num_tokens": 11349871.0, "reward": 0.6875, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.3167599439620972, "sampling/importance_sampling_ratio/mean": 0.9999128580093384, "sampling/importance_sampling_ratio/min": 0.7256494164466858, "sampling/sampling_logp_difference/max": 0.32068824768066406, "sampling/sampling_logp_difference/mean": 0.009066893719136715, "step": 515 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 755.0, "completions/mean_length": 640.28125, "completions/mean_terminated_length": 597.7083740234375, "completions/min_length": 369.0, "completions/min_terminated_length": 369.0, "entropy": 0.36829761415719986, "epoch": 0.2762312633832976, "frac_reward_zero_std": 0.5, "grad_norm": 0.012869792059063911, "learning_rate": 1e-05, "loss": 0.023, "num_tokens": 11374280.0, "reward": 0.8125, "reward_std": 0.249358132481575, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.4000791311264038, "sampling/importance_sampling_ratio/mean": 0.9999721646308899, "sampling/importance_sampling_ratio/min": 0.6369316577911377, "sampling/sampling_logp_difference/max": 0.4510929584503174, "sampling/sampling_logp_difference/mean": 0.011919982731342316, "step": 516 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00012242898810654879, "clip_ratio/low_min": 0.00012242898810654879, "clip_ratio/region_mean": 0.00012242898810654879, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 700.0, "completions/mean_length": 549.34375, "completions/mean_terminated_length": 508.85186767578125, "completions/min_length": 310.0, "completions/min_terminated_length": 310.0, "entropy": 0.4445907063782215, "epoch": 0.2767665952890792, "frac_reward_zero_std": 0.5, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0458, "num_tokens": 11396179.0, "reward": 0.75, "reward_std": 0.2587745785713196, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.44916832447052, "sampling/importance_sampling_ratio/mean": 1.0002517700195312, "sampling/importance_sampling_ratio/min": 0.7010741233825684, "sampling/sampling_logp_difference/max": 0.3709897994995117, "sampling/sampling_logp_difference/mean": 0.013188843615353107, "step": 517 }, { "clip_ratio/high_max": 5.303351645125076e-05, "clip_ratio/high_mean": 5.303351645125076e-05, "clip_ratio/low_mean": 0.00010527325866860338, "clip_ratio/low_min": 0.00010527325866860338, "clip_ratio/region_mean": 0.00015830677511985414, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 717.0, "completions/mean_length": 564.625, "completions/mean_terminated_length": 496.8333435058594, "completions/min_length": 263.0, "completions/min_terminated_length": 263.0, "entropy": 0.3561297971755266, "epoch": 0.2773019271948608, "frac_reward_zero_std": 0.0, "grad_norm": 0.013775930739939213, "learning_rate": 1e-05, "loss": 0.0579, "num_tokens": 11418127.0, "reward": 0.46875, "reward_std": 0.4944729208946228, "rewards/accuracy_reward/mean": 0.46875, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.437573790550232, "sampling/importance_sampling_ratio/mean": 0.9998301863670349, "sampling/importance_sampling_ratio/min": 0.5677351951599121, "sampling/sampling_logp_difference/max": 0.5661001205444336, "sampling/sampling_logp_difference/mean": 0.011913307942450047, "step": 518 }, { "clip_ratio/high_max": 5.1062092097708955e-05, "clip_ratio/high_mean": 5.1062092097708955e-05, "clip_ratio/low_mean": 0.00016235495422733948, "clip_ratio/low_min": 0.00016235495422733948, "clip_ratio/region_mean": 0.00021341704632504843, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 760.0, "completions/mean_length": 571.96875, "completions/mean_terminated_length": 535.6666870117188, "completions/min_length": 348.0, "completions/min_terminated_length": 348.0, "entropy": 0.41886864602565765, "epoch": 0.2778372591006424, "frac_reward_zero_std": 0.0, "grad_norm": 0.009069961495697498, "learning_rate": 1e-05, "loss": 0.0859, "num_tokens": 11440222.0, "reward": 0.53125, "reward_std": 0.4218915104866028, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.4745533466339111, "sampling/importance_sampling_ratio/mean": 1.0001682043075562, "sampling/importance_sampling_ratio/min": 0.5070000886917114, "sampling/sampling_logp_difference/max": 0.6792440414428711, "sampling/sampling_logp_difference/mean": 0.013481111265718937, "step": 519 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00020833496819250286, "clip_ratio/low_min": 0.00020833496819250286, "clip_ratio/region_mean": 0.00020833496819250286, "completions/clipped_ratio": 0.0, "completions/max_length": 591.0, "completions/max_terminated_length": 591.0, "completions/mean_length": 410.4375, "completions/mean_terminated_length": 410.4375, "completions/min_length": 268.0, "completions/min_terminated_length": 268.0, "entropy": 0.3321453146636486, "epoch": 0.278372591006424, "frac_reward_zero_std": 0.5, "grad_norm": 0.02371736243367195, "learning_rate": 1e-05, "loss": 0.0636, "num_tokens": 11456932.0, "reward": 0.65625, "reward_std": 0.22201895713806152, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.3492980003356934, "sampling/importance_sampling_ratio/mean": 0.9997901320457458, "sampling/importance_sampling_ratio/min": 0.756468653678894, "sampling/sampling_logp_difference/max": 0.29958438873291016, "sampling/sampling_logp_difference/mean": 0.010396013036370277, "step": 520 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.492091440828517e-05, "clip_ratio/low_min": 5.492091440828517e-05, "clip_ratio/region_mean": 5.492091440828517e-05, "completions/clipped_ratio": 0.3125, "completions/max_length": 768.0, "completions/max_terminated_length": 710.0, "completions/mean_length": 608.90625, "completions/mean_terminated_length": 536.5909423828125, "completions/min_length": 316.0, "completions/min_terminated_length": 316.0, "entropy": 0.3451676666736603, "epoch": 0.27890792291220556, "frac_reward_zero_std": 0.0, "grad_norm": 0.00869609136134386, "learning_rate": 1e-05, "loss": 0.0798, "num_tokens": 11480577.0, "reward": 0.53125, "reward_std": 0.4397946000099182, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.6381021738052368, "sampling/importance_sampling_ratio/mean": 0.9999784827232361, "sampling/importance_sampling_ratio/min": 0.7408223152160645, "sampling/sampling_logp_difference/max": 0.49353837966918945, "sampling/sampling_logp_difference/mean": 0.01023965235799551, "step": 521 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00025105642998823896, "clip_ratio/low_min": 0.00025105642998823896, "clip_ratio/region_mean": 0.00025105642998823896, "completions/clipped_ratio": 0.375, "completions/max_length": 768.0, "completions/max_terminated_length": 741.0, "completions/mean_length": 638.90625, "completions/mean_terminated_length": 561.4500122070312, "completions/min_length": 256.0, "completions/min_terminated_length": 256.0, "entropy": 0.44716671109199524, "epoch": 0.27944325481798715, "frac_reward_zero_std": 0.25, "grad_norm": 0.022217893972992897, "learning_rate": 1e-05, "loss": 0.0313, "num_tokens": 11504902.0, "reward": 0.46875, "reward_std": 0.3608423173427582, "rewards/accuracy_reward/mean": 0.46875, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.9299200773239136, "sampling/importance_sampling_ratio/mean": 1.0000885725021362, "sampling/importance_sampling_ratio/min": 0.5071318745613098, "sampling/sampling_logp_difference/max": 0.6789841651916504, "sampling/sampling_logp_difference/mean": 0.013543638400733471, "step": 522 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 640.0, "completions/mean_length": 486.8125, "completions/mean_terminated_length": 457.72412109375, "completions/min_length": 174.0, "completions/min_terminated_length": 174.0, "entropy": 0.34625769034028053, "epoch": 0.27997858672376874, "frac_reward_zero_std": 0.25, "grad_norm": 0.015269968658685684, "learning_rate": 1e-05, "loss": 0.0073, "num_tokens": 11523664.0, "reward": 0.65625, "reward_std": 0.3061639964580536, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.474115252494812, "sampling/importance_sampling_ratio/mean": 0.9997300505638123, "sampling/importance_sampling_ratio/min": 0.7066593766212463, "sampling/sampling_logp_difference/max": 0.3880579471588135, "sampling/sampling_logp_difference/mean": 0.010783224366605282, "step": 523 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0002079651203530375, "clip_ratio/low_min": 0.0002079651203530375, "clip_ratio/region_mean": 0.0002079651203530375, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 766.0, "completions/mean_length": 571.21875, "completions/mean_terminated_length": 516.1199951171875, "completions/min_length": 185.0, "completions/min_terminated_length": 185.0, "entropy": 0.4020249657332897, "epoch": 0.28051391862955033, "frac_reward_zero_std": 0.0, "grad_norm": 0.012099537998437881, "learning_rate": 1e-05, "loss": 0.0118, "num_tokens": 11545791.0, "reward": 0.53125, "reward_std": 0.521792471408844, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.4420913457870483, "sampling/importance_sampling_ratio/mean": 1.0004171133041382, "sampling/importance_sampling_ratio/min": 0.7396310567855835, "sampling/sampling_logp_difference/max": 0.36609435081481934, "sampling/sampling_logp_difference/mean": 0.012760459445416927, "step": 524 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 602.0, "completions/mean_length": 441.28125, "completions/mean_terminated_length": 419.5000305175781, "completions/min_length": 250.0, "completions/min_terminated_length": 250.0, "entropy": 0.31816796585917473, "epoch": 0.2810492505353319, "frac_reward_zero_std": 0.5, "grad_norm": 0.01995597779750824, "learning_rate": 1e-05, "loss": 0.0577, "num_tokens": 11563208.0, "reward": 0.90625, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.328745722770691, "sampling/importance_sampling_ratio/mean": 0.9999685287475586, "sampling/importance_sampling_ratio/min": 0.700685441493988, "sampling/sampling_logp_difference/max": 0.3556962013244629, "sampling/sampling_logp_difference/mean": 0.01083736028522253, "step": 525 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 9.703365503810346e-05, "clip_ratio/low_min": 9.703365503810346e-05, "clip_ratio/region_mean": 9.703365503810346e-05, "completions/clipped_ratio": 0.3125, "completions/max_length": 768.0, "completions/max_terminated_length": 760.0, "completions/mean_length": 625.875, "completions/mean_terminated_length": 561.2727661132812, "completions/min_length": 280.0, "completions/min_terminated_length": 280.0, "entropy": 0.5319276303052902, "epoch": 0.2815845824411135, "frac_reward_zero_std": 0.0, "grad_norm": 0.013550633564591408, "learning_rate": 1e-05, "loss": 0.1078, "num_tokens": 11586780.0, "reward": 0.40625, "reward_std": 0.521792471408844, "rewards/accuracy_reward/mean": 0.40625, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.4476656913757324, "sampling/importance_sampling_ratio/mean": 1.0002061128616333, "sampling/importance_sampling_ratio/min": 0.7269487380981445, "sampling/sampling_logp_difference/max": 0.3699524402618408, "sampling/sampling_logp_difference/mean": 0.0151499193161726, "step": 526 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 746.0, "completions/mean_length": 492.96875, "completions/mean_terminated_length": 484.0967712402344, "completions/min_length": 154.0, "completions/min_terminated_length": 154.0, "entropy": 0.3193536028265953, "epoch": 0.2821199143468951, "frac_reward_zero_std": 0.75, "grad_norm": 0.004949445836246014, "learning_rate": 1e-05, "loss": -0.0045, "num_tokens": 11606163.0, "reward": 0.9375, "reward_std": 0.1157275140285492, "rewards/accuracy_reward/mean": 0.9375, "rewards/accuracy_reward/std": 0.24593468010425568, "sampling/importance_sampling_ratio/max": 1.6154122352600098, "sampling/importance_sampling_ratio/mean": 0.9998866319656372, "sampling/importance_sampling_ratio/min": 0.6933082342147827, "sampling/sampling_logp_difference/max": 0.47959017753601074, "sampling/sampling_logp_difference/mean": 0.010449059307575226, "step": 527 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 736.0, "completions/mean_length": 453.375, "completions/mean_terminated_length": 443.2257995605469, "completions/min_length": 184.0, "completions/min_terminated_length": 184.0, "entropy": 0.28020163998007774, "epoch": 0.2826552462526767, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": -0.0057, "num_tokens": 11624119.0, "reward": 0.9375, "reward_std": 0.1157275140285492, "rewards/accuracy_reward/mean": 0.9375, "rewards/accuracy_reward/std": 0.24593468010425568, "sampling/importance_sampling_ratio/max": 1.3215128183364868, "sampling/importance_sampling_ratio/mean": 1.0000460147857666, "sampling/importance_sampling_ratio/min": 0.7348396182060242, "sampling/sampling_logp_difference/max": 0.308103084564209, "sampling/sampling_logp_difference/mean": 0.009280155412852764, "step": 528 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.166748789837584e-05, "clip_ratio/low_min": 6.166748789837584e-05, "clip_ratio/region_mean": 6.166748789837584e-05, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 736.0, "completions/mean_length": 500.96875, "completions/mean_terminated_length": 451.5185241699219, "completions/min_length": 210.0, "completions/min_terminated_length": 210.0, "entropy": 0.3511334862560034, "epoch": 0.2831905781584582, "frac_reward_zero_std": 0.5, "grad_norm": 0.014866421930491924, "learning_rate": 1e-05, "loss": 0.0598, "num_tokens": 11643870.0, "reward": 0.78125, "reward_std": 0.2630178928375244, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.4438130855560303, "sampling/importance_sampling_ratio/mean": 1.0003409385681152, "sampling/importance_sampling_ratio/min": 0.7352181077003479, "sampling/sampling_logp_difference/max": 0.36728763580322266, "sampling/sampling_logp_difference/mean": 0.011323046870529652, "step": 529 }, { "clip_ratio/high_max": 6.332320481305942e-05, "clip_ratio/high_mean": 6.332320481305942e-05, "clip_ratio/low_mean": 0.0001737024140311405, "clip_ratio/low_min": 0.0001737024140311405, "clip_ratio/region_mean": 0.00023702561884419993, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 684.0, "completions/mean_length": 535.71875, "completions/mean_terminated_length": 482.11541748046875, "completions/min_length": 336.0, "completions/min_terminated_length": 336.0, "entropy": 0.34200095385313034, "epoch": 0.2837259100642398, "frac_reward_zero_std": 0.5, "grad_norm": 0.004239071160554886, "learning_rate": 1e-05, "loss": 0.0319, "num_tokens": 11664997.0, "reward": 0.71875, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.9145855903625488, "sampling/importance_sampling_ratio/mean": 1.0002284049987793, "sampling/importance_sampling_ratio/min": 0.6010749936103821, "sampling/sampling_logp_difference/max": 0.6495012044906616, "sampling/sampling_logp_difference/mean": 0.011249025352299213, "step": 530 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.000150988700625021, "clip_ratio/low_min": 0.000150988700625021, "clip_ratio/region_mean": 0.000150988700625021, "completions/clipped_ratio": 0.0, "completions/max_length": 680.0, "completions/max_terminated_length": 680.0, "completions/mean_length": 419.21875, "completions/mean_terminated_length": 419.21875, "completions/min_length": 267.0, "completions/min_terminated_length": 267.0, "entropy": 0.32063016667962074, "epoch": 0.2842612419700214, "frac_reward_zero_std": 0.25, "grad_norm": 0.0190244410187006, "learning_rate": 1e-05, "loss": 0.0072, "num_tokens": 11682012.0, "reward": 0.8125, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.5228393077850342, "sampling/importance_sampling_ratio/mean": 1.0001142024993896, "sampling/importance_sampling_ratio/min": 0.6502863764762878, "sampling/sampling_logp_difference/max": 0.430342435836792, "sampling/sampling_logp_difference/mean": 0.011216445825994015, "step": 531 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 675.0, "completions/mean_length": 551.46875, "completions/mean_terminated_length": 511.370361328125, "completions/min_length": 320.0, "completions/min_terminated_length": 320.0, "entropy": 0.4353875555098057, "epoch": 0.284796573875803, "frac_reward_zero_std": 0.25, "grad_norm": 0.0205855555832386, "learning_rate": 1e-05, "loss": 0.0758, "num_tokens": 11703315.0, "reward": 0.65625, "reward_std": 0.3377465009689331, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.4349011182785034, "sampling/importance_sampling_ratio/mean": 1.0000529289245605, "sampling/importance_sampling_ratio/min": 0.6882743239402771, "sampling/sampling_logp_difference/max": 0.3735678195953369, "sampling/sampling_logp_difference/mean": 0.01333760004490614, "step": 532 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 7.05019774613902e-05, "clip_ratio/low_min": 7.05019774613902e-05, "clip_ratio/region_mean": 7.05019774613902e-05, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 754.0, "completions/mean_length": 528.09375, "completions/mean_terminated_length": 512.1000366210938, "completions/min_length": 250.0, "completions/min_terminated_length": 250.0, "entropy": 0.32369837909936905, "epoch": 0.28533190578158457, "frac_reward_zero_std": 0.25, "grad_norm": 0.012548979371786118, "learning_rate": 1e-05, "loss": 0.0006, "num_tokens": 11724038.0, "reward": 0.8125, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.4544031620025635, "sampling/importance_sampling_ratio/mean": 0.9998462796211243, "sampling/importance_sampling_ratio/min": 0.6570523977279663, "sampling/sampling_logp_difference/max": 0.41999149322509766, "sampling/sampling_logp_difference/mean": 0.010763757862150669, "step": 533 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.18242122780066e-05, "clip_ratio/low_min": 5.18242122780066e-05, "clip_ratio/region_mean": 5.18242122780066e-05, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 618.0, "completions/mean_length": 479.875, "completions/mean_terminated_length": 438.71429443359375, "completions/min_length": 326.0, "completions/min_terminated_length": 326.0, "entropy": 0.3744928799569607, "epoch": 0.28586723768736616, "frac_reward_zero_std": 0.25, "grad_norm": 0.029886197298765182, "learning_rate": 1e-05, "loss": 0.1306, "num_tokens": 11743138.0, "reward": 0.6875, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.4820278882980347, "sampling/importance_sampling_ratio/mean": 0.999498724937439, "sampling/importance_sampling_ratio/min": 0.6708641648292542, "sampling/sampling_logp_difference/max": 0.3991886377334595, "sampling/sampling_logp_difference/mean": 0.012439063750207424, "step": 534 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 742.0, "completions/mean_length": 490.1875, "completions/mean_terminated_length": 481.2257995605469, "completions/min_length": 213.0, "completions/min_terminated_length": 213.0, "entropy": 0.34023406356573105, "epoch": 0.28640256959314775, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0046, "num_tokens": 11762680.0, "reward": 0.78125, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.5184926986694336, "sampling/importance_sampling_ratio/mean": 0.9999358654022217, "sampling/importance_sampling_ratio/min": 0.7156133651733398, "sampling/sampling_logp_difference/max": 0.41771817207336426, "sampling/sampling_logp_difference/mean": 0.010797393508255482, "step": 535 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 702.0, "completions/max_terminated_length": 702.0, "completions/mean_length": 470.0625, "completions/mean_terminated_length": 470.0625, "completions/min_length": 249.0, "completions/min_terminated_length": 249.0, "entropy": 0.32570483535528183, "epoch": 0.28693790149892934, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": -0.0123, "num_tokens": 11781426.0, "reward": 0.96875, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.96875, "rewards/accuracy_reward/std": 0.1767766922712326, "sampling/importance_sampling_ratio/max": 1.465185523033142, "sampling/importance_sampling_ratio/mean": 0.9999537467956543, "sampling/importance_sampling_ratio/min": 0.6084758639335632, "sampling/sampling_logp_difference/max": 0.496798038482666, "sampling/sampling_logp_difference/mean": 0.011000039055943489, "step": 536 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00019886280279024504, "clip_ratio/low_min": 0.00019886280279024504, "clip_ratio/region_mean": 0.00019886280279024504, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 765.0, "completions/mean_length": 599.40625, "completions/mean_terminated_length": 581.9655151367188, "completions/min_length": 364.0, "completions/min_terminated_length": 364.0, "entropy": 0.2990373261272907, "epoch": 0.2874732334047109, "frac_reward_zero_std": 0.25, "grad_norm": 0.010818886570632458, "learning_rate": 1e-05, "loss": 0.0459, "num_tokens": 11804671.0, "reward": 0.5625, "reward_std": 0.38298875093460083, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.4307703971862793, "sampling/importance_sampling_ratio/mean": 1.0002832412719727, "sampling/importance_sampling_ratio/min": 0.64666748046875, "sampling/sampling_logp_difference/max": 0.43592309951782227, "sampling/sampling_logp_difference/mean": 0.01015778910368681, "step": 537 }, { "clip_ratio/high_max": 5.7313158322358504e-05, "clip_ratio/high_mean": 5.7313158322358504e-05, "clip_ratio/low_mean": 0.00017786582247936167, "clip_ratio/low_min": 0.00017786582247936167, "clip_ratio/region_mean": 0.00023517898080172017, "completions/clipped_ratio": 0.3125, "completions/max_length": 768.0, "completions/max_terminated_length": 768.0, "completions/mean_length": 593.46875, "completions/mean_terminated_length": 514.1363525390625, "completions/min_length": 283.0, "completions/min_terminated_length": 283.0, "entropy": 0.5299359522759914, "epoch": 0.2880085653104925, "frac_reward_zero_std": 0.25, "grad_norm": 0.006210138089954853, "learning_rate": 1e-05, "loss": -0.024, "num_tokens": 11828142.0, "reward": 0.4375, "reward_std": 0.3104073107242584, "rewards/accuracy_reward/mean": 0.4375, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.3749855756759644, "sampling/importance_sampling_ratio/mean": 1.0002870559692383, "sampling/importance_sampling_ratio/min": 0.6991753578186035, "sampling/sampling_logp_difference/max": 0.35785365104675293, "sampling/sampling_logp_difference/mean": 0.014098601415753365, "step": 538 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00019079378762398846, "clip_ratio/low_min": 0.00019079378762398846, "clip_ratio/region_mean": 0.00019079378762398846, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 751.0, "completions/mean_length": 449.6875, "completions/mean_terminated_length": 428.4666748046875, "completions/min_length": 161.0, "completions/min_terminated_length": 161.0, "entropy": 0.44281355291604996, "epoch": 0.2885438972162741, "frac_reward_zero_std": 0.0, "grad_norm": 0.021223677322268486, "learning_rate": 1e-05, "loss": -0.0587, "num_tokens": 11846244.0, "reward": 0.625, "reward_std": 0.4261348247528076, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.628201961517334, "sampling/importance_sampling_ratio/mean": 1.000152826309204, "sampling/importance_sampling_ratio/min": 0.5882433652877808, "sampling/sampling_logp_difference/max": 0.5306146144866943, "sampling/sampling_logp_difference/mean": 0.01380736194550991, "step": 539 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00012186438470962457, "clip_ratio/low_min": 0.00012186438470962457, "clip_ratio/region_mean": 0.00012186438470962457, "completions/clipped_ratio": 0.3125, "completions/max_length": 768.0, "completions/max_terminated_length": 761.0, "completions/mean_length": 584.3125, "completions/mean_terminated_length": 500.8182067871094, "completions/min_length": 320.0, "completions/min_terminated_length": 320.0, "entropy": 0.4238555245101452, "epoch": 0.2890792291220557, "frac_reward_zero_std": 0.25, "grad_norm": 0.008518840186297894, "learning_rate": 1e-05, "loss": 0.0765, "num_tokens": 11868750.0, "reward": 0.625, "reward_std": 0.3104073107242584, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.4809027910232544, "sampling/importance_sampling_ratio/mean": 0.9999627470970154, "sampling/importance_sampling_ratio/min": 0.7298859357833862, "sampling/sampling_logp_difference/max": 0.39265191555023193, "sampling/sampling_logp_difference/mean": 0.013026274740695953, "step": 540 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.415401053847745e-05, "clip_ratio/low_min": 4.415401053847745e-05, "clip_ratio/region_mean": 4.415401053847745e-05, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 715.0, "completions/mean_length": 559.9375, "completions/mean_terminated_length": 530.2142944335938, "completions/min_length": 345.0, "completions/min_terminated_length": 345.0, "entropy": 0.3965966682881117, "epoch": 0.2896145610278373, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0122, "num_tokens": 11890508.0, "reward": 0.78125, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.4583379030227661, "sampling/importance_sampling_ratio/mean": 1.000077247619629, "sampling/importance_sampling_ratio/min": 0.7309221625328064, "sampling/sampling_logp_difference/max": 0.37729740142822266, "sampling/sampling_logp_difference/mean": 0.011999874375760555, "step": 541 }, { "clip_ratio/high_max": 4.954419273417443e-05, "clip_ratio/high_mean": 4.954419273417443e-05, "clip_ratio/low_mean": 0.00013027618115302175, "clip_ratio/low_min": 0.00013027618115302175, "clip_ratio/region_mean": 0.00017982037388719618, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 722.0, "completions/mean_length": 515.375, "completions/mean_terminated_length": 431.16668701171875, "completions/min_length": 287.0, "completions/min_terminated_length": 287.0, "entropy": 0.43001551553606987, "epoch": 0.29014989293361887, "frac_reward_zero_std": 0.25, "grad_norm": 0.021864552050828934, "learning_rate": 1e-05, "loss": -0.0041, "num_tokens": 11911192.0, "reward": 0.59375, "reward_std": 0.3377464711666107, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.4208108186721802, "sampling/importance_sampling_ratio/mean": 1.0005956888198853, "sampling/importance_sampling_ratio/min": 0.7243146896362305, "sampling/sampling_logp_difference/max": 0.3512277603149414, "sampling/sampling_logp_difference/mean": 0.013221126049757004, "step": 542 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.28125, "completions/max_length": 768.0, "completions/max_terminated_length": 731.0, "completions/mean_length": 550.375, "completions/mean_terminated_length": 465.2174072265625, "completions/min_length": 201.0, "completions/min_terminated_length": 201.0, "entropy": 0.3935386463999748, "epoch": 0.2906852248394004, "frac_reward_zero_std": 0.25, "grad_norm": 0.012153339572250843, "learning_rate": 1e-05, "loss": 0.0078, "num_tokens": 11932948.0, "reward": 0.5, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5080004930496216, "sampling/importance_sampling_ratio/max": 1.5028960704803467, "sampling/importance_sampling_ratio/mean": 1.0005394220352173, "sampling/importance_sampling_ratio/min": 0.684719443321228, "sampling/sampling_logp_difference/max": 0.4073939323425293, "sampling/sampling_logp_difference/mean": 0.012549460865557194, "step": 543 }, { "clip_ratio/high_max": 6.811989442212507e-05, "clip_ratio/high_mean": 6.811989442212507e-05, "clip_ratio/low_mean": 5.6921675422927365e-05, "clip_ratio/low_min": 5.6921675422927365e-05, "clip_ratio/region_mean": 0.00012504156984505244, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 649.0, "completions/mean_length": 435.40625, "completions/mean_terminated_length": 424.6773986816406, "completions/min_length": 297.0, "completions/min_terminated_length": 297.0, "entropy": 0.3174975700676441, "epoch": 0.291220556745182, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": -0.0212, "num_tokens": 11950273.0, "reward": 0.9375, "reward_std": 0.1157275140285492, "rewards/accuracy_reward/mean": 0.9375, "rewards/accuracy_reward/std": 0.24593468010425568, "sampling/importance_sampling_ratio/max": 1.4301387071609497, "sampling/importance_sampling_ratio/mean": 0.9999814033508301, "sampling/importance_sampling_ratio/min": 0.7221091389656067, "sampling/sampling_logp_difference/max": 0.3577713966369629, "sampling/sampling_logp_difference/mean": 0.011437466368079185, "step": 544 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.720823901356198e-05, "clip_ratio/low_min": 5.720823901356198e-05, "clip_ratio/region_mean": 5.720823901356198e-05, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 673.0, "completions/mean_length": 585.5625, "completions/mean_terminated_length": 534.47998046875, "completions/min_length": 388.0, "completions/min_terminated_length": 388.0, "entropy": 0.40324000269174576, "epoch": 0.2917558886509636, "frac_reward_zero_std": 0.75, "grad_norm": 0.004175969399511814, "learning_rate": 1e-05, "loss": -0.002, "num_tokens": 11972979.0, "reward": 0.78125, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.4461041688919067, "sampling/importance_sampling_ratio/mean": 0.9998451471328735, "sampling/importance_sampling_ratio/min": 0.618733823299408, "sampling/sampling_logp_difference/max": 0.48008012771606445, "sampling/sampling_logp_difference/mean": 0.012317562475800514, "step": 545 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.638218888430856e-05, "clip_ratio/low_min": 4.638218888430856e-05, "clip_ratio/region_mean": 4.638218888430856e-05, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 695.0, "completions/mean_length": 557.90625, "completions/mean_terminated_length": 509.423095703125, "completions/min_length": 296.0, "completions/min_terminated_length": 296.0, "entropy": 0.3849346190690994, "epoch": 0.29229122055674517, "frac_reward_zero_std": 0.25, "grad_norm": 0.010279565118253231, "learning_rate": 1e-05, "loss": 0.0736, "num_tokens": 11994784.0, "reward": 0.71875, "reward_std": 0.3198433816432953, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.4003819227218628, "sampling/importance_sampling_ratio/mean": 0.9998975992202759, "sampling/importance_sampling_ratio/min": 0.10590694099664688, "sampling/sampling_logp_difference/max": 2.245194435119629, "sampling/sampling_logp_difference/mean": 0.012231167405843735, "step": 546 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 721.0, "completions/mean_length": 543.65625, "completions/mean_terminated_length": 502.1111145019531, "completions/min_length": 335.0, "completions/min_terminated_length": 335.0, "entropy": 0.30879425071179867, "epoch": 0.29282655246252676, "frac_reward_zero_std": 0.25, "grad_norm": 0.02512621320784092, "learning_rate": 1e-05, "loss": 0.0556, "num_tokens": 12015829.0, "reward": 0.84375, "reward_std": 0.3061639666557312, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.4133927822113037, "sampling/importance_sampling_ratio/mean": 0.9995831251144409, "sampling/importance_sampling_ratio/min": 0.7208872437477112, "sampling/sampling_logp_difference/max": 0.3459930419921875, "sampling/sampling_logp_difference/mean": 0.010667083784937859, "step": 547 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 741.0, "completions/mean_length": 557.625, "completions/mean_terminated_length": 543.6000366210938, "completions/min_length": 367.0, "completions/min_terminated_length": 367.0, "entropy": 0.3389309160411358, "epoch": 0.29336188436830835, "frac_reward_zero_std": 0.5, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0692, "num_tokens": 12038129.0, "reward": 0.9375, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.9375, "rewards/accuracy_reward/std": 0.24593468010425568, "sampling/importance_sampling_ratio/max": 1.4502341747283936, "sampling/importance_sampling_ratio/mean": 1.0001298189163208, "sampling/importance_sampling_ratio/min": 0.6379529237747192, "sampling/sampling_logp_difference/max": 0.4494907855987549, "sampling/sampling_logp_difference/mean": 0.011395558714866638, "step": 548 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.753106572432443e-05, "clip_ratio/low_min": 6.753106572432443e-05, "clip_ratio/region_mean": 6.753106572432443e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 649.0, "completions/max_terminated_length": 649.0, "completions/mean_length": 420.28125, "completions/mean_terminated_length": 420.28125, "completions/min_length": 169.0, "completions/min_terminated_length": 169.0, "entropy": 0.32912104576826096, "epoch": 0.29389721627408993, "frac_reward_zero_std": 0.25, "grad_norm": 0.011712674051523209, "learning_rate": 1e-05, "loss": -0.0219, "num_tokens": 12055554.0, "reward": 0.8125, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.5154039859771729, "sampling/importance_sampling_ratio/mean": 0.9998295903205872, "sampling/importance_sampling_ratio/min": 0.7116454243659973, "sampling/sampling_logp_difference/max": 0.4156820774078369, "sampling/sampling_logp_difference/mean": 0.011668115854263306, "step": 549 }, { "clip_ratio/high_max": 6.0299083997961134e-05, "clip_ratio/high_mean": 6.0299083997961134e-05, "clip_ratio/low_mean": 5.397236600401811e-05, "clip_ratio/low_min": 5.397236600401811e-05, "clip_ratio/region_mean": 0.00011427145000197925, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 757.0, "completions/mean_length": 483.0, "completions/mean_terminated_length": 453.5172424316406, "completions/min_length": 220.0, "completions/min_terminated_length": 220.0, "entropy": 0.34380580112338066, "epoch": 0.2944325481798715, "frac_reward_zero_std": 0.5, "grad_norm": 0.003323654644191265, "learning_rate": 1e-05, "loss": 0.057, "num_tokens": 12074282.0, "reward": 0.9375, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.9375, "rewards/accuracy_reward/std": 0.24593468010425568, "sampling/importance_sampling_ratio/max": 1.466320514678955, "sampling/importance_sampling_ratio/mean": 1.0004823207855225, "sampling/importance_sampling_ratio/min": 0.6793035268783569, "sampling/sampling_logp_difference/max": 0.3866872787475586, "sampling/sampling_logp_difference/mean": 0.011829284951090813, "step": 550 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 9.412650251761079e-05, "clip_ratio/low_min": 9.412650251761079e-05, "clip_ratio/region_mean": 9.412650251761079e-05, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 723.0, "completions/mean_length": 452.21875, "completions/mean_terminated_length": 442.0322570800781, "completions/min_length": 174.0, "completions/min_terminated_length": 174.0, "entropy": 0.3522225581109524, "epoch": 0.2949678800856531, "frac_reward_zero_std": 0.25, "grad_norm": 0.017959894612431526, "learning_rate": 1e-05, "loss": -0.0623, "num_tokens": 12092393.0, "reward": 0.65625, "reward_std": 0.3377465009689331, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.5850427150726318, "sampling/importance_sampling_ratio/mean": 0.999859631061554, "sampling/importance_sampling_ratio/min": 0.5930550694465637, "sampling/sampling_logp_difference/max": 0.522468090057373, "sampling/sampling_logp_difference/mean": 0.01135376188904047, "step": 551 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0001395868166582659, "clip_ratio/low_min": 0.0001395868166582659, "clip_ratio/region_mean": 0.0001395868166582659, "completions/clipped_ratio": 0.0, "completions/max_length": 618.0, "completions/max_terminated_length": 618.0, "completions/mean_length": 476.15625, "completions/mean_terminated_length": 476.15625, "completions/min_length": 291.0, "completions/min_terminated_length": 291.0, "entropy": 0.3376327082514763, "epoch": 0.2955032119914347, "frac_reward_zero_std": 0.75, "grad_norm": 0.009012979455292225, "learning_rate": 1e-05, "loss": 0.0022, "num_tokens": 12111486.0, "reward": 0.84375, "reward_std": 0.1293872892856598, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.3721379041671753, "sampling/importance_sampling_ratio/mean": 0.9999564290046692, "sampling/importance_sampling_ratio/min": 0.21763287484645844, "sampling/sampling_logp_difference/max": 1.5249457359313965, "sampling/sampling_logp_difference/mean": 0.011011257767677307, "step": 552 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 711.0, "completions/max_terminated_length": 711.0, "completions/mean_length": 455.09375, "completions/mean_terminated_length": 455.09375, "completions/min_length": 218.0, "completions/min_terminated_length": 218.0, "entropy": 0.3443738520145416, "epoch": 0.2960385438972163, "frac_reward_zero_std": 0.75, "grad_norm": 0.013324462808668613, "learning_rate": 1e-05, "loss": -0.0503, "num_tokens": 12129569.0, "reward": 0.9375, "reward_std": 0.1157275140285492, "rewards/accuracy_reward/mean": 0.9375, "rewards/accuracy_reward/std": 0.24593468010425568, "sampling/importance_sampling_ratio/max": 1.3456470966339111, "sampling/importance_sampling_ratio/mean": 0.9997557997703552, "sampling/importance_sampling_ratio/min": 0.7060806751251221, "sampling/sampling_logp_difference/max": 0.3480257987976074, "sampling/sampling_logp_difference/mean": 0.01204129308462143, "step": 553 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 727.0, "completions/mean_length": 455.0625, "completions/mean_terminated_length": 444.9677429199219, "completions/min_length": 213.0, "completions/min_terminated_length": 213.0, "entropy": 0.35447244346141815, "epoch": 0.2965738758029979, "frac_reward_zero_std": 0.5, "grad_norm": 0.005047038197517395, "learning_rate": 1e-05, "loss": 0.0414, "num_tokens": 12147731.0, "reward": 0.90625, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.4405754804611206, "sampling/importance_sampling_ratio/mean": 0.9999611973762512, "sampling/importance_sampling_ratio/min": 0.6882584095001221, "sampling/sampling_logp_difference/max": 0.37359094619750977, "sampling/sampling_logp_difference/mean": 0.011543944478034973, "step": 554 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0002068972826236859, "clip_ratio/low_min": 0.0002068972826236859, "clip_ratio/region_mean": 0.0002068972826236859, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 648.0, "completions/mean_length": 448.03125, "completions/mean_terminated_length": 437.70965576171875, "completions/min_length": 315.0, "completions/min_terminated_length": 315.0, "entropy": 0.41757863387465477, "epoch": 0.29710920770877947, "frac_reward_zero_std": 0.5, "grad_norm": 0.02503243274986744, "learning_rate": 1e-05, "loss": -0.0516, "num_tokens": 12166124.0, "reward": 0.71875, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.4345734119415283, "sampling/importance_sampling_ratio/mean": 1.0002120733261108, "sampling/importance_sampling_ratio/min": 0.6146836280822754, "sampling/sampling_logp_difference/max": 0.4866476058959961, "sampling/sampling_logp_difference/mean": 0.01272653415799141, "step": 555 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.715592124033719e-05, "clip_ratio/low_min": 5.715592124033719e-05, "clip_ratio/region_mean": 5.715592124033719e-05, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 731.0, "completions/mean_length": 518.125, "completions/mean_terminated_length": 460.4615478515625, "completions/min_length": 255.0, "completions/min_terminated_length": 255.0, "entropy": 0.3800526633858681, "epoch": 0.29764453961456105, "frac_reward_zero_std": 0.5, "grad_norm": 0.008308999240398407, "learning_rate": 1e-05, "loss": 0.039, "num_tokens": 12186216.0, "reward": 0.625, "reward_std": 0.2314550280570984, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.5458557605743408, "sampling/importance_sampling_ratio/mean": 0.9998978972434998, "sampling/importance_sampling_ratio/min": 0.5737590193748474, "sampling/sampling_logp_difference/max": 0.5555458068847656, "sampling/sampling_logp_difference/mean": 0.012542897835373878, "step": 556 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.745633850689046e-05, "clip_ratio/low_min": 4.745633850689046e-05, "clip_ratio/region_mean": 4.745633850689046e-05, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 736.0, "completions/mean_length": 548.15625, "completions/mean_terminated_length": 486.5999755859375, "completions/min_length": 299.0, "completions/min_terminated_length": 299.0, "entropy": 0.38176218047738075, "epoch": 0.2981798715203426, "frac_reward_zero_std": 0.25, "grad_norm": 0.024729037657380104, "learning_rate": 1e-05, "loss": 0.0284, "num_tokens": 12207485.0, "reward": 0.59375, "reward_std": 0.3061639964580536, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.4402064085006714, "sampling/importance_sampling_ratio/mean": 0.9997570514678955, "sampling/importance_sampling_ratio/min": 0.6999471187591553, "sampling/sampling_logp_difference/max": 0.36478638648986816, "sampling/sampling_logp_difference/mean": 0.012147819623351097, "step": 557 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.106497312430292e-05, "clip_ratio/low_min": 6.106497312430292e-05, "clip_ratio/region_mean": 6.106497312430292e-05, "completions/clipped_ratio": 0.28125, "completions/max_length": 768.0, "completions/max_terminated_length": 758.0, "completions/mean_length": 580.65625, "completions/mean_terminated_length": 507.34783935546875, "completions/min_length": 310.0, "completions/min_terminated_length": 310.0, "entropy": 0.35670357197523117, "epoch": 0.2987152034261242, "frac_reward_zero_std": 0.25, "grad_norm": 0.01215875893831253, "learning_rate": 1e-05, "loss": 0.0056, "num_tokens": 12229690.0, "reward": 0.4375, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.4375, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.3789316415786743, "sampling/importance_sampling_ratio/mean": 0.9998000860214233, "sampling/importance_sampling_ratio/min": 0.6110749840736389, "sampling/sampling_logp_difference/max": 0.4925355911254883, "sampling/sampling_logp_difference/mean": 0.01118595339357853, "step": 558 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00014841670781606808, "clip_ratio/low_min": 0.00014841670781606808, "clip_ratio/region_mean": 0.00014841670781606808, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 685.0, "completions/mean_length": 510.875, "completions/mean_terminated_length": 502.58062744140625, "completions/min_length": 225.0, "completions/min_terminated_length": 225.0, "entropy": 0.3493451811373234, "epoch": 0.29925053533190576, "frac_reward_zero_std": 0.0, "grad_norm": 0.006535404361784458, "learning_rate": 1e-05, "loss": -0.0476, "num_tokens": 12249870.0, "reward": 0.84375, "reward_std": 0.3808925747871399, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.3719756603240967, "sampling/importance_sampling_ratio/mean": 1.0001106262207031, "sampling/importance_sampling_ratio/min": 0.4449857175350189, "sampling/sampling_logp_difference/max": 0.8097131252288818, "sampling/sampling_logp_difference/mean": 0.011852250434458256, "step": 559 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 686.0, "completions/mean_length": 437.46875, "completions/mean_terminated_length": 426.8064270019531, "completions/min_length": 180.0, "completions/min_terminated_length": 180.0, "entropy": 0.32386602833867073, "epoch": 0.29978586723768735, "frac_reward_zero_std": 0.25, "grad_norm": 0.006508872844278812, "learning_rate": 1e-05, "loss": 0.0394, "num_tokens": 12267429.0, "reward": 0.90625, "reward_std": 0.2651650309562683, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.7509204149246216, "sampling/importance_sampling_ratio/mean": 1.0000189542770386, "sampling/importance_sampling_ratio/min": 0.6039376258850098, "sampling/sampling_logp_difference/max": 0.5601415634155273, "sampling/sampling_logp_difference/mean": 0.01119659747928381, "step": 560 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.28125, "completions/max_length": 768.0, "completions/max_terminated_length": 756.0, "completions/mean_length": 601.15625, "completions/mean_terminated_length": 535.8695678710938, "completions/min_length": 389.0, "completions/min_terminated_length": 389.0, "entropy": 0.36822259798645973, "epoch": 0.30032119914346894, "frac_reward_zero_std": 0.25, "grad_norm": 0.015724875032901764, "learning_rate": 1e-05, "loss": -0.0018, "num_tokens": 12289858.0, "reward": 0.71875, "reward_std": 0.3471629321575165, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.3015179634094238, "sampling/importance_sampling_ratio/mean": 0.9997707605361938, "sampling/importance_sampling_ratio/min": 0.6908484101295471, "sampling/sampling_logp_difference/max": 0.36983489990234375, "sampling/sampling_logp_difference/mean": 0.011337664909660816, "step": 561 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 9.808108370634727e-05, "clip_ratio/low_min": 9.808108370634727e-05, "clip_ratio/region_mean": 9.808108370634727e-05, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 761.0, "completions/mean_length": 540.5, "completions/mean_terminated_length": 516.9655151367188, "completions/min_length": 320.0, "completions/min_terminated_length": 320.0, "entropy": 0.31132769770920277, "epoch": 0.30085653104925053, "frac_reward_zero_std": 0.0, "grad_norm": 0.011534156277775764, "learning_rate": 1e-05, "loss": 0.0197, "num_tokens": 12311138.0, "reward": 0.5, "reward_std": 0.4492306709289551, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5080004930496216, "sampling/importance_sampling_ratio/max": 1.3694326877593994, "sampling/importance_sampling_ratio/mean": 1.0003710985183716, "sampling/importance_sampling_ratio/min": 0.7437523007392883, "sampling/sampling_logp_difference/max": 0.31439661979675293, "sampling/sampling_logp_difference/mean": 0.010466148145496845, "step": 562 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 8.724504004931077e-05, "clip_ratio/low_min": 8.724504004931077e-05, "clip_ratio/region_mean": 8.724504004931077e-05, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 741.0, "completions/mean_length": 568.84375, "completions/mean_terminated_length": 502.4583435058594, "completions/min_length": 223.0, "completions/min_terminated_length": 223.0, "entropy": 0.38482216745615005, "epoch": 0.3013918629550321, "frac_reward_zero_std": 0.25, "grad_norm": 0.007253366056829691, "learning_rate": 1e-05, "loss": 0.0984, "num_tokens": 12332877.0, "reward": 0.75, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.3173551559448242, "sampling/importance_sampling_ratio/mean": 0.9999594688415527, "sampling/importance_sampling_ratio/min": 0.6961379647254944, "sampling/sampling_logp_difference/max": 0.36220741271972656, "sampling/sampling_logp_difference/mean": 0.011871026828885078, "step": 563 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.34375, "completions/max_length": 768.0, "completions/max_terminated_length": 743.0, "completions/mean_length": 585.65625, "completions/mean_terminated_length": 490.1428527832031, "completions/min_length": 309.0, "completions/min_terminated_length": 309.0, "entropy": 0.5359249711036682, "epoch": 0.3019271948608137, "frac_reward_zero_std": 0.75, "grad_norm": 0.012351706624031067, "learning_rate": 1e-05, "loss": 0.0201, "num_tokens": 12355762.0, "reward": 0.59375, "reward_std": 0.1293872892856598, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.4052972793579102, "sampling/importance_sampling_ratio/mean": 1.000327229499817, "sampling/importance_sampling_ratio/min": 0.7129547595977783, "sampling/sampling_logp_difference/max": 0.34024882316589355, "sampling/sampling_logp_difference/mean": 0.016371745616197586, "step": 564 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00020264643535483629, "clip_ratio/low_min": 0.00020264643535483629, "clip_ratio/region_mean": 0.00020264643535483629, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 653.0, "completions/mean_length": 432.6875, "completions/mean_terminated_length": 421.8709411621094, "completions/min_length": 258.0, "completions/min_terminated_length": 258.0, "entropy": 0.3549719266593456, "epoch": 0.3024625267665953, "frac_reward_zero_std": 0.25, "grad_norm": 0.014779582619667053, "learning_rate": 1e-05, "loss": -0.0115, "num_tokens": 12373136.0, "reward": 0.53125, "reward_std": 0.3471629321575165, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.5077543258666992, "sampling/importance_sampling_ratio/mean": 0.999976396560669, "sampling/importance_sampling_ratio/min": 0.6832563877105713, "sampling/sampling_logp_difference/max": 0.41062140464782715, "sampling/sampling_logp_difference/mean": 0.011491380631923676, "step": 565 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 768.0, "completions/mean_length": 466.21875, "completions/mean_terminated_length": 456.4838562011719, "completions/min_length": 289.0, "completions/min_terminated_length": 289.0, "entropy": 0.28736421652138233, "epoch": 0.3029978586723769, "frac_reward_zero_std": 0.5, "grad_norm": 0.0026263478212058544, "learning_rate": 1e-05, "loss": 0.0274, "num_tokens": 12391703.0, "reward": 0.90625, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.6552960872650146, "sampling/importance_sampling_ratio/mean": 0.9999865889549255, "sampling/importance_sampling_ratio/min": 0.7178400158882141, "sampling/sampling_logp_difference/max": 0.5039799213409424, "sampling/sampling_logp_difference/mean": 0.00997740589082241, "step": 566 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 7.221259147627279e-05, "clip_ratio/low_min": 7.221259147627279e-05, "clip_ratio/region_mean": 7.221259147627279e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 652.0, "completions/max_terminated_length": 652.0, "completions/mean_length": 422.875, "completions/mean_terminated_length": 422.875, "completions/min_length": 204.0, "completions/min_terminated_length": 204.0, "entropy": 0.26030854880809784, "epoch": 0.3035331905781585, "frac_reward_zero_std": 0.5, "grad_norm": 0.022980762645602226, "learning_rate": 1e-05, "loss": -0.0205, "num_tokens": 12408923.0, "reward": 0.71875, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.4185572862625122, "sampling/importance_sampling_ratio/mean": 1.000119924545288, "sampling/importance_sampling_ratio/min": 0.7247578501701355, "sampling/sampling_logp_difference/max": 0.3496403694152832, "sampling/sampling_logp_difference/mean": 0.009362556971609592, "step": 567 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 682.0, "completions/mean_length": 404.40625, "completions/mean_terminated_length": 392.6773986816406, "completions/min_length": 116.0, "completions/min_terminated_length": 116.0, "entropy": 0.32211763970553875, "epoch": 0.30406852248394006, "frac_reward_zero_std": 0.75, "grad_norm": 0.006427176296710968, "learning_rate": 1e-05, "loss": 0.0127, "num_tokens": 12425112.0, "reward": 0.9375, "reward_std": 0.1157275140285492, "rewards/accuracy_reward/mean": 0.9375, "rewards/accuracy_reward/std": 0.24593468010425568, "sampling/importance_sampling_ratio/max": 1.3863672018051147, "sampling/importance_sampling_ratio/mean": 0.9999868273735046, "sampling/importance_sampling_ratio/min": 0.6178496479988098, "sampling/sampling_logp_difference/max": 0.4815101623535156, "sampling/sampling_logp_difference/mean": 0.011179612949490547, "step": 568 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 701.0, "completions/mean_length": 490.96875, "completions/mean_terminated_length": 462.3103332519531, "completions/min_length": 259.0, "completions/min_terminated_length": 259.0, "entropy": 0.3328098524361849, "epoch": 0.30460385438972165, "frac_reward_zero_std": 0.25, "grad_norm": 0.008083218708634377, "learning_rate": 1e-05, "loss": 0.0181, "num_tokens": 12444359.0, "reward": 0.6875, "reward_std": 0.3514062464237213, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.3812774419784546, "sampling/importance_sampling_ratio/mean": 1.0000039339065552, "sampling/importance_sampling_ratio/min": 0.6159681677818298, "sampling/sampling_logp_difference/max": 0.4845600128173828, "sampling/sampling_logp_difference/mean": 0.01154243666678667, "step": 569 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 7.977025961736217e-05, "clip_ratio/low_min": 7.977025961736217e-05, "clip_ratio/region_mean": 7.977025961736217e-05, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 764.0, "completions/mean_length": 456.0, "completions/mean_terminated_length": 435.20001220703125, "completions/min_length": 197.0, "completions/min_terminated_length": 197.0, "entropy": 0.4078424833714962, "epoch": 0.30513918629550324, "frac_reward_zero_std": 0.5, "grad_norm": 0.008885356597602367, "learning_rate": 1e-05, "loss": -0.0218, "num_tokens": 12463175.0, "reward": 0.6875, "reward_std": 0.249358132481575, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.70856773853302, "sampling/importance_sampling_ratio/mean": 0.9998947978019714, "sampling/importance_sampling_ratio/min": 0.5030553340911865, "sampling/sampling_logp_difference/max": 0.6870551109313965, "sampling/sampling_logp_difference/mean": 0.013486012816429138, "step": 570 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 641.0, "completions/mean_length": 477.34375, "completions/mean_terminated_length": 457.9667053222656, "completions/min_length": 250.0, "completions/min_terminated_length": 250.0, "entropy": 0.3029682785272598, "epoch": 0.3056745182012848, "frac_reward_zero_std": 0.5, "grad_norm": 0.014453751966357231, "learning_rate": 1e-05, "loss": 0.067, "num_tokens": 12482450.0, "reward": 0.65625, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.4614412784576416, "sampling/importance_sampling_ratio/mean": 0.9999015927314758, "sampling/importance_sampling_ratio/min": 0.7071356177330017, "sampling/sampling_logp_difference/max": 0.3794231414794922, "sampling/sampling_logp_difference/mean": 0.010116026736795902, "step": 571 }, { "clip_ratio/high_max": 0.0002342275038245134, "clip_ratio/high_mean": 0.0002342275038245134, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0002342275038245134, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 665.0, "completions/mean_length": 504.78125, "completions/mean_terminated_length": 467.1785888671875, "completions/min_length": 217.0, "completions/min_terminated_length": 217.0, "entropy": 0.38962068036198616, "epoch": 0.30620985010706636, "frac_reward_zero_std": 0.25, "grad_norm": 0.024353552609682083, "learning_rate": 1e-05, "loss": 0.0685, "num_tokens": 12502091.0, "reward": 0.625, "reward_std": 0.292504221200943, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.4311225414276123, "sampling/importance_sampling_ratio/mean": 1.0002267360687256, "sampling/importance_sampling_ratio/min": 0.4781179428100586, "sampling/sampling_logp_difference/max": 0.7378978729248047, "sampling/sampling_logp_difference/mean": 0.012861599214375019, "step": 572 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 634.0, "completions/mean_length": 432.90625, "completions/mean_terminated_length": 422.0967712402344, "completions/min_length": 223.0, "completions/min_terminated_length": 223.0, "entropy": 0.3488805815577507, "epoch": 0.30674518201284795, "frac_reward_zero_std": 0.5, "grad_norm": 0.002496341010555625, "learning_rate": 1e-05, "loss": 0.0121, "num_tokens": 12519144.0, "reward": 0.59375, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.5523734092712402, "sampling/importance_sampling_ratio/mean": 1.0000110864639282, "sampling/importance_sampling_ratio/min": 0.6951249241828918, "sampling/sampling_logp_difference/max": 0.4397850036621094, "sampling/sampling_logp_difference/mean": 0.01103675551712513, "step": 573 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0002104796549247112, "clip_ratio/low_min": 0.0002104796549247112, "clip_ratio/region_mean": 0.0002104796549247112, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 644.0, "completions/mean_length": 504.90625, "completions/mean_terminated_length": 456.1851806640625, "completions/min_length": 206.0, "completions/min_terminated_length": 206.0, "entropy": 0.36549049615859985, "epoch": 0.30728051391862954, "frac_reward_zero_std": 0.0, "grad_norm": 0.021736497059464455, "learning_rate": 1e-05, "loss": -0.0014, "num_tokens": 12539293.0, "reward": 0.71875, "reward_std": 0.4218915104866028, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.6297128200531006, "sampling/importance_sampling_ratio/mean": 0.9993993639945984, "sampling/importance_sampling_ratio/min": 0.676750659942627, "sampling/sampling_logp_difference/max": 0.4884037971496582, "sampling/sampling_logp_difference/mean": 0.012100109830498695, "step": 574 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.701015495811589e-05, "clip_ratio/low_min": 4.701015495811589e-05, "clip_ratio/region_mean": 4.701015495811589e-05, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 754.0, "completions/mean_length": 504.96875, "completions/mean_terminated_length": 467.39288330078125, "completions/min_length": 181.0, "completions/min_terminated_length": 181.0, "entropy": 0.4503924436867237, "epoch": 0.3078158458244111, "frac_reward_zero_std": 0.25, "grad_norm": 0.007269065361469984, "learning_rate": 1e-05, "loss": 0.0752, "num_tokens": 12559260.0, "reward": 0.65625, "reward_std": 0.3198433816432953, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.5005733966827393, "sampling/importance_sampling_ratio/mean": 1.0002752542495728, "sampling/importance_sampling_ratio/min": 0.6220771670341492, "sampling/sampling_logp_difference/max": 0.47469115257263184, "sampling/sampling_logp_difference/mean": 0.01278842706233263, "step": 575 }, { "clip_ratio/high_max": 7.237985118990764e-05, "clip_ratio/high_mean": 7.237985118990764e-05, "clip_ratio/low_mean": 0.00014186622865963727, "clip_ratio/low_min": 0.00014186622865963727, "clip_ratio/region_mean": 0.0002142460798495449, "completions/clipped_ratio": 0.0, "completions/max_length": 584.0, "completions/max_terminated_length": 584.0, "completions/mean_length": 425.34375, "completions/mean_terminated_length": 425.34375, "completions/min_length": 297.0, "completions/min_terminated_length": 297.0, "entropy": 0.412844717502594, "epoch": 0.3083511777301927, "frac_reward_zero_std": 0.5, "grad_norm": 0.0110018914565444, "learning_rate": 1e-05, "loss": -0.003, "num_tokens": 12576791.0, "reward": 0.8125, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.409265160560608, "sampling/importance_sampling_ratio/mean": 1.0001308917999268, "sampling/importance_sampling_ratio/min": 0.6833316683769226, "sampling/sampling_logp_difference/max": 0.38077497482299805, "sampling/sampling_logp_difference/mean": 0.013048562221229076, "step": 576 }, { "clip_ratio/high_max": 5.8548008382786065e-05, "clip_ratio/high_mean": 5.8548008382786065e-05, "clip_ratio/low_mean": 4.8809059080667794e-05, "clip_ratio/low_min": 4.8809059080667794e-05, "clip_ratio/region_mean": 0.00010735706746345386, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 768.0, "completions/mean_length": 506.9375, "completions/mean_terminated_length": 489.5333557128906, "completions/min_length": 263.0, "completions/min_terminated_length": 263.0, "entropy": 0.42174794897437096, "epoch": 0.3088865096359743, "frac_reward_zero_std": 0.0, "grad_norm": 0.03381386399269104, "learning_rate": 1e-05, "loss": 0.045, "num_tokens": 12596885.0, "reward": 0.5625, "reward_std": 0.4261348247528076, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.4981865882873535, "sampling/importance_sampling_ratio/mean": 1.000162959098816, "sampling/importance_sampling_ratio/min": 0.6529225707054138, "sampling/sampling_logp_difference/max": 0.4262967109680176, "sampling/sampling_logp_difference/mean": 0.012921232730150223, "step": 577 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00014186279440764338, "clip_ratio/low_min": 0.00014186279440764338, "clip_ratio/region_mean": 0.00014186279440764338, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 747.0, "completions/mean_length": 461.3125, "completions/mean_terminated_length": 440.86669921875, "completions/min_length": 194.0, "completions/min_terminated_length": 194.0, "entropy": 0.3352612443268299, "epoch": 0.3094218415417559, "frac_reward_zero_std": 0.75, "grad_norm": 0.009724777191877365, "learning_rate": 1e-05, "loss": 0.0122, "num_tokens": 12614999.0, "reward": 0.84375, "reward_std": 0.1293872892856598, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.4091874361038208, "sampling/importance_sampling_ratio/mean": 1.0004322528839111, "sampling/importance_sampling_ratio/min": 0.3782390058040619, "sampling/sampling_logp_difference/max": 0.97222900390625, "sampling/sampling_logp_difference/mean": 0.010968740098178387, "step": 578 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 629.0, "completions/mean_length": 485.65625, "completions/mean_terminated_length": 476.5483703613281, "completions/min_length": 296.0, "completions/min_terminated_length": 296.0, "entropy": 0.30073725804686546, "epoch": 0.3099571734475375, "frac_reward_zero_std": 0.5, "grad_norm": 0.019250115379691124, "learning_rate": 1e-05, "loss": -0.0194, "num_tokens": 12634788.0, "reward": 0.84375, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.3734782934188843, "sampling/importance_sampling_ratio/mean": 0.9999491572380066, "sampling/importance_sampling_ratio/min": 0.7112151980400085, "sampling/sampling_logp_difference/max": 0.34078025817871094, "sampling/sampling_logp_difference/mean": 0.010382591746747494, "step": 579 }, { "clip_ratio/high_max": 7.494004967156798e-05, "clip_ratio/high_mean": 7.494004967156798e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 7.494004967156798e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 659.0, "completions/max_terminated_length": 659.0, "completions/mean_length": 473.09375, "completions/mean_terminated_length": 473.09375, "completions/min_length": 288.0, "completions/min_terminated_length": 288.0, "entropy": 0.39355795457959175, "epoch": 0.31049250535331907, "frac_reward_zero_std": 0.25, "grad_norm": 0.0271613709628582, "learning_rate": 1e-05, "loss": 0.0115, "num_tokens": 12654271.0, "reward": 0.90625, "reward_std": 0.2651650309562683, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.3442903757095337, "sampling/importance_sampling_ratio/mean": 1.000354528427124, "sampling/importance_sampling_ratio/min": 0.6165125966072083, "sampling/sampling_logp_difference/max": 0.483676552772522, "sampling/sampling_logp_difference/mean": 0.012498216703534126, "step": 580 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 670.0, "completions/max_terminated_length": 670.0, "completions/mean_length": 457.03125, "completions/mean_terminated_length": 457.03125, "completions/min_length": 221.0, "completions/min_terminated_length": 221.0, "entropy": 0.30188281275331974, "epoch": 0.31102783725910066, "frac_reward_zero_std": 0.5, "grad_norm": 0.005763284396380186, "learning_rate": 1e-05, "loss": 0.0036, "num_tokens": 12672720.0, "reward": 0.9375, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.9375, "rewards/accuracy_reward/std": 0.24593468010425568, "sampling/importance_sampling_ratio/max": 1.3802168369293213, "sampling/importance_sampling_ratio/mean": 0.9999557137489319, "sampling/importance_sampling_ratio/min": 0.6441707611083984, "sampling/sampling_logp_difference/max": 0.4397914409637451, "sampling/sampling_logp_difference/mean": 0.009894377551972866, "step": 581 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.163155583431944e-05, "clip_ratio/low_min": 5.163155583431944e-05, "clip_ratio/region_mean": 5.163155583431944e-05, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 719.0, "completions/mean_length": 548.875, "completions/mean_terminated_length": 526.2069091796875, "completions/min_length": 264.0, "completions/min_terminated_length": 264.0, "entropy": 0.2989288941025734, "epoch": 0.31156316916488225, "frac_reward_zero_std": 0.25, "grad_norm": 0.005480234511196613, "learning_rate": 1e-05, "loss": 0.0605, "num_tokens": 12693604.0, "reward": 0.84375, "reward_std": 0.3198433816432953, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.4595129489898682, "sampling/importance_sampling_ratio/mean": 1.0002418756484985, "sampling/importance_sampling_ratio/min": 0.6605982780456543, "sampling/sampling_logp_difference/max": 0.414609432220459, "sampling/sampling_logp_difference/mean": 0.01005987636744976, "step": 582 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0001239581215486396, "clip_ratio/low_min": 0.0001239581215486396, "clip_ratio/region_mean": 0.0001239581215486396, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 670.0, "completions/mean_length": 473.5, "completions/mean_terminated_length": 431.4285888671875, "completions/min_length": 217.0, "completions/min_terminated_length": 217.0, "entropy": 0.39790476486086845, "epoch": 0.31209850107066384, "frac_reward_zero_std": 0.25, "grad_norm": 0.004399289842694998, "learning_rate": 1e-05, "loss": 0.0551, "num_tokens": 12712172.0, "reward": 0.65625, "reward_std": 0.3061639964580536, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.4628119468688965, "sampling/importance_sampling_ratio/mean": 0.9997876882553101, "sampling/importance_sampling_ratio/min": 0.6796148419380188, "sampling/sampling_logp_difference/max": 0.3862290382385254, "sampling/sampling_logp_difference/mean": 0.012877034954726696, "step": 583 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 707.0, "completions/mean_length": 423.0, "completions/mean_terminated_length": 400.0000305175781, "completions/min_length": 237.0, "completions/min_terminated_length": 237.0, "entropy": 0.35159216448664665, "epoch": 0.31263383297644537, "frac_reward_zero_std": 0.5, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.062, "num_tokens": 12729068.0, "reward": 0.9375, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.9375, "rewards/accuracy_reward/std": 0.24593468010425568, "sampling/importance_sampling_ratio/max": 1.4646342992782593, "sampling/importance_sampling_ratio/mean": 1.0000327825546265, "sampling/importance_sampling_ratio/min": 0.7577028870582581, "sampling/sampling_logp_difference/max": 0.3816056251525879, "sampling/sampling_logp_difference/mean": 0.011340050958096981, "step": 584 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 664.0, "completions/mean_length": 473.375, "completions/mean_terminated_length": 463.8709411621094, "completions/min_length": 238.0, "completions/min_terminated_length": 238.0, "entropy": 0.3590838871896267, "epoch": 0.31316916488222696, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0332, "num_tokens": 12747752.0, "reward": 0.9375, "reward_std": 0.1157275140285492, "rewards/accuracy_reward/mean": 0.9375, "rewards/accuracy_reward/std": 0.24593468010425568, "sampling/importance_sampling_ratio/max": 1.5954334735870361, "sampling/importance_sampling_ratio/mean": 0.9996042847633362, "sampling/importance_sampling_ratio/min": 0.6934288740158081, "sampling/sampling_logp_difference/max": 0.4671454429626465, "sampling/sampling_logp_difference/mean": 0.012334608472883701, "step": 585 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00010694679440348409, "clip_ratio/low_min": 0.00010694679440348409, "clip_ratio/region_mean": 0.00010694679440348409, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 714.0, "completions/mean_length": 568.90625, "completions/mean_terminated_length": 502.54168701171875, "completions/min_length": 361.0, "completions/min_terminated_length": 361.0, "entropy": 0.35023289918899536, "epoch": 0.31370449678800855, "frac_reward_zero_std": 0.5, "grad_norm": 0.0034463456831872463, "learning_rate": 1e-05, "loss": 0.0571, "num_tokens": 12769461.0, "reward": 0.71875, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.4596868753433228, "sampling/importance_sampling_ratio/mean": 1.0000150203704834, "sampling/importance_sampling_ratio/min": 0.6247786283493042, "sampling/sampling_logp_difference/max": 0.47035789489746094, "sampling/sampling_logp_difference/mean": 0.011078916490077972, "step": 586 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.214851989876479e-05, "clip_ratio/low_min": 5.214851989876479e-05, "clip_ratio/region_mean": 5.214851989876479e-05, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 758.0, "completions/mean_length": 576.09375, "completions/mean_terminated_length": 512.125, "completions/min_length": 254.0, "completions/min_terminated_length": 254.0, "entropy": 0.3751702047884464, "epoch": 0.31423982869379014, "frac_reward_zero_std": 0.5, "grad_norm": 0.009753352031111717, "learning_rate": 1e-05, "loss": 0.0168, "num_tokens": 12791336.0, "reward": 0.71875, "reward_std": 0.2630178928375244, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.380725383758545, "sampling/importance_sampling_ratio/mean": 0.9998529553413391, "sampling/importance_sampling_ratio/min": 0.7274522185325623, "sampling/sampling_logp_difference/max": 0.32260894775390625, "sampling/sampling_logp_difference/mean": 0.011273259297013283, "step": 587 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 644.0, "completions/max_terminated_length": 644.0, "completions/mean_length": 434.28125, "completions/mean_terminated_length": 434.28125, "completions/min_length": 210.0, "completions/min_terminated_length": 210.0, "entropy": 0.3039466142654419, "epoch": 0.3147751605995717, "frac_reward_zero_std": 0.75, "grad_norm": 0.0028925358783453703, "learning_rate": 1e-05, "loss": 0.0296, "num_tokens": 12808297.0, "reward": 0.96875, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.96875, "rewards/accuracy_reward/std": 0.1767766922712326, "sampling/importance_sampling_ratio/max": 1.4272725582122803, "sampling/importance_sampling_ratio/mean": 1.0001274347305298, "sampling/importance_sampling_ratio/min": 0.6404282450675964, "sampling/sampling_logp_difference/max": 0.4456181526184082, "sampling/sampling_logp_difference/mean": 0.009803217835724354, "step": 588 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 8.305647497763857e-05, "clip_ratio/low_min": 8.305647497763857e-05, "clip_ratio/region_mean": 8.305647497763857e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 558.0, "completions/max_terminated_length": 558.0, "completions/mean_length": 391.28125, "completions/mean_terminated_length": 391.28125, "completions/min_length": 185.0, "completions/min_terminated_length": 185.0, "entropy": 0.316328976303339, "epoch": 0.3153104925053533, "frac_reward_zero_std": 0.5, "grad_norm": 0.01038505882024765, "learning_rate": 1e-05, "loss": -0.0428, "num_tokens": 12824410.0, "reward": 0.8125, "reward_std": 0.249358132481575, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.3519842624664307, "sampling/importance_sampling_ratio/mean": 0.9998332262039185, "sampling/importance_sampling_ratio/min": 0.7554951906204224, "sampling/sampling_logp_difference/max": 0.3015732765197754, "sampling/sampling_logp_difference/mean": 0.010774289257824421, "step": 589 }, { "clip_ratio/high_max": 0.00013501863577403128, "clip_ratio/high_mean": 0.00013501863577403128, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00013501863577403128, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 704.0, "completions/mean_length": 523.03125, "completions/mean_terminated_length": 497.6896667480469, "completions/min_length": 326.0, "completions/min_terminated_length": 326.0, "entropy": 0.31476516649127007, "epoch": 0.3158458244111349, "frac_reward_zero_std": 0.25, "grad_norm": 0.027100680395960808, "learning_rate": 1e-05, "loss": 0.0568, "num_tokens": 12844723.0, "reward": 0.625, "reward_std": 0.2925041913986206, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.3829365968704224, "sampling/importance_sampling_ratio/mean": 1.0001263618469238, "sampling/importance_sampling_ratio/min": 0.781120777130127, "sampling/sampling_logp_difference/max": 0.32420921325683594, "sampling/sampling_logp_difference/mean": 0.010345133021473885, "step": 590 }, { "clip_ratio/high_max": 5.369415885070339e-05, "clip_ratio/high_mean": 5.369415885070339e-05, "clip_ratio/low_mean": 0.00013852096162736416, "clip_ratio/low_min": 0.00013852096162736416, "clip_ratio/region_mean": 0.00019221512047806755, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 709.0, "completions/mean_length": 502.28125, "completions/mean_terminated_length": 464.3214416503906, "completions/min_length": 255.0, "completions/min_terminated_length": 255.0, "entropy": 0.3764626607298851, "epoch": 0.3163811563169165, "frac_reward_zero_std": 0.25, "grad_norm": 0.011633516289293766, "learning_rate": 1e-05, "loss": 0.106, "num_tokens": 12864684.0, "reward": 0.71875, "reward_std": 0.3608423173427582, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.5665745735168457, "sampling/importance_sampling_ratio/mean": 1.0000176429748535, "sampling/importance_sampling_ratio/min": 0.6921218037605286, "sampling/sampling_logp_difference/max": 0.44889140129089355, "sampling/sampling_logp_difference/mean": 0.011256412602961063, "step": 591 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 663.0, "completions/max_terminated_length": 663.0, "completions/mean_length": 439.375, "completions/mean_terminated_length": 439.375, "completions/min_length": 213.0, "completions/min_terminated_length": 213.0, "entropy": 0.3273787572979927, "epoch": 0.3169164882226981, "frac_reward_zero_std": 0.5, "grad_norm": 0.003047094913199544, "learning_rate": 1e-05, "loss": 0.0143, "num_tokens": 12882304.0, "reward": 0.90625, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.386520266532898, "sampling/importance_sampling_ratio/mean": 1.0001561641693115, "sampling/importance_sampling_ratio/min": 0.7159668207168579, "sampling/sampling_logp_difference/max": 0.3341214656829834, "sampling/sampling_logp_difference/mean": 0.010446848347783089, "step": 592 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00021715218099416234, "clip_ratio/low_min": 0.00021715218099416234, "clip_ratio/region_mean": 0.00021715218099416234, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 730.0, "completions/mean_length": 501.5625, "completions/mean_terminated_length": 426.9599914550781, "completions/min_length": 250.0, "completions/min_terminated_length": 250.0, "entropy": 0.5103813372552395, "epoch": 0.31745182012847967, "frac_reward_zero_std": 0.0, "grad_norm": 0.011841914616525173, "learning_rate": 1e-05, "loss": 0.0229, "num_tokens": 12902210.0, "reward": 0.5, "reward_std": 0.4082317352294922, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5080004930496216, "sampling/importance_sampling_ratio/max": 1.6238415241241455, "sampling/importance_sampling_ratio/mean": 1.000093698501587, "sampling/importance_sampling_ratio/min": 0.7020272016525269, "sampling/sampling_logp_difference/max": 0.48479461669921875, "sampling/sampling_logp_difference/mean": 0.014888807199895382, "step": 593 }, { "clip_ratio/high_max": 6.760410906281322e-05, "clip_ratio/high_mean": 6.760410906281322e-05, "clip_ratio/low_mean": 6.760410906281322e-05, "clip_ratio/low_min": 6.760410906281322e-05, "clip_ratio/region_mean": 0.00013520821812562644, "completions/clipped_ratio": 0.28125, "completions/max_length": 768.0, "completions/max_terminated_length": 744.0, "completions/mean_length": 612.875, "completions/mean_terminated_length": 552.1739501953125, "completions/min_length": 351.0, "completions/min_terminated_length": 351.0, "entropy": 0.46455319225788116, "epoch": 0.31798715203426126, "frac_reward_zero_std": 0.0, "grad_norm": 0.009063370525836945, "learning_rate": 1e-05, "loss": 0.0475, "num_tokens": 12926246.0, "reward": 0.625, "reward_std": 0.4492306709289551, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.4822250604629517, "sampling/importance_sampling_ratio/mean": 0.9995887279510498, "sampling/importance_sampling_ratio/min": 0.14249230921268463, "sampling/sampling_logp_difference/max": 1.9484672546386719, "sampling/sampling_logp_difference/mean": 0.013255693949759007, "step": 594 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0002372380367887672, "clip_ratio/low_min": 0.0002372380367887672, "clip_ratio/region_mean": 0.0002372380367887672, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 743.0, "completions/mean_length": 546.875, "completions/mean_terminated_length": 505.9259338378906, "completions/min_length": 271.0, "completions/min_terminated_length": 271.0, "entropy": 0.3897904269397259, "epoch": 0.31852248394004284, "frac_reward_zero_std": 0.25, "grad_norm": 0.010369587689638138, "learning_rate": 1e-05, "loss": 0.0675, "num_tokens": 12947642.0, "reward": 0.65625, "reward_std": 0.3377464711666107, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9995553493499756, "sampling/importance_sampling_ratio/min": 0.6306949257850647, "sampling/sampling_logp_difference/max": 0.8285665512084961, "sampling/sampling_logp_difference/mean": 0.012024225667119026, "step": 595 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00015292173338821158, "clip_ratio/low_min": 0.00015292173338821158, "clip_ratio/region_mean": 0.00015292173338821158, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 756.0, "completions/mean_length": 491.84375, "completions/mean_terminated_length": 452.39288330078125, "completions/min_length": 273.0, "completions/min_terminated_length": 273.0, "entropy": 0.3545280061662197, "epoch": 0.31905781584582443, "frac_reward_zero_std": 0.25, "grad_norm": 0.012320420704782009, "learning_rate": 1e-05, "loss": 0.064, "num_tokens": 12967469.0, "reward": 0.5625, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.2975786924362183, "sampling/importance_sampling_ratio/mean": 1.0001546144485474, "sampling/importance_sampling_ratio/min": 0.7114093899726868, "sampling/sampling_logp_difference/max": 0.34050726890563965, "sampling/sampling_logp_difference/mean": 0.01130654476583004, "step": 596 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00014027773431735113, "clip_ratio/low_min": 0.00014027773431735113, "clip_ratio/region_mean": 0.00014027773431735113, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 693.0, "completions/mean_length": 533.34375, "completions/mean_terminated_length": 479.19232177734375, "completions/min_length": 213.0, "completions/min_terminated_length": 213.0, "entropy": 0.3938218764960766, "epoch": 0.319593147751606, "frac_reward_zero_std": 0.5, "grad_norm": 0.019227659329771996, "learning_rate": 1e-05, "loss": 0.0173, "num_tokens": 12988344.0, "reward": 0.46875, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.46875, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.4319961071014404, "sampling/importance_sampling_ratio/mean": 0.999968409538269, "sampling/importance_sampling_ratio/min": 0.6174544095993042, "sampling/sampling_logp_difference/max": 0.4821500778198242, "sampling/sampling_logp_difference/mean": 0.0121418172493577, "step": 597 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00012024539319099858, "clip_ratio/low_min": 0.00012024539319099858, "clip_ratio/region_mean": 0.00012024539319099858, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 767.0, "completions/mean_length": 532.59375, "completions/mean_terminated_length": 489.0, "completions/min_length": 255.0, "completions/min_terminated_length": 255.0, "entropy": 0.39213399589061737, "epoch": 0.32012847965738755, "frac_reward_zero_std": 0.0, "grad_norm": 0.02263588458299637, "learning_rate": 1e-05, "loss": -0.0016, "num_tokens": 13009667.0, "reward": 0.5625, "reward_std": 0.5081326961517334, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.452756404876709, "sampling/importance_sampling_ratio/mean": 1.0003527402877808, "sampling/importance_sampling_ratio/min": 0.6124302744865417, "sampling/sampling_logp_difference/max": 0.49032020568847656, "sampling/sampling_logp_difference/mean": 0.012032617814838886, "step": 598 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 712.0, "completions/mean_length": 509.9375, "completions/mean_terminated_length": 483.2413635253906, "completions/min_length": 189.0, "completions/min_terminated_length": 189.0, "entropy": 0.417438130825758, "epoch": 0.32066381156316914, "frac_reward_zero_std": 0.25, "grad_norm": 0.0060572451911866665, "learning_rate": 1e-05, "loss": 0.0458, "num_tokens": 13030321.0, "reward": 0.65625, "reward_std": 0.3061639666557312, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.3527554273605347, "sampling/importance_sampling_ratio/mean": 1.000324010848999, "sampling/importance_sampling_ratio/min": 0.6157258749008179, "sampling/sampling_logp_difference/max": 0.4849534034729004, "sampling/sampling_logp_difference/mean": 0.012862362898886204, "step": 599 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 688.0, "completions/mean_length": 420.5, "completions/mean_terminated_length": 409.2903137207031, "completions/min_length": 271.0, "completions/min_terminated_length": 271.0, "entropy": 0.3093317486345768, "epoch": 0.32119914346895073, "frac_reward_zero_std": 0.75, "grad_norm": 0.019723497331142426, "learning_rate": 1e-05, "loss": -0.0183, "num_tokens": 13047321.0, "reward": 0.9375, "reward_std": 0.1157275140285492, "rewards/accuracy_reward/mean": 0.9375, "rewards/accuracy_reward/std": 0.24593468010425568, "sampling/importance_sampling_ratio/max": 1.3038541078567505, "sampling/importance_sampling_ratio/mean": 0.9999650716781616, "sampling/importance_sampling_ratio/min": 0.7139387726783752, "sampling/sampling_logp_difference/max": 0.3369581699371338, "sampling/sampling_logp_difference/mean": 0.010076196864247322, "step": 600 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.907734546577558e-05, "clip_ratio/low_min": 4.907734546577558e-05, "clip_ratio/region_mean": 4.907734546577558e-05, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 768.0, "completions/mean_length": 570.65625, "completions/mean_terminated_length": 504.875, "completions/min_length": 281.0, "completions/min_terminated_length": 281.0, "entropy": 0.36661629751324654, "epoch": 0.3217344753747323, "frac_reward_zero_std": 0.25, "grad_norm": 0.011421416886150837, "learning_rate": 1e-05, "loss": 0.0478, "num_tokens": 13069318.0, "reward": 0.6875, "reward_std": 0.3745020925998688, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.6252936124801636, "sampling/importance_sampling_ratio/mean": 0.999879777431488, "sampling/importance_sampling_ratio/min": 0.5668175220489502, "sampling/sampling_logp_difference/max": 0.5677177906036377, "sampling/sampling_logp_difference/mean": 0.011020981706678867, "step": 601 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 638.0, "completions/mean_length": 489.625, "completions/mean_terminated_length": 460.82757568359375, "completions/min_length": 280.0, "completions/min_terminated_length": 280.0, "entropy": 0.3008575960993767, "epoch": 0.3222698072805139, "frac_reward_zero_std": 0.25, "grad_norm": 0.013026995584368706, "learning_rate": 1e-05, "loss": 0.0337, "num_tokens": 13088866.0, "reward": 0.8125, "reward_std": 0.3104073107242584, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.4073952436447144, "sampling/importance_sampling_ratio/mean": 0.9999853372573853, "sampling/importance_sampling_ratio/min": 0.6923598051071167, "sampling/sampling_logp_difference/max": 0.36764955520629883, "sampling/sampling_logp_difference/mean": 0.010159152559936047, "step": 602 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 685.0, "completions/mean_length": 492.4375, "completions/mean_terminated_length": 474.0666809082031, "completions/min_length": 360.0, "completions/min_terminated_length": 360.0, "entropy": 0.3233669772744179, "epoch": 0.3228051391862955, "frac_reward_zero_std": 0.5, "grad_norm": 0.004429081454873085, "learning_rate": 1e-05, "loss": 0.0324, "num_tokens": 13108544.0, "reward": 0.9375, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.9375, "rewards/accuracy_reward/std": 0.24593468010425568, "sampling/importance_sampling_ratio/max": 1.4388188123703003, "sampling/importance_sampling_ratio/mean": 1.0002284049987793, "sampling/importance_sampling_ratio/min": 0.7001235485076904, "sampling/sampling_logp_difference/max": 0.36382246017456055, "sampling/sampling_logp_difference/mean": 0.010552718304097652, "step": 603 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0001708725503704045, "clip_ratio/low_min": 0.0001708725503704045, "clip_ratio/region_mean": 0.0001708725503704045, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 766.0, "completions/mean_length": 545.34375, "completions/mean_terminated_length": 483.0, "completions/min_length": 254.0, "completions/min_terminated_length": 254.0, "entropy": 0.47946010157465935, "epoch": 0.3233404710920771, "frac_reward_zero_std": 0.25, "grad_norm": 0.031759437173604965, "learning_rate": 1e-05, "loss": -0.0158, "num_tokens": 13130379.0, "reward": 0.4375, "reward_std": 0.3104073107242584, "rewards/accuracy_reward/mean": 0.4375, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.2905904054641724, "sampling/importance_sampling_ratio/mean": 0.9998774528503418, "sampling/importance_sampling_ratio/min": 0.7416612505912781, "sampling/sampling_logp_difference/max": 0.2988626956939697, "sampling/sampling_logp_difference/mean": 0.013906342908740044, "step": 604 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00023261904789251275, "clip_ratio/low_min": 0.00023261904789251275, "clip_ratio/region_mean": 0.00023261904789251275, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 681.0, "completions/mean_length": 575.65625, "completions/mean_terminated_length": 511.54168701171875, "completions/min_length": 342.0, "completions/min_terminated_length": 342.0, "entropy": 0.42365530505776405, "epoch": 0.3238758029978587, "frac_reward_zero_std": 0.25, "grad_norm": 0.006852906662970781, "learning_rate": 1e-05, "loss": 0.0172, "num_tokens": 13152448.0, "reward": 0.71875, "reward_std": 0.2651650309562683, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.2941696643829346, "sampling/importance_sampling_ratio/mean": 1.0000784397125244, "sampling/importance_sampling_ratio/min": 0.7816289663314819, "sampling/sampling_logp_difference/max": 0.25786924362182617, "sampling/sampling_logp_difference/mean": 0.012645269744098186, "step": 605 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 654.0, "completions/mean_length": 478.6875, "completions/mean_terminated_length": 469.3548278808594, "completions/min_length": 266.0, "completions/min_terminated_length": 266.0, "entropy": 0.35248585045337677, "epoch": 0.32441113490364026, "frac_reward_zero_std": 0.75, "grad_norm": 0.0067518651485443115, "learning_rate": 1e-05, "loss": 0.0042, "num_tokens": 13171142.0, "reward": 0.9375, "reward_std": 0.1157275140285492, "rewards/accuracy_reward/mean": 0.9375, "rewards/accuracy_reward/std": 0.24593468010425568, "sampling/importance_sampling_ratio/max": 1.3850748538970947, "sampling/importance_sampling_ratio/mean": 0.999954104423523, "sampling/importance_sampling_ratio/min": 0.729388415813446, "sampling/sampling_logp_difference/max": 0.32575416564941406, "sampling/sampling_logp_difference/mean": 0.010965636000037193, "step": 606 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 718.0, "completions/mean_length": 494.78125, "completions/mean_terminated_length": 466.5172424316406, "completions/min_length": 231.0, "completions/min_terminated_length": 231.0, "entropy": 0.4147927723824978, "epoch": 0.32494646680942185, "frac_reward_zero_std": 0.75, "grad_norm": 0.00472274562343955, "learning_rate": 1e-05, "loss": 0.0092, "num_tokens": 13190343.0, "reward": 0.78125, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.4420902729034424, "sampling/importance_sampling_ratio/mean": 1.0001254081726074, "sampling/importance_sampling_ratio/min": 0.698153555393219, "sampling/sampling_logp_difference/max": 0.36609363555908203, "sampling/sampling_logp_difference/mean": 0.012433970347046852, "step": 607 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00015350939429481514, "clip_ratio/low_min": 0.00015350939429481514, "clip_ratio/region_mean": 0.00015350939429481514, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 761.0, "completions/mean_length": 491.25, "completions/mean_terminated_length": 399.0, "completions/min_length": 230.0, "completions/min_terminated_length": 230.0, "entropy": 0.353031437844038, "epoch": 0.32548179871520344, "frac_reward_zero_std": 0.25, "grad_norm": 0.011456039734184742, "learning_rate": 1e-05, "loss": 0.0453, "num_tokens": 13209543.0, "reward": 0.53125, "reward_std": 0.3471629321575165, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9996740818023682, "sampling/importance_sampling_ratio/min": 0.4374963045120239, "sampling/sampling_logp_difference/max": 1.7040212154388428, "sampling/sampling_logp_difference/mean": 0.010685842484235764, "step": 608 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0001578001247253269, "clip_ratio/low_min": 0.0001578001247253269, "clip_ratio/region_mean": 0.0001578001247253269, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 762.0, "completions/mean_length": 584.0, "completions/mean_terminated_length": 522.6666870117188, "completions/min_length": 297.0, "completions/min_terminated_length": 297.0, "entropy": 0.41221435368061066, "epoch": 0.32601713062098503, "frac_reward_zero_std": 0.5, "grad_norm": 0.007403896190226078, "learning_rate": 1e-05, "loss": 0.0575, "num_tokens": 13231847.0, "reward": 0.5625, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.4222649335861206, "sampling/importance_sampling_ratio/mean": 1.0001360177993774, "sampling/importance_sampling_ratio/min": 0.7329180240631104, "sampling/sampling_logp_difference/max": 0.3522505760192871, "sampling/sampling_logp_difference/mean": 0.012082134373486042, "step": 609 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 754.0, "completions/mean_length": 443.5625, "completions/mean_terminated_length": 421.933349609375, "completions/min_length": 219.0, "completions/min_terminated_length": 219.0, "entropy": 0.3271928485482931, "epoch": 0.3265524625267666, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0491, "num_tokens": 13249457.0, "reward": 0.9375, "reward_std": 0.1157275140285492, "rewards/accuracy_reward/mean": 0.9375, "rewards/accuracy_reward/std": 0.24593468010425568, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0004411935806274, "sampling/importance_sampling_ratio/min": 0.7110956907272339, "sampling/sampling_logp_difference/max": 0.8487842082977295, "sampling/sampling_logp_difference/mean": 0.010361955501139164, "step": 610 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 656.0, "completions/mean_length": 505.6875, "completions/mean_terminated_length": 488.20001220703125, "completions/min_length": 263.0, "completions/min_terminated_length": 263.0, "entropy": 0.2920081876218319, "epoch": 0.3270877944325482, "frac_reward_zero_std": 0.5, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0692, "num_tokens": 13269135.0, "reward": 0.875, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 1.4318369626998901, "sampling/importance_sampling_ratio/mean": 1.0002622604370117, "sampling/importance_sampling_ratio/min": 0.7309180498123169, "sampling/sampling_logp_difference/max": 0.35895824432373047, "sampling/sampling_logp_difference/mean": 0.009328608401119709, "step": 611 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00017318427126156166, "clip_ratio/low_min": 0.00017318427126156166, "clip_ratio/region_mean": 0.00017318427126156166, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 736.0, "completions/mean_length": 538.90625, "completions/mean_terminated_length": 496.4814758300781, "completions/min_length": 226.0, "completions/min_terminated_length": 226.0, "entropy": 0.32793927378952503, "epoch": 0.32762312633832974, "frac_reward_zero_std": 0.25, "grad_norm": 0.013294613920152187, "learning_rate": 1e-05, "loss": 0.0699, "num_tokens": 13290108.0, "reward": 0.53125, "reward_std": 0.378745436668396, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.4561046361923218, "sampling/importance_sampling_ratio/mean": 1.0001935958862305, "sampling/importance_sampling_ratio/min": 0.6996236443519592, "sampling/sampling_logp_difference/max": 0.3757648468017578, "sampling/sampling_logp_difference/mean": 0.010137120261788368, "step": 612 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 751.0, "completions/mean_length": 410.5625, "completions/mean_terminated_length": 399.0322570800781, "completions/min_length": 233.0, "completions/min_terminated_length": 233.0, "entropy": 0.37601885199546814, "epoch": 0.32815845824411133, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": -0.0261, "num_tokens": 13306742.0, "reward": 0.59375, "reward_std": 0.1293872892856598, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.3682092428207397, "sampling/importance_sampling_ratio/mean": 0.9998409748077393, "sampling/importance_sampling_ratio/min": 0.7513484358787537, "sampling/sampling_logp_difference/max": 0.31350278854370117, "sampling/sampling_logp_difference/mean": 0.011797838844358921, "step": 613 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.13346419413574e-05, "clip_ratio/low_min": 6.13346419413574e-05, "clip_ratio/region_mean": 6.13346419413574e-05, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 724.0, "completions/mean_length": 518.09375, "completions/mean_terminated_length": 482.39288330078125, "completions/min_length": 251.0, "completions/min_terminated_length": 251.0, "entropy": 0.3716732859611511, "epoch": 0.3286937901498929, "frac_reward_zero_std": 0.5, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0584, "num_tokens": 13326969.0, "reward": 0.5, "reward_std": 0.26726123690605164, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5080004930496216, "sampling/importance_sampling_ratio/max": 1.3558627367019653, "sampling/importance_sampling_ratio/mean": 1.0003249645233154, "sampling/importance_sampling_ratio/min": 0.629155158996582, "sampling/sampling_logp_difference/max": 0.46337735652923584, "sampling/sampling_logp_difference/mean": 0.011459769681096077, "step": 614 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00013365456834435463, "clip_ratio/low_min": 0.00013365456834435463, "clip_ratio/region_mean": 0.00013365456834435463, "completions/clipped_ratio": 0.0, "completions/max_length": 621.0, "completions/max_terminated_length": 621.0, "completions/mean_length": 429.375, "completions/mean_terminated_length": 429.375, "completions/min_length": 275.0, "completions/min_terminated_length": 275.0, "entropy": 0.34134896472096443, "epoch": 0.3292291220556745, "frac_reward_zero_std": 0.75, "grad_norm": 0.009187165647745132, "learning_rate": 1e-05, "loss": 0.0177, "num_tokens": 13344245.0, "reward": 0.90625, "reward_std": 0.1293872892856598, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.372597575187683, "sampling/importance_sampling_ratio/mean": 1.0002752542495728, "sampling/importance_sampling_ratio/min": 0.6704867482185364, "sampling/sampling_logp_difference/max": 0.3997514247894287, "sampling/sampling_logp_difference/mean": 0.011531875468790531, "step": 615 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00010604775889078155, "clip_ratio/low_min": 0.00010604775889078155, "clip_ratio/region_mean": 0.00010604775889078155, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 727.0, "completions/mean_length": 571.84375, "completions/mean_terminated_length": 551.5516967773438, "completions/min_length": 390.0, "completions/min_terminated_length": 390.0, "entropy": 0.25688663870096207, "epoch": 0.3297644539614561, "frac_reward_zero_std": 0.25, "grad_norm": 0.005713587626814842, "learning_rate": 1e-05, "loss": 0.0339, "num_tokens": 13366408.0, "reward": 0.6875, "reward_std": 0.2925041913986206, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.3288211822509766, "sampling/importance_sampling_ratio/mean": 0.9999892711639404, "sampling/importance_sampling_ratio/min": 0.7728637456893921, "sampling/sampling_logp_difference/max": 0.28429222106933594, "sampling/sampling_logp_difference/mean": 0.008276664651930332, "step": 616 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 705.0, "completions/mean_length": 514.59375, "completions/mean_terminated_length": 506.4193420410156, "completions/min_length": 320.0, "completions/min_terminated_length": 320.0, "entropy": 0.3054012767970562, "epoch": 0.3302997858672377, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 13386995.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.3915033340454102, "sampling/importance_sampling_ratio/mean": 1.0001293420791626, "sampling/importance_sampling_ratio/min": 0.7308658957481384, "sampling/sampling_logp_difference/max": 0.3303847312927246, "sampling/sampling_logp_difference/mean": 0.010608382523059845, "step": 617 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 680.0, "completions/max_terminated_length": 680.0, "completions/mean_length": 423.84375, "completions/mean_terminated_length": 423.84375, "completions/min_length": 227.0, "completions/min_terminated_length": 227.0, "entropy": 0.2652575895190239, "epoch": 0.33083511777301927, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 13404278.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.5266615152359009, "sampling/importance_sampling_ratio/mean": 0.9996988773345947, "sampling/importance_sampling_ratio/min": 0.7503836750984192, "sampling/sampling_logp_difference/max": 0.4230833053588867, "sampling/sampling_logp_difference/mean": 0.008782808668911457, "step": 618 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 766.0, "completions/mean_length": 479.375, "completions/mean_terminated_length": 460.13336181640625, "completions/min_length": 255.0, "completions/min_terminated_length": 255.0, "entropy": 0.2553304620087147, "epoch": 0.33137044967880086, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0158, "num_tokens": 13422898.0, "reward": 0.90625, "reward_std": 0.1293872892856598, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.3723410367965698, "sampling/importance_sampling_ratio/mean": 1.0000486373901367, "sampling/importance_sampling_ratio/min": 0.7063539624214172, "sampling/sampling_logp_difference/max": 0.3476388454437256, "sampling/sampling_logp_difference/mean": 0.00843969825655222, "step": 619 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 749.0, "completions/mean_length": 564.71875, "completions/mean_terminated_length": 527.0740966796875, "completions/min_length": 296.0, "completions/min_terminated_length": 296.0, "entropy": 0.3039424940943718, "epoch": 0.33190578158458245, "frac_reward_zero_std": 0.0, "grad_norm": 0.014260738156735897, "learning_rate": 1e-05, "loss": 0.0973, "num_tokens": 13444929.0, "reward": 0.5, "reward_std": 0.4671337604522705, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5080004930496216, "sampling/importance_sampling_ratio/max": 1.298768401145935, "sampling/importance_sampling_ratio/mean": 0.9998161792755127, "sampling/importance_sampling_ratio/min": 0.7113383412361145, "sampling/sampling_logp_difference/max": 0.3406071662902832, "sampling/sampling_logp_difference/mean": 0.009649362415075302, "step": 620 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 745.0, "completions/mean_length": 493.53125, "completions/mean_terminated_length": 475.2333679199219, "completions/min_length": 288.0, "completions/min_terminated_length": 288.0, "entropy": 0.37491171061992645, "epoch": 0.33244111349036404, "frac_reward_zero_std": 0.5, "grad_norm": 0.0036310877185314894, "learning_rate": 1e-05, "loss": 0.0184, "num_tokens": 13464506.0, "reward": 0.6875, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.4245522022247314, "sampling/importance_sampling_ratio/mean": 0.9994794130325317, "sampling/importance_sampling_ratio/min": 0.7612655758857727, "sampling/sampling_logp_difference/max": 0.35385751724243164, "sampling/sampling_logp_difference/mean": 0.011728239245712757, "step": 621 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 754.0, "completions/mean_length": 445.15625, "completions/mean_terminated_length": 434.7419128417969, "completions/min_length": 177.0, "completions/min_terminated_length": 177.0, "entropy": 0.3791900798678398, "epoch": 0.3329764453961456, "frac_reward_zero_std": 0.5, "grad_norm": 0.011801331304013729, "learning_rate": 1e-05, "loss": -0.0266, "num_tokens": 13482463.0, "reward": 0.78125, "reward_std": 0.2630178928375244, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.329850673675537, "sampling/importance_sampling_ratio/mean": 1.0001842975616455, "sampling/importance_sampling_ratio/min": 0.5571074485778809, "sampling/sampling_logp_difference/max": 0.5849971771240234, "sampling/sampling_logp_difference/mean": 0.011895262636244297, "step": 622 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 518.0, "completions/max_terminated_length": 518.0, "completions/mean_length": 343.84375, "completions/mean_terminated_length": 343.84375, "completions/min_length": 165.0, "completions/min_terminated_length": 165.0, "entropy": 0.28707359731197357, "epoch": 0.3335117773019272, "frac_reward_zero_std": 0.5, "grad_norm": 0.010176939889788628, "learning_rate": 1e-05, "loss": -0.0191, "num_tokens": 13496850.0, "reward": 0.84375, "reward_std": 0.22201895713806152, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.3418649435043335, "sampling/importance_sampling_ratio/mean": 0.9998665452003479, "sampling/importance_sampling_ratio/min": 0.6146644353866577, "sampling/sampling_logp_difference/max": 0.4866788387298584, "sampling/sampling_logp_difference/mean": 0.010198241099715233, "step": 623 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 740.0, "completions/mean_length": 485.59375, "completions/mean_terminated_length": 456.3793029785156, "completions/min_length": 128.0, "completions/min_terminated_length": 128.0, "entropy": 0.3566009998321533, "epoch": 0.3340471092077088, "frac_reward_zero_std": 0.5, "grad_norm": 0.004952109418809414, "learning_rate": 1e-05, "loss": -0.0494, "num_tokens": 13516069.0, "reward": 0.78125, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.4125614166259766, "sampling/importance_sampling_ratio/mean": 0.9998021125793457, "sampling/importance_sampling_ratio/min": 0.7557869553565979, "sampling/sampling_logp_difference/max": 0.34540462493896484, "sampling/sampling_logp_difference/mean": 0.011198865249752998, "step": 624 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 728.0, "completions/mean_length": 569.75, "completions/mean_terminated_length": 524.0, "completions/min_length": 332.0, "completions/min_terminated_length": 332.0, "entropy": 0.3010255694389343, "epoch": 0.33458244111349034, "frac_reward_zero_std": 0.5, "grad_norm": 0.002087995409965515, "learning_rate": 1e-05, "loss": -0.0104, "num_tokens": 13538709.0, "reward": 0.75, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.391015648841858, "sampling/importance_sampling_ratio/mean": 0.9999024868011475, "sampling/importance_sampling_ratio/min": 0.6822434067726135, "sampling/sampling_logp_difference/max": 0.382368803024292, "sampling/sampling_logp_difference/mean": 0.010051458142697811, "step": 625 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 765.0, "completions/max_terminated_length": 765.0, "completions/mean_length": 460.3125, "completions/mean_terminated_length": 460.3125, "completions/min_length": 253.0, "completions/min_terminated_length": 253.0, "entropy": 0.36738942563533783, "epoch": 0.3351177730192719, "frac_reward_zero_std": 0.0, "grad_norm": 0.018555957823991776, "learning_rate": 1e-05, "loss": 0.026, "num_tokens": 13556767.0, "reward": 0.6875, "reward_std": 0.4082317352294922, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.55997896194458, "sampling/importance_sampling_ratio/mean": 1.00010085105896, "sampling/importance_sampling_ratio/min": 0.6896438002586365, "sampling/sampling_logp_difference/max": 0.4446723461151123, "sampling/sampling_logp_difference/mean": 0.012395805679261684, "step": 626 }, { "clip_ratio/high_max": 7.916402682894841e-05, "clip_ratio/high_mean": 7.916402682894841e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 7.916402682894841e-05, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 747.0, "completions/mean_length": 528.84375, "completions/mean_terminated_length": 449.125, "completions/min_length": 241.0, "completions/min_terminated_length": 241.0, "entropy": 0.47731801494956017, "epoch": 0.3356531049250535, "frac_reward_zero_std": 0.0, "grad_norm": 0.0235456395894289, "learning_rate": 1e-05, "loss": 0.0211, "num_tokens": 13577474.0, "reward": 0.65625, "reward_std": 0.4218915104866028, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.8571079969406128, "sampling/importance_sampling_ratio/mean": 1.0001784563064575, "sampling/importance_sampling_ratio/min": 0.2742895483970642, "sampling/sampling_logp_difference/max": 1.2935709953308105, "sampling/sampling_logp_difference/mean": 0.013619859702885151, "step": 627 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 758.0, "completions/mean_length": 540.59375, "completions/mean_terminated_length": 525.433349609375, "completions/min_length": 245.0, "completions/min_terminated_length": 245.0, "entropy": 0.2912454418838024, "epoch": 0.3361884368308351, "frac_reward_zero_std": 0.5, "grad_norm": 0.02371685579419136, "learning_rate": 1e-05, "loss": 0.0311, "num_tokens": 13598141.0, "reward": 0.90625, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.3892697095870972, "sampling/importance_sampling_ratio/mean": 0.9999818205833435, "sampling/importance_sampling_ratio/min": 0.5333466529846191, "sampling/sampling_logp_difference/max": 0.6285836696624756, "sampling/sampling_logp_difference/mean": 0.009452036581933498, "step": 628 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.28125, "completions/max_length": 768.0, "completions/max_terminated_length": 687.0, "completions/mean_length": 596.15625, "completions/mean_terminated_length": 528.9130249023438, "completions/min_length": 364.0, "completions/min_terminated_length": 364.0, "entropy": 0.264277670532465, "epoch": 0.3367237687366167, "frac_reward_zero_std": 0.5, "grad_norm": 0.029641496017575264, "learning_rate": 1e-05, "loss": 0.0604, "num_tokens": 13620954.0, "reward": 0.6875, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.2616605758666992, "sampling/importance_sampling_ratio/mean": 1.0001510381698608, "sampling/importance_sampling_ratio/min": 0.6960204243659973, "sampling/sampling_logp_difference/max": 0.36237621307373047, "sampling/sampling_logp_difference/mean": 0.008369022980332375, "step": 629 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.439979188144207e-05, "clip_ratio/low_min": 6.439979188144207e-05, "clip_ratio/region_mean": 6.439979188144207e-05, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 736.0, "completions/mean_length": 472.0625, "completions/mean_terminated_length": 441.4482727050781, "completions/min_length": 212.0, "completions/min_terminated_length": 212.0, "entropy": 0.26084725745022297, "epoch": 0.3372591006423983, "frac_reward_zero_std": 0.5, "grad_norm": 0.02300383523106575, "learning_rate": 1e-05, "loss": 0.034, "num_tokens": 13639708.0, "reward": 0.75, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.2997791767120361, "sampling/importance_sampling_ratio/mean": 0.9998540282249451, "sampling/importance_sampling_ratio/min": 0.7616881728172302, "sampling/sampling_logp_difference/max": 0.2722179889678955, "sampling/sampling_logp_difference/mean": 0.00920176226645708, "step": 630 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 691.0, "completions/max_terminated_length": 691.0, "completions/mean_length": 426.5, "completions/mean_terminated_length": 426.5, "completions/min_length": 252.0, "completions/min_terminated_length": 252.0, "entropy": 0.311075109988451, "epoch": 0.33779443254817987, "frac_reward_zero_std": 0.5, "grad_norm": 0.006628968752920628, "learning_rate": 1e-05, "loss": -0.0028, "num_tokens": 13657076.0, "reward": 0.9375, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.9375, "rewards/accuracy_reward/std": 0.24593468010425568, "sampling/importance_sampling_ratio/max": 1.4387227296829224, "sampling/importance_sampling_ratio/mean": 0.9996896982192993, "sampling/importance_sampling_ratio/min": 0.6817339658737183, "sampling/sampling_logp_difference/max": 0.3831157684326172, "sampling/sampling_logp_difference/mean": 0.010619109496474266, "step": 631 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 729.0, "completions/mean_length": 509.125, "completions/mean_terminated_length": 500.774169921875, "completions/min_length": 321.0, "completions/min_terminated_length": 321.0, "entropy": 0.2965533994138241, "epoch": 0.33832976445396146, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 13676968.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.3712502717971802, "sampling/importance_sampling_ratio/mean": 1.0000545978546143, "sampling/importance_sampling_ratio/min": 0.7730183005332947, "sampling/sampling_logp_difference/max": 0.3157229423522949, "sampling/sampling_logp_difference/mean": 0.009084545075893402, "step": 632 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00012959235755261034, "clip_ratio/low_min": 0.00012959235755261034, "clip_ratio/region_mean": 0.00012959235755261034, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 724.0, "completions/mean_length": 524.375, "completions/mean_terminated_length": 499.17242431640625, "completions/min_length": 310.0, "completions/min_terminated_length": 310.0, "entropy": 0.41893286257982254, "epoch": 0.33886509635974305, "frac_reward_zero_std": 0.5, "grad_norm": 0.00824392307549715, "learning_rate": 1e-05, "loss": 0.0185, "num_tokens": 13698004.0, "reward": 0.625, "reward_std": 0.2314550280570984, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.4280670881271362, "sampling/importance_sampling_ratio/mean": 1.0004184246063232, "sampling/importance_sampling_ratio/min": 0.5557708740234375, "sampling/sampling_logp_difference/max": 0.5873992443084717, "sampling/sampling_logp_difference/mean": 0.012900300323963165, "step": 633 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 671.0, "completions/mean_length": 395.78125, "completions/mean_terminated_length": 383.774169921875, "completions/min_length": 186.0, "completions/min_terminated_length": 186.0, "entropy": 0.32786916568875313, "epoch": 0.33940042826552463, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": -0.002, "num_tokens": 13714381.0, "reward": 0.90625, "reward_std": 0.1293872892856598, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.3311372995376587, "sampling/importance_sampling_ratio/mean": 1.000272274017334, "sampling/importance_sampling_ratio/min": 0.7063815593719482, "sampling/sampling_logp_difference/max": 0.34759974479675293, "sampling/sampling_logp_difference/mean": 0.011236365884542465, "step": 634 }, { "clip_ratio/high_max": 6.55479816487059e-05, "clip_ratio/high_mean": 6.55479816487059e-05, "clip_ratio/low_mean": 0.00023448842330253683, "clip_ratio/low_min": 0.00023448842330253683, "clip_ratio/region_mean": 0.00030003640495124273, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 727.0, "completions/mean_length": 530.1875, "completions/mean_terminated_length": 450.91668701171875, "completions/min_length": 198.0, "completions/min_terminated_length": 198.0, "entropy": 0.354931078851223, "epoch": 0.3399357601713062, "frac_reward_zero_std": 0.25, "grad_norm": 0.021596211940050125, "learning_rate": 1e-05, "loss": -0.0105, "num_tokens": 13735019.0, "reward": 0.71875, "reward_std": 0.2651650309562683, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.5256561040878296, "sampling/importance_sampling_ratio/mean": 1.0000821352005005, "sampling/importance_sampling_ratio/min": 0.6678833961486816, "sampling/sampling_logp_difference/max": 0.4224245548248291, "sampling/sampling_logp_difference/mean": 0.01115508284419775, "step": 635 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 675.0, "completions/max_terminated_length": 675.0, "completions/mean_length": 429.15625, "completions/mean_terminated_length": 429.15625, "completions/min_length": 253.0, "completions/min_terminated_length": 253.0, "entropy": 0.29260409623384476, "epoch": 0.3404710920770878, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 13752848.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.4287768602371216, "sampling/importance_sampling_ratio/mean": 0.9997872114181519, "sampling/importance_sampling_ratio/min": 0.7665042281150818, "sampling/sampling_logp_difference/max": 0.35681867599487305, "sampling/sampling_logp_difference/mean": 0.009826496243476868, "step": 636 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00015769056699355133, "clip_ratio/low_min": 0.00015769056699355133, "clip_ratio/region_mean": 0.00015769056699355133, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 742.0, "completions/mean_length": 554.96875, "completions/mean_terminated_length": 540.7667236328125, "completions/min_length": 208.0, "completions/min_terminated_length": 208.0, "entropy": 0.3323060441762209, "epoch": 0.3410064239828694, "frac_reward_zero_std": 0.0, "grad_norm": 0.014778217300772667, "learning_rate": 1e-05, "loss": 0.0251, "num_tokens": 13775199.0, "reward": 0.59375, "reward_std": 0.4807935357093811, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.4481210708618164, "sampling/importance_sampling_ratio/mean": 1.000173568725586, "sampling/importance_sampling_ratio/min": 0.7709746956825256, "sampling/sampling_logp_difference/max": 0.3702669143676758, "sampling/sampling_logp_difference/mean": 0.011516406200826168, "step": 637 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 749.0, "completions/mean_length": 554.5625, "completions/mean_terminated_length": 505.3077087402344, "completions/min_length": 263.0, "completions/min_terminated_length": 263.0, "entropy": 0.45366695895791054, "epoch": 0.341541755888651, "frac_reward_zero_std": 0.5, "grad_norm": 0.015367541462182999, "learning_rate": 1e-05, "loss": -0.0083, "num_tokens": 13797153.0, "reward": 0.625, "reward_std": 0.2314550280570984, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.461901307106018, "sampling/importance_sampling_ratio/mean": 1.0005213022232056, "sampling/importance_sampling_ratio/min": 0.6465102434158325, "sampling/sampling_logp_difference/max": 0.43616628646850586, "sampling/sampling_logp_difference/mean": 0.013607689179480076, "step": 638 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.610259151784703e-05, "clip_ratio/low_min": 6.610259151784703e-05, "clip_ratio/region_mean": 6.610259151784703e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 682.0, "completions/max_terminated_length": 682.0, "completions/mean_length": 475.34375, "completions/mean_terminated_length": 475.34375, "completions/min_length": 289.0, "completions/min_terminated_length": 289.0, "entropy": 0.26533373817801476, "epoch": 0.3420770877944325, "frac_reward_zero_std": 0.25, "grad_norm": 0.026921937242150307, "learning_rate": 1e-05, "loss": 0.0135, "num_tokens": 13815828.0, "reward": 0.59375, "reward_std": 0.3198433816432953, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.4592902660369873, "sampling/importance_sampling_ratio/mean": 0.9997817873954773, "sampling/importance_sampling_ratio/min": 0.7529975771903992, "sampling/sampling_logp_difference/max": 0.37795019149780273, "sampling/sampling_logp_difference/mean": 0.009174923412501812, "step": 639 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 735.0, "completions/max_terminated_length": 735.0, "completions/mean_length": 410.84375, "completions/mean_terminated_length": 410.84375, "completions/min_length": 185.0, "completions/min_terminated_length": 185.0, "entropy": 0.3154573068022728, "epoch": 0.3426124197002141, "frac_reward_zero_std": 0.75, "grad_norm": 0.01859649457037449, "learning_rate": 1e-05, "loss": -0.0104, "num_tokens": 13832559.0, "reward": 0.9375, "reward_std": 0.1157275140285492, "rewards/accuracy_reward/mean": 0.9375, "rewards/accuracy_reward/std": 0.24593468010425568, "sampling/importance_sampling_ratio/max": 1.2813645601272583, "sampling/importance_sampling_ratio/mean": 1.0000611543655396, "sampling/importance_sampling_ratio/min": 0.700233519077301, "sampling/sampling_logp_difference/max": 0.3563413619995117, "sampling/sampling_logp_difference/mean": 0.010139619931578636, "step": 640 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 692.0, "completions/mean_length": 477.09375, "completions/mean_terminated_length": 467.70965576171875, "completions/min_length": 271.0, "completions/min_terminated_length": 271.0, "entropy": 0.2767404969781637, "epoch": 0.3431477516059957, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0419, "num_tokens": 13851330.0, "reward": 0.90625, "reward_std": 0.1293872892856598, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.3960591554641724, "sampling/importance_sampling_ratio/mean": 1.0000600814819336, "sampling/importance_sampling_ratio/min": 0.7296417951583862, "sampling/sampling_logp_difference/max": 0.33365345001220703, "sampling/sampling_logp_difference/mean": 0.00965786911547184, "step": 641 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.972288636257872e-05, "clip_ratio/low_min": 5.972288636257872e-05, "clip_ratio/region_mean": 5.972288636257872e-05, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 698.0, "completions/mean_length": 456.15625, "completions/mean_terminated_length": 423.89654541015625, "completions/min_length": 247.0, "completions/min_terminated_length": 247.0, "entropy": 0.3610168881714344, "epoch": 0.3436830835117773, "frac_reward_zero_std": 0.75, "grad_norm": 0.011776655912399292, "learning_rate": 1e-05, "loss": 0.0774, "num_tokens": 13869991.0, "reward": 0.84375, "reward_std": 0.1293872892856598, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.6153571605682373, "sampling/importance_sampling_ratio/mean": 1.0004236698150635, "sampling/importance_sampling_ratio/min": 0.7631751298904419, "sampling/sampling_logp_difference/max": 0.4795560836791992, "sampling/sampling_logp_difference/mean": 0.012264749966561794, "step": 642 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.3125, "completions/max_length": 768.0, "completions/max_terminated_length": 766.0, "completions/mean_length": 570.5, "completions/mean_terminated_length": 480.727294921875, "completions/min_length": 313.0, "completions/min_terminated_length": 313.0, "entropy": 0.4524570107460022, "epoch": 0.3442184154175589, "frac_reward_zero_std": 0.75, "grad_norm": 0.005275725852698088, "learning_rate": 1e-05, "loss": 0.0277, "num_tokens": 13892391.0, "reward": 0.6875, "reward_std": 0.1157275140285492, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.6950676441192627, "sampling/importance_sampling_ratio/mean": 0.9998581409454346, "sampling/importance_sampling_ratio/min": 0.6842971444129944, "sampling/sampling_logp_difference/max": 0.5277225971221924, "sampling/sampling_logp_difference/mean": 0.014087657444179058, "step": 643 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 741.0, "completions/mean_length": 432.8125, "completions/mean_terminated_length": 422.0, "completions/min_length": 141.0, "completions/min_terminated_length": 141.0, "entropy": 0.31366887129843235, "epoch": 0.34475374732334046, "frac_reward_zero_std": 0.75, "grad_norm": 0.012850224040448666, "learning_rate": 1e-05, "loss": 0.0203, "num_tokens": 13909697.0, "reward": 0.875, "reward_std": 0.13363061845302582, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 1.425286054611206, "sampling/importance_sampling_ratio/mean": 1.0000022649765015, "sampling/importance_sampling_ratio/min": 0.6593077182769775, "sampling/sampling_logp_difference/max": 0.41656494140625, "sampling/sampling_logp_difference/mean": 0.009947948157787323, "step": 644 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0001031969441100955, "clip_ratio/low_min": 0.0001031969441100955, "clip_ratio/region_mean": 0.0001031969441100955, "completions/clipped_ratio": 0.28125, "completions/max_length": 768.0, "completions/max_terminated_length": 758.0, "completions/mean_length": 544.3125, "completions/mean_terminated_length": 456.7826232910156, "completions/min_length": 251.0, "completions/min_terminated_length": 251.0, "entropy": 0.37259775772690773, "epoch": 0.34528907922912205, "frac_reward_zero_std": 0.25, "grad_norm": 0.010465526022017002, "learning_rate": 1e-05, "loss": 0.1164, "num_tokens": 13930667.0, "reward": 0.625, "reward_std": 0.3104073107242584, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.3667930364608765, "sampling/importance_sampling_ratio/mean": 1.0000056028366089, "sampling/importance_sampling_ratio/min": 0.6974546909332275, "sampling/sampling_logp_difference/max": 0.3603177070617676, "sampling/sampling_logp_difference/mean": 0.011153936386108398, "step": 645 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00011498959065647796, "clip_ratio/low_min": 0.00011498959065647796, "clip_ratio/region_mean": 0.00011498959065647796, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 700.0, "completions/mean_length": 492.8125, "completions/mean_terminated_length": 453.5000305175781, "completions/min_length": 236.0, "completions/min_terminated_length": 236.0, "entropy": 0.3283007889986038, "epoch": 0.34582441113490364, "frac_reward_zero_std": 0.5, "grad_norm": 0.008446267805993557, "learning_rate": 1e-05, "loss": 0.0214, "num_tokens": 13950317.0, "reward": 0.78125, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.4042552709579468, "sampling/importance_sampling_ratio/mean": 1.0003057718276978, "sampling/importance_sampling_ratio/min": 0.7099096775054932, "sampling/sampling_logp_difference/max": 0.3426175117492676, "sampling/sampling_logp_difference/mean": 0.011360456235706806, "step": 646 }, { "clip_ratio/high_max": 6.0444875998655334e-05, "clip_ratio/high_mean": 6.0444875998655334e-05, "clip_ratio/low_mean": 0.00010931084398180246, "clip_ratio/low_min": 0.00010931084398180246, "clip_ratio/region_mean": 0.0001697557199804578, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 694.0, "completions/mean_length": 546.28125, "completions/mean_terminated_length": 472.375, "completions/min_length": 244.0, "completions/min_terminated_length": 244.0, "entropy": 0.42438481748104095, "epoch": 0.34635974304068523, "frac_reward_zero_std": 0.0, "grad_norm": 0.0141336964443326, "learning_rate": 1e-05, "loss": -0.0179, "num_tokens": 13972102.0, "reward": 0.40625, "reward_std": 0.4218915104866028, "rewards/accuracy_reward/mean": 0.40625, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.6954357624053955, "sampling/importance_sampling_ratio/mean": 1.000226616859436, "sampling/importance_sampling_ratio/min": 0.6467899084091187, "sampling/sampling_logp_difference/max": 0.5279397964477539, "sampling/sampling_logp_difference/mean": 0.013065065257251263, "step": 647 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00013569513976108283, "clip_ratio/low_min": 0.00013569513976108283, "clip_ratio/region_mean": 0.00013569513976108283, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 742.0, "completions/mean_length": 524.4375, "completions/mean_terminated_length": 489.64288330078125, "completions/min_length": 266.0, "completions/min_terminated_length": 266.0, "entropy": 0.30432552844285965, "epoch": 0.3468950749464668, "frac_reward_zero_std": 0.25, "grad_norm": 0.004665049258619547, "learning_rate": 1e-05, "loss": 0.0898, "num_tokens": 13992652.0, "reward": 0.625, "reward_std": 0.3104073107242584, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.4343668222427368, "sampling/importance_sampling_ratio/mean": 1.0000685453414917, "sampling/importance_sampling_ratio/min": 0.6709504127502441, "sampling/sampling_logp_difference/max": 0.3990600109100342, "sampling/sampling_logp_difference/mean": 0.009907660074532032, "step": 648 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 742.0, "completions/mean_length": 475.53125, "completions/mean_terminated_length": 466.0967712402344, "completions/min_length": 279.0, "completions/min_terminated_length": 279.0, "entropy": 0.23276705294847488, "epoch": 0.3474304068522484, "frac_reward_zero_std": 0.5, "grad_norm": 0.009352444671094418, "learning_rate": 1e-05, "loss": 0.0069, "num_tokens": 14011093.0, "reward": 0.875, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 1.4106377363204956, "sampling/importance_sampling_ratio/mean": 1.0000784397125244, "sampling/importance_sampling_ratio/min": 0.7408989667892456, "sampling/sampling_logp_difference/max": 0.3440418243408203, "sampling/sampling_logp_difference/mean": 0.0077589633874595165, "step": 649 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00023136611707741395, "clip_ratio/low_min": 0.00023136611707741395, "clip_ratio/region_mean": 0.00023136611707741395, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 767.0, "completions/mean_length": 426.28125, "completions/mean_terminated_length": 415.258056640625, "completions/min_length": 213.0, "completions/min_terminated_length": 213.0, "entropy": 0.37946831807494164, "epoch": 0.34796573875803, "frac_reward_zero_std": 0.5, "grad_norm": 0.015928439795970917, "learning_rate": 1e-05, "loss": 0.0351, "num_tokens": 14028910.0, "reward": 0.78125, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.3176966905593872, "sampling/importance_sampling_ratio/mean": 0.9998493790626526, "sampling/importance_sampling_ratio/min": 0.7060791850090027, "sampling/sampling_logp_difference/max": 0.34802794456481934, "sampling/sampling_logp_difference/mean": 0.012711400166153908, "step": 650 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00010587761789793149, "clip_ratio/low_min": 0.00010587761789793149, "clip_ratio/region_mean": 0.00010587761789793149, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 732.0, "completions/mean_length": 561.4375, "completions/mean_terminated_length": 492.5833435058594, "completions/min_length": 230.0, "completions/min_terminated_length": 230.0, "entropy": 0.40715794265270233, "epoch": 0.3485010706638116, "frac_reward_zero_std": 0.5, "grad_norm": 0.009008923545479774, "learning_rate": 1e-05, "loss": 0.047, "num_tokens": 14051060.0, "reward": 0.40625, "reward_std": 0.22201895713806152, "rewards/accuracy_reward/mean": 0.40625, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.556815266609192, "sampling/importance_sampling_ratio/mean": 0.9994319081306458, "sampling/importance_sampling_ratio/min": 0.6722291707992554, "sampling/sampling_logp_difference/max": 0.4426422119140625, "sampling/sampling_logp_difference/mean": 0.01241252664476633, "step": 651 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 8.698677993379533e-05, "clip_ratio/low_min": 8.698677993379533e-05, "clip_ratio/region_mean": 8.698677993379533e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 733.0, "completions/max_terminated_length": 733.0, "completions/mean_length": 408.71875, "completions/mean_terminated_length": 408.71875, "completions/min_length": 202.0, "completions/min_terminated_length": 202.0, "entropy": 0.350768007338047, "epoch": 0.3490364025695932, "frac_reward_zero_std": 0.5, "grad_norm": 0.03018689714372158, "learning_rate": 1e-05, "loss": 0.0085, "num_tokens": 14067859.0, "reward": 0.8125, "reward_std": 0.249358132481575, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.3940702676773071, "sampling/importance_sampling_ratio/mean": 0.9998666048049927, "sampling/importance_sampling_ratio/min": 0.7021839022636414, "sampling/sampling_logp_difference/max": 0.3535599708557129, "sampling/sampling_logp_difference/mean": 0.011442244052886963, "step": 652 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 627.0, "completions/max_terminated_length": 627.0, "completions/mean_length": 525.9375, "completions/mean_terminated_length": 525.9375, "completions/min_length": 428.0, "completions/min_terminated_length": 428.0, "entropy": 0.26179109886288643, "epoch": 0.3495717344753747, "frac_reward_zero_std": 0.25, "grad_norm": 0.014005845412611961, "learning_rate": 1e-05, "loss": -0.0034, "num_tokens": 14088801.0, "reward": 0.6875, "reward_std": 0.3924052119255066, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.3349133729934692, "sampling/importance_sampling_ratio/mean": 0.9999707937240601, "sampling/importance_sampling_ratio/min": 0.7767595648765564, "sampling/sampling_logp_difference/max": 0.28886640071868896, "sampling/sampling_logp_difference/mean": 0.008738524280488491, "step": 653 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00011466631258372217, "clip_ratio/low_min": 0.00011466631258372217, "clip_ratio/region_mean": 0.00011466631258372217, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 585.0, "completions/mean_length": 486.71875, "completions/mean_terminated_length": 407.9599914550781, "completions/min_length": 271.0, "completions/min_terminated_length": 271.0, "entropy": 0.5003787465393543, "epoch": 0.3501070663811563, "frac_reward_zero_std": 0.5, "grad_norm": 0.028450779616832733, "learning_rate": 1e-05, "loss": 0.0732, "num_tokens": 14107896.0, "reward": 0.71875, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.380290150642395, "sampling/importance_sampling_ratio/mean": 0.9999485611915588, "sampling/importance_sampling_ratio/min": 0.5255972743034363, "sampling/sampling_logp_difference/max": 0.6432199478149414, "sampling/sampling_logp_difference/mean": 0.015035094693303108, "step": 654 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.720823901356198e-05, "clip_ratio/low_min": 5.720823901356198e-05, "clip_ratio/region_mean": 5.720823901356198e-05, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 671.0, "completions/mean_length": 469.28125, "completions/mean_terminated_length": 413.96295166015625, "completions/min_length": 237.0, "completions/min_terminated_length": 237.0, "entropy": 0.3441755622625351, "epoch": 0.3506423982869379, "frac_reward_zero_std": 0.25, "grad_norm": 0.0384136363863945, "learning_rate": 1e-05, "loss": 0.1092, "num_tokens": 14126193.0, "reward": 0.71875, "reward_std": 0.3198433816432953, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.4771353006362915, "sampling/importance_sampling_ratio/mean": 1.0000910758972168, "sampling/importance_sampling_ratio/min": 0.5364977121353149, "sampling/sampling_logp_difference/max": 0.6226930618286133, "sampling/sampling_logp_difference/mean": 0.011351614259183407, "step": 655 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.860290548298508e-05, "clip_ratio/low_min": 5.860290548298508e-05, "clip_ratio/region_mean": 5.860290548298508e-05, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 719.0, "completions/mean_length": 570.46875, "completions/mean_terminated_length": 550.0344848632812, "completions/min_length": 365.0, "completions/min_terminated_length": 365.0, "entropy": 0.3153909929096699, "epoch": 0.3511777301927195, "frac_reward_zero_std": 0.0, "grad_norm": 0.0221896693110466, "learning_rate": 1e-05, "loss": 0.028, "num_tokens": 14147664.0, "reward": 0.6875, "reward_std": 0.4808131456375122, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.607094645500183, "sampling/importance_sampling_ratio/mean": 1.0001755952835083, "sampling/importance_sampling_ratio/min": 0.7048884630203247, "sampling/sampling_logp_difference/max": 0.4744279384613037, "sampling/sampling_logp_difference/mean": 0.009652480483055115, "step": 656 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 662.0, "completions/max_terminated_length": 662.0, "completions/mean_length": 392.90625, "completions/mean_terminated_length": 392.90625, "completions/min_length": 185.0, "completions/min_terminated_length": 185.0, "entropy": 0.34484558179974556, "epoch": 0.35171306209850106, "frac_reward_zero_std": 0.5, "grad_norm": 0.012628809548914433, "learning_rate": 1e-05, "loss": 0.0514, "num_tokens": 14164893.0, "reward": 0.875, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 1.3562942743301392, "sampling/importance_sampling_ratio/mean": 0.999950647354126, "sampling/importance_sampling_ratio/min": 0.7395182251930237, "sampling/sampling_logp_difference/max": 0.30475616455078125, "sampling/sampling_logp_difference/mean": 0.011204061098396778, "step": 657 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 737.0, "completions/mean_length": 451.875, "completions/mean_terminated_length": 441.6773986816406, "completions/min_length": 264.0, "completions/min_terminated_length": 264.0, "entropy": 0.30138526670634747, "epoch": 0.35224839400428265, "frac_reward_zero_std": 0.5, "grad_norm": 0.014551598578691483, "learning_rate": 1e-05, "loss": 0.0002, "num_tokens": 14183577.0, "reward": 0.90625, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.8423817157745361, "sampling/importance_sampling_ratio/mean": 1.0001647472381592, "sampling/importance_sampling_ratio/min": 0.6992892622947693, "sampling/sampling_logp_difference/max": 0.6110591888427734, "sampling/sampling_logp_difference/mean": 0.010228762403130531, "step": 658 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00010833868509507738, "clip_ratio/low_min": 0.00010833868509507738, "clip_ratio/region_mean": 0.00010833868509507738, "completions/clipped_ratio": 0.28125, "completions/max_length": 768.0, "completions/max_terminated_length": 743.0, "completions/mean_length": 517.625, "completions/mean_terminated_length": 419.6521911621094, "completions/min_length": 276.0, "completions/min_terminated_length": 276.0, "entropy": 0.38055815920233727, "epoch": 0.35278372591006424, "frac_reward_zero_std": 0.25, "grad_norm": 0.008826745674014091, "learning_rate": 1e-05, "loss": 0.0248, "num_tokens": 14204133.0, "reward": 0.46875, "reward_std": 0.3061639964580536, "rewards/accuracy_reward/mean": 0.46875, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.4185012578964233, "sampling/importance_sampling_ratio/mean": 1.0001429319381714, "sampling/importance_sampling_ratio/min": 0.7214440703392029, "sampling/sampling_logp_difference/max": 0.34960079193115234, "sampling/sampling_logp_difference/mean": 0.011931713670492172, "step": 659 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00012206177780171856, "clip_ratio/low_min": 0.00012206177780171856, "clip_ratio/region_mean": 0.00012206177780171856, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 737.0, "completions/mean_length": 514.0625, "completions/mean_terminated_length": 497.13336181640625, "completions/min_length": 301.0, "completions/min_terminated_length": 301.0, "entropy": 0.2995508201420307, "epoch": 0.3533190578158458, "frac_reward_zero_std": 0.25, "grad_norm": 0.010350965894758701, "learning_rate": 1e-05, "loss": 0.0044, "num_tokens": 14224871.0, "reward": 0.65625, "reward_std": 0.3377464711666107, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.434417486190796, "sampling/importance_sampling_ratio/mean": 0.9998875260353088, "sampling/importance_sampling_ratio/min": 0.6783097386360168, "sampling/sampling_logp_difference/max": 0.3881511688232422, "sampling/sampling_logp_difference/mean": 0.009691186249256134, "step": 660 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 623.0, "completions/max_terminated_length": 623.0, "completions/mean_length": 433.25, "completions/mean_terminated_length": 433.25, "completions/min_length": 241.0, "completions/min_terminated_length": 241.0, "entropy": 0.32164764404296875, "epoch": 0.3538543897216274, "frac_reward_zero_std": 0.25, "grad_norm": 0.010363774374127388, "learning_rate": 1e-05, "loss": -0.0382, "num_tokens": 14242391.0, "reward": 0.75, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.413983941078186, "sampling/importance_sampling_ratio/mean": 1.0000758171081543, "sampling/importance_sampling_ratio/min": 0.6953238248825073, "sampling/sampling_logp_difference/max": 0.36337757110595703, "sampling/sampling_logp_difference/mean": 0.010926454328000546, "step": 661 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 713.0, "completions/max_terminated_length": 713.0, "completions/mean_length": 436.09375, "completions/mean_terminated_length": 436.09375, "completions/min_length": 221.0, "completions/min_terminated_length": 221.0, "entropy": 0.25281756557524204, "epoch": 0.354389721627409, "frac_reward_zero_std": 0.5, "grad_norm": 0.02384403720498085, "learning_rate": 1e-05, "loss": -0.0308, "num_tokens": 14259514.0, "reward": 0.9375, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.9375, "rewards/accuracy_reward/std": 0.24593468010425568, "sampling/importance_sampling_ratio/max": 1.2802929878234863, "sampling/importance_sampling_ratio/mean": 0.999589741230011, "sampling/importance_sampling_ratio/min": 0.6354244351387024, "sampling/sampling_logp_difference/max": 0.4534621238708496, "sampling/sampling_logp_difference/mean": 0.008979644626379013, "step": 662 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00021215934248175472, "clip_ratio/low_min": 0.00021215934248175472, "clip_ratio/region_mean": 0.00021215934248175472, "completions/clipped_ratio": 0.3125, "completions/max_length": 768.0, "completions/max_terminated_length": 686.0, "completions/mean_length": 560.6875, "completions/mean_terminated_length": 466.4545593261719, "completions/min_length": 242.0, "completions/min_terminated_length": 242.0, "entropy": 0.43268299102783203, "epoch": 0.3549250535331906, "frac_reward_zero_std": 0.0, "grad_norm": 0.020790722221136093, "learning_rate": 1e-05, "loss": 0.0202, "num_tokens": 14281560.0, "reward": 0.46875, "reward_std": 0.4397946000099182, "rewards/accuracy_reward/mean": 0.46875, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.4320964813232422, "sampling/importance_sampling_ratio/mean": 0.9997743964195251, "sampling/importance_sampling_ratio/min": 0.5930963158607483, "sampling/sampling_logp_difference/max": 0.5223984718322754, "sampling/sampling_logp_difference/mean": 0.013722680509090424, "step": 663 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 704.0, "completions/mean_length": 474.0, "completions/mean_terminated_length": 464.51611328125, "completions/min_length": 264.0, "completions/min_terminated_length": 264.0, "entropy": 0.4267589747905731, "epoch": 0.3554603854389722, "frac_reward_zero_std": 0.5, "grad_norm": 0.009368442930281162, "learning_rate": 1e-05, "loss": 0.0372, "num_tokens": 14301176.0, "reward": 0.84375, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.8331667184829712, "sampling/importance_sampling_ratio/mean": 0.9994924664497375, "sampling/importance_sampling_ratio/min": 0.5951104164123535, "sampling/sampling_logp_difference/max": 0.6060450077056885, "sampling/sampling_logp_difference/mean": 0.01349746622145176, "step": 664 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 7.79301772126928e-05, "clip_ratio/low_min": 7.79301772126928e-05, "clip_ratio/region_mean": 7.79301772126928e-05, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 652.0, "completions/mean_length": 418.25, "completions/mean_terminated_length": 394.933349609375, "completions/min_length": 141.0, "completions/min_terminated_length": 141.0, "entropy": 0.2833059933036566, "epoch": 0.35599571734475377, "frac_reward_zero_std": 0.75, "grad_norm": 0.008563783019781113, "learning_rate": 1e-05, "loss": 0.0533, "num_tokens": 14317992.0, "reward": 0.8125, "reward_std": 0.1157275140285492, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.484289288520813, "sampling/importance_sampling_ratio/mean": 1.0001963376998901, "sampling/importance_sampling_ratio/min": 0.6188217401504517, "sampling/sampling_logp_difference/max": 0.4799380302429199, "sampling/sampling_logp_difference/mean": 0.01006927527487278, "step": 665 }, { "clip_ratio/high_max": 5.397236600401811e-05, "clip_ratio/high_mean": 5.397236600401811e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 5.397236600401811e-05, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 737.0, "completions/mean_length": 561.25, "completions/mean_terminated_length": 531.7142944335938, "completions/min_length": 215.0, "completions/min_terminated_length": 215.0, "entropy": 0.29960734583437443, "epoch": 0.35653104925053536, "frac_reward_zero_std": 0.5, "grad_norm": 0.007625725120306015, "learning_rate": 1e-05, "loss": 0.0084, "num_tokens": 14339680.0, "reward": 0.875, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 1.432206392288208, "sampling/importance_sampling_ratio/mean": 1.0004332065582275, "sampling/importance_sampling_ratio/min": 0.7404946088790894, "sampling/sampling_logp_difference/max": 0.35921621322631836, "sampling/sampling_logp_difference/mean": 0.010498258285224438, "step": 666 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.000206250864721369, "clip_ratio/low_min": 0.000206250864721369, "clip_ratio/region_mean": 0.000206250864721369, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 615.0, "completions/mean_length": 427.5625, "completions/mean_terminated_length": 404.86669921875, "completions/min_length": 227.0, "completions/min_terminated_length": 227.0, "entropy": 0.3941080830991268, "epoch": 0.3570663811563169, "frac_reward_zero_std": 0.25, "grad_norm": 0.020041802898049355, "learning_rate": 1e-05, "loss": 0.0143, "num_tokens": 14356858.0, "reward": 0.65625, "reward_std": 0.38816186785697937, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.396597146987915, "sampling/importance_sampling_ratio/mean": 0.9999650120735168, "sampling/importance_sampling_ratio/min": 0.6440496444702148, "sampling/sampling_logp_difference/max": 0.4399794340133667, "sampling/sampling_logp_difference/mean": 0.012557687237858772, "step": 667 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.006727562635206e-05, "clip_ratio/low_min": 6.006727562635206e-05, "clip_ratio/region_mean": 6.006727562635206e-05, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 759.0, "completions/mean_length": 449.6875, "completions/mean_terminated_length": 439.4193420410156, "completions/min_length": 220.0, "completions/min_terminated_length": 220.0, "entropy": 0.2734356876462698, "epoch": 0.3576017130620985, "frac_reward_zero_std": 0.5, "grad_norm": 0.006712873931974173, "learning_rate": 1e-05, "loss": 0.0749, "num_tokens": 14374464.0, "reward": 0.9375, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.9375, "rewards/accuracy_reward/std": 0.24593468010425568, "sampling/importance_sampling_ratio/max": 1.2982434034347534, "sampling/importance_sampling_ratio/mean": 1.000036597251892, "sampling/importance_sampling_ratio/min": 0.7399297952651978, "sampling/sampling_logp_difference/max": 0.30119991302490234, "sampling/sampling_logp_difference/mean": 0.009010636247694492, "step": 668 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 614.0, "completions/max_terminated_length": 614.0, "completions/mean_length": 399.25, "completions/mean_terminated_length": 399.25, "completions/min_length": 258.0, "completions/min_terminated_length": 258.0, "entropy": 0.2868831194937229, "epoch": 0.35813704496788007, "frac_reward_zero_std": 0.75, "grad_norm": 0.004024685360491276, "learning_rate": 1e-05, "loss": -0.0013, "num_tokens": 14391328.0, "reward": 0.96875, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.96875, "rewards/accuracy_reward/std": 0.1767766922712326, "sampling/importance_sampling_ratio/max": 1.7804118394851685, "sampling/importance_sampling_ratio/mean": 0.9999974966049194, "sampling/importance_sampling_ratio/min": 0.749528706073761, "sampling/sampling_logp_difference/max": 0.5768446922302246, "sampling/sampling_logp_difference/mean": 0.009922800585627556, "step": 669 }, { "clip_ratio/high_max": 5.55308761249762e-05, "clip_ratio/high_mean": 5.55308761249762e-05, "clip_ratio/low_mean": 5.582849553320557e-05, "clip_ratio/low_min": 5.582849553320557e-05, "clip_ratio/region_mean": 0.00011135937165818177, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 752.0, "completions/mean_length": 503.875, "completions/mean_terminated_length": 415.8333435058594, "completions/min_length": 226.0, "completions/min_terminated_length": 226.0, "entropy": 0.4187277927994728, "epoch": 0.35867237687366166, "frac_reward_zero_std": 0.75, "grad_norm": 0.00617664260789752, "learning_rate": 1e-05, "loss": 0.0103, "num_tokens": 14411612.0, "reward": 0.6875, "reward_std": 0.1157275140285492, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.4836184978485107, "sampling/importance_sampling_ratio/mean": 1.0000054836273193, "sampling/importance_sampling_ratio/min": 0.610197126865387, "sampling/sampling_logp_difference/max": 0.4939732551574707, "sampling/sampling_logp_difference/mean": 0.012816681526601315, "step": 670 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.668934318237007e-05, "clip_ratio/low_min": 5.668934318237007e-05, "clip_ratio/region_mean": 5.668934318237007e-05, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 598.0, "completions/mean_length": 472.78125, "completions/mean_terminated_length": 442.2413635253906, "completions/min_length": 252.0, "completions/min_terminated_length": 252.0, "entropy": 0.28476603515446186, "epoch": 0.35920770877944325, "frac_reward_zero_std": 0.0, "grad_norm": 0.01838054321706295, "learning_rate": 1e-05, "loss": 0.0899, "num_tokens": 14430253.0, "reward": 0.78125, "reward_std": 0.4218915104866028, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.3775122165679932, "sampling/importance_sampling_ratio/mean": 0.9997398257255554, "sampling/importance_sampling_ratio/min": 0.6943086981773376, "sampling/sampling_logp_difference/max": 0.3648386001586914, "sampling/sampling_logp_difference/mean": 0.009384599514305592, "step": 671 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 681.0, "completions/mean_length": 456.875, "completions/mean_terminated_length": 446.83868408203125, "completions/min_length": 236.0, "completions/min_terminated_length": 236.0, "entropy": 0.3012051973491907, "epoch": 0.35974304068522484, "frac_reward_zero_std": 0.5, "grad_norm": 0.01845335215330124, "learning_rate": 1e-05, "loss": 0.0221, "num_tokens": 14448897.0, "reward": 0.875, "reward_std": 0.2314550280570984, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 1.390568494796753, "sampling/importance_sampling_ratio/mean": 1.000192642211914, "sampling/importance_sampling_ratio/min": 0.5708194375038147, "sampling/sampling_logp_difference/max": 0.5606822967529297, "sampling/sampling_logp_difference/mean": 0.010296937078237534, "step": 672 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 631.0, "completions/max_terminated_length": 631.0, "completions/mean_length": 393.03125, "completions/mean_terminated_length": 393.03125, "completions/min_length": 118.0, "completions/min_terminated_length": 118.0, "entropy": 0.27892127074301243, "epoch": 0.3602783725910064, "frac_reward_zero_std": 0.25, "grad_norm": 0.024538559839129448, "learning_rate": 1e-05, "loss": -0.0102, "num_tokens": 14465018.0, "reward": 0.90625, "reward_std": 0.2651650309562683, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.41757071018219, "sampling/importance_sampling_ratio/mean": 0.9998502731323242, "sampling/importance_sampling_ratio/min": 0.700309693813324, "sampling/sampling_logp_difference/max": 0.3562326431274414, "sampling/sampling_logp_difference/mean": 0.010319615714251995, "step": 673 }, { "clip_ratio/high_max": 6.558236782439053e-05, "clip_ratio/high_mean": 6.558236782439053e-05, "clip_ratio/low_mean": 5.653550397255458e-05, "clip_ratio/low_min": 5.653550397255458e-05, "clip_ratio/region_mean": 0.0001221178717969451, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 718.0, "completions/mean_length": 543.15625, "completions/mean_terminated_length": 501.5185241699219, "completions/min_length": 213.0, "completions/min_terminated_length": 213.0, "entropy": 0.4119257219135761, "epoch": 0.360813704496788, "frac_reward_zero_std": 0.25, "grad_norm": 0.02290952019393444, "learning_rate": 1e-05, "loss": 0.0779, "num_tokens": 14486527.0, "reward": 0.71875, "reward_std": 0.3198433816432953, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.5678411722183228, "sampling/importance_sampling_ratio/mean": 0.9999833703041077, "sampling/importance_sampling_ratio/min": 0.7061614990234375, "sampling/sampling_logp_difference/max": 0.44969964027404785, "sampling/sampling_logp_difference/mean": 0.01298696082085371, "step": 674 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 8.417508070124313e-05, "clip_ratio/low_min": 8.417508070124313e-05, "clip_ratio/region_mean": 8.417508070124313e-05, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 659.0, "completions/mean_length": 418.03125, "completions/mean_terminated_length": 406.7419128417969, "completions/min_length": 216.0, "completions/min_terminated_length": 216.0, "entropy": 0.3174128942191601, "epoch": 0.3613490364025696, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": -0.0409, "num_tokens": 14503728.0, "reward": 0.9375, "reward_std": 0.1157275140285492, "rewards/accuracy_reward/mean": 0.9375, "rewards/accuracy_reward/std": 0.24593468010425568, "sampling/importance_sampling_ratio/max": 1.3247625827789307, "sampling/importance_sampling_ratio/mean": 1.0001379251480103, "sampling/importance_sampling_ratio/min": 0.6964145302772522, "sampling/sampling_logp_difference/max": 0.36181020736694336, "sampling/sampling_logp_difference/mean": 0.01076096948236227, "step": 675 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0001639887923374772, "clip_ratio/low_min": 0.0001639887923374772, "clip_ratio/region_mean": 0.0001639887923374772, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 741.0, "completions/mean_length": 398.21875, "completions/mean_terminated_length": 386.2903137207031, "completions/min_length": 190.0, "completions/min_terminated_length": 190.0, "entropy": 0.42826948687434196, "epoch": 0.3618843683083512, "frac_reward_zero_std": 0.5, "grad_norm": 0.018404463306069374, "learning_rate": 1e-05, "loss": 0.0451, "num_tokens": 14520439.0, "reward": 0.90625, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.4523186683654785, "sampling/importance_sampling_ratio/mean": 1.0004849433898926, "sampling/importance_sampling_ratio/min": 0.5930233597755432, "sampling/sampling_logp_difference/max": 0.5225214958190918, "sampling/sampling_logp_difference/mean": 0.01220714021474123, "step": 676 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 677.0, "completions/mean_length": 527.5625, "completions/mean_terminated_length": 472.0769348144531, "completions/min_length": 200.0, "completions/min_terminated_length": 200.0, "entropy": 0.398250425234437, "epoch": 0.3624197002141328, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 14541169.0, "reward": 0.75, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.8422622680664062, "sampling/importance_sampling_ratio/mean": 1.0002211332321167, "sampling/importance_sampling_ratio/min": 0.6787064075469971, "sampling/sampling_logp_difference/max": 0.6109943389892578, "sampling/sampling_logp_difference/mean": 0.012101640924811363, "step": 677 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 694.0, "completions/mean_length": 454.25, "completions/mean_terminated_length": 444.1290283203125, "completions/min_length": 117.0, "completions/min_terminated_length": 117.0, "entropy": 0.2895977236330509, "epoch": 0.36295503211991437, "frac_reward_zero_std": 0.25, "grad_norm": 0.018193962052464485, "learning_rate": 1e-05, "loss": 0.0717, "num_tokens": 14558889.0, "reward": 0.875, "reward_std": 0.2925041913986206, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 1.408778429031372, "sampling/importance_sampling_ratio/mean": 0.9998857975006104, "sampling/importance_sampling_ratio/min": 0.715449333190918, "sampling/sampling_logp_difference/max": 0.34272289276123047, "sampling/sampling_logp_difference/mean": 0.009631810709834099, "step": 678 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 617.0, "completions/mean_length": 448.21875, "completions/mean_terminated_length": 437.9031982421875, "completions/min_length": 293.0, "completions/min_terminated_length": 293.0, "entropy": 0.2796888202428818, "epoch": 0.36349036402569596, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": -0.0073, "num_tokens": 14576816.0, "reward": 0.71875, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.6304254531860352, "sampling/importance_sampling_ratio/mean": 1.0001057386398315, "sampling/importance_sampling_ratio/min": 0.7328755855560303, "sampling/sampling_logp_difference/max": 0.48884105682373047, "sampling/sampling_logp_difference/mean": 0.010036729276180267, "step": 679 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 7.365940109593794e-05, "clip_ratio/low_min": 7.365940109593794e-05, "clip_ratio/region_mean": 7.365940109593794e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 655.0, "completions/max_terminated_length": 655.0, "completions/mean_length": 435.03125, "completions/mean_terminated_length": 435.03125, "completions/min_length": 171.0, "completions/min_terminated_length": 171.0, "entropy": 0.2705319318920374, "epoch": 0.3640256959314775, "frac_reward_zero_std": 0.75, "grad_norm": 0.005537100601941347, "learning_rate": 1e-05, "loss": -0.0327, "num_tokens": 14594233.0, "reward": 0.8125, "reward_std": 0.1157275140285492, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.7540359497070312, "sampling/importance_sampling_ratio/mean": 0.9998466968536377, "sampling/importance_sampling_ratio/min": 0.6739554405212402, "sampling/sampling_logp_difference/max": 0.5619193315505981, "sampling/sampling_logp_difference/mean": 0.009636824019253254, "step": 680 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.959911115700379e-05, "clip_ratio/low_min": 6.959911115700379e-05, "clip_ratio/region_mean": 6.959911115700379e-05, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 716.0, "completions/mean_length": 499.6875, "completions/mean_terminated_length": 461.357177734375, "completions/min_length": 341.0, "completions/min_terminated_length": 341.0, "entropy": 0.403135072439909, "epoch": 0.3645610278372591, "frac_reward_zero_std": 0.5, "grad_norm": 0.004972268361598253, "learning_rate": 1e-05, "loss": 0.012, "num_tokens": 14613935.0, "reward": 0.75, "reward_std": 0.2314550280570984, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.5715802907943726, "sampling/importance_sampling_ratio/mean": 0.9998313784599304, "sampling/importance_sampling_ratio/min": 0.7040213346481323, "sampling/sampling_logp_difference/max": 0.45208168029785156, "sampling/sampling_logp_difference/mean": 0.012608659453690052, "step": 681 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 7.022471982054412e-05, "clip_ratio/low_min": 7.022471982054412e-05, "clip_ratio/region_mean": 7.022471982054412e-05, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 713.0, "completions/mean_length": 498.59375, "completions/mean_terminated_length": 470.72412109375, "completions/min_length": 290.0, "completions/min_terminated_length": 290.0, "entropy": 0.3240054100751877, "epoch": 0.36509635974304067, "frac_reward_zero_std": 0.25, "grad_norm": 0.01842893287539482, "learning_rate": 1e-05, "loss": 0.0445, "num_tokens": 14634002.0, "reward": 0.6875, "reward_std": 0.38298875093460083, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.2981414794921875, "sampling/importance_sampling_ratio/mean": 0.9996616840362549, "sampling/importance_sampling_ratio/min": 0.731063961982727, "sampling/sampling_logp_difference/max": 0.31325435638427734, "sampling/sampling_logp_difference/mean": 0.010245262645184994, "step": 682 }, { "clip_ratio/high_max": 8.79662184161134e-05, "clip_ratio/high_mean": 8.79662184161134e-05, "clip_ratio/low_mean": 0.00017959770048037171, "clip_ratio/low_min": 0.00017959770048037171, "clip_ratio/region_mean": 0.0002675639188964851, "completions/clipped_ratio": 0.0, "completions/max_length": 531.0, "completions/max_terminated_length": 531.0, "completions/mean_length": 373.71875, "completions/mean_terminated_length": 373.71875, "completions/min_length": 207.0, "completions/min_terminated_length": 207.0, "entropy": 0.32563577592372894, "epoch": 0.36563169164882225, "frac_reward_zero_std": 0.5, "grad_norm": 0.015845371410250664, "learning_rate": 1e-05, "loss": 0.0122, "num_tokens": 14649825.0, "reward": 0.78125, "reward_std": 0.2630178928375244, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.3533612489700317, "sampling/importance_sampling_ratio/mean": 0.9999992251396179, "sampling/importance_sampling_ratio/min": 0.7251482605934143, "sampling/sampling_logp_difference/max": 0.3213791847229004, "sampling/sampling_logp_difference/mean": 0.011381244286894798, "step": 683 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 726.0, "completions/mean_length": 436.875, "completions/mean_terminated_length": 426.19354248046875, "completions/min_length": 245.0, "completions/min_terminated_length": 245.0, "entropy": 0.39201460406184196, "epoch": 0.36616702355460384, "frac_reward_zero_std": 0.25, "grad_norm": 0.01667793281376362, "learning_rate": 1e-05, "loss": 0.0147, "num_tokens": 14667477.0, "reward": 0.71875, "reward_std": 0.378745436668396, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.6575206518173218, "sampling/importance_sampling_ratio/mean": 0.9999275803565979, "sampling/importance_sampling_ratio/min": 0.739303469657898, "sampling/sampling_logp_difference/max": 0.5053229331970215, "sampling/sampling_logp_difference/mean": 0.012439138256013393, "step": 684 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 7.534659380326048e-05, "clip_ratio/low_min": 7.534659380326048e-05, "clip_ratio/region_mean": 7.534659380326048e-05, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 635.0, "completions/mean_length": 537.28125, "completions/mean_terminated_length": 472.67999267578125, "completions/min_length": 369.0, "completions/min_terminated_length": 369.0, "entropy": 0.42449041455984116, "epoch": 0.36670235546038543, "frac_reward_zero_std": 0.25, "grad_norm": 0.006100288592278957, "learning_rate": 1e-05, "loss": 0.0174, "num_tokens": 14688654.0, "reward": 0.46875, "reward_std": 0.2651650309562683, "rewards/accuracy_reward/mean": 0.46875, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.693973422050476, "sampling/importance_sampling_ratio/mean": 1.0002092123031616, "sampling/importance_sampling_ratio/min": 0.7011035680770874, "sampling/sampling_logp_difference/max": 0.5270768404006958, "sampling/sampling_logp_difference/mean": 0.012892551720142365, "step": 685 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0001289334468310699, "clip_ratio/low_min": 0.0001289334468310699, "clip_ratio/region_mean": 0.0001289334468310699, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 727.0, "completions/mean_length": 454.875, "completions/mean_terminated_length": 434.0000305175781, "completions/min_length": 177.0, "completions/min_terminated_length": 177.0, "entropy": 0.3620806783437729, "epoch": 0.367237687366167, "frac_reward_zero_std": 0.25, "grad_norm": 0.010676519945263863, "learning_rate": 1e-05, "loss": 0.0076, "num_tokens": 14706810.0, "reward": 0.65625, "reward_std": 0.3608423173427582, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.3780028820037842, "sampling/importance_sampling_ratio/mean": 0.9999836087226868, "sampling/importance_sampling_ratio/min": 0.7050690054893494, "sampling/sampling_logp_difference/max": 0.3494596481323242, "sampling/sampling_logp_difference/mean": 0.01154541689902544, "step": 686 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.456611299654469e-05, "clip_ratio/low_min": 6.456611299654469e-05, "clip_ratio/region_mean": 6.456611299654469e-05, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 617.0, "completions/mean_length": 425.8125, "completions/mean_terminated_length": 414.774169921875, "completions/min_length": 199.0, "completions/min_terminated_length": 199.0, "entropy": 0.31526566483080387, "epoch": 0.3677730192719486, "frac_reward_zero_std": 0.75, "grad_norm": 0.014808454550802708, "learning_rate": 1e-05, "loss": -0.0027, "num_tokens": 14723460.0, "reward": 0.875, "reward_std": 0.13363061845302582, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 1.5825399160385132, "sampling/importance_sampling_ratio/mean": 1.0002360343933105, "sampling/importance_sampling_ratio/min": 0.6621766686439514, "sampling/sampling_logp_difference/max": 0.4590311050415039, "sampling/sampling_logp_difference/mean": 0.010774160735309124, "step": 687 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0001803751802071929, "clip_ratio/low_min": 0.0001803751802071929, "clip_ratio/region_mean": 0.0001803751802071929, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 555.0, "completions/mean_length": 465.59375, "completions/mean_terminated_length": 395.8077087402344, "completions/min_length": 225.0, "completions/min_terminated_length": 225.0, "entropy": 0.36562727577984333, "epoch": 0.3683083511777302, "frac_reward_zero_std": 0.0, "grad_norm": 0.009232750162482262, "learning_rate": 1e-05, "loss": 0.0656, "num_tokens": 14741935.0, "reward": 0.65625, "reward_std": 0.3808925747871399, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.477591633796692, "sampling/importance_sampling_ratio/mean": 1.0000759363174438, "sampling/importance_sampling_ratio/min": 0.5472881197929382, "sampling/sampling_logp_difference/max": 0.6027798652648926, "sampling/sampling_logp_difference/mean": 0.012475117109715939, "step": 688 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.3125, "completions/max_length": 768.0, "completions/max_terminated_length": 698.0, "completions/mean_length": 549.25, "completions/mean_terminated_length": 449.8182067871094, "completions/min_length": 220.0, "completions/min_terminated_length": 220.0, "entropy": 0.4224976561963558, "epoch": 0.3688436830835118, "frac_reward_zero_std": 0.5, "grad_norm": 0.00646795891225338, "learning_rate": 1e-05, "loss": 0.0225, "num_tokens": 14764047.0, "reward": 0.625, "reward_std": 0.2314550280570984, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.6040548086166382, "sampling/importance_sampling_ratio/mean": 0.9998417496681213, "sampling/importance_sampling_ratio/min": 0.721508264541626, "sampling/sampling_logp_difference/max": 0.4725346565246582, "sampling/sampling_logp_difference/mean": 0.012980914674699306, "step": 689 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 746.0, "completions/mean_length": 451.125, "completions/mean_terminated_length": 430.0000305175781, "completions/min_length": 172.0, "completions/min_terminated_length": 172.0, "entropy": 0.3737363927066326, "epoch": 0.3693790149892934, "frac_reward_zero_std": 0.5, "grad_norm": 0.012766394764184952, "learning_rate": 1e-05, "loss": -0.0639, "num_tokens": 14782235.0, "reward": 0.6875, "reward_std": 0.249358132481575, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.4845302104949951, "sampling/importance_sampling_ratio/mean": 1.0001002550125122, "sampling/importance_sampling_ratio/min": 0.669922947883606, "sampling/sampling_logp_difference/max": 0.400592565536499, "sampling/sampling_logp_difference/mean": 0.012106068432331085, "step": 690 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.000156429159687832, "clip_ratio/low_min": 0.000156429159687832, "clip_ratio/region_mean": 0.000156429159687832, "completions/clipped_ratio": 0.28125, "completions/max_length": 768.0, "completions/max_terminated_length": 722.0, "completions/mean_length": 551.46875, "completions/mean_terminated_length": 466.7391357421875, "completions/min_length": 295.0, "completions/min_terminated_length": 295.0, "entropy": 0.5542093142867088, "epoch": 0.36991434689507496, "frac_reward_zero_std": 0.5, "grad_norm": 0.028639044612646103, "learning_rate": 1e-05, "loss": 0.045, "num_tokens": 14803578.0, "reward": 0.375, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.375, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.4161304235458374, "sampling/importance_sampling_ratio/mean": 1.0001494884490967, "sampling/importance_sampling_ratio/min": 0.6985909938812256, "sampling/sampling_logp_difference/max": 0.3586897850036621, "sampling/sampling_logp_difference/mean": 0.01604504883289337, "step": 691 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 749.0, "completions/mean_length": 537.0, "completions/mean_terminated_length": 513.1034545898438, "completions/min_length": 212.0, "completions/min_terminated_length": 212.0, "entropy": 0.2902977168560028, "epoch": 0.37044967880085655, "frac_reward_zero_std": 0.25, "grad_norm": 0.004602069966495037, "learning_rate": 1e-05, "loss": 0.0993, "num_tokens": 14824722.0, "reward": 0.78125, "reward_std": 0.3061639666557312, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.3806614875793457, "sampling/importance_sampling_ratio/mean": 0.9998685121536255, "sampling/importance_sampling_ratio/min": 0.6865155100822449, "sampling/sampling_logp_difference/max": 0.37612640857696533, "sampling/sampling_logp_difference/mean": 0.009488875977694988, "step": 692 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 7.275902316905558e-05, "clip_ratio/low_min": 7.275902316905558e-05, "clip_ratio/region_mean": 7.275902316905558e-05, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 670.0, "completions/mean_length": 421.90625, "completions/mean_terminated_length": 410.7419128417969, "completions/min_length": 188.0, "completions/min_terminated_length": 188.0, "entropy": 0.2983164694160223, "epoch": 0.37098501070663814, "frac_reward_zero_std": 0.25, "grad_norm": 0.00871491339057684, "learning_rate": 1e-05, "loss": 0.0066, "num_tokens": 14841839.0, "reward": 0.84375, "reward_std": 0.3061639964580536, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.3694186210632324, "sampling/importance_sampling_ratio/mean": 1.000069260597229, "sampling/importance_sampling_ratio/min": 0.7014452815055847, "sampling/sampling_logp_difference/max": 0.3546123504638672, "sampling/sampling_logp_difference/mean": 0.0099724680185318, "step": 693 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 734.0, "completions/mean_length": 555.375, "completions/mean_terminated_length": 495.8399963378906, "completions/min_length": 294.0, "completions/min_terminated_length": 294.0, "entropy": 0.3679488059133291, "epoch": 0.3715203426124197, "frac_reward_zero_std": 0.75, "grad_norm": 0.03551452234387398, "learning_rate": 1e-05, "loss": -0.0027, "num_tokens": 14863243.0, "reward": 0.78125, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.3860419988632202, "sampling/importance_sampling_ratio/mean": 0.9996388554573059, "sampling/importance_sampling_ratio/min": 0.5520268082618713, "sampling/sampling_logp_difference/max": 0.5941586494445801, "sampling/sampling_logp_difference/mean": 0.011161968111991882, "step": 694 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 705.0, "completions/max_terminated_length": 705.0, "completions/mean_length": 482.84375, "completions/mean_terminated_length": 482.84375, "completions/min_length": 266.0, "completions/min_terminated_length": 266.0, "entropy": 0.30257387086749077, "epoch": 0.37205567451820126, "frac_reward_zero_std": 0.25, "grad_norm": 0.007895922288298607, "learning_rate": 1e-05, "loss": -0.0119, "num_tokens": 14882014.0, "reward": 0.84375, "reward_std": 0.3198433816432953, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.2971529960632324, "sampling/importance_sampling_ratio/mean": 0.9995283484458923, "sampling/importance_sampling_ratio/min": 0.718858003616333, "sampling/sampling_logp_difference/max": 0.3300914764404297, "sampling/sampling_logp_difference/mean": 0.01028190553188324, "step": 695 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.085686618462205e-05, "clip_ratio/low_min": 6.085686618462205e-05, "clip_ratio/region_mean": 6.085686618462205e-05, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 764.0, "completions/mean_length": 514.09375, "completions/mean_terminated_length": 497.16668701171875, "completions/min_length": 360.0, "completions/min_terminated_length": 360.0, "entropy": 0.27033248357474804, "epoch": 0.37259100642398285, "frac_reward_zero_std": 0.5, "grad_norm": 0.00923229567706585, "learning_rate": 1e-05, "loss": 0.0065, "num_tokens": 14902369.0, "reward": 0.71875, "reward_std": 0.2630178928375244, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.4416664838790894, "sampling/importance_sampling_ratio/mean": 1.0003470182418823, "sampling/importance_sampling_ratio/min": 0.6697207689285278, "sampling/sampling_logp_difference/max": 0.4008944034576416, "sampling/sampling_logp_difference/mean": 0.009560334496200085, "step": 696 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00014302176714409143, "clip_ratio/low_min": 0.00014302176714409143, "clip_ratio/region_mean": 0.00014302176714409143, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 593.0, "completions/mean_length": 416.0, "completions/mean_terminated_length": 404.6451416015625, "completions/min_length": 222.0, "completions/min_terminated_length": 222.0, "entropy": 0.3604154847562313, "epoch": 0.37312633832976444, "frac_reward_zero_std": 0.25, "grad_norm": 0.02464476227760315, "learning_rate": 1e-05, "loss": 0.0041, "num_tokens": 14919249.0, "reward": 0.6875, "reward_std": 0.3745020925998688, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.4501818418502808, "sampling/importance_sampling_ratio/mean": 0.9995229244232178, "sampling/importance_sampling_ratio/min": 0.700311005115509, "sampling/sampling_logp_difference/max": 0.37168896198272705, "sampling/sampling_logp_difference/mean": 0.011596843600273132, "step": 697 }, { "clip_ratio/high_max": 7.42280317354016e-05, "clip_ratio/high_mean": 7.42280317354016e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 7.42280317354016e-05, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 763.0, "completions/mean_length": 498.5, "completions/mean_terminated_length": 489.8064270019531, "completions/min_length": 327.0, "completions/min_terminated_length": 327.0, "entropy": 0.3239190801978111, "epoch": 0.37366167023554603, "frac_reward_zero_std": 0.25, "grad_norm": 0.010498818010091782, "learning_rate": 1e-05, "loss": 0.03, "num_tokens": 14939201.0, "reward": 0.8125, "reward_std": 0.3104073107242584, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.4165321588516235, "sampling/importance_sampling_ratio/mean": 0.9996457695960999, "sampling/importance_sampling_ratio/min": 0.39121150970458984, "sampling/sampling_logp_difference/max": 0.9385068416595459, "sampling/sampling_logp_difference/mean": 0.010730743408203125, "step": 698 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 627.0, "completions/max_terminated_length": 627.0, "completions/mean_length": 414.5, "completions/mean_terminated_length": 414.5, "completions/min_length": 228.0, "completions/min_terminated_length": 228.0, "entropy": 0.2429221346974373, "epoch": 0.3741970021413276, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 14955817.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.348206639289856, "sampling/importance_sampling_ratio/mean": 1.0001676082611084, "sampling/importance_sampling_ratio/min": 0.6665967106819153, "sampling/sampling_logp_difference/max": 0.40557003021240234, "sampling/sampling_logp_difference/mean": 0.008237956091761589, "step": 699 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 496.0, "completions/max_terminated_length": 496.0, "completions/mean_length": 309.5, "completions/mean_terminated_length": 309.5, "completions/min_length": 167.0, "completions/min_terminated_length": 167.0, "entropy": 0.32242966815829277, "epoch": 0.3747323340471092, "frac_reward_zero_std": 0.5, "grad_norm": 0.007265747524797916, "learning_rate": 1e-05, "loss": -0.0232, "num_tokens": 14968929.0, "reward": 0.78125, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.2998929023742676, "sampling/importance_sampling_ratio/mean": 0.9999431371688843, "sampling/importance_sampling_ratio/min": 0.7561819553375244, "sampling/sampling_logp_difference/max": 0.2794732451438904, "sampling/sampling_logp_difference/mean": 0.010783866979181767, "step": 700 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 701.0, "completions/mean_length": 454.53125, "completions/mean_terminated_length": 444.4193420410156, "completions/min_length": 131.0, "completions/min_terminated_length": 131.0, "entropy": 0.2625401485711336, "epoch": 0.3752676659528908, "frac_reward_zero_std": 0.5, "grad_norm": 0.00635444326326251, "learning_rate": 1e-05, "loss": 0.0379, "num_tokens": 14987066.0, "reward": 0.75, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.4335325956344604, "sampling/importance_sampling_ratio/mean": 1.000280737876892, "sampling/importance_sampling_ratio/min": 0.6617458462715149, "sampling/sampling_logp_difference/max": 0.4128737449645996, "sampling/sampling_logp_difference/mean": 0.009299966506659985, "step": 701 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.1546390750445426e-05, "clip_ratio/low_min": 5.1546390750445426e-05, "clip_ratio/region_mean": 5.1546390750445426e-05, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 737.0, "completions/mean_length": 479.375, "completions/mean_terminated_length": 398.55999755859375, "completions/min_length": 212.0, "completions/min_terminated_length": 212.0, "entropy": 0.5339374206960201, "epoch": 0.3758029978586724, "frac_reward_zero_std": 0.5, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0073, "num_tokens": 15006518.0, "reward": 0.65625, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.397071123123169, "sampling/importance_sampling_ratio/mean": 0.999973714351654, "sampling/importance_sampling_ratio/min": 0.6090170741081238, "sampling/sampling_logp_difference/max": 0.4959089756011963, "sampling/sampling_logp_difference/mean": 0.014633250422775745, "step": 702 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 7.88146280683577e-05, "clip_ratio/low_min": 7.88146280683577e-05, "clip_ratio/region_mean": 7.88146280683577e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 586.0, "completions/max_terminated_length": 586.0, "completions/mean_length": 372.96875, "completions/mean_terminated_length": 372.96875, "completions/min_length": 217.0, "completions/min_terminated_length": 217.0, "entropy": 0.3855822868645191, "epoch": 0.37633832976445397, "frac_reward_zero_std": 0.5, "grad_norm": 0.019846171140670776, "learning_rate": 1e-05, "loss": 0.0753, "num_tokens": 15021965.0, "reward": 0.90625, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.2919931411743164, "sampling/importance_sampling_ratio/mean": 0.9998896718025208, "sampling/importance_sampling_ratio/min": 0.7070300579071045, "sampling/sampling_logp_difference/max": 0.346682071685791, "sampling/sampling_logp_difference/mean": 0.012079265899956226, "step": 703 }, { "clip_ratio/high_max": 5.087505269329995e-05, "clip_ratio/high_mean": 5.087505269329995e-05, "clip_ratio/low_mean": 0.00010730029680416919, "clip_ratio/low_min": 0.00010730029680416919, "clip_ratio/region_mean": 0.00015817534949746914, "completions/clipped_ratio": 0.375, "completions/max_length": 768.0, "completions/max_terminated_length": 733.0, "completions/mean_length": 601.5, "completions/mean_terminated_length": 501.6000061035156, "completions/min_length": 308.0, "completions/min_terminated_length": 308.0, "entropy": 0.5670550353825092, "epoch": 0.37687366167023556, "frac_reward_zero_std": 0.25, "grad_norm": 0.01242276281118393, "learning_rate": 1e-05, "loss": 0.0049, "num_tokens": 15045205.0, "reward": 0.34375, "reward_std": 0.378745436668396, "rewards/accuracy_reward/mean": 0.34375, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.5675126314163208, "sampling/importance_sampling_ratio/mean": 1.000095248222351, "sampling/importance_sampling_ratio/min": 0.7066628932952881, "sampling/sampling_logp_difference/max": 0.4494900703430176, "sampling/sampling_logp_difference/mean": 0.015393679030239582, "step": 704 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 700.0, "completions/max_terminated_length": 700.0, "completions/mean_length": 417.71875, "completions/mean_terminated_length": 417.71875, "completions/min_length": 219.0, "completions/min_terminated_length": 219.0, "entropy": 0.35111863166093826, "epoch": 0.37740899357601715, "frac_reward_zero_std": 0.5, "grad_norm": 0.01782170869410038, "learning_rate": 1e-05, "loss": 0.0099, "num_tokens": 15062500.0, "reward": 0.875, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 1.3412227630615234, "sampling/importance_sampling_ratio/mean": 0.9997622966766357, "sampling/importance_sampling_ratio/min": 0.7273291945457458, "sampling/sampling_logp_difference/max": 0.31837618350982666, "sampling/sampling_logp_difference/mean": 0.011413737200200558, "step": 705 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 668.0, "completions/mean_length": 383.84375, "completions/mean_terminated_length": 371.45159912109375, "completions/min_length": 128.0, "completions/min_terminated_length": 128.0, "entropy": 0.37662991508841515, "epoch": 0.37794432548179874, "frac_reward_zero_std": 0.75, "grad_norm": 0.007424172479659319, "learning_rate": 1e-05, "loss": 0.0188, "num_tokens": 15078527.0, "reward": 0.5625, "reward_std": 0.1157275140285492, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.3684529066085815, "sampling/importance_sampling_ratio/mean": 1.000205636024475, "sampling/importance_sampling_ratio/min": 0.44091466069221497, "sampling/sampling_logp_difference/max": 0.818903923034668, "sampling/sampling_logp_difference/mean": 0.01036691665649414, "step": 706 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 643.0, "completions/mean_length": 490.40625, "completions/mean_terminated_length": 471.9000244140625, "completions/min_length": 245.0, "completions/min_terminated_length": 245.0, "entropy": 0.30377865210175514, "epoch": 0.3784796573875803, "frac_reward_zero_std": 0.5, "grad_norm": 0.004978889599442482, "learning_rate": 1e-05, "loss": 0.0435, "num_tokens": 15097820.0, "reward": 0.8125, "reward_std": 0.249358132481575, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.451399564743042, "sampling/importance_sampling_ratio/mean": 0.9998151063919067, "sampling/importance_sampling_ratio/min": 0.6608973741531372, "sampling/sampling_logp_difference/max": 0.4141566753387451, "sampling/sampling_logp_difference/mean": 0.009623161517083645, "step": 707 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 630.0, "completions/max_terminated_length": 630.0, "completions/mean_length": 422.375, "completions/mean_terminated_length": 422.375, "completions/min_length": 256.0, "completions/min_terminated_length": 256.0, "entropy": 0.28235390968620777, "epoch": 0.37901498929336186, "frac_reward_zero_std": 0.25, "grad_norm": 0.00819606613367796, "learning_rate": 1e-05, "loss": 0.0011, "num_tokens": 15114736.0, "reward": 0.875, "reward_std": 0.292504221200943, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 1.3884201049804688, "sampling/importance_sampling_ratio/mean": 1.000165343284607, "sampling/importance_sampling_ratio/min": 0.33030667901039124, "sampling/sampling_logp_difference/max": 1.1077337265014648, "sampling/sampling_logp_difference/mean": 0.0095993522554636, "step": 708 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 8.069722389336675e-05, "clip_ratio/low_min": 8.069722389336675e-05, "clip_ratio/region_mean": 8.069722389336675e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 672.0, "completions/max_terminated_length": 672.0, "completions/mean_length": 390.78125, "completions/mean_terminated_length": 390.78125, "completions/min_length": 264.0, "completions/min_terminated_length": 264.0, "entropy": 0.3380414769053459, "epoch": 0.37955032119914345, "frac_reward_zero_std": 0.25, "grad_norm": 0.01816956326365471, "learning_rate": 1e-05, "loss": -0.0078, "num_tokens": 15131313.0, "reward": 0.84375, "reward_std": 0.3198433816432953, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.9131790399551392, "sampling/importance_sampling_ratio/mean": 0.9997246265411377, "sampling/importance_sampling_ratio/min": 0.7011801600456238, "sampling/sampling_logp_difference/max": 0.648766279220581, "sampling/sampling_logp_difference/mean": 0.011466284282505512, "step": 709 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00011698735397658311, "clip_ratio/low_min": 0.00011698735397658311, "clip_ratio/region_mean": 0.00011698735397658311, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 738.0, "completions/mean_length": 499.96875, "completions/mean_terminated_length": 461.6785888671875, "completions/min_length": 266.0, "completions/min_terminated_length": 266.0, "entropy": 0.35549603775143623, "epoch": 0.38008565310492504, "frac_reward_zero_std": 0.25, "grad_norm": 0.02748221345245838, "learning_rate": 1e-05, "loss": -0.022, "num_tokens": 15151992.0, "reward": 0.65625, "reward_std": 0.3198433816432953, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.4221206903457642, "sampling/importance_sampling_ratio/mean": 0.9998782277107239, "sampling/importance_sampling_ratio/min": 0.6979137659072876, "sampling/sampling_logp_difference/max": 0.35965967178344727, "sampling/sampling_logp_difference/mean": 0.011623858474195004, "step": 710 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 8.294625149574131e-05, "clip_ratio/low_min": 8.294625149574131e-05, "clip_ratio/region_mean": 8.294625149574131e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 709.0, "completions/max_terminated_length": 709.0, "completions/mean_length": 427.46875, "completions/mean_terminated_length": 427.46875, "completions/min_length": 186.0, "completions/min_terminated_length": 186.0, "entropy": 0.30363865941762924, "epoch": 0.3806209850107066, "frac_reward_zero_std": 0.5, "grad_norm": 0.00964261032640934, "learning_rate": 1e-05, "loss": 0.0051, "num_tokens": 15169079.0, "reward": 0.78125, "reward_std": 0.2630178928375244, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.4173977375030518, "sampling/importance_sampling_ratio/mean": 0.999652624130249, "sampling/importance_sampling_ratio/min": 0.698860228061676, "sampling/sampling_logp_difference/max": 0.358304500579834, "sampling/sampling_logp_difference/mean": 0.010039136745035648, "step": 711 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 750.0, "completions/mean_length": 545.0, "completions/mean_terminated_length": 482.55999755859375, "completions/min_length": 296.0, "completions/min_terminated_length": 296.0, "entropy": 0.44356758520007133, "epoch": 0.3811563169164882, "frac_reward_zero_std": 0.5, "grad_norm": 0.021267326548695564, "learning_rate": 1e-05, "loss": 0.0082, "num_tokens": 15190719.0, "reward": 0.75, "reward_std": 0.2587745785713196, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.4318623542785645, "sampling/importance_sampling_ratio/mean": 1.000069499015808, "sampling/importance_sampling_ratio/min": 0.7342823147773743, "sampling/sampling_logp_difference/max": 0.358975887298584, "sampling/sampling_logp_difference/mean": 0.013153758831322193, "step": 712 }, { "clip_ratio/high_max": 8.468834857922047e-05, "clip_ratio/high_mean": 8.468834857922047e-05, "clip_ratio/low_mean": 8.085381705313921e-05, "clip_ratio/low_min": 8.085381705313921e-05, "clip_ratio/region_mean": 0.00016554216563235968, "completions/clipped_ratio": 0.0, "completions/max_length": 643.0, "completions/max_terminated_length": 643.0, "completions/mean_length": 425.84375, "completions/mean_terminated_length": 425.84375, "completions/min_length": 209.0, "completions/min_terminated_length": 209.0, "entropy": 0.2659711726009846, "epoch": 0.3816916488222698, "frac_reward_zero_std": 0.5, "grad_norm": 0.00843278132379055, "learning_rate": 1e-05, "loss": -0.002, "num_tokens": 15207610.0, "reward": 0.71875, "reward_std": 0.2630178928375244, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.30759859085083, "sampling/importance_sampling_ratio/mean": 1.0002485513687134, "sampling/importance_sampling_ratio/min": 0.7000202536582947, "sampling/sampling_logp_difference/max": 0.3566460609436035, "sampling/sampling_logp_difference/mean": 0.009244011715054512, "step": 713 }, { "clip_ratio/high_max": 5.310110282152891e-05, "clip_ratio/high_mean": 5.310110282152891e-05, "clip_ratio/low_mean": 0.00016015375149436295, "clip_ratio/low_min": 0.00016015375149436295, "clip_ratio/region_mean": 0.00021325485431589186, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 731.0, "completions/mean_length": 507.75, "completions/mean_terminated_length": 490.4000244140625, "completions/min_length": 248.0, "completions/min_terminated_length": 248.0, "entropy": 0.28116846084594727, "epoch": 0.3822269807280514, "frac_reward_zero_std": 0.0, "grad_norm": 0.0116853853687644, "learning_rate": 1e-05, "loss": -0.0388, "num_tokens": 15227658.0, "reward": 0.78125, "reward_std": 0.4218915104866028, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.4448853731155396, "sampling/importance_sampling_ratio/mean": 1.000178337097168, "sampling/importance_sampling_ratio/min": 0.69992995262146, "sampling/sampling_logp_difference/max": 0.3680300712585449, "sampling/sampling_logp_difference/mean": 0.009712045080959797, "step": 714 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0001412651254213415, "clip_ratio/low_min": 0.0001412651254213415, "clip_ratio/region_mean": 0.0001412651254213415, "completions/clipped_ratio": 0.40625, "completions/max_length": 768.0, "completions/max_terminated_length": 762.0, "completions/mean_length": 578.90625, "completions/mean_terminated_length": 449.52630615234375, "completions/min_length": 262.0, "completions/min_terminated_length": 262.0, "entropy": 0.4918227419257164, "epoch": 0.382762312633833, "frac_reward_zero_std": 0.25, "grad_norm": 0.03207610547542572, "learning_rate": 1e-05, "loss": 0.0564, "num_tokens": 15250231.0, "reward": 0.5625, "reward_std": 0.292504221200943, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.3958628177642822, "sampling/importance_sampling_ratio/mean": 1.0001263618469238, "sampling/importance_sampling_ratio/min": 0.6963813304901123, "sampling/sampling_logp_difference/max": 0.36185789108276367, "sampling/sampling_logp_difference/mean": 0.014611990191042423, "step": 715 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 757.0, "completions/mean_length": 470.625, "completions/mean_terminated_length": 439.862060546875, "completions/min_length": 255.0, "completions/min_terminated_length": 255.0, "entropy": 0.3172050081193447, "epoch": 0.38329764453961457, "frac_reward_zero_std": 0.25, "grad_norm": 0.021208766847848892, "learning_rate": 1e-05, "loss": 0.0068, "num_tokens": 15268931.0, "reward": 0.78125, "reward_std": 0.3471629321575165, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.4504036903381348, "sampling/importance_sampling_ratio/mean": 1.000354528427124, "sampling/importance_sampling_ratio/min": 0.7036083340644836, "sampling/sampling_logp_difference/max": 0.3718419075012207, "sampling/sampling_logp_difference/mean": 0.010729103349149227, "step": 716 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 743.0, "completions/mean_length": 471.25, "completions/mean_terminated_length": 451.4667053222656, "completions/min_length": 252.0, "completions/min_terminated_length": 252.0, "entropy": 0.2803878542035818, "epoch": 0.38383297644539616, "frac_reward_zero_std": 0.5, "grad_norm": 0.005157788749784231, "learning_rate": 1e-05, "loss": 0.0383, "num_tokens": 15288067.0, "reward": 0.90625, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.3918088674545288, "sampling/importance_sampling_ratio/mean": 0.9998689293861389, "sampling/importance_sampling_ratio/min": 0.7035257816314697, "sampling/sampling_logp_difference/max": 0.3516507148742676, "sampling/sampling_logp_difference/mean": 0.009469738230109215, "step": 717 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 741.0, "completions/mean_length": 591.125, "completions/mean_terminated_length": 541.5999755859375, "completions/min_length": 326.0, "completions/min_terminated_length": 326.0, "entropy": 0.28335285373032093, "epoch": 0.38436830835117775, "frac_reward_zero_std": 0.5, "grad_norm": 0.00884544849395752, "learning_rate": 1e-05, "loss": 0.0188, "num_tokens": 15310607.0, "reward": 0.65625, "reward_std": 0.22201895713806152, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.476885437965393, "sampling/importance_sampling_ratio/mean": 0.9997802972793579, "sampling/importance_sampling_ratio/min": 0.5552170872688293, "sampling/sampling_logp_difference/max": 0.5883960723876953, "sampling/sampling_logp_difference/mean": 0.009259765967726707, "step": 718 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 755.0, "completions/mean_length": 470.5625, "completions/mean_terminated_length": 460.9677429199219, "completions/min_length": 176.0, "completions/min_terminated_length": 176.0, "entropy": 0.34930726885795593, "epoch": 0.38490364025695933, "frac_reward_zero_std": 0.5, "grad_norm": 0.022168725728988647, "learning_rate": 1e-05, "loss": 0.0259, "num_tokens": 15329457.0, "reward": 0.84375, "reward_std": 0.22201895713806152, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.4237807989120483, "sampling/importance_sampling_ratio/mean": 0.9998915791511536, "sampling/importance_sampling_ratio/min": 0.6903370022773743, "sampling/sampling_logp_difference/max": 0.3705754280090332, "sampling/sampling_logp_difference/mean": 0.010541636496782303, "step": 719 }, { "clip_ratio/high_max": 8.272667037090287e-05, "clip_ratio/high_mean": 8.272667037090287e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 8.272667037090287e-05, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 666.0, "completions/mean_length": 422.15625, "completions/mean_terminated_length": 411.0, "completions/min_length": 241.0, "completions/min_terminated_length": 241.0, "entropy": 0.2984021846204996, "epoch": 0.3854389721627409, "frac_reward_zero_std": 0.25, "grad_norm": 0.018680274486541748, "learning_rate": 1e-05, "loss": 0.1035, "num_tokens": 15346166.0, "reward": 0.78125, "reward_std": 0.3377465009689331, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.4660590887069702, "sampling/importance_sampling_ratio/mean": 0.9999105334281921, "sampling/importance_sampling_ratio/min": 0.5593701004981995, "sampling/sampling_logp_difference/max": 0.5809439420700073, "sampling/sampling_logp_difference/mean": 0.009945410303771496, "step": 720 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 689.0, "completions/max_terminated_length": 689.0, "completions/mean_length": 430.53125, "completions/mean_terminated_length": 430.53125, "completions/min_length": 158.0, "completions/min_terminated_length": 158.0, "entropy": 0.3552410937845707, "epoch": 0.3859743040685225, "frac_reward_zero_std": 0.5, "grad_norm": 0.008144088089466095, "learning_rate": 1e-05, "loss": -0.023, "num_tokens": 15363543.0, "reward": 0.84375, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.2996821403503418, "sampling/importance_sampling_ratio/mean": 1.000144124031067, "sampling/importance_sampling_ratio/min": 0.7797349691390991, "sampling/sampling_logp_difference/max": 0.26211977005004883, "sampling/sampling_logp_difference/mean": 0.010755490511655807, "step": 721 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 698.0, "completions/max_terminated_length": 698.0, "completions/mean_length": 449.15625, "completions/mean_terminated_length": 449.15625, "completions/min_length": 244.0, "completions/min_terminated_length": 244.0, "entropy": 0.3446524739265442, "epoch": 0.38650963597430404, "frac_reward_zero_std": 0.5, "grad_norm": 0.003281314391642809, "learning_rate": 1e-05, "loss": 0.0254, "num_tokens": 15381876.0, "reward": 0.78125, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.4003485441207886, "sampling/importance_sampling_ratio/mean": 0.999922513961792, "sampling/importance_sampling_ratio/min": 0.6165217757225037, "sampling/sampling_logp_difference/max": 0.4836616516113281, "sampling/sampling_logp_difference/mean": 0.011187918484210968, "step": 722 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 596.0, "completions/mean_length": 440.75, "completions/mean_terminated_length": 430.19354248046875, "completions/min_length": 180.0, "completions/min_terminated_length": 180.0, "entropy": 0.35840288922190666, "epoch": 0.38704496788008563, "frac_reward_zero_std": 0.5, "grad_norm": 0.013158319517970085, "learning_rate": 1e-05, "loss": -0.029, "num_tokens": 15400068.0, "reward": 0.84375, "reward_std": 0.22201895713806152, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.431912899017334, "sampling/importance_sampling_ratio/mean": 0.9998740553855896, "sampling/importance_sampling_ratio/min": 0.5467763543128967, "sampling/sampling_logp_difference/max": 0.6037154197692871, "sampling/sampling_logp_difference/mean": 0.011617033742368221, "step": 723 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00013079975178698078, "clip_ratio/low_min": 0.00013079975178698078, "clip_ratio/region_mean": 0.00013079975178698078, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 699.0, "completions/mean_length": 461.59375, "completions/mean_terminated_length": 451.70965576171875, "completions/min_length": 213.0, "completions/min_terminated_length": 213.0, "entropy": 0.41740207001566887, "epoch": 0.3875802997858672, "frac_reward_zero_std": 0.25, "grad_norm": 0.008753396570682526, "learning_rate": 1e-05, "loss": 0.028, "num_tokens": 15418471.0, "reward": 0.71875, "reward_std": 0.3198433816432953, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.7728537321090698, "sampling/importance_sampling_ratio/mean": 0.9997082948684692, "sampling/importance_sampling_ratio/min": 0.468240350484848, "sampling/sampling_logp_difference/max": 0.7587735652923584, "sampling/sampling_logp_difference/mean": 0.013727336190640926, "step": 724 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 698.0, "completions/max_terminated_length": 698.0, "completions/mean_length": 425.1875, "completions/mean_terminated_length": 425.1875, "completions/min_length": 245.0, "completions/min_terminated_length": 245.0, "entropy": 0.3067735768854618, "epoch": 0.3881156316916488, "frac_reward_zero_std": 0.75, "grad_norm": 0.016379397362470627, "learning_rate": 1e-05, "loss": 0.0134, "num_tokens": 15435469.0, "reward": 0.9375, "reward_std": 0.1157275140285492, "rewards/accuracy_reward/mean": 0.9375, "rewards/accuracy_reward/std": 0.24593468010425568, "sampling/importance_sampling_ratio/max": 1.302474021911621, "sampling/importance_sampling_ratio/mean": 1.000181794166565, "sampling/importance_sampling_ratio/min": 0.7234917879104614, "sampling/sampling_logp_difference/max": 0.3236660957336426, "sampling/sampling_logp_difference/mean": 0.010212586261332035, "step": 725 }, { "clip_ratio/high_max": 7.188039307948202e-05, "clip_ratio/high_mean": 7.188039307948202e-05, "clip_ratio/low_mean": 6.234413740457967e-05, "clip_ratio/low_min": 6.234413740457967e-05, "clip_ratio/region_mean": 0.0001342245304840617, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 617.0, "completions/mean_length": 469.875, "completions/mean_terminated_length": 427.2857360839844, "completions/min_length": 238.0, "completions/min_terminated_length": 238.0, "entropy": 0.33584076538681984, "epoch": 0.3886509635974304, "frac_reward_zero_std": 0.0, "grad_norm": 0.011967268772423267, "learning_rate": 1e-05, "loss": 0.1041, "num_tokens": 15453993.0, "reward": 0.71875, "reward_std": 0.4534739851951599, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.5494422912597656, "sampling/importance_sampling_ratio/mean": 0.9996713399887085, "sampling/importance_sampling_ratio/min": 0.5857813954353333, "sampling/sampling_logp_difference/max": 0.5348086357116699, "sampling/sampling_logp_difference/mean": 0.0111548425629735, "step": 726 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 674.0, "completions/mean_length": 429.0, "completions/mean_terminated_length": 418.06451416015625, "completions/min_length": 191.0, "completions/min_terminated_length": 191.0, "entropy": 0.322954460978508, "epoch": 0.389186295503212, "frac_reward_zero_std": 0.5, "grad_norm": 0.013946806080639362, "learning_rate": 1e-05, "loss": 0.0573, "num_tokens": 15471481.0, "reward": 0.84375, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.3577029705047607, "sampling/importance_sampling_ratio/mean": 1.000420331954956, "sampling/importance_sampling_ratio/min": 0.7600410580635071, "sampling/sampling_logp_difference/max": 0.30579423904418945, "sampling/sampling_logp_difference/mean": 0.011020390316843987, "step": 727 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00010679096158128232, "clip_ratio/low_min": 0.00010679096158128232, "clip_ratio/region_mean": 0.00010679096158128232, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 728.0, "completions/mean_length": 539.78125, "completions/mean_terminated_length": 497.5185241699219, "completions/min_length": 359.0, "completions/min_terminated_length": 359.0, "entropy": 0.2796822488307953, "epoch": 0.3897216274089936, "frac_reward_zero_std": 0.25, "grad_norm": 0.0059602754190564156, "learning_rate": 1e-05, "loss": 0.0727, "num_tokens": 15492594.0, "reward": 0.625, "reward_std": 0.3104073107242584, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.3901206254959106, "sampling/importance_sampling_ratio/mean": 0.9996997117996216, "sampling/importance_sampling_ratio/min": 0.7212743759155273, "sampling/sampling_logp_difference/max": 0.3293905258178711, "sampling/sampling_logp_difference/mean": 0.009473959915339947, "step": 728 }, { "clip_ratio/high_max": 5.56297272851225e-05, "clip_ratio/high_mean": 5.56297272851225e-05, "clip_ratio/low_mean": 0.000152136773976963, "clip_ratio/low_min": 0.000152136773976963, "clip_ratio/region_mean": 0.0002077665012620855, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 729.0, "completions/mean_length": 563.03125, "completions/mean_terminated_length": 494.7083435058594, "completions/min_length": 277.0, "completions/min_terminated_length": 277.0, "entropy": 0.4573715813457966, "epoch": 0.39025695931477516, "frac_reward_zero_std": 0.25, "grad_norm": 0.0070287566632032394, "learning_rate": 1e-05, "loss": 0.0796, "num_tokens": 15514627.0, "reward": 0.5625, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.6117656230926514, "sampling/importance_sampling_ratio/mean": 1.0001170635223389, "sampling/importance_sampling_ratio/min": 0.5874090194702148, "sampling/sampling_logp_difference/max": 0.5320339202880859, "sampling/sampling_logp_difference/mean": 0.014223533682525158, "step": 729 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 670.0, "completions/mean_length": 420.53125, "completions/mean_terminated_length": 409.32257080078125, "completions/min_length": 246.0, "completions/min_terminated_length": 246.0, "entropy": 0.31619079411029816, "epoch": 0.39079229122055675, "frac_reward_zero_std": 0.75, "grad_norm": 0.005202142987400293, "learning_rate": 1e-05, "loss": 0.058, "num_tokens": 15531860.0, "reward": 0.96875, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.96875, "rewards/accuracy_reward/std": 0.1767766922712326, "sampling/importance_sampling_ratio/max": 1.4242578744888306, "sampling/importance_sampling_ratio/mean": 1.000077247619629, "sampling/importance_sampling_ratio/min": 0.6973584294319153, "sampling/sampling_logp_difference/max": 0.3604557514190674, "sampling/sampling_logp_difference/mean": 0.010554124601185322, "step": 730 }, { "clip_ratio/high_max": 8.080155384959653e-05, "clip_ratio/high_mean": 8.080155384959653e-05, "clip_ratio/low_mean": 5.87406029808335e-05, "clip_ratio/low_min": 5.87406029808335e-05, "clip_ratio/region_mean": 0.00013954215683043003, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 585.0, "completions/mean_length": 427.125, "completions/mean_terminated_length": 416.1290283203125, "completions/min_length": 214.0, "completions/min_terminated_length": 214.0, "entropy": 0.3117987625300884, "epoch": 0.39132762312633834, "frac_reward_zero_std": 0.25, "grad_norm": 0.007393387146294117, "learning_rate": 1e-05, "loss": 0.0688, "num_tokens": 15549240.0, "reward": 0.90625, "reward_std": 0.2651650309562683, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.433593511581421, "sampling/importance_sampling_ratio/mean": 1.000236988067627, "sampling/importance_sampling_ratio/min": 0.7485927939414978, "sampling/sampling_logp_difference/max": 0.3601841926574707, "sampling/sampling_logp_difference/mean": 0.0106127904728055, "step": 731 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00013024853251408786, "clip_ratio/low_min": 0.00013024853251408786, "clip_ratio/region_mean": 0.00013024853251408786, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 691.0, "completions/mean_length": 482.625, "completions/mean_terminated_length": 441.857177734375, "completions/min_length": 271.0, "completions/min_terminated_length": 271.0, "entropy": 0.4086911641061306, "epoch": 0.39186295503211993, "frac_reward_zero_std": 0.25, "grad_norm": 0.028034420683979988, "learning_rate": 1e-05, "loss": 0.0302, "num_tokens": 15568316.0, "reward": 0.46875, "reward_std": 0.3198433816432953, "rewards/accuracy_reward/mean": 0.46875, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.4345265626907349, "sampling/importance_sampling_ratio/mean": 0.9996673464775085, "sampling/importance_sampling_ratio/min": 0.7197425961494446, "sampling/sampling_logp_difference/max": 0.36083483695983887, "sampling/sampling_logp_difference/mean": 0.012979852966964245, "step": 732 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00018101435853168368, "clip_ratio/low_min": 0.00018101435853168368, "clip_ratio/region_mean": 0.00018101435853168368, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 656.0, "completions/mean_length": 465.03125, "completions/mean_terminated_length": 433.6896667480469, "completions/min_length": 234.0, "completions/min_terminated_length": 234.0, "entropy": 0.39621807262301445, "epoch": 0.3923982869379015, "frac_reward_zero_std": 0.25, "grad_norm": 0.013066532090306282, "learning_rate": 1e-05, "loss": 0.0734, "num_tokens": 15587013.0, "reward": 0.625, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.4360840320587158, "sampling/importance_sampling_ratio/mean": 0.9997668862342834, "sampling/importance_sampling_ratio/min": 0.7086175084114075, "sampling/sampling_logp_difference/max": 0.3619198799133301, "sampling/sampling_logp_difference/mean": 0.012843320146203041, "step": 733 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 667.0, "completions/max_terminated_length": 667.0, "completions/mean_length": 457.84375, "completions/mean_terminated_length": 457.84375, "completions/min_length": 244.0, "completions/min_terminated_length": 244.0, "entropy": 0.34751833975315094, "epoch": 0.3929336188436831, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0152, "num_tokens": 15604888.0, "reward": 0.78125, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.579491138458252, "sampling/importance_sampling_ratio/mean": 1.0000112056732178, "sampling/importance_sampling_ratio/min": 0.743756115436554, "sampling/sampling_logp_difference/max": 0.45710277557373047, "sampling/sampling_logp_difference/mean": 0.010801946744322777, "step": 734 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00010974539327435195, "clip_ratio/low_min": 0.00010974539327435195, "clip_ratio/region_mean": 0.00010974539327435195, "completions/clipped_ratio": 0.3125, "completions/max_length": 768.0, "completions/max_terminated_length": 762.0, "completions/mean_length": 539.78125, "completions/mean_terminated_length": 436.04547119140625, "completions/min_length": 167.0, "completions/min_terminated_length": 167.0, "entropy": 0.3937859311699867, "epoch": 0.39346895074946464, "frac_reward_zero_std": 0.5, "grad_norm": 0.015017692930996418, "learning_rate": 1e-05, "loss": 0.0916, "num_tokens": 15626313.0, "reward": 0.5625, "reward_std": 0.2587745785713196, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.3079745769500732, "sampling/importance_sampling_ratio/mean": 1.0001164674758911, "sampling/importance_sampling_ratio/min": 0.6615281701087952, "sampling/sampling_logp_difference/max": 0.41320276260375977, "sampling/sampling_logp_difference/mean": 0.01178530603647232, "step": 735 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 617.0, "completions/mean_length": 464.96875, "completions/mean_terminated_length": 444.7666931152344, "completions/min_length": 245.0, "completions/min_terminated_length": 245.0, "entropy": 0.3442009389400482, "epoch": 0.39400428265524623, "frac_reward_zero_std": 0.25, "grad_norm": 0.017365379258990288, "learning_rate": 1e-05, "loss": -0.0806, "num_tokens": 15644872.0, "reward": 0.5625, "reward_std": 0.38298875093460083, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.4252296686172485, "sampling/importance_sampling_ratio/mean": 0.9997314214706421, "sampling/importance_sampling_ratio/min": 0.6978505849838257, "sampling/sampling_logp_difference/max": 0.35975027084350586, "sampling/sampling_logp_difference/mean": 0.011152451857924461, "step": 736 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 695.0, "completions/max_terminated_length": 695.0, "completions/mean_length": 411.125, "completions/mean_terminated_length": 411.125, "completions/min_length": 248.0, "completions/min_terminated_length": 248.0, "entropy": 0.3204801678657532, "epoch": 0.3945396145610278, "frac_reward_zero_std": 0.5, "grad_norm": 0.02090236358344555, "learning_rate": 1e-05, "loss": -0.0372, "num_tokens": 15661492.0, "reward": 0.875, "reward_std": 0.2314550280570984, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 1.4939148426055908, "sampling/importance_sampling_ratio/mean": 1.000043511390686, "sampling/importance_sampling_ratio/min": 0.7046853303909302, "sampling/sampling_logp_difference/max": 0.401400089263916, "sampling/sampling_logp_difference/mean": 0.010819543153047562, "step": 737 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 708.0, "completions/max_terminated_length": 708.0, "completions/mean_length": 460.34375, "completions/mean_terminated_length": 460.34375, "completions/min_length": 237.0, "completions/min_terminated_length": 237.0, "entropy": 0.2755957208573818, "epoch": 0.3950749464668094, "frac_reward_zero_std": 0.5, "grad_norm": 0.005674439016729593, "learning_rate": 1e-05, "loss": 0.019, "num_tokens": 15679663.0, "reward": 0.875, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 1.4159061908721924, "sampling/importance_sampling_ratio/mean": 0.9999711513519287, "sampling/importance_sampling_ratio/min": 0.7319160103797913, "sampling/sampling_logp_difference/max": 0.34776973724365234, "sampling/sampling_logp_difference/mean": 0.009373167529702187, "step": 738 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 509.0, "completions/mean_length": 399.625, "completions/mean_terminated_length": 387.7419128417969, "completions/min_length": 233.0, "completions/min_terminated_length": 233.0, "entropy": 0.32343524135649204, "epoch": 0.395610278372591, "frac_reward_zero_std": 0.75, "grad_norm": 0.015889469534158707, "learning_rate": 1e-05, "loss": 0.0031, "num_tokens": 15696403.0, "reward": 0.875, "reward_std": 0.13363061845302582, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 1.5283796787261963, "sampling/importance_sampling_ratio/mean": 0.9995576739311218, "sampling/importance_sampling_ratio/min": 0.702329695224762, "sampling/sampling_logp_difference/max": 0.4242081642150879, "sampling/sampling_logp_difference/mean": 0.011235849931836128, "step": 739 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.8809059080667794e-05, "clip_ratio/low_min": 4.8809059080667794e-05, "clip_ratio/region_mean": 4.8809059080667794e-05, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 739.0, "completions/mean_length": 526.40625, "completions/mean_terminated_length": 458.7599792480469, "completions/min_length": 265.0, "completions/min_terminated_length": 265.0, "entropy": 0.31673742085695267, "epoch": 0.3961456102783726, "frac_reward_zero_std": 0.25, "grad_norm": 0.028274651616811752, "learning_rate": 1e-05, "loss": 0.0256, "num_tokens": 15717128.0, "reward": 0.65625, "reward_std": 0.3061639666557312, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.3521307706832886, "sampling/importance_sampling_ratio/mean": 1.0007063150405884, "sampling/importance_sampling_ratio/min": 0.6952041387557983, "sampling/sampling_logp_difference/max": 0.36354970932006836, "sampling/sampling_logp_difference/mean": 0.01056370697915554, "step": 740 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 716.0, "completions/mean_length": 474.9375, "completions/mean_terminated_length": 433.0714416503906, "completions/min_length": 222.0, "completions/min_terminated_length": 222.0, "entropy": 0.35712743923068047, "epoch": 0.3966809421841542, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0037, "num_tokens": 15736694.0, "reward": 0.71875, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.4580076932907104, "sampling/importance_sampling_ratio/mean": 0.9994637966156006, "sampling/importance_sampling_ratio/min": 0.6935352683067322, "sampling/sampling_logp_difference/max": 0.37707090377807617, "sampling/sampling_logp_difference/mean": 0.012560808099806309, "step": 741 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 709.0, "completions/max_terminated_length": 709.0, "completions/mean_length": 423.5, "completions/mean_terminated_length": 423.5, "completions/min_length": 188.0, "completions/min_terminated_length": 188.0, "entropy": 0.2690687384456396, "epoch": 0.39721627408993576, "frac_reward_zero_std": 0.5, "grad_norm": 0.007789425551891327, "learning_rate": 1e-05, "loss": 0.0069, "num_tokens": 15753486.0, "reward": 0.90625, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.2923842668533325, "sampling/importance_sampling_ratio/mean": 0.9999104738235474, "sampling/importance_sampling_ratio/min": 0.5045144557952881, "sampling/sampling_logp_difference/max": 0.6841588020324707, "sampling/sampling_logp_difference/mean": 0.009072489105165005, "step": 742 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 756.0, "completions/max_terminated_length": 756.0, "completions/mean_length": 407.625, "completions/mean_terminated_length": 407.625, "completions/min_length": 197.0, "completions/min_terminated_length": 197.0, "entropy": 0.23075343668460846, "epoch": 0.39775160599571735, "frac_reward_zero_std": 0.75, "grad_norm": 0.011375130154192448, "learning_rate": 1e-05, "loss": -0.0061, "num_tokens": 15770138.0, "reward": 0.84375, "reward_std": 0.1293872892856598, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.285789966583252, "sampling/importance_sampling_ratio/mean": 0.9999356269836426, "sampling/importance_sampling_ratio/min": 0.6539907455444336, "sampling/sampling_logp_difference/max": 0.42466211318969727, "sampling/sampling_logp_difference/mean": 0.008017098531126976, "step": 743 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 743.0, "completions/max_terminated_length": 743.0, "completions/mean_length": 421.5, "completions/mean_terminated_length": 421.5, "completions/min_length": 215.0, "completions/min_terminated_length": 215.0, "entropy": 0.2794561889022589, "epoch": 0.39828693790149894, "frac_reward_zero_std": 0.5, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": -0.0023, "num_tokens": 15786994.0, "reward": 0.9375, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.9375, "rewards/accuracy_reward/std": 0.24593468010425568, "sampling/importance_sampling_ratio/max": 1.405788540840149, "sampling/importance_sampling_ratio/mean": 0.999731183052063, "sampling/importance_sampling_ratio/min": 0.729838490486145, "sampling/sampling_logp_difference/max": 0.34059834480285645, "sampling/sampling_logp_difference/mean": 0.010126659646630287, "step": 744 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00012459380013751797, "clip_ratio/low_min": 0.00012459380013751797, "clip_ratio/region_mean": 0.00012459380013751797, "completions/clipped_ratio": 0.0, "completions/max_length": 660.0, "completions/max_terminated_length": 660.0, "completions/mean_length": 430.78125, "completions/mean_terminated_length": 430.78125, "completions/min_length": 281.0, "completions/min_terminated_length": 281.0, "entropy": 0.2542971894145012, "epoch": 0.3988222698072805, "frac_reward_zero_std": 0.5, "grad_norm": 0.009447346441447735, "learning_rate": 1e-05, "loss": 0.0376, "num_tokens": 15804395.0, "reward": 0.8125, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.2802484035491943, "sampling/importance_sampling_ratio/mean": 0.9995415210723877, "sampling/importance_sampling_ratio/min": 0.5784639716148376, "sampling/sampling_logp_difference/max": 0.5473790168762207, "sampling/sampling_logp_difference/mean": 0.009644705802202225, "step": 745 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 562.0, "completions/max_terminated_length": 562.0, "completions/mean_length": 381.875, "completions/mean_terminated_length": 381.875, "completions/min_length": 230.0, "completions/min_terminated_length": 230.0, "entropy": 0.3233192600309849, "epoch": 0.3993576017130621, "frac_reward_zero_std": 0.25, "grad_norm": 0.008273539133369923, "learning_rate": 1e-05, "loss": 0.0202, "num_tokens": 15820103.0, "reward": 0.875, "reward_std": 0.292504221200943, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 1.4061501026153564, "sampling/importance_sampling_ratio/mean": 0.9998979568481445, "sampling/importance_sampling_ratio/min": 0.7168700695037842, "sampling/sampling_logp_difference/max": 0.34085559844970703, "sampling/sampling_logp_difference/mean": 0.009892796166241169, "step": 746 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00013287238834891468, "clip_ratio/low_min": 0.00013287238834891468, "clip_ratio/region_mean": 0.00013287238834891468, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 597.0, "completions/mean_length": 395.03125, "completions/mean_terminated_length": 383.0, "completions/min_length": 228.0, "completions/min_terminated_length": 228.0, "entropy": 0.3803809992969036, "epoch": 0.3998929336188437, "frac_reward_zero_std": 0.5, "grad_norm": 0.058856669813394547, "learning_rate": 1e-05, "loss": 0.0639, "num_tokens": 15836504.0, "reward": 0.84375, "reward_std": 0.22201895713806152, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.3565399646759033, "sampling/importance_sampling_ratio/mean": 0.9999346733093262, "sampling/importance_sampling_ratio/min": 0.7007579207420349, "sampling/sampling_logp_difference/max": 0.3555927276611328, "sampling/sampling_logp_difference/mean": 0.011545914225280285, "step": 747 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 628.0, "completions/mean_length": 410.8125, "completions/mean_terminated_length": 399.2903137207031, "completions/min_length": 202.0, "completions/min_terminated_length": 202.0, "entropy": 0.3029617629945278, "epoch": 0.4004282655246253, "frac_reward_zero_std": 0.5, "grad_norm": 0.014787835069000721, "learning_rate": 1e-05, "loss": 0.0361, "num_tokens": 15853162.0, "reward": 0.8125, "reward_std": 0.249358132481575, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.4724562168121338, "sampling/importance_sampling_ratio/mean": 0.9996786117553711, "sampling/importance_sampling_ratio/min": 0.6643556356430054, "sampling/sampling_logp_difference/max": 0.4089376926422119, "sampling/sampling_logp_difference/mean": 0.010183299891650677, "step": 748 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00017294470671913587, "clip_ratio/low_min": 0.00017294470671913587, "clip_ratio/region_mean": 0.00017294470671913587, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 733.0, "completions/mean_length": 538.6875, "completions/mean_terminated_length": 474.47998046875, "completions/min_length": 223.0, "completions/min_terminated_length": 223.0, "entropy": 0.3989364579319954, "epoch": 0.4009635974304068, "frac_reward_zero_std": 0.25, "grad_norm": 0.008750291541218758, "learning_rate": 1e-05, "loss": 0.0115, "num_tokens": 15874688.0, "reward": 0.4375, "reward_std": 0.3104073107242584, "rewards/accuracy_reward/mean": 0.4375, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.420766830444336, "sampling/importance_sampling_ratio/mean": 0.9997806549072266, "sampling/importance_sampling_ratio/min": 0.7073161602020264, "sampling/sampling_logp_difference/max": 0.3511967658996582, "sampling/sampling_logp_difference/mean": 0.012256912887096405, "step": 749 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.765682726632804e-05, "clip_ratio/low_min": 5.765682726632804e-05, "clip_ratio/region_mean": 5.765682726632804e-05, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 665.0, "completions/mean_length": 493.03125, "completions/mean_terminated_length": 453.7500305175781, "completions/min_length": 258.0, "completions/min_terminated_length": 258.0, "entropy": 0.26635151728987694, "epoch": 0.4014989293361884, "frac_reward_zero_std": 0.25, "grad_norm": 0.013368546962738037, "learning_rate": 1e-05, "loss": 0.064, "num_tokens": 15894089.0, "reward": 0.5, "reward_std": 0.3514062464237213, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5080004930496216, "sampling/importance_sampling_ratio/max": 1.5822577476501465, "sampling/importance_sampling_ratio/mean": 1.000051736831665, "sampling/importance_sampling_ratio/min": 0.2953316867351532, "sampling/sampling_logp_difference/max": 1.219656229019165, "sampling/sampling_logp_difference/mean": 0.00905274972319603, "step": 750 }, { "clip_ratio/high_max": 7.802746404195204e-05, "clip_ratio/high_mean": 7.802746404195204e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 7.802746404195204e-05, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 723.0, "completions/mean_length": 481.25, "completions/mean_terminated_length": 462.13336181640625, "completions/min_length": 301.0, "completions/min_terminated_length": 301.0, "entropy": 0.3769327774643898, "epoch": 0.40203426124197, "frac_reward_zero_std": 0.25, "grad_norm": 0.0483182817697525, "learning_rate": 1e-05, "loss": 0.0122, "num_tokens": 15912921.0, "reward": 0.6875, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.4769721031188965, "sampling/importance_sampling_ratio/mean": 0.9998060464859009, "sampling/importance_sampling_ratio/min": 0.6666865944862366, "sampling/sampling_logp_difference/max": 0.4054352045059204, "sampling/sampling_logp_difference/mean": 0.012469490990042686, "step": 751 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00016942126967478544, "clip_ratio/low_min": 0.00016942126967478544, "clip_ratio/region_mean": 0.00016942126967478544, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 628.0, "completions/mean_length": 485.03125, "completions/mean_terminated_length": 432.629638671875, "completions/min_length": 295.0, "completions/min_terminated_length": 295.0, "entropy": 0.38218012638390064, "epoch": 0.4025695931477516, "frac_reward_zero_std": 0.5, "grad_norm": 0.012462539598345757, "learning_rate": 1e-05, "loss": 0.0415, "num_tokens": 15932250.0, "reward": 0.75, "reward_std": 0.2314550280570984, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0002398490905762, "sampling/importance_sampling_ratio/min": 0.48620763421058655, "sampling/sampling_logp_difference/max": 0.7410974502563477, "sampling/sampling_logp_difference/mean": 0.011902324855327606, "step": 752 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.6484408560208976e-05, "clip_ratio/low_min": 5.6484408560208976e-05, "clip_ratio/region_mean": 5.6484408560208976e-05, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 645.0, "completions/mean_length": 455.4375, "completions/mean_terminated_length": 434.60003662109375, "completions/min_length": 241.0, "completions/min_terminated_length": 241.0, "entropy": 0.40006024576723576, "epoch": 0.4031049250535332, "frac_reward_zero_std": 0.5, "grad_norm": 0.010189701803028584, "learning_rate": 1e-05, "loss": 0.0842, "num_tokens": 15950536.0, "reward": 0.8125, "reward_std": 0.2587745785713196, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.5445659160614014, "sampling/importance_sampling_ratio/mean": 1.0000519752502441, "sampling/importance_sampling_ratio/min": 0.7623915076255798, "sampling/sampling_logp_difference/max": 0.43474292755126953, "sampling/sampling_logp_difference/mean": 0.011419408023357391, "step": 753 }, { "clip_ratio/high_max": 5.7444853155175224e-05, "clip_ratio/high_mean": 5.7444853155175224e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 5.7444853155175224e-05, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 703.0, "completions/mean_length": 559.5, "completions/mean_terminated_length": 490.0, "completions/min_length": 299.0, "completions/min_terminated_length": 299.0, "entropy": 0.4570811428129673, "epoch": 0.40364025695931477, "frac_reward_zero_std": 0.25, "grad_norm": 0.014431962743401527, "learning_rate": 1e-05, "loss": -0.0126, "num_tokens": 15972584.0, "reward": 0.5625, "reward_std": 0.3471825420856476, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.5669465065002441, "sampling/importance_sampling_ratio/mean": 1.0001393556594849, "sampling/importance_sampling_ratio/min": 0.686478316783905, "sampling/sampling_logp_difference/max": 0.4491288661956787, "sampling/sampling_logp_difference/mean": 0.015018288977444172, "step": 754 }, { "clip_ratio/high_max": 6.316321378108114e-05, "clip_ratio/high_mean": 6.316321378108114e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 6.316321378108114e-05, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 731.0, "completions/mean_length": 432.0625, "completions/mean_terminated_length": 354.5384826660156, "completions/min_length": 150.0, "completions/min_terminated_length": 150.0, "entropy": 0.39246325194835663, "epoch": 0.40417558886509636, "frac_reward_zero_std": 0.25, "grad_norm": 0.014133045449852943, "learning_rate": 1e-05, "loss": -0.0695, "num_tokens": 15989626.0, "reward": 0.625, "reward_std": 0.3104073107242584, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.4653111696243286, "sampling/importance_sampling_ratio/mean": 1.0005357265472412, "sampling/importance_sampling_ratio/min": 0.7505085468292236, "sampling/sampling_logp_difference/max": 0.3820676803588867, "sampling/sampling_logp_difference/mean": 0.011345033533871174, "step": 755 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 690.0, "completions/mean_length": 484.5625, "completions/mean_terminated_length": 475.4193420410156, "completions/min_length": 328.0, "completions/min_terminated_length": 328.0, "entropy": 0.35822298005223274, "epoch": 0.40471092077087795, "frac_reward_zero_std": 0.5, "grad_norm": 0.028463730588555336, "learning_rate": 1e-05, "loss": 0.0381, "num_tokens": 16009412.0, "reward": 0.78125, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.6963238716125488, "sampling/importance_sampling_ratio/mean": 0.9997175931930542, "sampling/importance_sampling_ratio/min": 0.5386660695075989, "sampling/sampling_logp_difference/max": 0.618659496307373, "sampling/sampling_logp_difference/mean": 0.011470532976090908, "step": 756 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 556.0, "completions/max_terminated_length": 556.0, "completions/mean_length": 408.375, "completions/mean_terminated_length": 408.375, "completions/min_length": 224.0, "completions/min_terminated_length": 224.0, "entropy": 0.27229262702167034, "epoch": 0.40524625267665954, "frac_reward_zero_std": 0.5, "grad_norm": 0.02580346167087555, "learning_rate": 1e-05, "loss": 0.0344, "num_tokens": 16026240.0, "reward": 0.9375, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.9375, "rewards/accuracy_reward/std": 0.24593468010425568, "sampling/importance_sampling_ratio/max": 1.4563825130462646, "sampling/importance_sampling_ratio/mean": 0.9996753931045532, "sampling/importance_sampling_ratio/min": 0.7031433582305908, "sampling/sampling_logp_difference/max": 0.37595558166503906, "sampling/sampling_logp_difference/mean": 0.009219473227858543, "step": 757 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 736.0, "completions/max_terminated_length": 736.0, "completions/mean_length": 414.59375, "completions/mean_terminated_length": 414.59375, "completions/min_length": 150.0, "completions/min_terminated_length": 150.0, "entropy": 0.29809033684432507, "epoch": 0.4057815845824411, "frac_reward_zero_std": 0.25, "grad_norm": 0.005617144517600536, "learning_rate": 1e-05, "loss": -0.0031, "num_tokens": 16042971.0, "reward": 0.71875, "reward_std": 0.3471629321575165, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.2700101137161255, "sampling/importance_sampling_ratio/mean": 1.000062346458435, "sampling/importance_sampling_ratio/min": 0.706326961517334, "sampling/sampling_logp_difference/max": 0.34767699241638184, "sampling/sampling_logp_difference/mean": 0.009595252573490143, "step": 758 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.901960710412823e-05, "clip_ratio/low_min": 4.901960710412823e-05, "clip_ratio/region_mean": 4.901960710412823e-05, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 620.0, "completions/mean_length": 532.28125, "completions/mean_terminated_length": 466.2799987792969, "completions/min_length": 270.0, "completions/min_terminated_length": 270.0, "entropy": 0.3346907217055559, "epoch": 0.4063169164882227, "frac_reward_zero_std": 0.25, "grad_norm": 0.015848277136683464, "learning_rate": 1e-05, "loss": 0.0345, "num_tokens": 16064428.0, "reward": 0.53125, "reward_std": 0.378745436668396, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.3548301458358765, "sampling/importance_sampling_ratio/mean": 0.9993706941604614, "sampling/importance_sampling_ratio/min": 0.6472514271736145, "sampling/sampling_logp_difference/max": 0.43502044677734375, "sampling/sampling_logp_difference/mean": 0.010415096767246723, "step": 759 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.706008571200073e-05, "clip_ratio/low_min": 6.706008571200073e-05, "clip_ratio/region_mean": 6.706008571200073e-05, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 679.0, "completions/mean_length": 523.0625, "completions/mean_terminated_length": 497.72412109375, "completions/min_length": 238.0, "completions/min_terminated_length": 238.0, "entropy": 0.2949142698198557, "epoch": 0.4068522483940043, "frac_reward_zero_std": 0.25, "grad_norm": 0.008824774995446205, "learning_rate": 1e-05, "loss": -0.0136, "num_tokens": 16084710.0, "reward": 0.375, "reward_std": 0.3104073107242584, "rewards/accuracy_reward/mean": 0.375, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.624711275100708, "sampling/importance_sampling_ratio/mean": 1.0001001358032227, "sampling/importance_sampling_ratio/min": 0.7189587950706482, "sampling/sampling_logp_difference/max": 0.48533010482788086, "sampling/sampling_logp_difference/mean": 0.009899305179715157, "step": 760 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.441880784928799e-05, "clip_ratio/low_min": 5.441880784928799e-05, "clip_ratio/region_mean": 5.441880784928799e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 632.0, "completions/max_terminated_length": 632.0, "completions/mean_length": 480.40625, "completions/mean_terminated_length": 480.40625, "completions/min_length": 278.0, "completions/min_terminated_length": 278.0, "entropy": 0.27928940765559673, "epoch": 0.4073875802997859, "frac_reward_zero_std": 0.0, "grad_norm": 0.012388971634209156, "learning_rate": 1e-05, "loss": -0.0378, "num_tokens": 16103971.0, "reward": 0.59375, "reward_std": 0.3987956643104553, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.424300193786621, "sampling/importance_sampling_ratio/mean": 0.9999654293060303, "sampling/importance_sampling_ratio/min": 0.5367613434791565, "sampling/sampling_logp_difference/max": 0.622201681137085, "sampling/sampling_logp_difference/mean": 0.009574158117175102, "step": 761 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 716.0, "completions/mean_length": 509.625, "completions/mean_terminated_length": 482.89654541015625, "completions/min_length": 252.0, "completions/min_terminated_length": 252.0, "entropy": 0.2994512524455786, "epoch": 0.4079229122055675, "frac_reward_zero_std": 0.5, "grad_norm": 0.008833318948745728, "learning_rate": 1e-05, "loss": 0.0644, "num_tokens": 16123567.0, "reward": 0.84375, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.5718441009521484, "sampling/importance_sampling_ratio/mean": 1.0001444816589355, "sampling/importance_sampling_ratio/min": 0.5895460844039917, "sampling/sampling_logp_difference/max": 0.5284023284912109, "sampling/sampling_logp_difference/mean": 0.009947758167982101, "step": 762 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 701.0, "completions/mean_length": 444.34375, "completions/mean_terminated_length": 433.9031982421875, "completions/min_length": 157.0, "completions/min_terminated_length": 157.0, "entropy": 0.27685064263641834, "epoch": 0.408458244111349, "frac_reward_zero_std": 0.5, "grad_norm": 0.003286470426246524, "learning_rate": 1e-05, "loss": 0.065, "num_tokens": 16141594.0, "reward": 0.84375, "reward_std": 0.22201895713806152, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.2818659543991089, "sampling/importance_sampling_ratio/mean": 1.0000402927398682, "sampling/importance_sampling_ratio/min": 0.7070391774177551, "sampling/sampling_logp_difference/max": 0.34666919708251953, "sampling/sampling_logp_difference/mean": 0.009597057476639748, "step": 763 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.339598283171654e-05, "clip_ratio/low_min": 5.339598283171654e-05, "clip_ratio/region_mean": 5.339598283171654e-05, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 737.0, "completions/mean_length": 501.71875, "completions/mean_terminated_length": 474.17242431640625, "completions/min_length": 293.0, "completions/min_terminated_length": 293.0, "entropy": 0.4013344496488571, "epoch": 0.4089935760171306, "frac_reward_zero_std": 0.5, "grad_norm": 0.023167669773101807, "learning_rate": 1e-05, "loss": -0.0163, "num_tokens": 16161313.0, "reward": 0.84375, "reward_std": 0.22201895713806152, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.3985191583633423, "sampling/importance_sampling_ratio/mean": 1.000298261642456, "sampling/importance_sampling_ratio/min": 0.7825467586517334, "sampling/sampling_logp_difference/max": 0.33541393280029297, "sampling/sampling_logp_difference/mean": 0.012038334272801876, "step": 764 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 747.0, "completions/max_terminated_length": 747.0, "completions/mean_length": 438.0625, "completions/mean_terminated_length": 438.0625, "completions/min_length": 255.0, "completions/min_terminated_length": 255.0, "entropy": 0.30539108626544476, "epoch": 0.4095289079229122, "frac_reward_zero_std": 0.75, "grad_norm": 0.016800880432128906, "learning_rate": 1e-05, "loss": -0.0384, "num_tokens": 16179387.0, "reward": 0.90625, "reward_std": 0.1293872892856598, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.8013707399368286, "sampling/importance_sampling_ratio/mean": 0.999898374080658, "sampling/importance_sampling_ratio/min": 0.6412967443466187, "sampling/sampling_logp_difference/max": 0.588547945022583, "sampling/sampling_logp_difference/mean": 0.009695417247712612, "step": 765 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0001732737509883009, "clip_ratio/low_min": 0.0001732737509883009, "clip_ratio/region_mean": 0.0001732737509883009, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 743.0, "completions/mean_length": 562.71875, "completions/mean_terminated_length": 524.7037353515625, "completions/min_length": 314.0, "completions/min_terminated_length": 314.0, "entropy": 0.29583428986370564, "epoch": 0.4100642398286938, "frac_reward_zero_std": 0.25, "grad_norm": 0.009501498192548752, "learning_rate": 1e-05, "loss": 0.0282, "num_tokens": 16202186.0, "reward": 0.59375, "reward_std": 0.3198433816432953, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.4362612962722778, "sampling/importance_sampling_ratio/mean": 0.999686062335968, "sampling/importance_sampling_ratio/min": 0.48731547594070435, "sampling/sampling_logp_difference/max": 0.7188435792922974, "sampling/sampling_logp_difference/mean": 0.010522641241550446, "step": 766 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 692.0, "completions/mean_length": 507.53125, "completions/mean_terminated_length": 480.5862121582031, "completions/min_length": 243.0, "completions/min_terminated_length": 243.0, "entropy": 0.3411188870668411, "epoch": 0.41059957173447537, "frac_reward_zero_std": 0.5, "grad_norm": 0.018128888681530952, "learning_rate": 1e-05, "loss": 0.0475, "num_tokens": 16222331.0, "reward": 0.78125, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.4630672931671143, "sampling/importance_sampling_ratio/mean": 1.0001051425933838, "sampling/importance_sampling_ratio/min": 0.6867654323577881, "sampling/sampling_logp_difference/max": 0.3805351257324219, "sampling/sampling_logp_difference/mean": 0.011262279003858566, "step": 767 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 673.0, "completions/max_terminated_length": 673.0, "completions/mean_length": 398.9375, "completions/mean_terminated_length": 398.9375, "completions/min_length": 205.0, "completions/min_terminated_length": 205.0, "entropy": 0.2790275290608406, "epoch": 0.41113490364025695, "frac_reward_zero_std": 0.75, "grad_norm": 0.005060677882283926, "learning_rate": 1e-05, "loss": -0.0046, "num_tokens": 16238849.0, "reward": 0.96875, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.96875, "rewards/accuracy_reward/std": 0.1767766922712326, "sampling/importance_sampling_ratio/max": 1.435973048210144, "sampling/importance_sampling_ratio/mean": 0.9995302557945251, "sampling/importance_sampling_ratio/min": 0.7087464928627014, "sampling/sampling_logp_difference/max": 0.36184263229370117, "sampling/sampling_logp_difference/mean": 0.009966474957764149, "step": 768 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 728.0, "completions/mean_length": 457.4375, "completions/mean_terminated_length": 436.7333679199219, "completions/min_length": 176.0, "completions/min_terminated_length": 176.0, "entropy": 0.294871186837554, "epoch": 0.41167023554603854, "frac_reward_zero_std": 0.5, "grad_norm": 0.009670751169323921, "learning_rate": 1e-05, "loss": 0.0044, "num_tokens": 16256663.0, "reward": 0.8125, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.3708109855651855, "sampling/importance_sampling_ratio/mean": 0.9998509287834167, "sampling/importance_sampling_ratio/min": 0.7468981146812439, "sampling/sampling_logp_difference/max": 0.3154025077819824, "sampling/sampling_logp_difference/mean": 0.009731477126479149, "step": 769 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00013690898776985705, "clip_ratio/low_min": 0.00013690898776985705, "clip_ratio/region_mean": 0.00013690898776985705, "completions/clipped_ratio": 0.0, "completions/max_length": 741.0, "completions/max_terminated_length": 741.0, "completions/mean_length": 455.5, "completions/mean_terminated_length": 455.5, "completions/min_length": 255.0, "completions/min_terminated_length": 255.0, "entropy": 0.34996984153985977, "epoch": 0.41220556745182013, "frac_reward_zero_std": 0.25, "grad_norm": 0.005920007824897766, "learning_rate": 1e-05, "loss": -0.0093, "num_tokens": 16275295.0, "reward": 0.59375, "reward_std": 0.3471629321575165, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.4538800716400146, "sampling/importance_sampling_ratio/mean": 0.9999443292617798, "sampling/importance_sampling_ratio/min": 0.7225494980812073, "sampling/sampling_logp_difference/max": 0.3742358684539795, "sampling/sampling_logp_difference/mean": 0.011001553386449814, "step": 770 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 753.0, "completions/mean_length": 469.46875, "completions/mean_terminated_length": 449.5666809082031, "completions/min_length": 279.0, "completions/min_terminated_length": 279.0, "entropy": 0.33411771804094315, "epoch": 0.4127408993576017, "frac_reward_zero_std": 0.5, "grad_norm": 0.015054845251142979, "learning_rate": 1e-05, "loss": -0.0378, "num_tokens": 16294782.0, "reward": 0.5625, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.363189697265625, "sampling/importance_sampling_ratio/mean": 1.0000303983688354, "sampling/importance_sampling_ratio/min": 0.6979761719703674, "sampling/sampling_logp_difference/max": 0.35957032442092896, "sampling/sampling_logp_difference/mean": 0.011455297470092773, "step": 771 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 607.0, "completions/max_terminated_length": 607.0, "completions/mean_length": 435.90625, "completions/mean_terminated_length": 435.90625, "completions/min_length": 228.0, "completions/min_terminated_length": 228.0, "entropy": 0.27202615328133106, "epoch": 0.4132762312633833, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": -0.0109, "num_tokens": 16313067.0, "reward": 0.59375, "reward_std": 0.1293872892856598, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.378258228302002, "sampling/importance_sampling_ratio/mean": 0.9994197487831116, "sampling/importance_sampling_ratio/min": 0.6492050290107727, "sampling/sampling_logp_difference/max": 0.4320066571235657, "sampling/sampling_logp_difference/mean": 0.009395753964781761, "step": 772 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 595.0, "completions/max_terminated_length": 595.0, "completions/mean_length": 373.5625, "completions/mean_terminated_length": 373.5625, "completions/min_length": 158.0, "completions/min_terminated_length": 158.0, "entropy": 0.29544753581285477, "epoch": 0.4138115631691649, "frac_reward_zero_std": 0.75, "grad_norm": 0.003570796921849251, "learning_rate": 1e-05, "loss": 0.0375, "num_tokens": 16328933.0, "reward": 0.96875, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.96875, "rewards/accuracy_reward/std": 0.1767766922712326, "sampling/importance_sampling_ratio/max": 1.3228645324707031, "sampling/importance_sampling_ratio/mean": 1.0000734329223633, "sampling/importance_sampling_ratio/min": 0.7217989563941956, "sampling/sampling_logp_difference/max": 0.326008677482605, "sampling/sampling_logp_difference/mean": 0.00953470729291439, "step": 773 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 9.433962259208784e-05, "clip_ratio/low_min": 9.433962259208784e-05, "clip_ratio/region_mean": 9.433962259208784e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 664.0, "completions/max_terminated_length": 664.0, "completions/mean_length": 371.3125, "completions/mean_terminated_length": 371.3125, "completions/min_length": 191.0, "completions/min_terminated_length": 191.0, "entropy": 0.3113366477191448, "epoch": 0.4143468950749465, "frac_reward_zero_std": 0.25, "grad_norm": 0.013471437618136406, "learning_rate": 1e-05, "loss": -0.0216, "num_tokens": 16344423.0, "reward": 0.6875, "reward_std": 0.3514062464237213, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.3202178478240967, "sampling/importance_sampling_ratio/mean": 0.9994405508041382, "sampling/importance_sampling_ratio/min": 0.7053040266036987, "sampling/sampling_logp_difference/max": 0.34912633895874023, "sampling/sampling_logp_difference/mean": 0.01060332078486681, "step": 774 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 7.802746404195204e-05, "clip_ratio/low_min": 7.802746404195204e-05, "clip_ratio/region_mean": 7.802746404195204e-05, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 665.0, "completions/mean_length": 437.0625, "completions/mean_terminated_length": 415.0000305175781, "completions/min_length": 200.0, "completions/min_terminated_length": 200.0, "entropy": 0.43285075947642326, "epoch": 0.4148822269807281, "frac_reward_zero_std": 0.5, "grad_norm": 0.02249554730951786, "learning_rate": 1e-05, "loss": -0.0058, "num_tokens": 16362817.0, "reward": 0.59375, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.5441429615020752, "sampling/importance_sampling_ratio/mean": 0.999977171421051, "sampling/importance_sampling_ratio/min": 0.6840009093284607, "sampling/sampling_logp_difference/max": 0.43446898460388184, "sampling/sampling_logp_difference/mean": 0.013381147757172585, "step": 775 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00015147724479902536, "clip_ratio/low_min": 0.00015147724479902536, "clip_ratio/region_mean": 0.00015147724479902536, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 746.0, "completions/mean_length": 542.15625, "completions/mean_terminated_length": 490.0384826660156, "completions/min_length": 267.0, "completions/min_terminated_length": 267.0, "entropy": 0.3197234198451042, "epoch": 0.41541755888650966, "frac_reward_zero_std": 0.25, "grad_norm": 0.018198829144239426, "learning_rate": 1e-05, "loss": -0.0302, "num_tokens": 16384078.0, "reward": 0.5625, "reward_std": 0.3745020925998688, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.501399040222168, "sampling/importance_sampling_ratio/mean": 0.9999122023582458, "sampling/importance_sampling_ratio/min": 0.7645636200904846, "sampling/sampling_logp_difference/max": 0.40639734268188477, "sampling/sampling_logp_difference/mean": 0.010852006264030933, "step": 776 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 9.164222865365446e-05, "clip_ratio/low_min": 9.164222865365446e-05, "clip_ratio/region_mean": 9.164222865365446e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 723.0, "completions/max_terminated_length": 723.0, "completions/mean_length": 434.5625, "completions/mean_terminated_length": 434.5625, "completions/min_length": 203.0, "completions/min_terminated_length": 203.0, "entropy": 0.3417657408863306, "epoch": 0.4159528907922912, "frac_reward_zero_std": 0.25, "grad_norm": 0.025946030393242836, "learning_rate": 1e-05, "loss": 0.0231, "num_tokens": 16401288.0, "reward": 0.59375, "reward_std": 0.3198433816432953, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.5710872411727905, "sampling/importance_sampling_ratio/mean": 0.9997629523277283, "sampling/importance_sampling_ratio/min": 0.6923314332962036, "sampling/sampling_logp_difference/max": 0.4517679214477539, "sampling/sampling_logp_difference/mean": 0.011955196969211102, "step": 777 }, { "clip_ratio/high_max": 6.443299207603559e-05, "clip_ratio/high_mean": 6.443299207603559e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 6.443299207603559e-05, "completions/clipped_ratio": 0.3125, "completions/max_length": 768.0, "completions/max_terminated_length": 699.0, "completions/mean_length": 553.21875, "completions/mean_terminated_length": 455.5909118652344, "completions/min_length": 149.0, "completions/min_terminated_length": 149.0, "entropy": 0.4989879857748747, "epoch": 0.4164882226980728, "frac_reward_zero_std": 0.5, "grad_norm": 0.02220899611711502, "learning_rate": 1e-05, "loss": 0.0271, "num_tokens": 16422823.0, "reward": 0.34375, "reward_std": 0.22201895713806152, "rewards/accuracy_reward/mean": 0.34375, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.999857485294342, "sampling/importance_sampling_ratio/min": 0.7533624768257141, "sampling/sampling_logp_difference/max": 0.7256240844726562, "sampling/sampling_logp_difference/mean": 0.013844046741724014, "step": 778 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 521.0, "completions/mean_length": 362.0625, "completions/mean_terminated_length": 348.9677429199219, "completions/min_length": 197.0, "completions/min_terminated_length": 197.0, "entropy": 0.34428347274661064, "epoch": 0.4170235546038544, "frac_reward_zero_std": 0.25, "grad_norm": 0.027528373524546623, "learning_rate": 1e-05, "loss": -0.0426, "num_tokens": 16438033.0, "reward": 0.78125, "reward_std": 0.3377465009689331, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.345342993736267, "sampling/importance_sampling_ratio/mean": 0.9997936487197876, "sampling/importance_sampling_ratio/min": 0.7098269462585449, "sampling/sampling_logp_difference/max": 0.34273409843444824, "sampling/sampling_logp_difference/mean": 0.011760414578020573, "step": 779 }, { "clip_ratio/high_max": 9.735202183946967e-05, "clip_ratio/high_mean": 9.735202183946967e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 9.735202183946967e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 762.0, "completions/max_terminated_length": 762.0, "completions/mean_length": 412.21875, "completions/mean_terminated_length": 412.21875, "completions/min_length": 258.0, "completions/min_terminated_length": 258.0, "entropy": 0.29027098417282104, "epoch": 0.41755888650963596, "frac_reward_zero_std": 0.75, "grad_norm": 0.016562609001994133, "learning_rate": 1e-05, "loss": 0.025, "num_tokens": 16454424.0, "reward": 0.875, "reward_std": 0.13363061845302582, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 1.4183530807495117, "sampling/importance_sampling_ratio/mean": 0.9999801516532898, "sampling/importance_sampling_ratio/min": 0.724010169506073, "sampling/sampling_logp_difference/max": 0.34949636459350586, "sampling/sampling_logp_difference/mean": 0.010326000861823559, "step": 780 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 721.0, "completions/mean_length": 494.25, "completions/mean_terminated_length": 455.14288330078125, "completions/min_length": 258.0, "completions/min_terminated_length": 258.0, "entropy": 0.34014772810041904, "epoch": 0.41809421841541755, "frac_reward_zero_std": 0.75, "grad_norm": 0.005914956796914339, "learning_rate": 1e-05, "loss": 0.0079, "num_tokens": 16474184.0, "reward": 0.78125, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.3385014533996582, "sampling/importance_sampling_ratio/mean": 1.0002925395965576, "sampling/importance_sampling_ratio/min": 0.7563097476959229, "sampling/sampling_logp_difference/max": 0.2915506362915039, "sampling/sampling_logp_difference/mean": 0.011034255847334862, "step": 781 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 680.0, "completions/mean_length": 493.46875, "completions/mean_terminated_length": 475.16668701171875, "completions/min_length": 345.0, "completions/min_terminated_length": 345.0, "entropy": 0.3546532765030861, "epoch": 0.41862955032119914, "frac_reward_zero_std": 0.5, "grad_norm": 0.014526438899338245, "learning_rate": 1e-05, "loss": 0.0861, "num_tokens": 16493543.0, "reward": 0.8125, "reward_std": 0.249358132481575, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.4465943574905396, "sampling/importance_sampling_ratio/mean": 0.9996225237846375, "sampling/importance_sampling_ratio/min": 0.7496525049209595, "sampling/sampling_logp_difference/max": 0.3692120313644409, "sampling/sampling_logp_difference/mean": 0.01135631650686264, "step": 782 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 738.0, "completions/mean_length": 487.0, "completions/mean_terminated_length": 468.2666931152344, "completions/min_length": 191.0, "completions/min_terminated_length": 191.0, "entropy": 0.287403654307127, "epoch": 0.41916488222698073, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0461, "num_tokens": 16513167.0, "reward": 0.90625, "reward_std": 0.1293872892856598, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.3896218538284302, "sampling/importance_sampling_ratio/mean": 1.0000495910644531, "sampling/importance_sampling_ratio/min": 0.6388620734214783, "sampling/sampling_logp_difference/max": 0.44806671142578125, "sampling/sampling_logp_difference/mean": 0.01001773588359356, "step": 783 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00010822511103469878, "clip_ratio/low_min": 0.00010822511103469878, "clip_ratio/region_mean": 0.00010822511103469878, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 589.0, "completions/mean_length": 460.625, "completions/mean_terminated_length": 403.7037048339844, "completions/min_length": 228.0, "completions/min_terminated_length": 228.0, "entropy": 0.4263018108904362, "epoch": 0.4197002141327623, "frac_reward_zero_std": 0.25, "grad_norm": 0.028048962354660034, "learning_rate": 1e-05, "loss": 0.015, "num_tokens": 16532059.0, "reward": 0.8125, "reward_std": 0.3104073107242584, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.5398837327957153, "sampling/importance_sampling_ratio/mean": 1.0002316236495972, "sampling/importance_sampling_ratio/min": 0.7910860180854797, "sampling/sampling_logp_difference/max": 0.43170690536499023, "sampling/sampling_logp_difference/mean": 0.01344273705035448, "step": 784 }, { "clip_ratio/high_max": 7.846829976188019e-05, "clip_ratio/high_mean": 7.846829976188019e-05, "clip_ratio/low_mean": 0.000147733255289495, "clip_ratio/low_min": 0.000147733255289495, "clip_ratio/region_mean": 0.00022620155505137518, "completions/clipped_ratio": 0.0, "completions/max_length": 550.0, "completions/max_terminated_length": 550.0, "completions/mean_length": 394.84375, "completions/mean_terminated_length": 394.84375, "completions/min_length": 191.0, "completions/min_terminated_length": 191.0, "entropy": 0.30854377523064613, "epoch": 0.4202355460385439, "frac_reward_zero_std": 0.25, "grad_norm": 0.03182852640748024, "learning_rate": 1e-05, "loss": 0.0189, "num_tokens": 16548814.0, "reward": 0.84375, "reward_std": 0.3061639964580536, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.4465000629425049, "sampling/importance_sampling_ratio/mean": 1.000160813331604, "sampling/importance_sampling_ratio/min": 0.599141001701355, "sampling/sampling_logp_difference/max": 0.5122582912445068, "sampling/sampling_logp_difference/mean": 0.010697192512452602, "step": 785 }, { "clip_ratio/high_max": 8.934953802963719e-05, "clip_ratio/high_mean": 8.934953802963719e-05, "clip_ratio/low_mean": 5.6357079301960766e-05, "clip_ratio/low_min": 5.6357079301960766e-05, "clip_ratio/region_mean": 0.00014570661733159795, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 540.0, "completions/mean_length": 397.125, "completions/mean_terminated_length": 385.1612854003906, "completions/min_length": 232.0, "completions/min_terminated_length": 232.0, "entropy": 0.29431120678782463, "epoch": 0.4207708779443255, "frac_reward_zero_std": 0.5, "grad_norm": 0.006559995003044605, "learning_rate": 1e-05, "loss": 0.0352, "num_tokens": 16565282.0, "reward": 0.90625, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.673102855682373, "sampling/importance_sampling_ratio/mean": 1.0000230073928833, "sampling/importance_sampling_ratio/min": 0.5996344089508057, "sampling/sampling_logp_difference/max": 0.5146799087524414, "sampling/sampling_logp_difference/mean": 0.010234890505671501, "step": 786 }, { "clip_ratio/high_max": 7.318500865949318e-05, "clip_ratio/high_mean": 7.318500865949318e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 7.318500865949318e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 700.0, "completions/max_terminated_length": 700.0, "completions/mean_length": 358.0, "completions/mean_terminated_length": 358.0, "completions/min_length": 173.0, "completions/min_terminated_length": 173.0, "entropy": 0.31584992073476315, "epoch": 0.4213062098501071, "frac_reward_zero_std": 0.5, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0464, "num_tokens": 16580154.0, "reward": 0.90625, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.5974339246749878, "sampling/importance_sampling_ratio/mean": 0.9998888969421387, "sampling/importance_sampling_ratio/min": 0.6470592021942139, "sampling/sampling_logp_difference/max": 0.4683985710144043, "sampling/sampling_logp_difference/mean": 0.01112749520689249, "step": 787 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 613.0, "completions/mean_length": 408.875, "completions/mean_terminated_length": 397.2903137207031, "completions/min_length": 185.0, "completions/min_terminated_length": 185.0, "entropy": 0.2817312330007553, "epoch": 0.42184154175588867, "frac_reward_zero_std": 0.25, "grad_norm": 0.04354003816843033, "learning_rate": 1e-05, "loss": 0.0319, "num_tokens": 16596990.0, "reward": 0.90625, "reward_std": 0.2651650309562683, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.485916018486023, "sampling/importance_sampling_ratio/mean": 0.9996706247329712, "sampling/importance_sampling_ratio/min": 0.7777830362319946, "sampling/sampling_logp_difference/max": 0.39603137969970703, "sampling/sampling_logp_difference/mean": 0.009876232594251633, "step": 788 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 643.0, "completions/mean_length": 466.59375, "completions/mean_terminated_length": 456.8709411621094, "completions/min_length": 257.0, "completions/min_terminated_length": 257.0, "entropy": 0.38442598655819893, "epoch": 0.42237687366167026, "frac_reward_zero_std": 0.5, "grad_norm": 0.008006955496966839, "learning_rate": 1e-05, "loss": -0.0064, "num_tokens": 16616201.0, "reward": 0.40625, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.40625, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.6210137605667114, "sampling/importance_sampling_ratio/mean": 1.000440239906311, "sampling/importance_sampling_ratio/min": 0.7053622007369995, "sampling/sampling_logp_difference/max": 0.48305177688598633, "sampling/sampling_logp_difference/mean": 0.01286507397890091, "step": 789 }, { "clip_ratio/high_max": 6.503642362076789e-05, "clip_ratio/high_mean": 6.503642362076789e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 6.503642362076789e-05, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 682.0, "completions/mean_length": 440.125, "completions/mean_terminated_length": 379.40740966796875, "completions/min_length": 180.0, "completions/min_terminated_length": 180.0, "entropy": 0.3840542919933796, "epoch": 0.4229122055674518, "frac_reward_zero_std": 0.5, "grad_norm": 0.021103430539369583, "learning_rate": 1e-05, "loss": -0.0078, "num_tokens": 16634061.0, "reward": 0.8125, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.3124192953109741, "sampling/importance_sampling_ratio/mean": 0.9999834895133972, "sampling/importance_sampling_ratio/min": 0.5593166947364807, "sampling/sampling_logp_difference/max": 0.5810394287109375, "sampling/sampling_logp_difference/mean": 0.012441441416740417, "step": 790 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.036261063651182e-05, "clip_ratio/low_min": 5.036261063651182e-05, "clip_ratio/region_mean": 5.036261063651182e-05, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 708.0, "completions/mean_length": 549.8125, "completions/mean_terminated_length": 509.40740966796875, "completions/min_length": 292.0, "completions/min_terminated_length": 292.0, "entropy": 0.4020552970468998, "epoch": 0.4234475374732334, "frac_reward_zero_std": 0.5, "grad_norm": 0.023112032562494278, "learning_rate": 1e-05, "loss": -0.0065, "num_tokens": 16655855.0, "reward": 0.75, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.3809304237365723, "sampling/importance_sampling_ratio/mean": 0.999682605266571, "sampling/importance_sampling_ratio/min": 0.7346566915512085, "sampling/sampling_logp_difference/max": 0.3227574825286865, "sampling/sampling_logp_difference/mean": 0.012609808705747128, "step": 791 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00011947887469432317, "clip_ratio/low_min": 0.00011947887469432317, "clip_ratio/region_mean": 0.00011947887469432317, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 760.0, "completions/mean_length": 484.46875, "completions/mean_terminated_length": 465.5666809082031, "completions/min_length": 235.0, "completions/min_terminated_length": 235.0, "entropy": 0.4358667880296707, "epoch": 0.42398286937901497, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": -0.0032, "num_tokens": 16674934.0, "reward": 0.84375, "reward_std": 0.1293872892856598, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.3661179542541504, "sampling/importance_sampling_ratio/mean": 0.9998723864555359, "sampling/importance_sampling_ratio/min": 0.782680332660675, "sampling/sampling_logp_difference/max": 0.31197309494018555, "sampling/sampling_logp_difference/mean": 0.013119467534124851, "step": 792 }, { "clip_ratio/high_max": 8.400537626584992e-05, "clip_ratio/high_mean": 8.400537626584992e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 8.400537626584992e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 511.0, "completions/max_terminated_length": 511.0, "completions/mean_length": 358.9375, "completions/mean_terminated_length": 358.9375, "completions/min_length": 159.0, "completions/min_terminated_length": 159.0, "entropy": 0.25597029365599155, "epoch": 0.42451820128479656, "frac_reward_zero_std": 0.75, "grad_norm": 0.0038700823206454515, "learning_rate": 1e-05, "loss": 0.0041, "num_tokens": 16689908.0, "reward": 0.96875, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.96875, "rewards/accuracy_reward/std": 0.1767766922712326, "sampling/importance_sampling_ratio/max": 1.420012354850769, "sampling/importance_sampling_ratio/mean": 0.9999418258666992, "sampling/importance_sampling_ratio/min": 0.7710124254226685, "sampling/sampling_logp_difference/max": 0.3506655693054199, "sampling/sampling_logp_difference/mean": 0.008947829715907574, "step": 793 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 533.0, "completions/max_terminated_length": 533.0, "completions/mean_length": 339.84375, "completions/mean_terminated_length": 339.84375, "completions/min_length": 162.0, "completions/min_terminated_length": 162.0, "entropy": 0.26448273472487926, "epoch": 0.42505353319057815, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": -0.0002, "num_tokens": 16704335.0, "reward": 0.9375, "reward_std": 0.1157275140285492, "rewards/accuracy_reward/mean": 0.9375, "rewards/accuracy_reward/std": 0.24593468010425568, "sampling/importance_sampling_ratio/max": 1.359431266784668, "sampling/importance_sampling_ratio/mean": 0.9999831318855286, "sampling/importance_sampling_ratio/min": 0.7856153845787048, "sampling/sampling_logp_difference/max": 0.3070664405822754, "sampling/sampling_logp_difference/mean": 0.009440883062779903, "step": 794 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 669.0, "completions/max_terminated_length": 669.0, "completions/mean_length": 433.25, "completions/mean_terminated_length": 433.25, "completions/min_length": 244.0, "completions/min_terminated_length": 244.0, "entropy": 0.3539182171225548, "epoch": 0.42558886509635974, "frac_reward_zero_std": 0.0, "grad_norm": 0.015658603981137276, "learning_rate": 1e-05, "loss": -0.0086, "num_tokens": 16721991.0, "reward": 0.6875, "reward_std": 0.4355512857437134, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.6626696586608887, "sampling/importance_sampling_ratio/mean": 0.9999551773071289, "sampling/importance_sampling_ratio/min": 0.6779660582542419, "sampling/sampling_logp_difference/max": 0.5084245800971985, "sampling/sampling_logp_difference/mean": 0.01210436224937439, "step": 795 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 7.886435196269304e-05, "clip_ratio/low_min": 7.886435196269304e-05, "clip_ratio/region_mean": 7.886435196269304e-05, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 717.0, "completions/mean_length": 424.0, "completions/mean_terminated_length": 412.9031982421875, "completions/min_length": 190.0, "completions/min_terminated_length": 190.0, "entropy": 0.3245571702718735, "epoch": 0.4261241970021413, "frac_reward_zero_std": 0.5, "grad_norm": 0.005203715991228819, "learning_rate": 1e-05, "loss": 0.0258, "num_tokens": 16739199.0, "reward": 0.84375, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.3860565423965454, "sampling/importance_sampling_ratio/mean": 1.0002363920211792, "sampling/importance_sampling_ratio/min": 0.7304226756095886, "sampling/sampling_logp_difference/max": 0.3264627456665039, "sampling/sampling_logp_difference/mean": 0.01050733681768179, "step": 796 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 662.0, "completions/mean_length": 370.96875, "completions/mean_terminated_length": 358.1612854003906, "completions/min_length": 145.0, "completions/min_terminated_length": 145.0, "entropy": 0.3662017323076725, "epoch": 0.4266595289079229, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0327, "num_tokens": 16754558.0, "reward": 0.90625, "reward_std": 0.1293872892856598, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.4157353639602661, "sampling/importance_sampling_ratio/mean": 1.0003262758255005, "sampling/importance_sampling_ratio/min": 0.7653398513793945, "sampling/sampling_logp_difference/max": 0.34764909744262695, "sampling/sampling_logp_difference/mean": 0.011724627576768398, "step": 797 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.468066592584364e-05, "clip_ratio/low_min": 5.468066592584364e-05, "clip_ratio/region_mean": 5.468066592584364e-05, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 702.0, "completions/mean_length": 455.65625, "completions/mean_terminated_length": 445.58062744140625, "completions/min_length": 303.0, "completions/min_terminated_length": 303.0, "entropy": 0.3370545245707035, "epoch": 0.4271948608137045, "frac_reward_zero_std": 0.5, "grad_norm": 0.0043823388405144215, "learning_rate": 1e-05, "loss": -0.0209, "num_tokens": 16772859.0, "reward": 0.6875, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.4160512685775757, "sampling/importance_sampling_ratio/mean": 1.0002046823501587, "sampling/importance_sampling_ratio/min": 0.6508194804191589, "sampling/sampling_logp_difference/max": 0.4295229911804199, "sampling/sampling_logp_difference/mean": 0.010617252439260483, "step": 798 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.989458441035822e-05, "clip_ratio/low_min": 5.989458441035822e-05, "clip_ratio/region_mean": 5.989458441035822e-05, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 737.0, "completions/mean_length": 543.09375, "completions/mean_terminated_length": 519.8275756835938, "completions/min_length": 277.0, "completions/min_terminated_length": 277.0, "entropy": 0.3294404186308384, "epoch": 0.4277301927194861, "frac_reward_zero_std": 0.5, "grad_norm": 0.007699902169406414, "learning_rate": 1e-05, "loss": 0.0544, "num_tokens": 16794214.0, "reward": 0.875, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 1.4315295219421387, "sampling/importance_sampling_ratio/mean": 1.0000755786895752, "sampling/importance_sampling_ratio/min": 0.6101194024085999, "sampling/sampling_logp_difference/max": 0.49410057067871094, "sampling/sampling_logp_difference/mean": 0.011136038228869438, "step": 799 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 587.0, "completions/max_terminated_length": 587.0, "completions/mean_length": 407.21875, "completions/mean_terminated_length": 407.21875, "completions/min_length": 203.0, "completions/min_terminated_length": 203.0, "entropy": 0.29633622989058495, "epoch": 0.4282655246252677, "frac_reward_zero_std": 0.0, "grad_norm": 0.02443593367934227, "learning_rate": 1e-05, "loss": 0.0021, "num_tokens": 16811309.0, "reward": 0.84375, "reward_std": 0.3808925747871399, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.593308448791504, "sampling/importance_sampling_ratio/mean": 1.000170350074768, "sampling/importance_sampling_ratio/min": 0.7495781779289246, "sampling/sampling_logp_difference/max": 0.46581268310546875, "sampling/sampling_logp_difference/mean": 0.010436818934977055, "step": 800 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 8.132726361509413e-05, "clip_ratio/low_min": 8.132726361509413e-05, "clip_ratio/region_mean": 8.132726361509413e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 653.0, "completions/max_terminated_length": 653.0, "completions/mean_length": 452.96875, "completions/mean_terminated_length": 452.96875, "completions/min_length": 256.0, "completions/min_terminated_length": 256.0, "entropy": 0.2979501448571682, "epoch": 0.42880085653104927, "frac_reward_zero_std": 0.0, "grad_norm": 0.028952719643712044, "learning_rate": 1e-05, "loss": -0.0137, "num_tokens": 16829436.0, "reward": 0.65625, "reward_std": 0.4628904461860657, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.3987958431243896, "sampling/importance_sampling_ratio/mean": 1.0001310110092163, "sampling/importance_sampling_ratio/min": 0.6112988591194153, "sampling/sampling_logp_difference/max": 0.4921693801879883, "sampling/sampling_logp_difference/mean": 0.010467601008713245, "step": 801 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.757715189247392e-05, "clip_ratio/low_min": 5.757715189247392e-05, "clip_ratio/region_mean": 5.757715189247392e-05, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 640.0, "completions/mean_length": 482.3125, "completions/mean_terminated_length": 473.0967712402344, "completions/min_length": 294.0, "completions/min_terminated_length": 294.0, "entropy": 0.32549588568508625, "epoch": 0.42933618843683086, "frac_reward_zero_std": 0.25, "grad_norm": 0.007285809610038996, "learning_rate": 1e-05, "loss": 0.0304, "num_tokens": 16849150.0, "reward": 0.78125, "reward_std": 0.3061639964580536, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.3400046825408936, "sampling/importance_sampling_ratio/mean": 0.9999621510505676, "sampling/importance_sampling_ratio/min": 0.6875501871109009, "sampling/sampling_logp_difference/max": 0.3746204376220703, "sampling/sampling_logp_difference/mean": 0.01079532690346241, "step": 802 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 658.0, "completions/mean_length": 434.59375, "completions/mean_terminated_length": 423.83868408203125, "completions/min_length": 215.0, "completions/min_terminated_length": 215.0, "entropy": 0.30391672998666763, "epoch": 0.42987152034261245, "frac_reward_zero_std": 0.75, "grad_norm": 0.022445475682616234, "learning_rate": 1e-05, "loss": 0.0314, "num_tokens": 16866753.0, "reward": 0.9375, "reward_std": 0.1157275140285492, "rewards/accuracy_reward/mean": 0.9375, "rewards/accuracy_reward/std": 0.24593468010425568, "sampling/importance_sampling_ratio/max": 1.5056238174438477, "sampling/importance_sampling_ratio/mean": 0.9998089075088501, "sampling/importance_sampling_ratio/min": 0.6181923747062683, "sampling/sampling_logp_difference/max": 0.4809556007385254, "sampling/sampling_logp_difference/mean": 0.009911777451634407, "step": 803 }, { "clip_ratio/high_max": 7.405213546007872e-05, "clip_ratio/high_mean": 7.405213546007872e-05, "clip_ratio/low_mean": 0.00016379839144065045, "clip_ratio/low_min": 0.00016379839144065045, "clip_ratio/region_mean": 0.00023785052690072916, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 737.0, "completions/mean_length": 438.78125, "completions/mean_terminated_length": 377.8148193359375, "completions/min_length": 159.0, "completions/min_terminated_length": 159.0, "entropy": 0.3829710967838764, "epoch": 0.430406852248394, "frac_reward_zero_std": 0.5, "grad_norm": 0.019850274547934532, "learning_rate": 1e-05, "loss": 0.0196, "num_tokens": 16884810.0, "reward": 0.8125, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.5259016752243042, "sampling/importance_sampling_ratio/mean": 1.0003832578659058, "sampling/importance_sampling_ratio/min": 0.732100248336792, "sampling/sampling_logp_difference/max": 0.42258548736572266, "sampling/sampling_logp_difference/mean": 0.012629854492843151, "step": 804 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 690.0, "completions/mean_length": 419.46875, "completions/mean_terminated_length": 408.2257995605469, "completions/min_length": 166.0, "completions/min_terminated_length": 166.0, "entropy": 0.2942862119525671, "epoch": 0.43094218415417557, "frac_reward_zero_std": 0.5, "grad_norm": 0.020406479015946388, "learning_rate": 1e-05, "loss": 0.0575, "num_tokens": 16901569.0, "reward": 0.84375, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.4498920440673828, "sampling/importance_sampling_ratio/mean": 0.9996421933174133, "sampling/importance_sampling_ratio/min": 0.697530210018158, "sampling/sampling_logp_difference/max": 0.3714890480041504, "sampling/sampling_logp_difference/mean": 0.009771422483026981, "step": 805 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 550.0, "completions/mean_length": 350.125, "completions/mean_terminated_length": 336.6451416015625, "completions/min_length": 171.0, "completions/min_terminated_length": 171.0, "entropy": 0.3032227195799351, "epoch": 0.43147751605995716, "frac_reward_zero_std": 0.5, "grad_norm": 0.007902158424258232, "learning_rate": 1e-05, "loss": -0.0176, "num_tokens": 16916125.0, "reward": 0.75, "reward_std": 0.2314550280570984, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.4670381546020508, "sampling/importance_sampling_ratio/mean": 0.9999794363975525, "sampling/importance_sampling_ratio/min": 0.7704863548278809, "sampling/sampling_logp_difference/max": 0.38324546813964844, "sampling/sampling_logp_difference/mean": 0.010330552235245705, "step": 806 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 692.0, "completions/max_terminated_length": 692.0, "completions/mean_length": 439.5, "completions/mean_terminated_length": 439.5, "completions/min_length": 221.0, "completions/min_terminated_length": 221.0, "entropy": 0.3029349409043789, "epoch": 0.43201284796573874, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 16933789.0, "reward": 0.75, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.4466451406478882, "sampling/importance_sampling_ratio/mean": 1.0001295804977417, "sampling/importance_sampling_ratio/min": 0.6920611262321472, "sampling/sampling_logp_difference/max": 0.3692471981048584, "sampling/sampling_logp_difference/mean": 0.010409644804894924, "step": 807 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 676.0, "completions/max_terminated_length": 676.0, "completions/mean_length": 407.03125, "completions/mean_terminated_length": 407.03125, "completions/min_length": 281.0, "completions/min_terminated_length": 281.0, "entropy": 0.39045247435569763, "epoch": 0.43254817987152033, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0006, "num_tokens": 16950534.0, "reward": 0.9375, "reward_std": 0.1157275140285492, "rewards/accuracy_reward/mean": 0.9375, "rewards/accuracy_reward/std": 0.24593468010425568, "sampling/importance_sampling_ratio/max": 1.4458879232406616, "sampling/importance_sampling_ratio/mean": 1.0001180171966553, "sampling/importance_sampling_ratio/min": 0.6843310594558716, "sampling/sampling_logp_difference/max": 0.37931346893310547, "sampling/sampling_logp_difference/mean": 0.012879184447228909, "step": 808 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 764.0, "completions/mean_length": 530.875, "completions/mean_terminated_length": 497.0000305175781, "completions/min_length": 324.0, "completions/min_terminated_length": 324.0, "entropy": 0.3060468230396509, "epoch": 0.4330835117773019, "frac_reward_zero_std": 0.0, "grad_norm": 0.008273949846625328, "learning_rate": 1e-05, "loss": 0.0921, "num_tokens": 16971354.0, "reward": 0.78125, "reward_std": 0.4355708956718445, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.3393288850784302, "sampling/importance_sampling_ratio/mean": 0.999944806098938, "sampling/importance_sampling_ratio/min": 0.6818230748176575, "sampling/sampling_logp_difference/max": 0.38298511505126953, "sampling/sampling_logp_difference/mean": 0.010227049700915813, "step": 809 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.231306178960949e-05, "clip_ratio/low_min": 6.231306178960949e-05, "clip_ratio/region_mean": 6.231306178960949e-05, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 760.0, "completions/mean_length": 455.9375, "completions/mean_terminated_length": 445.8709411621094, "completions/min_length": 153.0, "completions/min_terminated_length": 153.0, "entropy": 0.2802313603460789, "epoch": 0.4336188436830835, "frac_reward_zero_std": 0.25, "grad_norm": 0.005037680268287659, "learning_rate": 1e-05, "loss": 0.0307, "num_tokens": 16989528.0, "reward": 0.84375, "reward_std": 0.3198433816432953, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.3644779920578003, "sampling/importance_sampling_ratio/mean": 0.9999107122421265, "sampling/importance_sampling_ratio/min": 0.7234307527542114, "sampling/sampling_logp_difference/max": 0.32375049591064453, "sampling/sampling_logp_difference/mean": 0.009951348416507244, "step": 810 }, { "clip_ratio/high_max": 5.5163283832371235e-05, "clip_ratio/high_mean": 5.5163283832371235e-05, "clip_ratio/low_mean": 0.00012614567822311074, "clip_ratio/low_min": 0.00012614567822311074, "clip_ratio/region_mean": 0.00018130896205548197, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 729.0, "completions/mean_length": 474.0, "completions/mean_terminated_length": 432.0000305175781, "completions/min_length": 133.0, "completions/min_terminated_length": 133.0, "entropy": 0.3423337507992983, "epoch": 0.4341541755888651, "frac_reward_zero_std": 0.5, "grad_norm": 0.012094578705728054, "learning_rate": 1e-05, "loss": 0.0034, "num_tokens": 17008328.0, "reward": 0.65625, "reward_std": 0.22201895713806152, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.4612789154052734, "sampling/importance_sampling_ratio/mean": 1.000400424003601, "sampling/importance_sampling_ratio/min": 0.6412340998649597, "sampling/sampling_logp_difference/max": 0.44436073303222656, "sampling/sampling_logp_difference/mean": 0.011007783003151417, "step": 811 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.9465768825029954e-05, "clip_ratio/low_min": 4.9465768825029954e-05, "clip_ratio/region_mean": 4.9465768825029954e-05, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 722.0, "completions/mean_length": 513.875, "completions/mean_terminated_length": 487.5862121582031, "completions/min_length": 272.0, "completions/min_terminated_length": 272.0, "entropy": 0.27780028618872166, "epoch": 0.4346895074946467, "frac_reward_zero_std": 0.25, "grad_norm": 0.019300971180200577, "learning_rate": 1e-05, "loss": 0.0486, "num_tokens": 17028564.0, "reward": 0.78125, "reward_std": 0.3061639964580536, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.4398139715194702, "sampling/importance_sampling_ratio/mean": 1.0000369548797607, "sampling/importance_sampling_ratio/min": 0.6995759010314941, "sampling/sampling_logp_difference/max": 0.3645138740539551, "sampling/sampling_logp_difference/mean": 0.009919270873069763, "step": 812 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0001410715631209314, "clip_ratio/low_min": 0.0001410715631209314, "clip_ratio/region_mean": 0.0001410715631209314, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 740.0, "completions/mean_length": 492.4375, "completions/mean_terminated_length": 441.40740966796875, "completions/min_length": 205.0, "completions/min_terminated_length": 205.0, "entropy": 0.3248841241002083, "epoch": 0.4352248394004283, "frac_reward_zero_std": 0.5, "grad_norm": 0.017952993512153625, "learning_rate": 1e-05, "loss": 0.0108, "num_tokens": 17047946.0, "reward": 0.34375, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.34375, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.3786996603012085, "sampling/importance_sampling_ratio/mean": 0.9999850988388062, "sampling/importance_sampling_ratio/min": 0.6992431879043579, "sampling/sampling_logp_difference/max": 0.3577566146850586, "sampling/sampling_logp_difference/mean": 0.010284186340868473, "step": 813 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00015852885553613305, "clip_ratio/low_min": 0.00015852885553613305, "clip_ratio/region_mean": 0.00015852885553613305, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 505.0, "completions/mean_length": 371.15625, "completions/mean_terminated_length": 358.3548278808594, "completions/min_length": 145.0, "completions/min_terminated_length": 145.0, "entropy": 0.3041163217276335, "epoch": 0.43576017130620986, "frac_reward_zero_std": 0.0, "grad_norm": 0.01717095822095871, "learning_rate": 1e-05, "loss": -0.0204, "num_tokens": 17063431.0, "reward": 0.6875, "reward_std": 0.4671337604522705, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.3836407661437988, "sampling/importance_sampling_ratio/mean": 1.0002788305282593, "sampling/importance_sampling_ratio/min": 0.5404121279716492, "sampling/sampling_logp_difference/max": 0.6154232025146484, "sampling/sampling_logp_difference/mean": 0.011140824295580387, "step": 814 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 756.0, "completions/max_terminated_length": 756.0, "completions/mean_length": 437.84375, "completions/mean_terminated_length": 437.84375, "completions/min_length": 138.0, "completions/min_terminated_length": 138.0, "entropy": 0.3168862573802471, "epoch": 0.43629550321199145, "frac_reward_zero_std": 0.25, "grad_norm": 0.01972295716404915, "learning_rate": 1e-05, "loss": -0.0744, "num_tokens": 17081010.0, "reward": 0.71875, "reward_std": 0.3471629321575165, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.4323437213897705, "sampling/importance_sampling_ratio/mean": 0.9998100399971008, "sampling/importance_sampling_ratio/min": 0.7168599963188171, "sampling/sampling_logp_difference/max": 0.3593120574951172, "sampling/sampling_logp_difference/mean": 0.010707537643611431, "step": 815 }, { "clip_ratio/high_max": 7.51653642510064e-05, "clip_ratio/high_mean": 7.51653642510064e-05, "clip_ratio/low_mean": 8.941345004132017e-05, "clip_ratio/low_min": 8.941345004132017e-05, "clip_ratio/region_mean": 0.00016457881429232657, "completions/clipped_ratio": 0.0, "completions/max_length": 531.0, "completions/max_terminated_length": 531.0, "completions/mean_length": 376.03125, "completions/mean_terminated_length": 376.03125, "completions/min_length": 229.0, "completions/min_terminated_length": 229.0, "entropy": 0.3296228162944317, "epoch": 0.43683083511777304, "frac_reward_zero_std": 0.5, "grad_norm": 0.02471880428493023, "learning_rate": 1e-05, "loss": -0.0007, "num_tokens": 17096931.0, "reward": 0.875, "reward_std": 0.2314550280570984, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 1.3028932809829712, "sampling/importance_sampling_ratio/mean": 0.9998487234115601, "sampling/importance_sampling_ratio/min": 0.6383896470069885, "sampling/sampling_logp_difference/max": 0.4488065242767334, "sampling/sampling_logp_difference/mean": 0.012072399258613586, "step": 816 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 757.0, "completions/max_terminated_length": 757.0, "completions/mean_length": 365.21875, "completions/mean_terminated_length": 365.21875, "completions/min_length": 173.0, "completions/min_terminated_length": 173.0, "entropy": 0.3616967312991619, "epoch": 0.43736616702355463, "frac_reward_zero_std": 0.5, "grad_norm": 0.009743794798851013, "learning_rate": 1e-05, "loss": 0.0026, "num_tokens": 17112514.0, "reward": 0.9375, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.9375, "rewards/accuracy_reward/std": 0.24593468010425568, "sampling/importance_sampling_ratio/max": 1.7478456497192383, "sampling/importance_sampling_ratio/mean": 0.9996093511581421, "sampling/importance_sampling_ratio/min": 0.6976439356803894, "sampling/sampling_logp_difference/max": 0.5583839416503906, "sampling/sampling_logp_difference/mean": 0.012133362703025341, "step": 817 }, { "clip_ratio/high_max": 6.237524939933792e-05, "clip_ratio/high_mean": 6.237524939933792e-05, "clip_ratio/low_mean": 0.00017773100989870727, "clip_ratio/low_min": 0.00017773100989870727, "clip_ratio/region_mean": 0.0002401062592980452, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 715.0, "completions/mean_length": 513.21875, "completions/mean_terminated_length": 505.0, "completions/min_length": 351.0, "completions/min_terminated_length": 351.0, "entropy": 0.32191819325089455, "epoch": 0.43790149892933616, "frac_reward_zero_std": 0.25, "grad_norm": 0.03233141452074051, "learning_rate": 1e-05, "loss": 0.003, "num_tokens": 17133425.0, "reward": 0.6875, "reward_std": 0.292504221200943, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.5630940198898315, "sampling/importance_sampling_ratio/mean": 0.9998918771743774, "sampling/importance_sampling_ratio/min": 0.662834107875824, "sampling/sampling_logp_difference/max": 0.4466671943664551, "sampling/sampling_logp_difference/mean": 0.011077848263084888, "step": 818 }, { "clip_ratio/high_max": 5.6357079301960766e-05, "clip_ratio/high_mean": 5.6357079301960766e-05, "clip_ratio/low_mean": 0.00011214663391001523, "clip_ratio/low_min": 0.00011214663391001523, "clip_ratio/region_mean": 0.000168503713211976, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 747.0, "completions/mean_length": 500.0625, "completions/mean_terminated_length": 472.3448181152344, "completions/min_length": 299.0, "completions/min_terminated_length": 299.0, "entropy": 0.2890834603458643, "epoch": 0.43843683083511775, "frac_reward_zero_std": 0.5, "grad_norm": 0.015631988644599915, "learning_rate": 1e-05, "loss": -0.0032, "num_tokens": 17153331.0, "reward": 0.71875, "reward_std": 0.2630178928375244, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.3648066520690918, "sampling/importance_sampling_ratio/mean": 1.0000524520874023, "sampling/importance_sampling_ratio/min": 0.5533369779586792, "sampling/sampling_logp_difference/max": 0.5917880535125732, "sampling/sampling_logp_difference/mean": 0.01031559333205223, "step": 819 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 701.0, "completions/max_terminated_length": 701.0, "completions/mean_length": 449.59375, "completions/mean_terminated_length": 449.59375, "completions/min_length": 307.0, "completions/min_terminated_length": 307.0, "entropy": 0.3009734135121107, "epoch": 0.43897216274089934, "frac_reward_zero_std": 0.25, "grad_norm": 0.008712420240044594, "learning_rate": 1e-05, "loss": -0.0699, "num_tokens": 17171550.0, "reward": 0.5625, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.999958872795105, "sampling/importance_sampling_ratio/min": 0.6119124293327332, "sampling/sampling_logp_difference/max": 0.7268886566162109, "sampling/sampling_logp_difference/mean": 0.010710617527365685, "step": 820 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00019959078053943813, "clip_ratio/low_min": 0.00019959078053943813, "clip_ratio/region_mean": 0.00019959078053943813, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 735.0, "completions/mean_length": 605.75, "completions/mean_terminated_length": 551.6666870117188, "completions/min_length": 304.0, "completions/min_terminated_length": 304.0, "entropy": 0.5200091004371643, "epoch": 0.43950749464668093, "frac_reward_zero_std": 0.0, "grad_norm": 0.012178665027022362, "learning_rate": 1e-05, "loss": 0.0472, "num_tokens": 17196494.0, "reward": 0.3125, "reward_std": 0.4492306709289551, "rewards/accuracy_reward/mean": 0.3125, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.5874097347259521, "sampling/importance_sampling_ratio/mean": 0.9998118877410889, "sampling/importance_sampling_ratio/min": 0.17467598617076874, "sampling/sampling_logp_difference/max": 1.7448225021362305, "sampling/sampling_logp_difference/mean": 0.015267375856637955, "step": 821 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 641.0, "completions/mean_length": 439.96875, "completions/mean_terminated_length": 418.10003662109375, "completions/min_length": 193.0, "completions/min_terminated_length": 193.0, "entropy": 0.29386604204773903, "epoch": 0.4400428265524625, "frac_reward_zero_std": 0.75, "grad_norm": 0.022035328671336174, "learning_rate": 1e-05, "loss": 0.029, "num_tokens": 17214245.0, "reward": 0.90625, "reward_std": 0.1293872892856598, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.2908525466918945, "sampling/importance_sampling_ratio/mean": 0.9993357062339783, "sampling/importance_sampling_ratio/min": 0.6960011720657349, "sampling/sampling_logp_difference/max": 0.36240386962890625, "sampling/sampling_logp_difference/mean": 0.010311836376786232, "step": 822 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 628.0, "completions/mean_length": 534.46875, "completions/mean_terminated_length": 456.625, "completions/min_length": 275.0, "completions/min_terminated_length": 275.0, "entropy": 0.42147471755743027, "epoch": 0.4405781584582441, "frac_reward_zero_std": 0.75, "grad_norm": 0.0033640230540186167, "learning_rate": 1e-05, "loss": 0.0025, "num_tokens": 17235044.0, "reward": 0.71875, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.5563052892684937, "sampling/importance_sampling_ratio/mean": 1.0000025033950806, "sampling/importance_sampling_ratio/min": 0.7584670782089233, "sampling/sampling_logp_difference/max": 0.44231462478637695, "sampling/sampling_logp_difference/mean": 0.013455349020659924, "step": 823 }, { "clip_ratio/high_max": 6.786102312617004e-05, "clip_ratio/high_mean": 6.786102312617004e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 6.786102312617004e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 665.0, "completions/max_terminated_length": 665.0, "completions/mean_length": 451.75, "completions/mean_terminated_length": 451.75, "completions/min_length": 244.0, "completions/min_terminated_length": 244.0, "entropy": 0.26561981998384, "epoch": 0.4411134903640257, "frac_reward_zero_std": 0.5, "grad_norm": 0.007244923152029514, "learning_rate": 1e-05, "loss": 0.0117, "num_tokens": 17252796.0, "reward": 0.875, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 1.662221908569336, "sampling/importance_sampling_ratio/mean": 0.9998593926429749, "sampling/importance_sampling_ratio/min": 0.7401193380355835, "sampling/sampling_logp_difference/max": 0.5081552267074585, "sampling/sampling_logp_difference/mean": 0.008950168266892433, "step": 824 }, { "clip_ratio/high_max": 4.950494985678233e-05, "clip_ratio/high_mean": 4.950494985678233e-05, "clip_ratio/low_mean": 5.169561700313352e-05, "clip_ratio/low_min": 5.169561700313352e-05, "clip_ratio/region_mean": 0.00010120056685991585, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 758.0, "completions/mean_length": 567.09375, "completions/mean_terminated_length": 510.8399963378906, "completions/min_length": 296.0, "completions/min_terminated_length": 296.0, "entropy": 0.39459048956632614, "epoch": 0.4416488222698073, "frac_reward_zero_std": 0.25, "grad_norm": 0.009630724787712097, "learning_rate": 1e-05, "loss": 0.0559, "num_tokens": 17274927.0, "reward": 0.6875, "reward_std": 0.3514062464237213, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.6130503416061401, "sampling/importance_sampling_ratio/mean": 1.0000581741333008, "sampling/importance_sampling_ratio/min": 0.6157658100128174, "sampling/sampling_logp_difference/max": 0.48488855361938477, "sampling/sampling_logp_difference/mean": 0.012862004339694977, "step": 825 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 585.0, "completions/mean_length": 405.375, "completions/mean_terminated_length": 393.6773986816406, "completions/min_length": 259.0, "completions/min_terminated_length": 259.0, "entropy": 0.34616445004940033, "epoch": 0.4421841541755889, "frac_reward_zero_std": 0.5, "grad_norm": 0.016571659594774246, "learning_rate": 1e-05, "loss": 0.0699, "num_tokens": 17291267.0, "reward": 0.75, "reward_std": 0.2314550280570984, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.00043523311615, "sampling/importance_sampling_ratio/min": 0.6178522706031799, "sampling/sampling_logp_difference/max": 0.9045839309692383, "sampling/sampling_logp_difference/mean": 0.011966786347329617, "step": 826 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0001231828791787848, "clip_ratio/low_min": 0.0001231828791787848, "clip_ratio/region_mean": 0.0001231828791787848, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 736.0, "completions/mean_length": 441.28125, "completions/mean_terminated_length": 430.7419128417969, "completions/min_length": 232.0, "completions/min_terminated_length": 232.0, "entropy": 0.49824395403265953, "epoch": 0.44271948608137046, "frac_reward_zero_std": 0.5, "grad_norm": 0.018703874200582504, "learning_rate": 1e-05, "loss": -0.0075, "num_tokens": 17309068.0, "reward": 0.59375, "reward_std": 0.22201895713806152, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.9725242853164673, "sampling/importance_sampling_ratio/mean": 1.0002188682556152, "sampling/importance_sampling_ratio/min": 0.7081458568572998, "sampling/sampling_logp_difference/max": 0.679314136505127, "sampling/sampling_logp_difference/mean": 0.014864131808280945, "step": 827 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 8.579272252973169e-05, "clip_ratio/low_min": 8.579272252973169e-05, "clip_ratio/region_mean": 8.579272252973169e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 646.0, "completions/max_terminated_length": 646.0, "completions/mean_length": 373.3125, "completions/mean_terminated_length": 373.3125, "completions/min_length": 145.0, "completions/min_terminated_length": 145.0, "entropy": 0.37105197459459305, "epoch": 0.44325481798715205, "frac_reward_zero_std": 0.25, "grad_norm": 0.04320777580142021, "learning_rate": 1e-05, "loss": -0.0142, "num_tokens": 17324638.0, "reward": 0.75, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.4403419494628906, "sampling/importance_sampling_ratio/mean": 1.0000241994857788, "sampling/importance_sampling_ratio/min": 0.4905491769313812, "sampling/sampling_logp_difference/max": 0.7122297286987305, "sampling/sampling_logp_difference/mean": 0.012443293817341328, "step": 828 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 743.0, "completions/mean_length": 482.625, "completions/mean_terminated_length": 473.4193420410156, "completions/min_length": 274.0, "completions/min_terminated_length": 274.0, "entropy": 0.23537281714379787, "epoch": 0.44379014989293364, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 17343866.0, "reward": 0.75, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.5877076387405396, "sampling/importance_sampling_ratio/mean": 1.0002354383468628, "sampling/importance_sampling_ratio/min": 0.7195164561271667, "sampling/sampling_logp_difference/max": 0.46229124069213867, "sampling/sampling_logp_difference/mean": 0.008366691879928112, "step": 829 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 633.0, "completions/mean_length": 470.03125, "completions/mean_terminated_length": 414.85186767578125, "completions/min_length": 209.0, "completions/min_terminated_length": 209.0, "entropy": 0.4646605346351862, "epoch": 0.4443254817987152, "frac_reward_zero_std": 0.25, "grad_norm": 0.02096404694020748, "learning_rate": 1e-05, "loss": 0.0515, "num_tokens": 17363595.0, "reward": 0.6875, "reward_std": 0.3745020925998688, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.5607069730758667, "sampling/importance_sampling_ratio/mean": 0.9995356798171997, "sampling/importance_sampling_ratio/min": 0.5868687629699707, "sampling/sampling_logp_difference/max": 0.5329539775848389, "sampling/sampling_logp_difference/mean": 0.014165048487484455, "step": 830 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.4656757129123434e-05, "clip_ratio/low_min": 5.4656757129123434e-05, "clip_ratio/region_mean": 5.4656757129123434e-05, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 704.0, "completions/mean_length": 514.15625, "completions/mean_terminated_length": 467.1481628417969, "completions/min_length": 167.0, "completions/min_terminated_length": 167.0, "entropy": 0.29630786553025246, "epoch": 0.44486081370449676, "frac_reward_zero_std": 0.25, "grad_norm": 0.022731341421604156, "learning_rate": 1e-05, "loss": 0.0357, "num_tokens": 17384136.0, "reward": 0.75, "reward_std": 0.3514062464237213, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.4652239084243774, "sampling/importance_sampling_ratio/mean": 1.0000675916671753, "sampling/importance_sampling_ratio/min": 0.6110819578170776, "sampling/sampling_logp_difference/max": 0.4925241470336914, "sampling/sampling_logp_difference/mean": 0.009815549477934837, "step": 831 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 664.0, "completions/max_terminated_length": 664.0, "completions/mean_length": 418.3125, "completions/mean_terminated_length": 418.3125, "completions/min_length": 124.0, "completions/min_terminated_length": 124.0, "entropy": 0.30645952746272087, "epoch": 0.44539614561027835, "frac_reward_zero_std": 0.75, "grad_norm": 0.006999002303928137, "learning_rate": 1e-05, "loss": 0.0061, "num_tokens": 17400818.0, "reward": 0.9375, "reward_std": 0.1157275140285492, "rewards/accuracy_reward/mean": 0.9375, "rewards/accuracy_reward/std": 0.24593468010425568, "sampling/importance_sampling_ratio/max": 1.4104315042495728, "sampling/importance_sampling_ratio/mean": 1.0002522468566895, "sampling/importance_sampling_ratio/min": 0.7386618256568909, "sampling/sampling_logp_difference/max": 0.34389567375183105, "sampling/sampling_logp_difference/mean": 0.011039087548851967, "step": 832 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 755.0, "completions/max_terminated_length": 755.0, "completions/mean_length": 473.6875, "completions/mean_terminated_length": 473.6875, "completions/min_length": 176.0, "completions/min_terminated_length": 176.0, "entropy": 0.24417481571435928, "epoch": 0.44593147751605994, "frac_reward_zero_std": 0.75, "grad_norm": 0.004943369887769222, "learning_rate": 1e-05, "loss": 0.0137, "num_tokens": 17419960.0, "reward": 0.96875, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.96875, "rewards/accuracy_reward/std": 0.1767766922712326, "sampling/importance_sampling_ratio/max": 1.3704149723052979, "sampling/importance_sampling_ratio/mean": 1.000150203704834, "sampling/importance_sampling_ratio/min": 0.6169997453689575, "sampling/sampling_logp_difference/max": 0.4828866720199585, "sampling/sampling_logp_difference/mean": 0.009111866354942322, "step": 833 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 643.0, "completions/mean_length": 480.25, "completions/mean_terminated_length": 450.4827575683594, "completions/min_length": 304.0, "completions/min_terminated_length": 304.0, "entropy": 0.46574817411601543, "epoch": 0.4464668094218415, "frac_reward_zero_std": 0.5, "grad_norm": 0.004699389915913343, "learning_rate": 1e-05, "loss": -0.0046, "num_tokens": 17439064.0, "reward": 0.5625, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.4031789302825928, "sampling/importance_sampling_ratio/mean": 1.0000865459442139, "sampling/importance_sampling_ratio/min": 0.6470046043395996, "sampling/sampling_logp_difference/max": 0.43540191650390625, "sampling/sampling_logp_difference/mean": 0.014360631816089153, "step": 834 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00010442862912896089, "clip_ratio/low_min": 0.00010442862912896089, "clip_ratio/region_mean": 0.00010442862912896089, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 765.0, "completions/mean_length": 525.25, "completions/mean_terminated_length": 457.2799987792969, "completions/min_length": 229.0, "completions/min_terminated_length": 229.0, "entropy": 0.2754701804369688, "epoch": 0.4470021413276231, "frac_reward_zero_std": 0.25, "grad_norm": 0.01315237581729889, "learning_rate": 1e-05, "loss": 0.0115, "num_tokens": 17459720.0, "reward": 0.65625, "reward_std": 0.3608423173427582, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.4207916259765625, "sampling/importance_sampling_ratio/mean": 0.9995573163032532, "sampling/importance_sampling_ratio/min": 0.5681164860725403, "sampling/sampling_logp_difference/max": 0.5654287338256836, "sampling/sampling_logp_difference/mean": 0.01060456968843937, "step": 835 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 529.0, "completions/max_terminated_length": 529.0, "completions/mean_length": 383.125, "completions/mean_terminated_length": 383.125, "completions/min_length": 216.0, "completions/min_terminated_length": 216.0, "entropy": 0.2697168178856373, "epoch": 0.4475374732334047, "frac_reward_zero_std": 0.5, "grad_norm": 0.048002421855926514, "learning_rate": 1e-05, "loss": -0.025, "num_tokens": 17475268.0, "reward": 0.9375, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.9375, "rewards/accuracy_reward/std": 0.24593468010425568, "sampling/importance_sampling_ratio/max": 1.454479455947876, "sampling/importance_sampling_ratio/mean": 1.00013267993927, "sampling/importance_sampling_ratio/min": 0.7839727401733398, "sampling/sampling_logp_difference/max": 0.3746480941772461, "sampling/sampling_logp_difference/mean": 0.009478505700826645, "step": 836 }, { "clip_ratio/high_max": 5.439512460725382e-05, "clip_ratio/high_mean": 5.439512460725382e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 5.439512460725382e-05, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 692.0, "completions/mean_length": 518.375, "completions/mean_terminated_length": 472.1481628417969, "completions/min_length": 307.0, "completions/min_terminated_length": 307.0, "entropy": 0.40216017700731754, "epoch": 0.4480728051391863, "frac_reward_zero_std": 0.25, "grad_norm": 0.013944534584879875, "learning_rate": 1e-05, "loss": 0.0475, "num_tokens": 17495768.0, "reward": 0.6875, "reward_std": 0.3514062762260437, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.513625979423523, "sampling/importance_sampling_ratio/mean": 1.0001559257507324, "sampling/importance_sampling_ratio/min": 0.24801397323608398, "sampling/sampling_logp_difference/max": 1.3942701816558838, "sampling/sampling_logp_difference/mean": 0.013445986434817314, "step": 837 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 695.0, "completions/max_terminated_length": 695.0, "completions/mean_length": 502.8125, "completions/mean_terminated_length": 502.8125, "completions/min_length": 326.0, "completions/min_terminated_length": 326.0, "entropy": 0.2928779497742653, "epoch": 0.4486081370449679, "frac_reward_zero_std": 0.25, "grad_norm": 0.0052935900166630745, "learning_rate": 1e-05, "loss": 0.0092, "num_tokens": 17515722.0, "reward": 0.84375, "reward_std": 0.3061639964580536, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.4502661228179932, "sampling/importance_sampling_ratio/mean": 0.9999672770500183, "sampling/importance_sampling_ratio/min": 0.7170757055282593, "sampling/sampling_logp_difference/max": 0.3717470169067383, "sampling/sampling_logp_difference/mean": 0.010355195961892605, "step": 838 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.115459837019444e-05, "clip_ratio/low_min": 6.115459837019444e-05, "clip_ratio/region_mean": 6.115459837019444e-05, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 615.0, "completions/mean_length": 420.59375, "completions/mean_terminated_length": 397.433349609375, "completions/min_length": 218.0, "completions/min_terminated_length": 218.0, "entropy": 0.3525313138961792, "epoch": 0.44914346895074947, "frac_reward_zero_std": 0.5, "grad_norm": 0.020515669137239456, "learning_rate": 1e-05, "loss": 0.024, "num_tokens": 17533253.0, "reward": 0.84375, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.8683252334594727, "sampling/importance_sampling_ratio/mean": 0.9998475909233093, "sampling/importance_sampling_ratio/min": 0.5709669589996338, "sampling/sampling_logp_difference/max": 0.6250424385070801, "sampling/sampling_logp_difference/mean": 0.01299965288490057, "step": 839 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 733.0, "completions/mean_length": 493.65625, "completions/mean_terminated_length": 484.8064270019531, "completions/min_length": 168.0, "completions/min_terminated_length": 168.0, "entropy": 0.22328947111964226, "epoch": 0.44967880085653106, "frac_reward_zero_std": 0.75, "grad_norm": 0.005702413152903318, "learning_rate": 1e-05, "loss": -0.038, "num_tokens": 17553082.0, "reward": 0.90625, "reward_std": 0.1293872892856598, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.2993196249008179, "sampling/importance_sampling_ratio/mean": 0.9999359846115112, "sampling/importance_sampling_ratio/min": 0.6806808710098267, "sampling/sampling_logp_difference/max": 0.3846616744995117, "sampling/sampling_logp_difference/mean": 0.008328845724463463, "step": 840 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 740.0, "completions/mean_length": 546.34375, "completions/mean_terminated_length": 505.2962951660156, "completions/min_length": 212.0, "completions/min_terminated_length": 212.0, "entropy": 0.3904264606535435, "epoch": 0.45021413276231265, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 17575165.0, "reward": 0.5, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5080004930496216, "sampling/importance_sampling_ratio/max": 1.4561699628829956, "sampling/importance_sampling_ratio/mean": 0.9996989965438843, "sampling/importance_sampling_ratio/min": 0.6488072872161865, "sampling/sampling_logp_difference/max": 0.432619571685791, "sampling/sampling_logp_difference/mean": 0.013349946588277817, "step": 841 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0001283141755266115, "clip_ratio/low_min": 0.0001283141755266115, "clip_ratio/region_mean": 0.0001283141755266115, "completions/clipped_ratio": 0.0, "completions/max_length": 719.0, "completions/max_terminated_length": 719.0, "completions/mean_length": 456.59375, "completions/mean_terminated_length": 456.59375, "completions/min_length": 212.0, "completions/min_terminated_length": 212.0, "entropy": 0.28330995328724384, "epoch": 0.45074946466809424, "frac_reward_zero_std": 0.5, "grad_norm": 0.034676454961299896, "learning_rate": 1e-05, "loss": -0.0453, "num_tokens": 17593376.0, "reward": 0.71875, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.404496431350708, "sampling/importance_sampling_ratio/mean": 0.9998428225517273, "sampling/importance_sampling_ratio/min": 0.6960118412971497, "sampling/sampling_logp_difference/max": 0.36238861083984375, "sampling/sampling_logp_difference/mean": 0.010486968792974949, "step": 842 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 535.0, "completions/mean_length": 401.90625, "completions/mean_terminated_length": 390.0967712402344, "completions/min_length": 163.0, "completions/min_terminated_length": 163.0, "entropy": 0.25354092195630074, "epoch": 0.4512847965738758, "frac_reward_zero_std": 0.5, "grad_norm": 0.013399692252278328, "learning_rate": 1e-05, "loss": 0.074, "num_tokens": 17609973.0, "reward": 0.8125, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.4726992845535278, "sampling/importance_sampling_ratio/mean": 0.999975860118866, "sampling/importance_sampling_ratio/min": 0.6956537961959839, "sampling/sampling_logp_difference/max": 0.3870968818664551, "sampling/sampling_logp_difference/mean": 0.009599726647138596, "step": 843 }, { "clip_ratio/high_max": 6.551362457685173e-05, "clip_ratio/high_mean": 6.551362457685173e-05, "clip_ratio/low_mean": 0.0002793692401610315, "clip_ratio/low_min": 0.0002793692401610315, "clip_ratio/region_mean": 0.0003448828647378832, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 663.0, "completions/mean_length": 437.28125, "completions/mean_terminated_length": 403.0689697265625, "completions/min_length": 164.0, "completions/min_terminated_length": 164.0, "entropy": 0.4792197309434414, "epoch": 0.4518201284796574, "frac_reward_zero_std": 0.5, "grad_norm": 0.040485985577106476, "learning_rate": 1e-05, "loss": 0.0784, "num_tokens": 17627462.0, "reward": 0.75, "reward_std": 0.2314550280570984, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.9082411527633667, "sampling/importance_sampling_ratio/mean": 1.0005013942718506, "sampling/importance_sampling_ratio/min": 0.6889159083366394, "sampling/sampling_logp_difference/max": 0.6461819410324097, "sampling/sampling_logp_difference/mean": 0.016048213467001915, "step": 844 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.2126772061455995e-05, "clip_ratio/low_min": 5.2126772061455995e-05, "clip_ratio/region_mean": 5.2126772061455995e-05, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 749.0, "completions/mean_length": 484.0, "completions/mean_terminated_length": 465.0666809082031, "completions/min_length": 191.0, "completions/min_terminated_length": 191.0, "entropy": 0.3404336478561163, "epoch": 0.45235546038543895, "frac_reward_zero_std": 0.75, "grad_norm": 0.018093738704919815, "learning_rate": 1e-05, "loss": 0.0077, "num_tokens": 17647278.0, "reward": 0.90625, "reward_std": 0.1293872892856598, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.6443889141082764, "sampling/importance_sampling_ratio/mean": 0.9996999502182007, "sampling/importance_sampling_ratio/min": 0.6987136602401733, "sampling/sampling_logp_difference/max": 0.49736881256103516, "sampling/sampling_logp_difference/mean": 0.01183375995606184, "step": 845 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00012124221029807813, "clip_ratio/low_min": 0.00012124221029807813, "clip_ratio/region_mean": 0.00012124221029807813, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 718.0, "completions/mean_length": 517.59375, "completions/mean_terminated_length": 509.51611328125, "completions/min_length": 280.0, "completions/min_terminated_length": 280.0, "entropy": 0.28196615166962147, "epoch": 0.45289079229122053, "frac_reward_zero_std": 0.5, "grad_norm": 0.00997235719114542, "learning_rate": 1e-05, "loss": -0.035, "num_tokens": 17667409.0, "reward": 0.71875, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.5072767734527588, "sampling/importance_sampling_ratio/mean": 1.0000327825546265, "sampling/importance_sampling_ratio/min": 0.7333748936653137, "sampling/sampling_logp_difference/max": 0.41030454635620117, "sampling/sampling_logp_difference/mean": 0.010541322641074657, "step": 846 }, { "clip_ratio/high_max": 6.698820652673021e-05, "clip_ratio/high_mean": 6.698820652673021e-05, "clip_ratio/low_mean": 0.00011798779814853333, "clip_ratio/low_min": 0.00011798779814853333, "clip_ratio/region_mean": 0.00018497600467526354, "completions/clipped_ratio": 0.0, "completions/max_length": 727.0, "completions/max_terminated_length": 727.0, "completions/mean_length": 475.46875, "completions/mean_terminated_length": 475.46875, "completions/min_length": 219.0, "completions/min_terminated_length": 219.0, "entropy": 0.34865502640604973, "epoch": 0.4534261241970021, "frac_reward_zero_std": 0.5, "grad_norm": 0.004660570994019508, "learning_rate": 1e-05, "loss": 0.0368, "num_tokens": 17686608.0, "reward": 0.8125, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.470431923866272, "sampling/importance_sampling_ratio/mean": 1.0003079175949097, "sampling/importance_sampling_ratio/min": 0.7072607278823853, "sampling/sampling_logp_difference/max": 0.3855562210083008, "sampling/sampling_logp_difference/mean": 0.011790947988629341, "step": 847 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00015960823293426074, "clip_ratio/low_min": 0.00015960823293426074, "clip_ratio/region_mean": 0.00015960823293426074, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 744.0, "completions/mean_length": 590.625, "completions/mean_terminated_length": 565.2857666015625, "completions/min_length": 344.0, "completions/min_terminated_length": 344.0, "entropy": 0.33926576375961304, "epoch": 0.4539614561027837, "frac_reward_zero_std": 0.5, "grad_norm": 0.01985170878469944, "learning_rate": 1e-05, "loss": -0.0024, "num_tokens": 17710020.0, "reward": 0.75, "reward_std": 0.2587745785713196, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.8691842555999756, "sampling/importance_sampling_ratio/mean": 1.0000360012054443, "sampling/importance_sampling_ratio/min": 0.6046698093414307, "sampling/sampling_logp_difference/max": 0.6255021095275879, "sampling/sampling_logp_difference/mean": 0.012216202914714813, "step": 848 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 552.0, "completions/max_terminated_length": 552.0, "completions/mean_length": 412.21875, "completions/mean_terminated_length": 412.21875, "completions/min_length": 143.0, "completions/min_terminated_length": 143.0, "entropy": 0.2868776898831129, "epoch": 0.4544967880085653, "frac_reward_zero_std": 0.0, "grad_norm": 0.013748974539339542, "learning_rate": 1e-05, "loss": -0.0423, "num_tokens": 17726899.0, "reward": 0.71875, "reward_std": 0.4628904461860657, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.3464021682739258, "sampling/importance_sampling_ratio/mean": 1.0001211166381836, "sampling/importance_sampling_ratio/min": 0.7243185043334961, "sampling/sampling_logp_difference/max": 0.3225240707397461, "sampling/sampling_logp_difference/mean": 0.010825544595718384, "step": 849 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.994963769102469e-05, "clip_ratio/low_min": 6.994963769102469e-05, "clip_ratio/region_mean": 6.994963769102469e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 752.0, "completions/max_terminated_length": 752.0, "completions/mean_length": 442.65625, "completions/mean_terminated_length": 442.65625, "completions/min_length": 160.0, "completions/min_terminated_length": 160.0, "entropy": 0.21593933179974556, "epoch": 0.4550321199143469, "frac_reward_zero_std": 0.25, "grad_norm": 0.006543995812535286, "learning_rate": 1e-05, "loss": -0.0158, "num_tokens": 17744664.0, "reward": 0.75, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.5651553869247437, "sampling/importance_sampling_ratio/mean": 0.9995262622833252, "sampling/importance_sampling_ratio/min": 0.624371349811554, "sampling/sampling_logp_difference/max": 0.4710099697113037, "sampling/sampling_logp_difference/mean": 0.00816783681511879, "step": 850 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.952380888629705e-05, "clip_ratio/low_min": 5.952380888629705e-05, "clip_ratio/region_mean": 5.952380888629705e-05, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 721.0, "completions/mean_length": 520.90625, "completions/mean_terminated_length": 495.3448181152344, "completions/min_length": 272.0, "completions/min_terminated_length": 272.0, "entropy": 0.3044566884636879, "epoch": 0.4555674518201285, "frac_reward_zero_std": 0.5, "grad_norm": 0.02042035572230816, "learning_rate": 1e-05, "loss": 0.0118, "num_tokens": 17765173.0, "reward": 0.84375, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.4871103763580322, "sampling/importance_sampling_ratio/mean": 1.0001747608184814, "sampling/importance_sampling_ratio/min": 0.6212068796157837, "sampling/sampling_logp_difference/max": 0.4760911464691162, "sampling/sampling_logp_difference/mean": 0.011123652569949627, "step": 851 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.124448555056006e-05, "clip_ratio/low_min": 6.124448555056006e-05, "clip_ratio/region_mean": 6.124448555056006e-05, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 670.0, "completions/mean_length": 454.25, "completions/mean_terminated_length": 444.1290283203125, "completions/min_length": 283.0, "completions/min_terminated_length": 283.0, "entropy": 0.2763819079846144, "epoch": 0.45610278372591007, "frac_reward_zero_std": 0.5, "grad_norm": 0.013571832329034805, "learning_rate": 1e-05, "loss": 0.0155, "num_tokens": 17783517.0, "reward": 0.65625, "reward_std": 0.22201895713806152, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.4096670150756836, "sampling/importance_sampling_ratio/mean": 1.0002992153167725, "sampling/importance_sampling_ratio/min": 0.682148277759552, "sampling/sampling_logp_difference/max": 0.3825082778930664, "sampling/sampling_logp_difference/mean": 0.010273137129843235, "step": 852 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 592.0, "completions/mean_length": 418.5625, "completions/mean_terminated_length": 395.2666931152344, "completions/min_length": 207.0, "completions/min_terminated_length": 207.0, "entropy": 0.32329537719488144, "epoch": 0.45663811563169165, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 17800271.0, "reward": 0.75, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.5444774627685547, "sampling/importance_sampling_ratio/mean": 1.0000261068344116, "sampling/importance_sampling_ratio/min": 0.6960228085517883, "sampling/sampling_logp_difference/max": 0.43468570709228516, "sampling/sampling_logp_difference/mean": 0.01200791634619236, "step": 853 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00014849321451038122, "clip_ratio/low_min": 0.00014849321451038122, "clip_ratio/region_mean": 0.00014849321451038122, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 760.0, "completions/mean_length": 436.34375, "completions/mean_terminated_length": 425.6451416015625, "completions/min_length": 237.0, "completions/min_terminated_length": 237.0, "entropy": 0.3880876265466213, "epoch": 0.45717344753747324, "frac_reward_zero_std": 0.25, "grad_norm": 0.009868266992270947, "learning_rate": 1e-05, "loss": 0.0124, "num_tokens": 17818162.0, "reward": 0.71875, "reward_std": 0.3198433816432953, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.4362332820892334, "sampling/importance_sampling_ratio/mean": 0.9998043179512024, "sampling/importance_sampling_ratio/min": 0.5585951805114746, "sampling/sampling_logp_difference/max": 0.5823302268981934, "sampling/sampling_logp_difference/mean": 0.01380274910479784, "step": 854 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00013056294847046956, "clip_ratio/low_min": 0.00013056294847046956, "clip_ratio/region_mean": 0.00013056294847046956, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 759.0, "completions/mean_length": 474.1875, "completions/mean_terminated_length": 454.60003662109375, "completions/min_length": 254.0, "completions/min_terminated_length": 254.0, "entropy": 0.3601721115410328, "epoch": 0.45770877944325483, "frac_reward_zero_std": 0.25, "grad_norm": 0.009773055091500282, "learning_rate": 1e-05, "loss": 0.0507, "num_tokens": 17836960.0, "reward": 0.75, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.5590063333511353, "sampling/importance_sampling_ratio/mean": 0.9999298453330994, "sampling/importance_sampling_ratio/min": 0.6473665833473206, "sampling/sampling_logp_difference/max": 0.4440486431121826, "sampling/sampling_logp_difference/mean": 0.012558458372950554, "step": 855 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.846585554536432e-05, "clip_ratio/low_min": 5.846585554536432e-05, "clip_ratio/region_mean": 5.846585554536432e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 639.0, "completions/max_terminated_length": 639.0, "completions/mean_length": 448.5, "completions/mean_terminated_length": 448.5, "completions/min_length": 279.0, "completions/min_terminated_length": 279.0, "entropy": 0.2852116785943508, "epoch": 0.4582441113490364, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": -0.0109, "num_tokens": 17854680.0, "reward": 0.9375, "reward_std": 0.1157275140285492, "rewards/accuracy_reward/mean": 0.9375, "rewards/accuracy_reward/std": 0.24593468010425568, "sampling/importance_sampling_ratio/max": 1.6794205904006958, "sampling/importance_sampling_ratio/mean": 1.0000758171081543, "sampling/importance_sampling_ratio/min": 0.665300190448761, "sampling/sampling_logp_difference/max": 0.5184488296508789, "sampling/sampling_logp_difference/mean": 0.01116214320063591, "step": 856 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 766.0, "completions/max_terminated_length": 766.0, "completions/mean_length": 421.6875, "completions/mean_terminated_length": 421.6875, "completions/min_length": 231.0, "completions/min_terminated_length": 231.0, "entropy": 0.29658959805965424, "epoch": 0.458779443254818, "frac_reward_zero_std": 0.5, "grad_norm": 0.021478310227394104, "learning_rate": 1e-05, "loss": -0.0052, "num_tokens": 17871486.0, "reward": 0.84375, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.834847331047058, "sampling/importance_sampling_ratio/mean": 1.0001215934753418, "sampling/importance_sampling_ratio/min": 0.6851791143417358, "sampling/sampling_logp_difference/max": 0.6069612503051758, "sampling/sampling_logp_difference/mean": 0.01114323828369379, "step": 857 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.139802669873461e-05, "clip_ratio/low_min": 5.139802669873461e-05, "clip_ratio/region_mean": 5.139802669873461e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 762.0, "completions/max_terminated_length": 762.0, "completions/mean_length": 456.9375, "completions/mean_terminated_length": 456.9375, "completions/min_length": 263.0, "completions/min_terminated_length": 263.0, "entropy": 0.28951916098594666, "epoch": 0.4593147751605996, "frac_reward_zero_std": 0.5, "grad_norm": 0.003334330627694726, "learning_rate": 1e-05, "loss": 0.0173, "num_tokens": 17890260.0, "reward": 0.8125, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.7477837800979614, "sampling/importance_sampling_ratio/mean": 0.999552845954895, "sampling/importance_sampling_ratio/min": 0.6054671406745911, "sampling/sampling_logp_difference/max": 0.558348536491394, "sampling/sampling_logp_difference/mean": 0.011437330394983292, "step": 858 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 687.0, "completions/mean_length": 457.9375, "completions/mean_terminated_length": 447.9354553222656, "completions/min_length": 189.0, "completions/min_terminated_length": 189.0, "entropy": 0.34119654819369316, "epoch": 0.45985010706638113, "frac_reward_zero_std": 0.0, "grad_norm": 0.019065650179982185, "learning_rate": 1e-05, "loss": 0.0065, "num_tokens": 17908162.0, "reward": 0.78125, "reward_std": 0.4218915104866028, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.4342588186264038, "sampling/importance_sampling_ratio/mean": 1.0001591444015503, "sampling/importance_sampling_ratio/min": 0.6432639956474304, "sampling/sampling_logp_difference/max": 0.44120001792907715, "sampling/sampling_logp_difference/mean": 0.011838923208415508, "step": 859 }, { "clip_ratio/high_max": 6.309944728855044e-05, "clip_ratio/high_mean": 6.309944728855044e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 6.309944728855044e-05, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 592.0, "completions/mean_length": 436.65625, "completions/mean_terminated_length": 414.5666809082031, "completions/min_length": 243.0, "completions/min_terminated_length": 243.0, "entropy": 0.32465410977602005, "epoch": 0.4603854389721627, "frac_reward_zero_std": 0.25, "grad_norm": 0.006189401727169752, "learning_rate": 1e-05, "loss": 0.0391, "num_tokens": 17925495.0, "reward": 0.90625, "reward_std": 0.2651650309562683, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.4466276168823242, "sampling/importance_sampling_ratio/mean": 0.9999515414237976, "sampling/importance_sampling_ratio/min": 0.6255346536636353, "sampling/sampling_logp_difference/max": 0.4691486358642578, "sampling/sampling_logp_difference/mean": 0.012850682251155376, "step": 860 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 756.0, "completions/mean_length": 520.90625, "completions/mean_terminated_length": 475.1481628417969, "completions/min_length": 288.0, "completions/min_terminated_length": 288.0, "entropy": 0.29688988626003265, "epoch": 0.4609207708779443, "frac_reward_zero_std": 0.5, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0018, "num_tokens": 17945460.0, "reward": 0.75, "reward_std": 0.2587745785713196, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.4301317930221558, "sampling/importance_sampling_ratio/mean": 0.9998509883880615, "sampling/importance_sampling_ratio/min": 0.6935265064239502, "sampling/sampling_logp_difference/max": 0.3659658432006836, "sampling/sampling_logp_difference/mean": 0.01100583653897047, "step": 861 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 763.0, "completions/mean_length": 527.34375, "completions/mean_terminated_length": 502.4482727050781, "completions/min_length": 345.0, "completions/min_terminated_length": 345.0, "entropy": 0.25307290256023407, "epoch": 0.4614561027837259, "frac_reward_zero_std": 0.25, "grad_norm": 0.005747991148382425, "learning_rate": 1e-05, "loss": 0.0376, "num_tokens": 17965943.0, "reward": 0.78125, "reward_std": 0.3377464711666107, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.5645287036895752, "sampling/importance_sampling_ratio/mean": 0.9996405243873596, "sampling/importance_sampling_ratio/min": 0.6969373822212219, "sampling/sampling_logp_difference/max": 0.4475846290588379, "sampling/sampling_logp_difference/mean": 0.010516190901398659, "step": 862 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.28125, "completions/max_length": 768.0, "completions/max_terminated_length": 754.0, "completions/mean_length": 547.1875, "completions/mean_terminated_length": 460.7826232910156, "completions/min_length": 291.0, "completions/min_terminated_length": 291.0, "entropy": 0.3906844034790993, "epoch": 0.4619914346895075, "frac_reward_zero_std": 0.0, "grad_norm": 0.02722861059010029, "learning_rate": 1e-05, "loss": 0.0244, "num_tokens": 17987221.0, "reward": 0.59375, "reward_std": 0.4218915104866028, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.6273584365844727, "sampling/importance_sampling_ratio/mean": 1.000257134437561, "sampling/importance_sampling_ratio/min": 0.6665515303611755, "sampling/sampling_logp_difference/max": 0.48695802688598633, "sampling/sampling_logp_difference/mean": 0.013111263513565063, "step": 863 }, { "clip_ratio/high_max": 0.00010442773782415316, "clip_ratio/high_mean": 0.00010442773782415316, "clip_ratio/low_mean": 7.018529140623286e-05, "clip_ratio/low_min": 7.018529140623286e-05, "clip_ratio/region_mean": 0.00017461302923038602, "completions/clipped_ratio": 0.0, "completions/max_length": 619.0, "completions/max_terminated_length": 619.0, "completions/mean_length": 353.9375, "completions/mean_terminated_length": 353.9375, "completions/min_length": 218.0, "completions/min_terminated_length": 218.0, "entropy": 0.2766990512609482, "epoch": 0.4625267665952891, "frac_reward_zero_std": 0.5, "grad_norm": 0.005675820168107748, "learning_rate": 1e-05, "loss": -0.0002, "num_tokens": 18001803.0, "reward": 0.875, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 1.403257966041565, "sampling/importance_sampling_ratio/mean": 0.9998831152915955, "sampling/importance_sampling_ratio/min": 0.6779201626777649, "sampling/sampling_logp_difference/max": 0.38872575759887695, "sampling/sampling_logp_difference/mean": 0.01106823980808258, "step": 864 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 633.0, "completions/mean_length": 468.375, "completions/mean_terminated_length": 425.5714416503906, "completions/min_length": 250.0, "completions/min_terminated_length": 250.0, "entropy": 0.35452619194984436, "epoch": 0.46306209850107066, "frac_reward_zero_std": 0.5, "grad_norm": 0.014592459425330162, "learning_rate": 1e-05, "loss": 0.0309, "num_tokens": 18020807.0, "reward": 0.78125, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.8562095165252686, "sampling/importance_sampling_ratio/mean": 0.9994197487831116, "sampling/importance_sampling_ratio/min": 0.3569437563419342, "sampling/sampling_logp_difference/max": 1.030177116394043, "sampling/sampling_logp_difference/mean": 0.013007337227463722, "step": 865 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 741.0, "completions/mean_length": 453.625, "completions/mean_terminated_length": 443.4838562011719, "completions/min_length": 262.0, "completions/min_terminated_length": 262.0, "entropy": 0.261502867564559, "epoch": 0.46359743040685225, "frac_reward_zero_std": 0.5, "grad_norm": 0.008076409809291363, "learning_rate": 1e-05, "loss": 0.0763, "num_tokens": 18038851.0, "reward": 0.875, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 1.4596995115280151, "sampling/importance_sampling_ratio/mean": 0.9996519088745117, "sampling/importance_sampling_ratio/min": 0.6547124981880188, "sampling/sampling_logp_difference/max": 0.4235590696334839, "sampling/sampling_logp_difference/mean": 0.010198540985584259, "step": 866 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 749.0, "completions/mean_length": 510.65625, "completions/mean_terminated_length": 502.3548278808594, "completions/min_length": 343.0, "completions/min_terminated_length": 343.0, "entropy": 0.35885627567768097, "epoch": 0.46413276231263384, "frac_reward_zero_std": 0.5, "grad_norm": 0.009353122673928738, "learning_rate": 1e-05, "loss": 0.0001, "num_tokens": 18058928.0, "reward": 0.90625, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.6648247241973877, "sampling/importance_sampling_ratio/mean": 0.999761700630188, "sampling/importance_sampling_ratio/min": 0.4991922974586487, "sampling/sampling_logp_difference/max": 0.6947638988494873, "sampling/sampling_logp_difference/mean": 0.012562422081828117, "step": 867 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00011859582446049899, "clip_ratio/low_min": 0.00011859582446049899, "clip_ratio/region_mean": 0.00011859582446049899, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 695.0, "completions/mean_length": 454.375, "completions/mean_terminated_length": 444.258056640625, "completions/min_length": 293.0, "completions/min_terminated_length": 293.0, "entropy": 0.29925916343927383, "epoch": 0.46466809421841543, "frac_reward_zero_std": 0.5, "grad_norm": 0.005252790171653032, "learning_rate": 1e-05, "loss": 0.0172, "num_tokens": 18077300.0, "reward": 0.65625, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.480759620666504, "sampling/importance_sampling_ratio/mean": 0.9997273683547974, "sampling/importance_sampling_ratio/min": 0.6663631200790405, "sampling/sampling_logp_difference/max": 0.40592050552368164, "sampling/sampling_logp_difference/mean": 0.012263418175280094, "step": 868 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 678.0, "completions/mean_length": 450.65625, "completions/mean_terminated_length": 429.5000305175781, "completions/min_length": 239.0, "completions/min_terminated_length": 239.0, "entropy": 0.2597888894379139, "epoch": 0.465203426124197, "frac_reward_zero_std": 0.75, "grad_norm": 0.018060805276036263, "learning_rate": 1e-05, "loss": 0.0066, "num_tokens": 18095649.0, "reward": 0.9375, "reward_std": 0.1157275140285492, "rewards/accuracy_reward/mean": 0.9375, "rewards/accuracy_reward/std": 0.24593468010425568, "sampling/importance_sampling_ratio/max": 1.722191333770752, "sampling/importance_sampling_ratio/mean": 1.0004228353500366, "sampling/importance_sampling_ratio/min": 0.7221609354019165, "sampling/sampling_logp_difference/max": 0.5435974597930908, "sampling/sampling_logp_difference/mean": 0.010789213702082634, "step": 869 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 713.0, "completions/mean_length": 603.78125, "completions/mean_terminated_length": 580.3214721679688, "completions/min_length": 379.0, "completions/min_terminated_length": 379.0, "entropy": 0.26773758232593536, "epoch": 0.4657387580299786, "frac_reward_zero_std": 0.25, "grad_norm": 0.016819944605231285, "learning_rate": 1e-05, "loss": 0.0553, "num_tokens": 18119234.0, "reward": 0.75, "reward_std": 0.2925041913986206, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.7391873598098755, "sampling/importance_sampling_ratio/mean": 0.999520480632782, "sampling/importance_sampling_ratio/min": 0.6458948850631714, "sampling/sampling_logp_difference/max": 0.5534179210662842, "sampling/sampling_logp_difference/mean": 0.009992285631597042, "step": 870 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 767.0, "completions/mean_length": 463.5, "completions/mean_terminated_length": 432.0, "completions/min_length": 290.0, "completions/min_terminated_length": 290.0, "entropy": 0.28490811958909035, "epoch": 0.4662740899357602, "frac_reward_zero_std": 0.0, "grad_norm": 0.01612742803990841, "learning_rate": 1e-05, "loss": 0.0194, "num_tokens": 18138290.0, "reward": 0.6875, "reward_std": 0.4355512857437134, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.589380145072937, "sampling/importance_sampling_ratio/mean": 0.9997609853744507, "sampling/importance_sampling_ratio/min": 0.6430565714836121, "sampling/sampling_logp_difference/max": 0.46334409713745117, "sampling/sampling_logp_difference/mean": 0.012222224846482277, "step": 871 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 736.0, "completions/max_terminated_length": 736.0, "completions/mean_length": 516.40625, "completions/mean_terminated_length": 516.40625, "completions/min_length": 352.0, "completions/min_terminated_length": 352.0, "entropy": 0.2322842124849558, "epoch": 0.4668094218415418, "frac_reward_zero_std": 0.75, "grad_norm": 0.006723431404680014, "learning_rate": 1e-05, "loss": 0.0083, "num_tokens": 18158759.0, "reward": 0.90625, "reward_std": 0.1293872892856598, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.440084457397461, "sampling/importance_sampling_ratio/mean": 1.0000141859054565, "sampling/importance_sampling_ratio/min": 0.685984194278717, "sampling/sampling_logp_difference/max": 0.37690067291259766, "sampling/sampling_logp_difference/mean": 0.009048039093613625, "step": 872 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.4324205848388374e-05, "clip_ratio/low_min": 5.4324205848388374e-05, "clip_ratio/region_mean": 5.4324205848388374e-05, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 703.0, "completions/mean_length": 497.125, "completions/mean_terminated_length": 458.4285888671875, "completions/min_length": 291.0, "completions/min_terminated_length": 291.0, "entropy": 0.4040381032973528, "epoch": 0.4673447537473233, "frac_reward_zero_std": 0.25, "grad_norm": 0.018366919830441475, "learning_rate": 1e-05, "loss": 0.0542, "num_tokens": 18178155.0, "reward": 0.6875, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.440218448638916, "sampling/importance_sampling_ratio/mean": 0.9998831748962402, "sampling/importance_sampling_ratio/min": 0.5919627547264099, "sampling/sampling_logp_difference/max": 0.5243115425109863, "sampling/sampling_logp_difference/mean": 0.013409332372248173, "step": 873 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 734.0, "completions/max_terminated_length": 734.0, "completions/mean_length": 383.75, "completions/mean_terminated_length": 383.75, "completions/min_length": 228.0, "completions/min_terminated_length": 228.0, "entropy": 0.22543956339359283, "epoch": 0.4678800856531049, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 18193835.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0002186298370361, "sampling/importance_sampling_ratio/min": 0.5272040367126465, "sampling/sampling_logp_difference/max": 0.8156676292419434, "sampling/sampling_logp_difference/mean": 0.010029658675193787, "step": 874 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0001817653828766197, "clip_ratio/low_min": 0.0001817653828766197, "clip_ratio/region_mean": 0.0001817653828766197, "completions/clipped_ratio": 0.375, "completions/max_length": 768.0, "completions/max_terminated_length": 765.0, "completions/mean_length": 586.28125, "completions/mean_terminated_length": 477.25, "completions/min_length": 288.0, "completions/min_terminated_length": 288.0, "entropy": 0.3310044165700674, "epoch": 0.4684154175588865, "frac_reward_zero_std": 0.0, "grad_norm": 0.00836801528930664, "learning_rate": 1e-05, "loss": 0.0389, "num_tokens": 18216260.0, "reward": 0.53125, "reward_std": 0.3808925747871399, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.4283980131149292, "sampling/importance_sampling_ratio/mean": 0.9998232126235962, "sampling/importance_sampling_ratio/min": 0.6860008239746094, "sampling/sampling_logp_difference/max": 0.3768763542175293, "sampling/sampling_logp_difference/mean": 0.011945049278438091, "step": 875 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 649.0, "completions/mean_length": 447.59375, "completions/mean_terminated_length": 437.258056640625, "completions/min_length": 256.0, "completions/min_terminated_length": 256.0, "entropy": 0.33078383654356003, "epoch": 0.4689507494646681, "frac_reward_zero_std": 0.75, "grad_norm": 0.006502924952656031, "learning_rate": 1e-05, "loss": 0.0448, "num_tokens": 18234087.0, "reward": 0.6875, "reward_std": 0.1157275140285492, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.5555464029312134, "sampling/importance_sampling_ratio/mean": 1.0001003742218018, "sampling/importance_sampling_ratio/min": 0.5430275797843933, "sampling/sampling_logp_difference/max": 0.6105952262878418, "sampling/sampling_logp_difference/mean": 0.012804846279323101, "step": 876 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 761.0, "completions/mean_length": 557.6875, "completions/mean_terminated_length": 487.5833435058594, "completions/min_length": 297.0, "completions/min_terminated_length": 297.0, "entropy": 0.23251344822347164, "epoch": 0.46948608137044967, "frac_reward_zero_std": 0.25, "grad_norm": 0.003787594148889184, "learning_rate": 1e-05, "loss": 0.0259, "num_tokens": 18255877.0, "reward": 0.75, "reward_std": 0.292504221200943, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.658536672592163, "sampling/importance_sampling_ratio/mean": 0.9999313950538635, "sampling/importance_sampling_ratio/min": 0.6560901999473572, "sampling/sampling_logp_difference/max": 0.5059356689453125, "sampling/sampling_logp_difference/mean": 0.009480309672653675, "step": 877 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 564.0, "completions/max_terminated_length": 564.0, "completions/mean_length": 425.1875, "completions/mean_terminated_length": 425.1875, "completions/min_length": 245.0, "completions/min_terminated_length": 245.0, "entropy": 0.2553399093449116, "epoch": 0.47002141327623126, "frac_reward_zero_std": 0.75, "grad_norm": 0.0038415465969592333, "learning_rate": 1e-05, "loss": -0.0391, "num_tokens": 18273027.0, "reward": 0.96875, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.96875, "rewards/accuracy_reward/std": 0.1767766922712326, "sampling/importance_sampling_ratio/max": 1.5385226011276245, "sampling/importance_sampling_ratio/mean": 0.9999315142631531, "sampling/importance_sampling_ratio/min": 0.6782143115997314, "sampling/sampling_logp_difference/max": 0.43082261085510254, "sampling/sampling_logp_difference/mean": 0.010707005858421326, "step": 878 }, { "clip_ratio/high_max": 5.118755143485032e-05, "clip_ratio/high_mean": 5.118755143485032e-05, "clip_ratio/low_mean": 0.00014566063691745512, "clip_ratio/low_min": 0.00014566063691745512, "clip_ratio/region_mean": 0.00019684818835230544, "completions/clipped_ratio": 0.28125, "completions/max_length": 768.0, "completions/max_terminated_length": 746.0, "completions/mean_length": 550.90625, "completions/mean_terminated_length": 465.95654296875, "completions/min_length": 264.0, "completions/min_terminated_length": 264.0, "entropy": 0.44744790345430374, "epoch": 0.47055674518201285, "frac_reward_zero_std": 0.5, "grad_norm": 0.007869189605116844, "learning_rate": 1e-05, "loss": 0.0408, "num_tokens": 18294632.0, "reward": 0.59375, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.5502578020095825, "sampling/importance_sampling_ratio/mean": 0.9996722936630249, "sampling/importance_sampling_ratio/min": 0.69932621717453, "sampling/sampling_logp_difference/max": 0.43842124938964844, "sampling/sampling_logp_difference/mean": 0.015066186897456646, "step": 879 }, { "clip_ratio/high_max": 6.067961294320412e-05, "clip_ratio/high_mean": 6.067961294320412e-05, "clip_ratio/low_mean": 6.067961294320412e-05, "clip_ratio/low_min": 6.067961294320412e-05, "clip_ratio/region_mean": 0.00012135922588640824, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 727.0, "completions/mean_length": 528.96875, "completions/mean_terminated_length": 473.8077087402344, "completions/min_length": 226.0, "completions/min_terminated_length": 226.0, "entropy": 0.40720242634415627, "epoch": 0.47109207708779444, "frac_reward_zero_std": 0.25, "grad_norm": 0.019042128697037697, "learning_rate": 1e-05, "loss": 0.0633, "num_tokens": 18315839.0, "reward": 0.5625, "reward_std": 0.2925041913986206, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.3979129791259766, "sampling/importance_sampling_ratio/mean": 0.9999843239784241, "sampling/importance_sampling_ratio/min": 0.6468991041183472, "sampling/sampling_logp_difference/max": 0.4355649948120117, "sampling/sampling_logp_difference/mean": 0.013768176548182964, "step": 880 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 652.0, "completions/max_terminated_length": 652.0, "completions/mean_length": 434.59375, "completions/mean_terminated_length": 434.59375, "completions/min_length": 236.0, "completions/min_terminated_length": 236.0, "entropy": 0.2813435569405556, "epoch": 0.471627408993576, "frac_reward_zero_std": 0.5, "grad_norm": 0.010080586187541485, "learning_rate": 1e-05, "loss": -0.0253, "num_tokens": 18333386.0, "reward": 0.875, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 1.568997859954834, "sampling/importance_sampling_ratio/mean": 1.000688076019287, "sampling/importance_sampling_ratio/min": 0.6839987635612488, "sampling/sampling_logp_difference/max": 0.450437068939209, "sampling/sampling_logp_difference/mean": 0.011479225009679794, "step": 881 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00010245901648886502, "clip_ratio/low_min": 0.00010245901648886502, "clip_ratio/region_mean": 0.00010245901648886502, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 728.0, "completions/mean_length": 559.75, "completions/mean_terminated_length": 511.69232177734375, "completions/min_length": 346.0, "completions/min_terminated_length": 346.0, "entropy": 0.31052958965301514, "epoch": 0.4721627408993576, "frac_reward_zero_std": 0.75, "grad_norm": 0.009617138653993607, "learning_rate": 1e-05, "loss": 0.011, "num_tokens": 18355410.0, "reward": 0.59375, "reward_std": 0.1293872892856598, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.640962839126587, "sampling/importance_sampling_ratio/mean": 0.9999303221702576, "sampling/importance_sampling_ratio/min": 0.695469081401825, "sampling/sampling_logp_difference/max": 0.4952831268310547, "sampling/sampling_logp_difference/mean": 0.01131723914295435, "step": 882 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 543.0, "completions/max_terminated_length": 543.0, "completions/mean_length": 383.9375, "completions/mean_terminated_length": 383.9375, "completions/min_length": 190.0, "completions/min_terminated_length": 190.0, "entropy": 0.25118929147720337, "epoch": 0.4726980728051392, "frac_reward_zero_std": 0.5, "grad_norm": 0.01852281764149666, "learning_rate": 1e-05, "loss": 0.0192, "num_tokens": 18371328.0, "reward": 0.71875, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.9445576667785645, "sampling/importance_sampling_ratio/mean": 1.000382900238037, "sampling/importance_sampling_ratio/min": 0.6961176991462708, "sampling/sampling_logp_difference/max": 0.6650345325469971, "sampling/sampling_logp_difference/mean": 0.010267798788845539, "step": 883 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 557.0, "completions/max_terminated_length": 557.0, "completions/mean_length": 344.65625, "completions/mean_terminated_length": 344.65625, "completions/min_length": 223.0, "completions/min_terminated_length": 223.0, "entropy": 0.2916817367076874, "epoch": 0.4732334047109208, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0106, "num_tokens": 18385613.0, "reward": 0.96875, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.96875, "rewards/accuracy_reward/std": 0.1767766922712326, "sampling/importance_sampling_ratio/max": 1.7127189636230469, "sampling/importance_sampling_ratio/mean": 0.9995903968811035, "sampling/importance_sampling_ratio/min": 0.7017140984535217, "sampling/sampling_logp_difference/max": 0.5380821228027344, "sampling/sampling_logp_difference/mean": 0.011689784936606884, "step": 884 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 603.0, "completions/mean_length": 440.9375, "completions/mean_terminated_length": 419.13336181640625, "completions/min_length": 293.0, "completions/min_terminated_length": 293.0, "entropy": 0.33362437784671783, "epoch": 0.4737687366167024, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0481, "num_tokens": 18403219.0, "reward": 0.875, "reward_std": 0.13363061845302582, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 1.8563076257705688, "sampling/importance_sampling_ratio/mean": 1.0002373456954956, "sampling/importance_sampling_ratio/min": 0.6271288394927979, "sampling/sampling_logp_difference/max": 0.6185894012451172, "sampling/sampling_logp_difference/mean": 0.012039019726216793, "step": 885 }, { "clip_ratio/high_max": 5.5358723329845816e-05, "clip_ratio/high_mean": 5.5358723329845816e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 5.5358723329845816e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 751.0, "completions/max_terminated_length": 751.0, "completions/mean_length": 503.09375, "completions/mean_terminated_length": 503.09375, "completions/min_length": 288.0, "completions/min_terminated_length": 288.0, "entropy": 0.32552289217710495, "epoch": 0.4743040685224839, "frac_reward_zero_std": 0.0, "grad_norm": 0.01108392421156168, "learning_rate": 1e-05, "loss": 0.0072, "num_tokens": 18423718.0, "reward": 0.75, "reward_std": 0.4261348247528076, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.4468801021575928, "sampling/importance_sampling_ratio/mean": 0.9990369081497192, "sampling/importance_sampling_ratio/min": 0.5851335525512695, "sampling/sampling_logp_difference/max": 0.5359151363372803, "sampling/sampling_logp_difference/mean": 0.012145783752202988, "step": 886 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 742.0, "completions/mean_length": 453.34375, "completions/mean_terminated_length": 443.19354248046875, "completions/min_length": 259.0, "completions/min_terminated_length": 259.0, "entropy": 0.29032275825738907, "epoch": 0.4748394004282655, "frac_reward_zero_std": 0.5, "grad_norm": 0.007234309334307909, "learning_rate": 1e-05, "loss": 0.0227, "num_tokens": 18441785.0, "reward": 0.90625, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.438271403312683, "sampling/importance_sampling_ratio/mean": 1.000157117843628, "sampling/importance_sampling_ratio/min": 0.7075569033622742, "sampling/sampling_logp_difference/max": 0.36344194412231445, "sampling/sampling_logp_difference/mean": 0.010901025496423244, "step": 887 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 759.0, "completions/max_terminated_length": 759.0, "completions/mean_length": 442.96875, "completions/mean_terminated_length": 442.96875, "completions/min_length": 247.0, "completions/min_terminated_length": 247.0, "entropy": 0.23408159986138344, "epoch": 0.4753747323340471, "frac_reward_zero_std": 0.5, "grad_norm": 0.005237105768173933, "learning_rate": 1e-05, "loss": 0.0134, "num_tokens": 18459912.0, "reward": 0.78125, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.5362108945846558, "sampling/importance_sampling_ratio/mean": 1.0000702142715454, "sampling/importance_sampling_ratio/min": 0.6879655718803406, "sampling/sampling_logp_difference/max": 0.429318904876709, "sampling/sampling_logp_difference/mean": 0.009599080309271812, "step": 888 }, { "clip_ratio/high_max": 0.00010968405695166439, "clip_ratio/high_mean": 0.00010968405695166439, "clip_ratio/low_mean": 5.887894440093078e-05, "clip_ratio/low_min": 5.887894440093078e-05, "clip_ratio/region_mean": 0.00016856300135259517, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 720.0, "completions/mean_length": 539.34375, "completions/mean_terminated_length": 524.1000366210938, "completions/min_length": 293.0, "completions/min_terminated_length": 293.0, "entropy": 0.30345601215958595, "epoch": 0.4759100642398287, "frac_reward_zero_std": 0.0, "grad_norm": 0.019864946603775024, "learning_rate": 1e-05, "loss": 0.0569, "num_tokens": 18481267.0, "reward": 0.65625, "reward_std": 0.4628904461860657, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.72467839717865, "sampling/importance_sampling_ratio/mean": 1.0002552270889282, "sampling/importance_sampling_ratio/min": 0.4199666678905487, "sampling/sampling_logp_difference/max": 0.8675799369812012, "sampling/sampling_logp_difference/mean": 0.012069141492247581, "step": 889 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 705.0, "completions/mean_length": 542.40625, "completions/mean_terminated_length": 527.36669921875, "completions/min_length": 338.0, "completions/min_terminated_length": 338.0, "entropy": 0.26934675872325897, "epoch": 0.47644539614561027, "frac_reward_zero_std": 0.5, "grad_norm": 0.01202559657394886, "learning_rate": 1e-05, "loss": 0.0332, "num_tokens": 18502776.0, "reward": 0.875, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 1.3873212337493896, "sampling/importance_sampling_ratio/mean": 0.9999104738235474, "sampling/importance_sampling_ratio/min": 0.6175732016563416, "sampling/sampling_logp_difference/max": 0.48195767402648926, "sampling/sampling_logp_difference/mean": 0.010256065987050533, "step": 890 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 714.0, "completions/max_terminated_length": 714.0, "completions/mean_length": 384.53125, "completions/mean_terminated_length": 384.53125, "completions/min_length": 219.0, "completions/min_terminated_length": 219.0, "entropy": 0.28182504139840603, "epoch": 0.47698072805139186, "frac_reward_zero_std": 0.75, "grad_norm": 0.004495512694120407, "learning_rate": 1e-05, "loss": 0.0301, "num_tokens": 18518745.0, "reward": 0.96875, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.96875, "rewards/accuracy_reward/std": 0.1767766922712326, "sampling/importance_sampling_ratio/max": 1.8272864818572998, "sampling/importance_sampling_ratio/mean": 1.0004711151123047, "sampling/importance_sampling_ratio/min": 0.5638600587844849, "sampling/sampling_logp_difference/max": 0.6028320789337158, "sampling/sampling_logp_difference/mean": 0.011364540085196495, "step": 891 }, { "clip_ratio/high_max": 5.5654498282819986e-05, "clip_ratio/high_mean": 5.5654498282819986e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 5.5654498282819986e-05, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 715.0, "completions/mean_length": 521.09375, "completions/mean_terminated_length": 504.63336181640625, "completions/min_length": 332.0, "completions/min_terminated_length": 332.0, "entropy": 0.2611116748303175, "epoch": 0.47751605995717344, "frac_reward_zero_std": 0.25, "grad_norm": 0.010341753251850605, "learning_rate": 1e-05, "loss": 0.076, "num_tokens": 18539180.0, "reward": 0.8125, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.443167805671692, "sampling/importance_sampling_ratio/mean": 0.9995496273040771, "sampling/importance_sampling_ratio/min": 0.43118199706077576, "sampling/sampling_logp_difference/max": 0.8412249684333801, "sampling/sampling_logp_difference/mean": 0.010036716237664223, "step": 892 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00022684068244416267, "clip_ratio/low_min": 0.00022684068244416267, "clip_ratio/region_mean": 0.00022684068244416267, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 759.0, "completions/mean_length": 497.125, "completions/mean_terminated_length": 446.96295166015625, "completions/min_length": 213.0, "completions/min_terminated_length": 213.0, "entropy": 0.22813864424824715, "epoch": 0.47805139186295503, "frac_reward_zero_std": 0.75, "grad_norm": 0.007934284396469593, "learning_rate": 1e-05, "loss": 0.0066, "num_tokens": 18559168.0, "reward": 0.84375, "reward_std": 0.1293872892856598, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.7819143533706665, "sampling/importance_sampling_ratio/mean": 1.0001704692840576, "sampling/importance_sampling_ratio/min": 0.6121436953544617, "sampling/sampling_logp_difference/max": 0.5776882171630859, "sampling/sampling_logp_difference/mean": 0.009797493927180767, "step": 893 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 718.0, "completions/max_terminated_length": 718.0, "completions/mean_length": 439.46875, "completions/mean_terminated_length": 439.46875, "completions/min_length": 213.0, "completions/min_terminated_length": 213.0, "entropy": 0.3192239012569189, "epoch": 0.4785867237687366, "frac_reward_zero_std": 0.25, "grad_norm": 0.020979948341846466, "learning_rate": 1e-05, "loss": -0.0382, "num_tokens": 18576847.0, "reward": 0.6875, "reward_std": 0.3745020925998688, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.503618597984314, "sampling/importance_sampling_ratio/mean": 0.9997166395187378, "sampling/importance_sampling_ratio/min": 0.6501674056053162, "sampling/sampling_logp_difference/max": 0.43052542209625244, "sampling/sampling_logp_difference/mean": 0.011881766840815544, "step": 894 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00014281420590123162, "clip_ratio/low_min": 0.00014281420590123162, "clip_ratio/region_mean": 0.00014281420590123162, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 607.0, "completions/mean_length": 406.78125, "completions/mean_terminated_length": 369.4137878417969, "completions/min_length": 259.0, "completions/min_terminated_length": 259.0, "entropy": 0.35287977010011673, "epoch": 0.4791220556745182, "frac_reward_zero_std": 0.5, "grad_norm": 0.010944359004497528, "learning_rate": 1e-05, "loss": -0.0141, "num_tokens": 18593472.0, "reward": 0.65625, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.6390905380249023, "sampling/importance_sampling_ratio/mean": 1.0002306699752808, "sampling/importance_sampling_ratio/min": 0.7132964134216309, "sampling/sampling_logp_difference/max": 0.4941415786743164, "sampling/sampling_logp_difference/mean": 0.013677633367478848, "step": 895 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 659.0, "completions/max_terminated_length": 659.0, "completions/mean_length": 437.5, "completions/mean_terminated_length": 437.5, "completions/min_length": 257.0, "completions/min_terminated_length": 257.0, "entropy": 0.2677403874695301, "epoch": 0.4796573875802998, "frac_reward_zero_std": 0.25, "grad_norm": 0.008695017546415329, "learning_rate": 1e-05, "loss": 0.0143, "num_tokens": 18611384.0, "reward": 0.8125, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.4183658361434937, "sampling/importance_sampling_ratio/mean": 0.9993807673454285, "sampling/importance_sampling_ratio/min": 0.6329377293586731, "sampling/sampling_logp_difference/max": 0.45738327503204346, "sampling/sampling_logp_difference/mean": 0.010964793153107166, "step": 896 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 608.0, "completions/max_terminated_length": 608.0, "completions/mean_length": 433.28125, "completions/mean_terminated_length": 433.28125, "completions/min_length": 302.0, "completions/min_terminated_length": 302.0, "entropy": 0.299940075725317, "epoch": 0.4801927194860814, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": -0.0108, "num_tokens": 18628953.0, "reward": 0.96875, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.96875, "rewards/accuracy_reward/std": 0.1767766922712326, "sampling/importance_sampling_ratio/max": 1.4305717945098877, "sampling/importance_sampling_ratio/mean": 1.0000569820404053, "sampling/importance_sampling_ratio/min": 0.7020472884178162, "sampling/sampling_logp_difference/max": 0.3580741882324219, "sampling/sampling_logp_difference/mean": 0.0108436094596982, "step": 897 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 684.0, "completions/mean_length": 391.71875, "completions/mean_terminated_length": 379.58062744140625, "completions/min_length": 214.0, "completions/min_terminated_length": 214.0, "entropy": 0.34211036562919617, "epoch": 0.480728051391863, "frac_reward_zero_std": 0.5, "grad_norm": 0.007606659084558487, "learning_rate": 1e-05, "loss": 0.0051, "num_tokens": 18645168.0, "reward": 0.90625, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.6583902835845947, "sampling/importance_sampling_ratio/mean": 1.0000258684158325, "sampling/importance_sampling_ratio/min": 0.6280116438865662, "sampling/sampling_logp_difference/max": 0.5058474540710449, "sampling/sampling_logp_difference/mean": 0.013064631260931492, "step": 898 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 671.0, "completions/max_terminated_length": 671.0, "completions/mean_length": 518.28125, "completions/mean_terminated_length": 518.28125, "completions/min_length": 320.0, "completions/min_terminated_length": 320.0, "entropy": 0.2641910742968321, "epoch": 0.48126338329764456, "frac_reward_zero_std": 0.5, "grad_norm": 0.003466410096734762, "learning_rate": 1e-05, "loss": -0.0089, "num_tokens": 18665681.0, "reward": 0.9375, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.9375, "rewards/accuracy_reward/std": 0.24593468010425568, "sampling/importance_sampling_ratio/max": 1.4305813312530518, "sampling/importance_sampling_ratio/mean": 1.000329613685608, "sampling/importance_sampling_ratio/min": 0.6659950017929077, "sampling/sampling_logp_difference/max": 0.40647315979003906, "sampling/sampling_logp_difference/mean": 0.010509542189538479, "step": 899 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 682.0, "completions/max_terminated_length": 682.0, "completions/mean_length": 446.15625, "completions/mean_terminated_length": 446.15625, "completions/min_length": 302.0, "completions/min_terminated_length": 302.0, "entropy": 0.29691479727625847, "epoch": 0.4817987152034261, "frac_reward_zero_std": 0.75, "grad_norm": 0.007280933205038309, "learning_rate": 1e-05, "loss": -0.0126, "num_tokens": 18683910.0, "reward": 0.96875, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.96875, "rewards/accuracy_reward/std": 0.1767766922712326, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0001438856124878, "sampling/importance_sampling_ratio/min": 0.6874964237213135, "sampling/sampling_logp_difference/max": 0.8669991493225098, "sampling/sampling_logp_difference/mean": 0.011581506580114365, "step": 900 }, { "clip_ratio/high_max": 7.720815483480692e-05, "clip_ratio/high_mean": 7.720815483480692e-05, "clip_ratio/low_mean": 0.00012594458530656993, "clip_ratio/low_min": 0.00012594458530656993, "clip_ratio/region_mean": 0.00020315274014137685, "completions/clipped_ratio": 0.0, "completions/max_length": 685.0, "completions/max_terminated_length": 685.0, "completions/mean_length": 452.9375, "completions/mean_terminated_length": 452.9375, "completions/min_length": 276.0, "completions/min_terminated_length": 276.0, "entropy": 0.2824072800576687, "epoch": 0.4823340471092077, "frac_reward_zero_std": 0.5, "grad_norm": 0.006068243179470301, "learning_rate": 1e-05, "loss": -0.0023, "num_tokens": 18702316.0, "reward": 0.875, "reward_std": 0.2314550280570984, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 1.819222331047058, "sampling/importance_sampling_ratio/mean": 1.000252604484558, "sampling/importance_sampling_ratio/min": 0.4884468913078308, "sampling/sampling_logp_difference/max": 0.716524600982666, "sampling/sampling_logp_difference/mean": 0.01148994266986847, "step": 901 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 721.0, "completions/mean_length": 546.6875, "completions/mean_terminated_length": 515.0714721679688, "completions/min_length": 274.0, "completions/min_terminated_length": 274.0, "entropy": 0.3716841824352741, "epoch": 0.4828693790149893, "frac_reward_zero_std": 0.25, "grad_norm": 0.009000168181955814, "learning_rate": 1e-05, "loss": 0.0521, "num_tokens": 18723578.0, "reward": 0.40625, "reward_std": 0.3377464711666107, "rewards/accuracy_reward/mean": 0.40625, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9998276233673096, "sampling/importance_sampling_ratio/min": 0.6062214970588684, "sampling/sampling_logp_difference/max": 0.7087397575378418, "sampling/sampling_logp_difference/mean": 0.01339913159608841, "step": 902 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00016592920292168856, "clip_ratio/low_min": 0.00016592920292168856, "clip_ratio/region_mean": 0.00016592920292168856, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 740.0, "completions/mean_length": 533.375, "completions/mean_terminated_length": 479.23077392578125, "completions/min_length": 199.0, "completions/min_terminated_length": 199.0, "entropy": 0.23833789117634296, "epoch": 0.48340471092077086, "frac_reward_zero_std": 0.5, "grad_norm": 0.005179865751415491, "learning_rate": 1e-05, "loss": 0.0308, "num_tokens": 18745302.0, "reward": 0.78125, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.599319577217102, "sampling/importance_sampling_ratio/mean": 1.0000009536743164, "sampling/importance_sampling_ratio/min": 0.702337920665741, "sampling/sampling_logp_difference/max": 0.46957826614379883, "sampling/sampling_logp_difference/mean": 0.009353325702250004, "step": 903 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 638.0, "completions/max_terminated_length": 638.0, "completions/mean_length": 440.5, "completions/mean_terminated_length": 440.5, "completions/min_length": 294.0, "completions/min_terminated_length": 294.0, "entropy": 0.21238282695412636, "epoch": 0.48394004282655245, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 18762782.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.434478998184204, "sampling/importance_sampling_ratio/mean": 0.999982476234436, "sampling/importance_sampling_ratio/min": 0.6461242437362671, "sampling/sampling_logp_difference/max": 0.4367635250091553, "sampling/sampling_logp_difference/mean": 0.008587619289755821, "step": 904 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0002008641604334116, "clip_ratio/low_min": 0.0002008641604334116, "clip_ratio/region_mean": 0.0002008641604334116, "completions/clipped_ratio": 0.0, "completions/max_length": 722.0, "completions/max_terminated_length": 722.0, "completions/mean_length": 462.03125, "completions/mean_terminated_length": 462.03125, "completions/min_length": 297.0, "completions/min_terminated_length": 297.0, "entropy": 0.3878296688199043, "epoch": 0.48447537473233404, "frac_reward_zero_std": 0.5, "grad_norm": 0.005296432413160801, "learning_rate": 1e-05, "loss": 0.013, "num_tokens": 18781703.0, "reward": 0.8125, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0001065731048584, "sampling/importance_sampling_ratio/min": 0.7359341979026794, "sampling/sampling_logp_difference/max": 0.7660646438598633, "sampling/sampling_logp_difference/mean": 0.013224497437477112, "step": 905 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00017650244990363717, "clip_ratio/low_min": 0.00017650244990363717, "clip_ratio/region_mean": 0.00017650244990363717, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 615.0, "completions/mean_length": 478.0, "completions/mean_terminated_length": 424.2962951660156, "completions/min_length": 225.0, "completions/min_terminated_length": 225.0, "entropy": 0.3274830374866724, "epoch": 0.48501070663811563, "frac_reward_zero_std": 0.5, "grad_norm": 0.009160587564110756, "learning_rate": 1e-05, "loss": 0.0395, "num_tokens": 18800815.0, "reward": 0.75, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.8400568962097168, "sampling/importance_sampling_ratio/mean": 1.0003446340560913, "sampling/importance_sampling_ratio/min": 0.6211739778518677, "sampling/sampling_logp_difference/max": 0.6097965240478516, "sampling/sampling_logp_difference/mean": 0.011642465367913246, "step": 906 }, { "clip_ratio/high_max": 6.041565939085558e-05, "clip_ratio/high_mean": 6.041565939085558e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 6.041565939085558e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 657.0, "completions/max_terminated_length": 657.0, "completions/mean_length": 475.34375, "completions/mean_terminated_length": 475.34375, "completions/min_length": 301.0, "completions/min_terminated_length": 301.0, "entropy": 0.202494403347373, "epoch": 0.4855460385438972, "frac_reward_zero_std": 0.75, "grad_norm": 0.012039056979119778, "learning_rate": 1e-05, "loss": 0.025, "num_tokens": 18819354.0, "reward": 0.84375, "reward_std": 0.1293872892856598, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.6875321865081787, "sampling/importance_sampling_ratio/mean": 0.9998688697814941, "sampling/importance_sampling_ratio/min": 0.43752893805503845, "sampling/sampling_logp_difference/max": 0.8266124725341797, "sampling/sampling_logp_difference/mean": 0.00876142829656601, "step": 907 }, { "clip_ratio/high_max": 5.6999542721314356e-05, "clip_ratio/high_mean": 5.6999542721314356e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 5.6999542721314356e-05, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 748.0, "completions/mean_length": 501.03125, "completions/mean_terminated_length": 492.4193420410156, "completions/min_length": 267.0, "completions/min_terminated_length": 267.0, "entropy": 0.31029269844293594, "epoch": 0.4860813704496788, "frac_reward_zero_std": 0.25, "grad_norm": 0.014903610572218895, "learning_rate": 1e-05, "loss": -0.0225, "num_tokens": 18839035.0, "reward": 0.46875, "reward_std": 0.3471629321575165, "rewards/accuracy_reward/mean": 0.46875, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0003976821899414, "sampling/importance_sampling_ratio/min": 0.6708811521530151, "sampling/sampling_logp_difference/max": 0.7926762104034424, "sampling/sampling_logp_difference/mean": 0.011670663021504879, "step": 908 }, { "clip_ratio/high_max": 6.572029087692499e-05, "clip_ratio/high_mean": 6.572029087692499e-05, "clip_ratio/low_mean": 0.0001427755632903427, "clip_ratio/low_min": 0.0001427755632903427, "clip_ratio/region_mean": 0.00020849585416726768, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 648.0, "completions/mean_length": 479.4375, "completions/mean_terminated_length": 460.20001220703125, "completions/min_length": 320.0, "completions/min_terminated_length": 320.0, "entropy": 0.35903792828321457, "epoch": 0.4866167023554604, "frac_reward_zero_std": 0.25, "grad_norm": 0.008260451257228851, "learning_rate": 1e-05, "loss": 0.0072, "num_tokens": 18859217.0, "reward": 0.53125, "reward_std": 0.2651650309562683, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.6009653806686401, "sampling/importance_sampling_ratio/mean": 0.9997239112854004, "sampling/importance_sampling_ratio/min": 0.661887526512146, "sampling/sampling_logp_difference/max": 0.47060680389404297, "sampling/sampling_logp_difference/mean": 0.013056580908596516, "step": 909 }, { "clip_ratio/high_max": 6.35485484963283e-05, "clip_ratio/high_mean": 6.35485484963283e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 6.35485484963283e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 750.0, "completions/max_terminated_length": 750.0, "completions/mean_length": 483.1875, "completions/mean_terminated_length": 483.1875, "completions/min_length": 259.0, "completions/min_terminated_length": 259.0, "entropy": 0.2662975788116455, "epoch": 0.487152034261242, "frac_reward_zero_std": 0.5, "grad_norm": 0.003052448621019721, "learning_rate": 1e-05, "loss": 0.0272, "num_tokens": 18878551.0, "reward": 0.90625, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.4324358701705933, "sampling/importance_sampling_ratio/mean": 1.000091552734375, "sampling/importance_sampling_ratio/min": 0.7687998414039612, "sampling/sampling_logp_difference/max": 0.3593764305114746, "sampling/sampling_logp_difference/mean": 0.010334242135286331, "step": 910 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 755.0, "completions/mean_length": 514.90625, "completions/mean_terminated_length": 488.72412109375, "completions/min_length": 228.0, "completions/min_terminated_length": 228.0, "entropy": 0.2492699082940817, "epoch": 0.4876873661670236, "frac_reward_zero_std": 0.5, "grad_norm": 0.014037317596375942, "learning_rate": 1e-05, "loss": 0.0038, "num_tokens": 18898660.0, "reward": 0.78125, "reward_std": 0.2630178928375244, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.437874436378479, "sampling/importance_sampling_ratio/mean": 0.9996024370193481, "sampling/importance_sampling_ratio/min": 0.5474321842193604, "sampling/sampling_logp_difference/max": 0.6025166511535645, "sampling/sampling_logp_difference/mean": 0.009555388242006302, "step": 911 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 750.0, "completions/max_terminated_length": 750.0, "completions/mean_length": 510.5625, "completions/mean_terminated_length": 510.5625, "completions/min_length": 327.0, "completions/min_terminated_length": 327.0, "entropy": 0.30975694209337234, "epoch": 0.48822269807280516, "frac_reward_zero_std": 0.25, "grad_norm": 0.013880766928195953, "learning_rate": 1e-05, "loss": -0.0137, "num_tokens": 18919062.0, "reward": 0.40625, "reward_std": 0.3471629321575165, "rewards/accuracy_reward/mean": 0.40625, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.5324641466140747, "sampling/importance_sampling_ratio/mean": 1.0001721382141113, "sampling/importance_sampling_ratio/min": 0.6259236931800842, "sampling/sampling_logp_difference/max": 0.46852684020996094, "sampling/sampling_logp_difference/mean": 0.012039105407893658, "step": 912 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.206164132687263e-05, "clip_ratio/low_min": 5.206164132687263e-05, "clip_ratio/region_mean": 5.206164132687263e-05, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 718.0, "completions/mean_length": 529.375, "completions/mean_terminated_length": 504.6896667480469, "completions/min_length": 271.0, "completions/min_terminated_length": 271.0, "entropy": 0.333847489207983, "epoch": 0.48875802997858675, "frac_reward_zero_std": 0.25, "grad_norm": 0.03074287250638008, "learning_rate": 1e-05, "loss": 0.0924, "num_tokens": 18940234.0, "reward": 0.875, "reward_std": 0.292504221200943, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 1.6131587028503418, "sampling/importance_sampling_ratio/mean": 1.0000418424606323, "sampling/importance_sampling_ratio/min": 0.6844300031661987, "sampling/sampling_logp_difference/max": 0.4781942367553711, "sampling/sampling_logp_difference/mean": 0.012957409024238586, "step": 913 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00013158086221665144, "clip_ratio/low_min": 0.00013158086221665144, "clip_ratio/region_mean": 0.00013158086221665144, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 665.0, "completions/mean_length": 464.09375, "completions/mean_terminated_length": 454.2903137207031, "completions/min_length": 243.0, "completions/min_terminated_length": 243.0, "entropy": 0.30686079524457455, "epoch": 0.4892933618843683, "frac_reward_zero_std": 0.25, "grad_norm": 0.01318507269024849, "learning_rate": 1e-05, "loss": -0.0319, "num_tokens": 18958749.0, "reward": 0.625, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.36928129196167, "sampling/importance_sampling_ratio/mean": 0.9999698400497437, "sampling/importance_sampling_ratio/min": 0.5420946478843689, "sampling/sampling_logp_difference/max": 0.6123147010803223, "sampling/sampling_logp_difference/mean": 0.011734270490705967, "step": 914 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.259389192564413e-05, "clip_ratio/low_min": 6.259389192564413e-05, "clip_ratio/region_mean": 6.259389192564413e-05, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 708.0, "completions/mean_length": 476.53125, "completions/mean_terminated_length": 457.10003662109375, "completions/min_length": 206.0, "completions/min_terminated_length": 206.0, "entropy": 0.31297939643263817, "epoch": 0.48982869379014987, "frac_reward_zero_std": 0.5, "grad_norm": 0.0388018861413002, "learning_rate": 1e-05, "loss": 0.066, "num_tokens": 18978182.0, "reward": 0.6875, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.7275769710540771, "sampling/importance_sampling_ratio/mean": 0.9997548460960388, "sampling/importance_sampling_ratio/min": 0.6455977559089661, "sampling/sampling_logp_difference/max": 0.5467197895050049, "sampling/sampling_logp_difference/mean": 0.011493703350424767, "step": 915 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 694.0, "completions/mean_length": 435.0, "completions/mean_terminated_length": 424.258056640625, "completions/min_length": 228.0, "completions/min_terminated_length": 228.0, "entropy": 0.28156609646975994, "epoch": 0.49036402569593146, "frac_reward_zero_std": 0.75, "grad_norm": 0.0062019904144108295, "learning_rate": 1e-05, "loss": 0.0003, "num_tokens": 18995990.0, "reward": 0.96875, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.96875, "rewards/accuracy_reward/std": 0.1767766922712326, "sampling/importance_sampling_ratio/max": 1.5014770030975342, "sampling/importance_sampling_ratio/mean": 0.9995688199996948, "sampling/importance_sampling_ratio/min": 0.6919264197349548, "sampling/sampling_logp_difference/max": 0.4064493179321289, "sampling/sampling_logp_difference/mean": 0.010881432332098484, "step": 916 }, { "clip_ratio/high_max": 4.7063251258805394e-05, "clip_ratio/high_mean": 4.7063251258805394e-05, "clip_ratio/low_mean": 0.00015507238276768476, "clip_ratio/low_min": 0.00015507238276768476, "clip_ratio/region_mean": 0.00020213563402649015, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 768.0, "completions/mean_length": 575.9375, "completions/mean_terminated_length": 522.1599731445312, "completions/min_length": 365.0, "completions/min_terminated_length": 365.0, "entropy": 0.24633987620472908, "epoch": 0.49089935760171305, "frac_reward_zero_std": 0.25, "grad_norm": 0.007041865028440952, "learning_rate": 1e-05, "loss": 0.0363, "num_tokens": 19018220.0, "reward": 0.71875, "reward_std": 0.3198433816432953, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.662911057472229, "sampling/importance_sampling_ratio/mean": 1.000190258026123, "sampling/importance_sampling_ratio/min": 0.7511054873466492, "sampling/sampling_logp_difference/max": 0.5085697174072266, "sampling/sampling_logp_difference/mean": 0.009644015692174435, "step": 917 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 611.0, "completions/max_terminated_length": 611.0, "completions/mean_length": 423.3125, "completions/mean_terminated_length": 423.3125, "completions/min_length": 283.0, "completions/min_terminated_length": 283.0, "entropy": 0.2914689928293228, "epoch": 0.49143468950749464, "frac_reward_zero_std": 0.75, "grad_norm": 0.006776359397917986, "learning_rate": 1e-05, "loss": 0.0137, "num_tokens": 19035510.0, "reward": 0.9375, "reward_std": 0.1157275140285492, "rewards/accuracy_reward/mean": 0.9375, "rewards/accuracy_reward/std": 0.24593468010425568, "sampling/importance_sampling_ratio/max": 1.555033564567566, "sampling/importance_sampling_ratio/mean": 0.9998643398284912, "sampling/importance_sampling_ratio/min": 0.6702762842178345, "sampling/sampling_logp_difference/max": 0.4414970874786377, "sampling/sampling_logp_difference/mean": 0.011066603474318981, "step": 918 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 718.0, "completions/mean_length": 491.1875, "completions/mean_terminated_length": 482.258056640625, "completions/min_length": 312.0, "completions/min_terminated_length": 312.0, "entropy": 0.28279813192784786, "epoch": 0.4919700214132762, "frac_reward_zero_std": 0.5, "grad_norm": 0.00830013770610094, "learning_rate": 1e-05, "loss": 0.0235, "num_tokens": 19055164.0, "reward": 0.625, "reward_std": 0.2314550280570984, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.8878000974655151, "sampling/importance_sampling_ratio/mean": 1.0000550746917725, "sampling/importance_sampling_ratio/min": 0.7158579230308533, "sampling/sampling_logp_difference/max": 0.6354122161865234, "sampling/sampling_logp_difference/mean": 0.00993894599378109, "step": 919 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.375, "completions/max_length": 768.0, "completions/max_terminated_length": 662.0, "completions/mean_length": 618.90625, "completions/mean_terminated_length": 529.4500122070312, "completions/min_length": 404.0, "completions/min_terminated_length": 404.0, "entropy": 0.5270113684237003, "epoch": 0.4925053533190578, "frac_reward_zero_std": 0.5, "grad_norm": 0.010443360544741154, "learning_rate": 1e-05, "loss": 0.0142, "num_tokens": 19079321.0, "reward": 0.25, "reward_std": 0.26726123690605164, "rewards/accuracy_reward/mean": 0.25, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.5007963180541992, "sampling/importance_sampling_ratio/mean": 1.000497817993164, "sampling/importance_sampling_ratio/min": 0.6487916111946106, "sampling/sampling_logp_difference/max": 0.4326436519622803, "sampling/sampling_logp_difference/mean": 0.01748787797987461, "step": 920 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 681.0, "completions/max_terminated_length": 681.0, "completions/mean_length": 489.4375, "completions/mean_terminated_length": 489.4375, "completions/min_length": 283.0, "completions/min_terminated_length": 283.0, "entropy": 0.23764795064926147, "epoch": 0.4930406852248394, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": -0.0094, "num_tokens": 19099007.0, "reward": 0.8125, "reward_std": 0.1157275140285492, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.5893279314041138, "sampling/importance_sampling_ratio/mean": 0.9997685551643372, "sampling/importance_sampling_ratio/min": 0.667430579662323, "sampling/sampling_logp_difference/max": 0.46331119537353516, "sampling/sampling_logp_difference/mean": 0.009766027331352234, "step": 921 }, { "clip_ratio/high_max": 5.765682726632804e-05, "clip_ratio/high_mean": 5.765682726632804e-05, "clip_ratio/low_mean": 0.00015271549636963755, "clip_ratio/low_min": 0.00015271549636963755, "clip_ratio/region_mean": 0.00021037232363596559, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 714.0, "completions/mean_length": 569.6875, "completions/mean_terminated_length": 532.9629516601562, "completions/min_length": 345.0, "completions/min_terminated_length": 345.0, "entropy": 0.3499591462314129, "epoch": 0.493576017130621, "frac_reward_zero_std": 0.0, "grad_norm": 0.019014639779925346, "learning_rate": 1e-05, "loss": 0.0438, "num_tokens": 19121261.0, "reward": 0.5625, "reward_std": 0.48503684997558594, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.464830994606018, "sampling/importance_sampling_ratio/mean": 1.0005414485931396, "sampling/importance_sampling_ratio/min": 0.650802731513977, "sampling/sampling_logp_difference/max": 0.4295487403869629, "sampling/sampling_logp_difference/mean": 0.012619705870747566, "step": 922 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0001937806373462081, "clip_ratio/low_min": 0.0001937806373462081, "clip_ratio/region_mean": 0.0001937806373462081, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 752.0, "completions/mean_length": 538.0625, "completions/mean_terminated_length": 473.67999267578125, "completions/min_length": 201.0, "completions/min_terminated_length": 201.0, "entropy": 0.39270614087581635, "epoch": 0.4941113490364026, "frac_reward_zero_std": 0.5, "grad_norm": 0.013273780234158039, "learning_rate": 1e-05, "loss": 0.0109, "num_tokens": 19142279.0, "reward": 0.65625, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.4174294471740723, "sampling/importance_sampling_ratio/mean": 0.9999698400497437, "sampling/importance_sampling_ratio/min": 0.6977755427360535, "sampling/sampling_logp_difference/max": 0.35985779762268066, "sampling/sampling_logp_difference/mean": 0.013609446585178375, "step": 923 }, { "clip_ratio/high_max": 5.776340185548179e-05, "clip_ratio/high_mean": 5.776340185548179e-05, "clip_ratio/low_mean": 0.00017970944463741034, "clip_ratio/low_min": 0.00017970944463741034, "clip_ratio/region_mean": 0.00023747284649289213, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 759.0, "completions/mean_length": 539.53125, "completions/mean_terminated_length": 497.22222900390625, "completions/min_length": 269.0, "completions/min_terminated_length": 269.0, "entropy": 0.2898131124675274, "epoch": 0.49464668094218417, "frac_reward_zero_std": 0.0, "grad_norm": 0.029068931937217712, "learning_rate": 1e-05, "loss": 0.0778, "num_tokens": 19163376.0, "reward": 0.6875, "reward_std": 0.4355512857437134, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.4534404277801514, "sampling/importance_sampling_ratio/mean": 1.000167727470398, "sampling/importance_sampling_ratio/min": 0.6783241629600525, "sampling/sampling_logp_difference/max": 0.38812994956970215, "sampling/sampling_logp_difference/mean": 0.010268832556903362, "step": 924 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00022056644229451194, "clip_ratio/low_min": 0.00022056644229451194, "clip_ratio/region_mean": 0.00022056644229451194, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 697.0, "completions/mean_length": 522.84375, "completions/mean_terminated_length": 477.4444580078125, "completions/min_length": 264.0, "completions/min_terminated_length": 264.0, "entropy": 0.3398930039256811, "epoch": 0.49518201284796576, "frac_reward_zero_std": 0.5, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0146, "num_tokens": 19184131.0, "reward": 0.75, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.4498913288116455, "sampling/importance_sampling_ratio/mean": 1.000004768371582, "sampling/importance_sampling_ratio/min": 0.6497147679328918, "sampling/sampling_logp_difference/max": 0.43122178316116333, "sampling/sampling_logp_difference/mean": 0.012509508058428764, "step": 925 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 749.0, "completions/mean_length": 534.0, "completions/mean_terminated_length": 509.7930908203125, "completions/min_length": 356.0, "completions/min_terminated_length": 356.0, "entropy": 0.32699492760002613, "epoch": 0.49571734475374735, "frac_reward_zero_std": 0.5, "grad_norm": 0.013125520199537277, "learning_rate": 1e-05, "loss": 0.0375, "num_tokens": 19205331.0, "reward": 0.8125, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.4404257535934448, "sampling/importance_sampling_ratio/mean": 0.9994783401489258, "sampling/importance_sampling_ratio/min": 0.6050943732261658, "sampling/sampling_logp_difference/max": 0.5023708343505859, "sampling/sampling_logp_difference/mean": 0.012279679998755455, "step": 926 }, { "clip_ratio/high_max": 6.348400347633287e-05, "clip_ratio/high_mean": 6.348400347633287e-05, "clip_ratio/low_mean": 0.00016947766562225297, "clip_ratio/low_min": 0.00016947766562225297, "clip_ratio/region_mean": 0.00023296166909858584, "completions/clipped_ratio": 0.0, "completions/max_length": 757.0, "completions/max_terminated_length": 757.0, "completions/mean_length": 435.3125, "completions/mean_terminated_length": 435.3125, "completions/min_length": 126.0, "completions/min_terminated_length": 126.0, "entropy": 0.340041421353817, "epoch": 0.49625267665952894, "frac_reward_zero_std": 0.5, "grad_norm": 0.011166700161993504, "learning_rate": 1e-05, "loss": 0.0299, "num_tokens": 19222317.0, "reward": 0.78125, "reward_std": 0.2630178928375244, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.612455129623413, "sampling/importance_sampling_ratio/mean": 0.9999621510505676, "sampling/importance_sampling_ratio/min": 0.739499568939209, "sampling/sampling_logp_difference/max": 0.47775793075561523, "sampling/sampling_logp_difference/mean": 0.011311161331832409, "step": 927 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00012195505041745491, "clip_ratio/low_min": 0.00012195505041745491, "clip_ratio/region_mean": 0.00012195505041745491, "completions/clipped_ratio": 0.3125, "completions/max_length": 768.0, "completions/max_terminated_length": 750.0, "completions/mean_length": 547.53125, "completions/mean_terminated_length": 447.3182067871094, "completions/min_length": 267.0, "completions/min_terminated_length": 267.0, "entropy": 0.5082920771092176, "epoch": 0.49678800856531047, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": -0.0019, "num_tokens": 19244222.0, "reward": 0.53125, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.6648000478744507, "sampling/importance_sampling_ratio/mean": 0.9998617768287659, "sampling/importance_sampling_ratio/min": 0.6047614812850952, "sampling/sampling_logp_difference/max": 0.5097050666809082, "sampling/sampling_logp_difference/mean": 0.017021482810378075, "step": 928 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.752851600642316e-05, "clip_ratio/low_min": 4.752851600642316e-05, "clip_ratio/region_mean": 4.752851600642316e-05, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 670.0, "completions/mean_length": 518.25, "completions/mean_terminated_length": 472.0, "completions/min_length": 144.0, "completions/min_terminated_length": 144.0, "entropy": 0.3265595231205225, "epoch": 0.49732334047109206, "frac_reward_zero_std": 0.5, "grad_norm": 0.007692792918533087, "learning_rate": 1e-05, "loss": 0.0579, "num_tokens": 19264606.0, "reward": 0.75, "reward_std": 0.2314550280570984, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.456028938293457, "sampling/importance_sampling_ratio/mean": 0.9998459815979004, "sampling/importance_sampling_ratio/min": 0.7026187777519226, "sampling/sampling_logp_difference/max": 0.37571287155151367, "sampling/sampling_logp_difference/mean": 0.011296630837023258, "step": 929 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 692.0, "completions/max_terminated_length": 692.0, "completions/mean_length": 437.59375, "completions/mean_terminated_length": 437.59375, "completions/min_length": 269.0, "completions/min_terminated_length": 269.0, "entropy": 0.2326170839369297, "epoch": 0.49785867237687365, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0066, "num_tokens": 19281905.0, "reward": 0.71875, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.4700236320495605, "sampling/importance_sampling_ratio/mean": 0.999994158744812, "sampling/importance_sampling_ratio/min": 0.6835604310035706, "sampling/sampling_logp_difference/max": 0.38527846336364746, "sampling/sampling_logp_difference/mean": 0.009263720363378525, "step": 930 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00013016723823966458, "clip_ratio/low_min": 0.00013016723823966458, "clip_ratio/region_mean": 0.00013016723823966458, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 738.0, "completions/mean_length": 499.78125, "completions/mean_terminated_length": 461.46429443359375, "completions/min_length": 214.0, "completions/min_terminated_length": 214.0, "entropy": 0.32206207886338234, "epoch": 0.49839400428265523, "frac_reward_zero_std": 0.0, "grad_norm": 0.014004467986524105, "learning_rate": 1e-05, "loss": -0.0124, "num_tokens": 19301706.0, "reward": 0.40625, "reward_std": 0.4534739851951599, "rewards/accuracy_reward/mean": 0.40625, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.7464927434921265, "sampling/importance_sampling_ratio/mean": 1.0003650188446045, "sampling/importance_sampling_ratio/min": 0.5425254702568054, "sampling/sampling_logp_difference/max": 0.6115202903747559, "sampling/sampling_logp_difference/mean": 0.011845179833471775, "step": 931 }, { "clip_ratio/high_max": 6.118453165981919e-05, "clip_ratio/high_mean": 6.118453165981919e-05, "clip_ratio/low_mean": 0.00010584674964775331, "clip_ratio/low_min": 0.00010584674964775331, "clip_ratio/region_mean": 0.0001670312813075725, "completions/clipped_ratio": 0.0, "completions/max_length": 727.0, "completions/max_terminated_length": 727.0, "completions/mean_length": 468.125, "completions/mean_terminated_length": 468.125, "completions/min_length": 158.0, "completions/min_terminated_length": 158.0, "entropy": 0.45232093334198, "epoch": 0.4989293361884368, "frac_reward_zero_std": 0.0, "grad_norm": 0.011814016848802567, "learning_rate": 1e-05, "loss": 0.0338, "num_tokens": 19320934.0, "reward": 0.5, "reward_std": 0.4671337604522705, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5080004930496216, "sampling/importance_sampling_ratio/max": 1.6969050168991089, "sampling/importance_sampling_ratio/mean": 0.9999136328697205, "sampling/importance_sampling_ratio/min": 0.3159007728099823, "sampling/sampling_logp_difference/max": 1.152327060699463, "sampling/sampling_logp_difference/mean": 0.014239568263292313, "step": 932 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 745.0, "completions/mean_length": 491.125, "completions/mean_terminated_length": 472.66668701171875, "completions/min_length": 291.0, "completions/min_terminated_length": 291.0, "entropy": 0.2690283786505461, "epoch": 0.4994646680942184, "frac_reward_zero_std": 0.25, "grad_norm": 0.02261507138609886, "learning_rate": 1e-05, "loss": 0.015, "num_tokens": 19340258.0, "reward": 0.8125, "reward_std": 0.3471825420856476, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.5850526094436646, "sampling/importance_sampling_ratio/mean": 0.9998255372047424, "sampling/importance_sampling_ratio/min": 0.6380040049552917, "sampling/sampling_logp_difference/max": 0.4606175422668457, "sampling/sampling_logp_difference/mean": 0.01035875640809536, "step": 933 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.02008042531088e-05, "clip_ratio/low_min": 5.02008042531088e-05, "clip_ratio/region_mean": 5.02008042531088e-05, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 765.0, "completions/mean_length": 501.8125, "completions/mean_terminated_length": 474.2758483886719, "completions/min_length": 270.0, "completions/min_terminated_length": 270.0, "entropy": 0.26222812198102474, "epoch": 0.5, "frac_reward_zero_std": 0.5, "grad_norm": 0.013308855704963207, "learning_rate": 1e-05, "loss": 0.0239, "num_tokens": 19359980.0, "reward": 0.6875, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.5375230312347412, "sampling/importance_sampling_ratio/mean": 1.0001217126846313, "sampling/importance_sampling_ratio/min": 0.7298194766044617, "sampling/sampling_logp_difference/max": 0.4301726818084717, "sampling/sampling_logp_difference/mean": 0.01018164400011301, "step": 934 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.446622137445956e-05, "clip_ratio/low_min": 6.446622137445956e-05, "clip_ratio/region_mean": 6.446622137445956e-05, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 761.0, "completions/mean_length": 494.28125, "completions/mean_terminated_length": 417.6399841308594, "completions/min_length": 259.0, "completions/min_terminated_length": 259.0, "entropy": 0.2620402257889509, "epoch": 0.5005353319057816, "frac_reward_zero_std": 0.75, "grad_norm": 0.010162093676626682, "learning_rate": 1e-05, "loss": -0.0005, "num_tokens": 19379717.0, "reward": 0.84375, "reward_std": 0.1293872892856598, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.4264450073242188, "sampling/importance_sampling_ratio/mean": 1.0002689361572266, "sampling/importance_sampling_ratio/min": 0.6831101179122925, "sampling/sampling_logp_difference/max": 0.3810991942882538, "sampling/sampling_logp_difference/mean": 0.010737966746091843, "step": 935 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 711.0, "completions/mean_length": 439.28125, "completions/mean_terminated_length": 428.6773986816406, "completions/min_length": 228.0, "completions/min_terminated_length": 228.0, "entropy": 0.2794332280755043, "epoch": 0.5010706638115632, "frac_reward_zero_std": 0.25, "grad_norm": 0.010731112211942673, "learning_rate": 1e-05, "loss": 0.0324, "num_tokens": 19397078.0, "reward": 0.71875, "reward_std": 0.3198433816432953, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.6577776670455933, "sampling/importance_sampling_ratio/mean": 1.0000596046447754, "sampling/importance_sampling_ratio/min": 0.6590961217880249, "sampling/sampling_logp_difference/max": 0.5054779052734375, "sampling/sampling_logp_difference/mean": 0.010796277783811092, "step": 936 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.713211769238114e-05, "clip_ratio/low_min": 6.713211769238114e-05, "clip_ratio/region_mean": 6.713211769238114e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 662.0, "completions/max_terminated_length": 662.0, "completions/mean_length": 485.65625, "completions/mean_terminated_length": 485.65625, "completions/min_length": 220.0, "completions/min_terminated_length": 220.0, "entropy": 0.2834157757461071, "epoch": 0.5016059957173448, "frac_reward_zero_std": 0.0, "grad_norm": 0.0138821080327034, "learning_rate": 1e-05, "loss": 0.0267, "num_tokens": 19416451.0, "reward": 0.625, "reward_std": 0.4261348247528076, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.4637072086334229, "sampling/importance_sampling_ratio/mean": 0.9997812509536743, "sampling/importance_sampling_ratio/min": 0.680621862411499, "sampling/sampling_logp_difference/max": 0.3847484588623047, "sampling/sampling_logp_difference/mean": 0.011037930846214294, "step": 937 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 583.0, "completions/max_terminated_length": 583.0, "completions/mean_length": 398.28125, "completions/mean_terminated_length": 398.28125, "completions/min_length": 201.0, "completions/min_terminated_length": 201.0, "entropy": 0.29254633747041225, "epoch": 0.5021413276231264, "frac_reward_zero_std": 0.75, "grad_norm": 0.010301598347723484, "learning_rate": 1e-05, "loss": 0.0259, "num_tokens": 19433308.0, "reward": 0.9375, "reward_std": 0.1157275140285492, "rewards/accuracy_reward/mean": 0.9375, "rewards/accuracy_reward/std": 0.24593468010425568, "sampling/importance_sampling_ratio/max": 1.8035612106323242, "sampling/importance_sampling_ratio/mean": 1.0004361867904663, "sampling/importance_sampling_ratio/min": 0.6567971110343933, "sampling/sampling_logp_difference/max": 0.5897631645202637, "sampling/sampling_logp_difference/mean": 0.012045230716466904, "step": 938 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 7.246376480907202e-05, "clip_ratio/low_min": 7.246376480907202e-05, "clip_ratio/region_mean": 7.246376480907202e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 603.0, "completions/max_terminated_length": 603.0, "completions/mean_length": 406.9375, "completions/mean_terminated_length": 406.9375, "completions/min_length": 286.0, "completions/min_terminated_length": 286.0, "entropy": 0.2541348449885845, "epoch": 0.5026766595289079, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": -0.0059, "num_tokens": 19449730.0, "reward": 0.84375, "reward_std": 0.1293872892856598, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.466042399406433, "sampling/importance_sampling_ratio/mean": 0.9999431371688843, "sampling/importance_sampling_ratio/min": 0.757443368434906, "sampling/sampling_logp_difference/max": 0.3825664520263672, "sampling/sampling_logp_difference/mean": 0.01026766374707222, "step": 939 }, { "clip_ratio/high_max": 6.883259629830718e-05, "clip_ratio/high_mean": 6.883259629830718e-05, "clip_ratio/low_mean": 0.00010640990876709111, "clip_ratio/low_min": 0.00010640990876709111, "clip_ratio/region_mean": 0.0001752425050653983, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 718.0, "completions/mean_length": 527.90625, "completions/mean_terminated_length": 493.607177734375, "completions/min_length": 343.0, "completions/min_terminated_length": 343.0, "entropy": 0.3679649531841278, "epoch": 0.5032119914346895, "frac_reward_zero_std": 0.5, "grad_norm": 0.004698208533227444, "learning_rate": 1e-05, "loss": 0.0624, "num_tokens": 19470351.0, "reward": 0.71875, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.4841936826705933, "sampling/importance_sampling_ratio/mean": 0.9998572468757629, "sampling/importance_sampling_ratio/min": 0.7141680121421814, "sampling/sampling_logp_difference/max": 0.39487171173095703, "sampling/sampling_logp_difference/mean": 0.012638048268854618, "step": 940 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.868132186355069e-05, "clip_ratio/low_min": 6.868132186355069e-05, "clip_ratio/region_mean": 6.868132186355069e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 766.0, "completions/max_terminated_length": 766.0, "completions/mean_length": 463.40625, "completions/mean_terminated_length": 463.40625, "completions/min_length": 267.0, "completions/min_terminated_length": 267.0, "entropy": 0.2769796848297119, "epoch": 0.5037473233404711, "frac_reward_zero_std": 0.5, "grad_norm": 0.030556363984942436, "learning_rate": 1e-05, "loss": -0.0122, "num_tokens": 19489316.0, "reward": 0.84375, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.4240692853927612, "sampling/importance_sampling_ratio/mean": 1.0002126693725586, "sampling/importance_sampling_ratio/min": 0.6985477209091187, "sampling/sampling_logp_difference/max": 0.3587517738342285, "sampling/sampling_logp_difference/mean": 0.010646170936524868, "step": 941 }, { "clip_ratio/high_max": 6.041565939085558e-05, "clip_ratio/high_mean": 6.041565939085558e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 6.041565939085558e-05, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 679.0, "completions/mean_length": 457.21875, "completions/mean_terminated_length": 425.0689697265625, "completions/min_length": 284.0, "completions/min_terminated_length": 284.0, "entropy": 0.28365115635097027, "epoch": 0.5042826552462527, "frac_reward_zero_std": 0.5, "grad_norm": 0.011648779734969139, "learning_rate": 1e-05, "loss": -0.0076, "num_tokens": 19507731.0, "reward": 0.6875, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.7110072374343872, "sampling/importance_sampling_ratio/mean": 1.0001929998397827, "sampling/importance_sampling_ratio/min": 0.6931325793266296, "sampling/sampling_logp_difference/max": 0.5370821952819824, "sampling/sampling_logp_difference/mean": 0.011281576938927174, "step": 942 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 613.0, "completions/max_terminated_length": 613.0, "completions/mean_length": 436.84375, "completions/mean_terminated_length": 436.84375, "completions/min_length": 326.0, "completions/min_terminated_length": 326.0, "entropy": 0.2924124225974083, "epoch": 0.5048179871520343, "frac_reward_zero_std": 0.5, "grad_norm": 0.0055651580914855, "learning_rate": 1e-05, "loss": 0.0339, "num_tokens": 19525430.0, "reward": 0.9375, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.9375, "rewards/accuracy_reward/std": 0.24593468010425568, "sampling/importance_sampling_ratio/max": 1.46877121925354, "sampling/importance_sampling_ratio/mean": 0.9997760057449341, "sampling/importance_sampling_ratio/min": 0.5539229512214661, "sampling/sampling_logp_difference/max": 0.5907297134399414, "sampling/sampling_logp_difference/mean": 0.011159236542880535, "step": 943 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.297229265328497e-05, "clip_ratio/low_min": 6.297229265328497e-05, "clip_ratio/region_mean": 6.297229265328497e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 608.0, "completions/max_terminated_length": 608.0, "completions/mean_length": 429.96875, "completions/mean_terminated_length": 429.96875, "completions/min_length": 299.0, "completions/min_terminated_length": 299.0, "entropy": 0.24778622947633266, "epoch": 0.5053533190578159, "frac_reward_zero_std": 0.5, "grad_norm": 0.012465167790651321, "learning_rate": 1e-05, "loss": 0.019, "num_tokens": 19542381.0, "reward": 0.84375, "reward_std": 0.22201895713806152, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.5475302934646606, "sampling/importance_sampling_ratio/mean": 1.0002715587615967, "sampling/importance_sampling_ratio/min": 0.6991207003593445, "sampling/sampling_logp_difference/max": 0.4366602897644043, "sampling/sampling_logp_difference/mean": 0.010228806175291538, "step": 944 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 700.0, "completions/mean_length": 467.03125, "completions/mean_terminated_length": 446.9666748046875, "completions/min_length": 259.0, "completions/min_terminated_length": 259.0, "entropy": 0.2548218183219433, "epoch": 0.5058886509635975, "frac_reward_zero_std": 0.5, "grad_norm": 0.027409628033638, "learning_rate": 1e-05, "loss": 0.0826, "num_tokens": 19560566.0, "reward": 0.90625, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.4531505107879639, "sampling/importance_sampling_ratio/mean": 1.0004417896270752, "sampling/importance_sampling_ratio/min": 0.7483683228492737, "sampling/sampling_logp_difference/max": 0.3737339973449707, "sampling/sampling_logp_difference/mean": 0.009670830331742764, "step": 945 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00010070791176985949, "clip_ratio/low_min": 0.00010070791176985949, "clip_ratio/region_mean": 0.00010070791176985949, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 756.0, "completions/mean_length": 554.53125, "completions/mean_terminated_length": 515.0, "completions/min_length": 297.0, "completions/min_terminated_length": 297.0, "entropy": 0.41932306811213493, "epoch": 0.5064239828693791, "frac_reward_zero_std": 0.25, "grad_norm": 0.019317878410220146, "learning_rate": 1e-05, "loss": 0.0388, "num_tokens": 19581975.0, "reward": 0.53125, "reward_std": 0.3608423173427582, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.4860639572143555, "sampling/importance_sampling_ratio/mean": 1.0001702308654785, "sampling/importance_sampling_ratio/min": 0.6995219588279724, "sampling/sampling_logp_difference/max": 0.39613091945648193, "sampling/sampling_logp_difference/mean": 0.013263840228319168, "step": 946 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 761.0, "completions/mean_length": 411.28125, "completions/mean_terminated_length": 387.5000305175781, "completions/min_length": 244.0, "completions/min_terminated_length": 244.0, "entropy": 0.322050666436553, "epoch": 0.5069593147751607, "frac_reward_zero_std": 0.75, "grad_norm": 0.036013711243867874, "learning_rate": 1e-05, "loss": -0.0156, "num_tokens": 19598336.0, "reward": 0.8125, "reward_std": 0.1157275140285492, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.5235679149627686, "sampling/importance_sampling_ratio/mean": 1.0000337362289429, "sampling/importance_sampling_ratio/min": 0.7071281671524048, "sampling/sampling_logp_difference/max": 0.4210548400878906, "sampling/sampling_logp_difference/mean": 0.012662426568567753, "step": 947 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 631.0, "completions/mean_length": 483.09375, "completions/mean_terminated_length": 473.9031982421875, "completions/min_length": 352.0, "completions/min_terminated_length": 352.0, "entropy": 0.2943005654960871, "epoch": 0.5074946466809421, "frac_reward_zero_std": 0.75, "grad_norm": 0.011393197812139988, "learning_rate": 1e-05, "loss": 0.0245, "num_tokens": 19617427.0, "reward": 0.875, "reward_std": 0.13363061845302582, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 1.4300520420074463, "sampling/importance_sampling_ratio/mean": 1.0001416206359863, "sampling/importance_sampling_ratio/min": 0.7061121463775635, "sampling/sampling_logp_difference/max": 0.3577108383178711, "sampling/sampling_logp_difference/mean": 0.011147226206958294, "step": 948 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00010816605572472326, "clip_ratio/low_min": 0.00010816605572472326, "clip_ratio/region_mean": 0.00010816605572472326, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 659.0, "completions/mean_length": 535.28125, "completions/mean_terminated_length": 502.0357360839844, "completions/min_length": 324.0, "completions/min_terminated_length": 324.0, "entropy": 0.34803836792707443, "epoch": 0.5080299785867237, "frac_reward_zero_std": 0.25, "grad_norm": 0.011546298861503601, "learning_rate": 1e-05, "loss": 0.032, "num_tokens": 19638516.0, "reward": 0.625, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.7856738567352295, "sampling/importance_sampling_ratio/mean": 0.9996559619903564, "sampling/importance_sampling_ratio/min": 0.6398385763168335, "sampling/sampling_logp_difference/max": 0.5797958374023438, "sampling/sampling_logp_difference/mean": 0.01192948967218399, "step": 949 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 694.0, "completions/max_terminated_length": 694.0, "completions/mean_length": 500.0, "completions/mean_terminated_length": 500.0, "completions/min_length": 291.0, "completions/min_terminated_length": 291.0, "entropy": 0.32023950666189194, "epoch": 0.5085653104925053, "frac_reward_zero_std": 0.75, "grad_norm": 0.003911130130290985, "learning_rate": 1e-05, "loss": -0.0045, "num_tokens": 19658284.0, "reward": 0.96875, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.96875, "rewards/accuracy_reward/std": 0.1767766922712326, "sampling/importance_sampling_ratio/max": 1.4485037326812744, "sampling/importance_sampling_ratio/mean": 0.9998983144760132, "sampling/importance_sampling_ratio/min": 0.7037187218666077, "sampling/sampling_logp_difference/max": 0.3705310821533203, "sampling/sampling_logp_difference/mean": 0.011814751662313938, "step": 950 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00013728720659855753, "clip_ratio/low_min": 0.00013728720659855753, "clip_ratio/region_mean": 0.00013728720659855753, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 618.0, "completions/mean_length": 412.875, "completions/mean_terminated_length": 401.4193420410156, "completions/min_length": 239.0, "completions/min_terminated_length": 239.0, "entropy": 0.3241432998329401, "epoch": 0.5091006423982869, "frac_reward_zero_std": 0.5, "grad_norm": 0.037214525043964386, "learning_rate": 1e-05, "loss": 0.0511, "num_tokens": 19675016.0, "reward": 0.90625, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.4139505624771118, "sampling/importance_sampling_ratio/mean": 1.0000523328781128, "sampling/importance_sampling_ratio/min": 0.7438786029815674, "sampling/sampling_logp_difference/max": 0.3463876247406006, "sampling/sampling_logp_difference/mean": 0.011365550570189953, "step": 951 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 741.0, "completions/mean_length": 480.53125, "completions/mean_terminated_length": 461.36669921875, "completions/min_length": 240.0, "completions/min_terminated_length": 240.0, "entropy": 0.254257895052433, "epoch": 0.5096359743040685, "frac_reward_zero_std": 0.75, "grad_norm": 0.01980079524219036, "learning_rate": 1e-05, "loss": 0.0157, "num_tokens": 19693849.0, "reward": 0.9375, "reward_std": 0.1157275140285492, "rewards/accuracy_reward/mean": 0.9375, "rewards/accuracy_reward/std": 0.24593468010425568, "sampling/importance_sampling_ratio/max": 1.4840246438980103, "sampling/importance_sampling_ratio/mean": 1.0001052618026733, "sampling/importance_sampling_ratio/min": 0.7634907364845276, "sampling/sampling_logp_difference/max": 0.3947577476501465, "sampling/sampling_logp_difference/mean": 0.010038191452622414, "step": 952 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 708.0, "completions/mean_length": 457.96875, "completions/mean_terminated_length": 425.89654541015625, "completions/min_length": 256.0, "completions/min_terminated_length": 256.0, "entropy": 0.41520047187805176, "epoch": 0.5101713062098501, "frac_reward_zero_std": 0.5, "grad_norm": 0.0030097414273768663, "learning_rate": 1e-05, "loss": 0.0197, "num_tokens": 19712288.0, "reward": 0.6875, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.3846614360809326, "sampling/importance_sampling_ratio/mean": 1.0002185106277466, "sampling/importance_sampling_ratio/min": 0.6740763783454895, "sampling/sampling_logp_difference/max": 0.3944118618965149, "sampling/sampling_logp_difference/mean": 0.012530251406133175, "step": 953 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.000273627694696188, "clip_ratio/low_min": 0.000273627694696188, "clip_ratio/region_mean": 0.000273627694696188, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 750.0, "completions/mean_length": 518.5625, "completions/mean_terminated_length": 482.9285888671875, "completions/min_length": 296.0, "completions/min_terminated_length": 296.0, "entropy": 0.324110297486186, "epoch": 0.5107066381156317, "frac_reward_zero_std": 0.5, "grad_norm": 0.005960370879620314, "learning_rate": 1e-05, "loss": 0.0614, "num_tokens": 19733346.0, "reward": 0.78125, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.5565340518951416, "sampling/importance_sampling_ratio/mean": 1.0002589225769043, "sampling/importance_sampling_ratio/min": 0.7417091727256775, "sampling/sampling_logp_difference/max": 0.4424614906311035, "sampling/sampling_logp_difference/mean": 0.011185092851519585, "step": 954 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00012192071153549477, "clip_ratio/low_min": 0.00012192071153549477, "clip_ratio/region_mean": 0.00012192071153549477, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 674.0, "completions/mean_length": 465.0, "completions/mean_terminated_length": 455.2257995605469, "completions/min_length": 259.0, "completions/min_terminated_length": 259.0, "entropy": 0.28435055166482925, "epoch": 0.5112419700214133, "frac_reward_zero_std": 0.5, "grad_norm": 0.014217003248631954, "learning_rate": 1e-05, "loss": -0.0178, "num_tokens": 19752074.0, "reward": 0.75, "reward_std": 0.2587745785713196, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.476675033569336, "sampling/importance_sampling_ratio/mean": 0.9998407363891602, "sampling/importance_sampling_ratio/min": 0.6523876190185547, "sampling/sampling_logp_difference/max": 0.42711639404296875, "sampling/sampling_logp_difference/mean": 0.010454796254634857, "step": 955 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 575.0, "completions/max_terminated_length": 575.0, "completions/mean_length": 416.21875, "completions/mean_terminated_length": 416.21875, "completions/min_length": 331.0, "completions/min_terminated_length": 331.0, "entropy": 0.2670032139867544, "epoch": 0.5117773019271948, "frac_reward_zero_std": 0.5, "grad_norm": 0.013444775715470314, "learning_rate": 1e-05, "loss": 0.0405, "num_tokens": 19768905.0, "reward": 0.8125, "reward_std": 0.249358132481575, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.599617838859558, "sampling/importance_sampling_ratio/mean": 0.9995540380477905, "sampling/importance_sampling_ratio/min": 0.6160816550254822, "sampling/sampling_logp_difference/max": 0.4843757152557373, "sampling/sampling_logp_difference/mean": 0.010016650892794132, "step": 956 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 617.0, "completions/max_terminated_length": 617.0, "completions/mean_length": 470.03125, "completions/mean_terminated_length": 470.03125, "completions/min_length": 293.0, "completions/min_terminated_length": 293.0, "entropy": 0.2323832232505083, "epoch": 0.5123126338329764, "frac_reward_zero_std": 0.75, "grad_norm": 0.016881059855222702, "learning_rate": 1e-05, "loss": 0.0197, "num_tokens": 19787314.0, "reward": 0.90625, "reward_std": 0.1293872892856598, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.5071568489074707, "sampling/importance_sampling_ratio/mean": 1.0001609325408936, "sampling/importance_sampling_ratio/min": 0.5303636193275452, "sampling/sampling_logp_difference/max": 0.6341924667358398, "sampling/sampling_logp_difference/mean": 0.009288148023188114, "step": 957 }, { "clip_ratio/high_max": 0.00012261857045814395, "clip_ratio/high_mean": 0.00012261857045814395, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00012261857045814395, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 642.0, "completions/mean_length": 490.78125, "completions/mean_terminated_length": 472.3000183105469, "completions/min_length": 296.0, "completions/min_terminated_length": 296.0, "entropy": 0.26723513193428516, "epoch": 0.512847965738758, "frac_reward_zero_std": 0.5, "grad_norm": 0.00785796344280243, "learning_rate": 1e-05, "loss": -0.0331, "num_tokens": 19806971.0, "reward": 0.90625, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.8052798509597778, "sampling/importance_sampling_ratio/mean": 1.0002069473266602, "sampling/importance_sampling_ratio/min": 0.30172789096832275, "sampling/sampling_logp_difference/max": 1.1982296705245972, "sampling/sampling_logp_difference/mean": 0.010774476453661919, "step": 958 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 760.0, "completions/mean_length": 473.15625, "completions/mean_terminated_length": 463.6451416015625, "completions/min_length": 223.0, "completions/min_terminated_length": 223.0, "entropy": 0.2213722411543131, "epoch": 0.5133832976445396, "frac_reward_zero_std": 0.75, "grad_norm": 0.015048615634441376, "learning_rate": 1e-05, "loss": -0.0126, "num_tokens": 19825768.0, "reward": 0.90625, "reward_std": 0.1293872892856598, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.4348511695861816, "sampling/importance_sampling_ratio/mean": 0.9997027516365051, "sampling/importance_sampling_ratio/min": 0.6895332932472229, "sampling/sampling_logp_difference/max": 0.37174034118652344, "sampling/sampling_logp_difference/mean": 0.00852168258279562, "step": 959 }, { "clip_ratio/high_max": 5.602868623100221e-05, "clip_ratio/high_mean": 5.602868623100221e-05, "clip_ratio/low_mean": 0.0001959605870069936, "clip_ratio/low_min": 0.0001959605870069936, "clip_ratio/region_mean": 0.0002519892732379958, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 758.0, "completions/mean_length": 603.8125, "completions/mean_terminated_length": 565.923095703125, "completions/min_length": 221.0, "completions/min_terminated_length": 221.0, "entropy": 0.38207705318927765, "epoch": 0.5139186295503212, "frac_reward_zero_std": 0.25, "grad_norm": 0.022195030003786087, "learning_rate": 1e-05, "loss": 0.0283, "num_tokens": 19849026.0, "reward": 0.34375, "reward_std": 0.3061639964580536, "rewards/accuracy_reward/mean": 0.34375, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.4360729455947876, "sampling/importance_sampling_ratio/mean": 0.9999850392341614, "sampling/importance_sampling_ratio/min": 0.6669304966926575, "sampling/sampling_logp_difference/max": 0.4050694704055786, "sampling/sampling_logp_difference/mean": 0.013118598610162735, "step": 960 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00012831228377763182, "clip_ratio/low_min": 0.00012831228377763182, "clip_ratio/region_mean": 0.00012831228377763182, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 609.0, "completions/mean_length": 466.34375, "completions/mean_terminated_length": 456.6128845214844, "completions/min_length": 287.0, "completions/min_terminated_length": 287.0, "entropy": 0.24108667485415936, "epoch": 0.5144539614561028, "frac_reward_zero_std": 0.5, "grad_norm": 0.004858960397541523, "learning_rate": 1e-05, "loss": 0.0065, "num_tokens": 19867597.0, "reward": 0.84375, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.5404936075210571, "sampling/importance_sampling_ratio/mean": 0.999920129776001, "sampling/importance_sampling_ratio/min": 0.7229648232460022, "sampling/sampling_logp_difference/max": 0.43210291862487793, "sampling/sampling_logp_difference/mean": 0.009461749345064163, "step": 961 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00011436413478804752, "clip_ratio/low_min": 0.00011436413478804752, "clip_ratio/region_mean": 0.00011436413478804752, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 763.0, "completions/mean_length": 535.6875, "completions/mean_terminated_length": 511.6551818847656, "completions/min_length": 280.0, "completions/min_terminated_length": 280.0, "entropy": 0.3625764474272728, "epoch": 0.5149892933618844, "frac_reward_zero_std": 0.5, "grad_norm": 0.010485700331628323, "learning_rate": 1e-05, "loss": -0.0269, "num_tokens": 19888667.0, "reward": 0.59375, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.6535862684249878, "sampling/importance_sampling_ratio/mean": 0.9999732971191406, "sampling/importance_sampling_ratio/min": 0.6269338726997375, "sampling/sampling_logp_difference/max": 0.5029463768005371, "sampling/sampling_logp_difference/mean": 0.012667607516050339, "step": 962 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 628.0, "completions/max_terminated_length": 628.0, "completions/mean_length": 434.15625, "completions/mean_terminated_length": 434.15625, "completions/min_length": 251.0, "completions/min_terminated_length": 251.0, "entropy": 0.2798925694078207, "epoch": 0.515524625267666, "frac_reward_zero_std": 0.75, "grad_norm": 0.004387347027659416, "learning_rate": 1e-05, "loss": 0.0161, "num_tokens": 19906608.0, "reward": 0.96875, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.96875, "rewards/accuracy_reward/std": 0.1767766922712326, "sampling/importance_sampling_ratio/max": 1.5710426568984985, "sampling/importance_sampling_ratio/mean": 1.0001790523529053, "sampling/importance_sampling_ratio/min": 0.7475011348724365, "sampling/sampling_logp_difference/max": 0.4517395496368408, "sampling/sampling_logp_difference/mean": 0.01011952105909586, "step": 963 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 746.0, "completions/mean_length": 481.1875, "completions/mean_terminated_length": 471.9354553222656, "completions/min_length": 295.0, "completions/min_terminated_length": 295.0, "entropy": 0.2522803172469139, "epoch": 0.5160599571734475, "frac_reward_zero_std": 0.5, "grad_norm": 0.005507873836904764, "learning_rate": 1e-05, "loss": 0.0394, "num_tokens": 19925614.0, "reward": 0.9375, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.9375, "rewards/accuracy_reward/std": 0.24593468010425568, "sampling/importance_sampling_ratio/max": 1.3990647792816162, "sampling/importance_sampling_ratio/mean": 0.9996039271354675, "sampling/importance_sampling_ratio/min": 0.7229023575782776, "sampling/sampling_logp_difference/max": 0.3358039855957031, "sampling/sampling_logp_difference/mean": 0.009419592097401619, "step": 964 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00023852916638134047, "clip_ratio/low_min": 0.00023852916638134047, "clip_ratio/region_mean": 0.00023852916638134047, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 734.0, "completions/mean_length": 531.875, "completions/mean_terminated_length": 498.14288330078125, "completions/min_length": 182.0, "completions/min_terminated_length": 182.0, "entropy": 0.29346702992916107, "epoch": 0.5165952890792291, "frac_reward_zero_std": 0.25, "grad_norm": 0.017893539741635323, "learning_rate": 1e-05, "loss": 0.0876, "num_tokens": 19946362.0, "reward": 0.78125, "reward_std": 0.3377464711666107, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.4782878160476685, "sampling/importance_sampling_ratio/mean": 1.000130295753479, "sampling/importance_sampling_ratio/min": 0.6992011666297913, "sampling/sampling_logp_difference/max": 0.39088451862335205, "sampling/sampling_logp_difference/mean": 0.01042831502854824, "step": 965 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.256518124951981e-05, "clip_ratio/low_min": 5.256518124951981e-05, "clip_ratio/region_mean": 5.256518124951981e-05, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 741.0, "completions/mean_length": 462.3125, "completions/mean_terminated_length": 430.6896667480469, "completions/min_length": 257.0, "completions/min_terminated_length": 257.0, "entropy": 0.23513570055365562, "epoch": 0.5171306209850107, "frac_reward_zero_std": 0.5, "grad_norm": 0.013849371112883091, "learning_rate": 1e-05, "loss": 0.0202, "num_tokens": 19965212.0, "reward": 0.8125, "reward_std": 0.249358132481575, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.5709261894226074, "sampling/importance_sampling_ratio/mean": 1.0000474452972412, "sampling/importance_sampling_ratio/min": 0.6098437309265137, "sampling/sampling_logp_difference/max": 0.4945526123046875, "sampling/sampling_logp_difference/mean": 0.009246601723134518, "step": 966 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 725.0, "completions/max_terminated_length": 725.0, "completions/mean_length": 479.25, "completions/mean_terminated_length": 479.25, "completions/min_length": 274.0, "completions/min_terminated_length": 274.0, "entropy": 0.22078409604728222, "epoch": 0.5176659528907923, "frac_reward_zero_std": 0.75, "grad_norm": 0.018964843824505806, "learning_rate": 1e-05, "loss": -0.0053, "num_tokens": 19984596.0, "reward": 0.8125, "reward_std": 0.1157275140285492, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.3845479488372803, "sampling/importance_sampling_ratio/mean": 0.9999772310256958, "sampling/importance_sampling_ratio/min": 0.7026873230934143, "sampling/sampling_logp_difference/max": 0.3528432846069336, "sampling/sampling_logp_difference/mean": 0.008462992496788502, "step": 967 }, { "clip_ratio/high_max": 5.623032120638527e-05, "clip_ratio/high_mean": 5.623032120638527e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 5.623032120638527e-05, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 741.0, "completions/mean_length": 576.4375, "completions/mean_terminated_length": 532.2307739257812, "completions/min_length": 343.0, "completions/min_terminated_length": 343.0, "entropy": 0.3101378045976162, "epoch": 0.5182012847965739, "frac_reward_zero_std": 0.25, "grad_norm": 0.027649912983179092, "learning_rate": 1e-05, "loss": -0.017, "num_tokens": 20007330.0, "reward": 0.5625, "reward_std": 0.292504221200943, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.3878732919692993, "sampling/importance_sampling_ratio/mean": 0.9999299645423889, "sampling/importance_sampling_ratio/min": 0.6943771839141846, "sampling/sampling_logp_difference/max": 0.36473989486694336, "sampling/sampling_logp_difference/mean": 0.01125099416822195, "step": 968 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.3125, "completions/max_length": 768.0, "completions/max_terminated_length": 648.0, "completions/mean_length": 598.0, "completions/mean_terminated_length": 520.727294921875, "completions/min_length": 383.0, "completions/min_terminated_length": 383.0, "entropy": 0.3771156594157219, "epoch": 0.5187366167023555, "frac_reward_zero_std": 0.25, "grad_norm": 0.007927303202450275, "learning_rate": 1e-05, "loss": 0.0808, "num_tokens": 20030898.0, "reward": 0.625, "reward_std": 0.3104073107242584, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.4083565473556519, "sampling/importance_sampling_ratio/mean": 0.9997958540916443, "sampling/importance_sampling_ratio/min": 0.7026858329772949, "sampling/sampling_logp_difference/max": 0.3528454303741455, "sampling/sampling_logp_difference/mean": 0.012313827872276306, "step": 969 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 510.0, "completions/max_terminated_length": 510.0, "completions/mean_length": 348.3125, "completions/mean_terminated_length": 348.3125, "completions/min_length": 203.0, "completions/min_terminated_length": 203.0, "entropy": 0.24957561120390892, "epoch": 0.5192719486081371, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 20045676.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.4911633729934692, "sampling/importance_sampling_ratio/mean": 0.9995911717414856, "sampling/importance_sampling_ratio/min": 0.7303797006607056, "sampling/sampling_logp_difference/max": 0.39955663681030273, "sampling/sampling_logp_difference/mean": 0.009428753517568111, "step": 970 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00010276515240548179, "clip_ratio/low_min": 0.00010276515240548179, "clip_ratio/region_mean": 0.00010276515240548179, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 757.0, "completions/mean_length": 555.59375, "completions/mean_terminated_length": 533.6206665039062, "completions/min_length": 208.0, "completions/min_terminated_length": 208.0, "entropy": 0.2982930038124323, "epoch": 0.5198072805139187, "frac_reward_zero_std": 0.25, "grad_norm": 0.032353516668081284, "learning_rate": 1e-05, "loss": 0.0612, "num_tokens": 20067407.0, "reward": 0.75, "reward_std": 0.292504221200943, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.4002083539962769, "sampling/importance_sampling_ratio/mean": 0.9995220899581909, "sampling/importance_sampling_ratio/min": 0.6951965689659119, "sampling/sampling_logp_difference/max": 0.36356067657470703, "sampling/sampling_logp_difference/mean": 0.01075439341366291, "step": 971 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 562.0, "completions/mean_length": 468.8125, "completions/mean_terminated_length": 369.0833435058594, "completions/min_length": 158.0, "completions/min_terminated_length": 158.0, "entropy": 0.3024728912860155, "epoch": 0.5203426124197003, "frac_reward_zero_std": 0.5, "grad_norm": 0.0036959454882889986, "learning_rate": 1e-05, "loss": 0.0278, "num_tokens": 20085825.0, "reward": 0.75, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9998359680175781, "sampling/importance_sampling_ratio/min": 0.6323160529136658, "sampling/sampling_logp_difference/max": 0.7227559089660645, "sampling/sampling_logp_difference/mean": 0.010977193713188171, "step": 972 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00014327518510981463, "clip_ratio/low_min": 0.00014327518510981463, "clip_ratio/region_mean": 0.00014327518510981463, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 750.0, "completions/mean_length": 505.0, "completions/mean_terminated_length": 444.3077087402344, "completions/min_length": 215.0, "completions/min_terminated_length": 215.0, "entropy": 0.336171830072999, "epoch": 0.5208779443254818, "frac_reward_zero_std": 0.5, "grad_norm": 0.0036234515719115734, "learning_rate": 1e-05, "loss": 0.0217, "num_tokens": 20105985.0, "reward": 0.8125, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.6552815437316895, "sampling/importance_sampling_ratio/mean": 1.000277042388916, "sampling/importance_sampling_ratio/min": 0.6642733812332153, "sampling/sampling_logp_difference/max": 0.5039710998535156, "sampling/sampling_logp_difference/mean": 0.012055602855980396, "step": 973 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 560.0, "completions/max_terminated_length": 560.0, "completions/mean_length": 381.59375, "completions/mean_terminated_length": 381.59375, "completions/min_length": 223.0, "completions/min_terminated_length": 223.0, "entropy": 0.246996833011508, "epoch": 0.5214132762312634, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": -0.0135, "num_tokens": 20121892.0, "reward": 0.96875, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.96875, "rewards/accuracy_reward/std": 0.1767766922712326, "sampling/importance_sampling_ratio/max": 1.428590178489685, "sampling/importance_sampling_ratio/mean": 1.0002020597457886, "sampling/importance_sampling_ratio/min": 0.6473625302314758, "sampling/sampling_logp_difference/max": 0.4348487854003906, "sampling/sampling_logp_difference/mean": 0.00973924994468689, "step": 974 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00029816240930813365, "clip_ratio/low_min": 0.00029816240930813365, "clip_ratio/region_mean": 0.00029816240930813365, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 675.0, "completions/mean_length": 506.1875, "completions/mean_terminated_length": 488.7333679199219, "completions/min_length": 203.0, "completions/min_terminated_length": 203.0, "entropy": 0.26473431661725044, "epoch": 0.521948608137045, "frac_reward_zero_std": 0.25, "grad_norm": 0.014563528820872307, "learning_rate": 1e-05, "loss": -0.0015, "num_tokens": 20142122.0, "reward": 0.46875, "reward_std": 0.3377464711666107, "rewards/accuracy_reward/mean": 0.46875, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.560551404953003, "sampling/importance_sampling_ratio/mean": 0.9999781250953674, "sampling/importance_sampling_ratio/min": 0.7204043865203857, "sampling/sampling_logp_difference/max": 0.4450392723083496, "sampling/sampling_logp_difference/mean": 0.009984633885324001, "step": 975 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 747.0, "completions/max_terminated_length": 747.0, "completions/mean_length": 459.96875, "completions/mean_terminated_length": 459.96875, "completions/min_length": 203.0, "completions/min_terminated_length": 203.0, "entropy": 0.29980884678661823, "epoch": 0.5224839400428265, "frac_reward_zero_std": 0.75, "grad_norm": 0.02271120250225067, "learning_rate": 1e-05, "loss": 0.0217, "num_tokens": 20160417.0, "reward": 0.9375, "reward_std": 0.1157275140285492, "rewards/accuracy_reward/mean": 0.9375, "rewards/accuracy_reward/std": 0.24593468010425568, "sampling/importance_sampling_ratio/max": 1.4021810293197632, "sampling/importance_sampling_ratio/mean": 0.9993802309036255, "sampling/importance_sampling_ratio/min": 0.7222098708152771, "sampling/sampling_logp_difference/max": 0.3380289077758789, "sampling/sampling_logp_difference/mean": 0.010800668969750404, "step": 976 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0002829060031217523, "clip_ratio/low_min": 0.0002829060031217523, "clip_ratio/region_mean": 0.0002829060031217523, "completions/clipped_ratio": 0.28125, "completions/max_length": 768.0, "completions/max_terminated_length": 753.0, "completions/mean_length": 670.375, "completions/mean_terminated_length": 632.1739501953125, "completions/min_length": 301.0, "completions/min_terminated_length": 301.0, "entropy": 0.28090241737663746, "epoch": 0.5230192719486081, "frac_reward_zero_std": 0.0, "grad_norm": 0.014829407446086407, "learning_rate": 1e-05, "loss": 0.0415, "num_tokens": 20186549.0, "reward": 0.4375, "reward_std": 0.5081326961517334, "rewards/accuracy_reward/mean": 0.4375, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.5468014478683472, "sampling/importance_sampling_ratio/mean": 1.0000035762786865, "sampling/importance_sampling_ratio/min": 0.6892077922821045, "sampling/sampling_logp_difference/max": 0.4361891746520996, "sampling/sampling_logp_difference/mean": 0.00957837700843811, "step": 977 }, { "clip_ratio/high_max": 7.97193861217238e-05, "clip_ratio/high_mean": 7.97193861217238e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 7.97193861217238e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 714.0, "completions/max_terminated_length": 714.0, "completions/mean_length": 454.5, "completions/mean_terminated_length": 454.5, "completions/min_length": 251.0, "completions/min_terminated_length": 251.0, "entropy": 0.254776893183589, "epoch": 0.5235546038543897, "frac_reward_zero_std": 0.75, "grad_norm": 0.0076627591624855995, "learning_rate": 1e-05, "loss": 0.0114, "num_tokens": 20204333.0, "reward": 0.9375, "reward_std": 0.1157275140285492, "rewards/accuracy_reward/mean": 0.9375, "rewards/accuracy_reward/std": 0.24593468010425568, "sampling/importance_sampling_ratio/max": 1.5157309770584106, "sampling/importance_sampling_ratio/mean": 1.0001399517059326, "sampling/importance_sampling_ratio/min": 0.6700972318649292, "sampling/sampling_logp_difference/max": 0.41589784622192383, "sampling/sampling_logp_difference/mean": 0.009625574573874474, "step": 978 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 703.0, "completions/mean_length": 571.0625, "completions/mean_terminated_length": 525.6154174804688, "completions/min_length": 342.0, "completions/min_terminated_length": 342.0, "entropy": 0.37119146436452866, "epoch": 0.5240899357601713, "frac_reward_zero_std": 0.25, "grad_norm": 0.010141514241695404, "learning_rate": 1e-05, "loss": 0.0273, "num_tokens": 20226567.0, "reward": 0.71875, "reward_std": 0.3198433816432953, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.7598707675933838, "sampling/importance_sampling_ratio/mean": 1.0000534057617188, "sampling/importance_sampling_ratio/min": 0.6091075539588928, "sampling/sampling_logp_difference/max": 0.5652403831481934, "sampling/sampling_logp_difference/mean": 0.012942535802721977, "step": 979 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 719.0, "completions/mean_length": 505.125, "completions/mean_terminated_length": 496.6451416015625, "completions/min_length": 344.0, "completions/min_terminated_length": 344.0, "entropy": 0.3061270248144865, "epoch": 0.5246252676659529, "frac_reward_zero_std": 0.5, "grad_norm": 0.021014241501688957, "learning_rate": 1e-05, "loss": 0.0419, "num_tokens": 20246699.0, "reward": 0.90625, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.4478200674057007, "sampling/importance_sampling_ratio/mean": 0.999735414981842, "sampling/importance_sampling_ratio/min": 0.6959509253501892, "sampling/sampling_logp_difference/max": 0.3700590133666992, "sampling/sampling_logp_difference/mean": 0.01126969326287508, "step": 980 }, { "clip_ratio/high_max": 5.9326055634301156e-05, "clip_ratio/high_mean": 5.9326055634301156e-05, "clip_ratio/low_mean": 0.00011280367834842764, "clip_ratio/low_min": 0.00011280367834842764, "clip_ratio/region_mean": 0.0001721297339827288, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 716.0, "completions/mean_length": 507.53125, "completions/mean_terminated_length": 490.16668701171875, "completions/min_length": 309.0, "completions/min_terminated_length": 309.0, "entropy": 0.3268727343529463, "epoch": 0.5251605995717344, "frac_reward_zero_std": 0.25, "grad_norm": 0.03915729746222496, "learning_rate": 1e-05, "loss": 0.0258, "num_tokens": 20266476.0, "reward": 0.75, "reward_std": 0.3514062464237213, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.5074266195297241, "sampling/importance_sampling_ratio/mean": 1.0002350807189941, "sampling/importance_sampling_ratio/min": 0.6859396696090698, "sampling/sampling_logp_difference/max": 0.4104039669036865, "sampling/sampling_logp_difference/mean": 0.012186874635517597, "step": 981 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 593.0, "completions/max_terminated_length": 593.0, "completions/mean_length": 372.65625, "completions/mean_terminated_length": 372.65625, "completions/min_length": 191.0, "completions/min_terminated_length": 191.0, "entropy": 0.28994522243738174, "epoch": 0.525695931477516, "frac_reward_zero_std": 0.5, "grad_norm": 0.025411561131477356, "learning_rate": 1e-05, "loss": 0.0112, "num_tokens": 20281681.0, "reward": 0.90625, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.4376773834228516, "sampling/importance_sampling_ratio/mean": 0.9996423125267029, "sampling/importance_sampling_ratio/min": 0.65763258934021, "sampling/sampling_logp_difference/max": 0.41910886764526367, "sampling/sampling_logp_difference/mean": 0.010259519331157207, "step": 982 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 8.777290713624097e-05, "clip_ratio/low_min": 8.777290713624097e-05, "clip_ratio/region_mean": 8.777290713624097e-05, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 732.0, "completions/mean_length": 573.15625, "completions/mean_terminated_length": 508.2083435058594, "completions/min_length": 292.0, "completions/min_terminated_length": 292.0, "entropy": 0.36684974282979965, "epoch": 0.5262312633832976, "frac_reward_zero_std": 0.25, "grad_norm": 0.016475502401590347, "learning_rate": 1e-05, "loss": 0.0193, "num_tokens": 20304206.0, "reward": 0.6875, "reward_std": 0.3745020925998688, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.6101723909378052, "sampling/importance_sampling_ratio/mean": 1.0001775026321411, "sampling/importance_sampling_ratio/min": 0.7203891277313232, "sampling/sampling_logp_difference/max": 0.47634124755859375, "sampling/sampling_logp_difference/mean": 0.012516574002802372, "step": 983 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 747.0, "completions/mean_length": 553.25, "completions/mean_terminated_length": 531.0344848632812, "completions/min_length": 315.0, "completions/min_terminated_length": 315.0, "entropy": 0.3875699080526829, "epoch": 0.5267665952890792, "frac_reward_zero_std": 0.5, "grad_norm": 0.015934111550450325, "learning_rate": 1e-05, "loss": -0.0034, "num_tokens": 20325894.0, "reward": 0.8125, "reward_std": 0.249358132481575, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.4636443853378296, "sampling/importance_sampling_ratio/mean": 1.000114917755127, "sampling/importance_sampling_ratio/min": 0.7714143991470337, "sampling/sampling_logp_difference/max": 0.38092947006225586, "sampling/sampling_logp_difference/mean": 0.012341463938355446, "step": 984 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00011558142432477325, "clip_ratio/low_min": 0.00011558142432477325, "clip_ratio/region_mean": 0.00011558142432477325, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 700.0, "completions/mean_length": 467.0, "completions/mean_terminated_length": 411.25927734375, "completions/min_length": 221.0, "completions/min_terminated_length": 221.0, "entropy": 0.3518456779420376, "epoch": 0.5273019271948608, "frac_reward_zero_std": 0.75, "grad_norm": 0.004745997954159975, "learning_rate": 1e-05, "loss": 0.0115, "num_tokens": 20344414.0, "reward": 0.78125, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.723758339881897, "sampling/importance_sampling_ratio/mean": 1.0002174377441406, "sampling/importance_sampling_ratio/min": 0.6284816265106201, "sampling/sampling_logp_difference/max": 0.5445070266723633, "sampling/sampling_logp_difference/mean": 0.013397024013102055, "step": 985 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 737.0, "completions/max_terminated_length": 737.0, "completions/mean_length": 409.0, "completions/mean_terminated_length": 409.0, "completions/min_length": 242.0, "completions/min_terminated_length": 242.0, "entropy": 0.259233757853508, "epoch": 0.5278372591006424, "frac_reward_zero_std": 0.5, "grad_norm": 0.03837405517697334, "learning_rate": 1e-05, "loss": 0.0469, "num_tokens": 20361838.0, "reward": 0.9375, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.9375, "rewards/accuracy_reward/std": 0.24593468010425568, "sampling/importance_sampling_ratio/max": 1.679805040359497, "sampling/importance_sampling_ratio/mean": 1.0000888109207153, "sampling/importance_sampling_ratio/min": 0.6981421113014221, "sampling/sampling_logp_difference/max": 0.5186777114868164, "sampling/sampling_logp_difference/mean": 0.010746989399194717, "step": 986 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 756.0, "completions/mean_length": 545.28125, "completions/mean_terminated_length": 513.4642944335938, "completions/min_length": 306.0, "completions/min_terminated_length": 306.0, "entropy": 0.33587929606437683, "epoch": 0.528372591006424, "frac_reward_zero_std": 0.0, "grad_norm": 0.02470543049275875, "learning_rate": 1e-05, "loss": 0.0127, "num_tokens": 20383519.0, "reward": 0.4375, "reward_std": 0.4492306709289551, "rewards/accuracy_reward/mean": 0.4375, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.6979705095291138, "sampling/importance_sampling_ratio/mean": 0.9997697472572327, "sampling/importance_sampling_ratio/min": 0.6274191737174988, "sampling/sampling_logp_difference/max": 0.5294337272644043, "sampling/sampling_logp_difference/mean": 0.012272883206605911, "step": 987 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 724.0, "completions/mean_length": 486.5, "completions/mean_terminated_length": 467.7333679199219, "completions/min_length": 290.0, "completions/min_terminated_length": 290.0, "entropy": 0.2786070182919502, "epoch": 0.5289079229122056, "frac_reward_zero_std": 0.5, "grad_norm": 0.006380059290677309, "learning_rate": 1e-05, "loss": 0.0128, "num_tokens": 20402767.0, "reward": 0.90625, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.5402277708053589, "sampling/importance_sampling_ratio/mean": 0.99989914894104, "sampling/importance_sampling_ratio/min": 0.7686281800270081, "sampling/sampling_logp_difference/max": 0.4319303035736084, "sampling/sampling_logp_difference/mean": 0.010861505754292011, "step": 988 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 647.0, "completions/max_terminated_length": 647.0, "completions/mean_length": 396.0, "completions/mean_terminated_length": 396.0, "completions/min_length": 255.0, "completions/min_terminated_length": 255.0, "entropy": 0.23258525133132935, "epoch": 0.5294432548179872, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 20419511.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.4397772550582886, "sampling/importance_sampling_ratio/mean": 1.000404715538025, "sampling/importance_sampling_ratio/min": 0.691949188709259, "sampling/sampling_logp_difference/max": 0.3682427406311035, "sampling/sampling_logp_difference/mean": 0.010228743776679039, "step": 989 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 620.0, "completions/max_terminated_length": 620.0, "completions/mean_length": 438.3125, "completions/mean_terminated_length": 438.3125, "completions/min_length": 238.0, "completions/min_terminated_length": 238.0, "entropy": 0.33510708808898926, "epoch": 0.5299785867237687, "frac_reward_zero_std": 0.5, "grad_norm": 0.008542737923562527, "learning_rate": 1e-05, "loss": -0.0146, "num_tokens": 20437337.0, "reward": 0.75, "reward_std": 0.2314550280570984, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.405011534690857, "sampling/importance_sampling_ratio/mean": 0.9998782277107239, "sampling/importance_sampling_ratio/min": 0.7477208971977234, "sampling/sampling_logp_difference/max": 0.3400454521179199, "sampling/sampling_logp_difference/mean": 0.012023129500448704, "step": 990 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 727.0, "completions/mean_length": 529.34375, "completions/mean_terminated_length": 521.6451416015625, "completions/min_length": 231.0, "completions/min_terminated_length": 231.0, "entropy": 0.28788173757493496, "epoch": 0.5305139186295503, "frac_reward_zero_std": 0.25, "grad_norm": 0.00872648973017931, "learning_rate": 1e-05, "loss": 0.026, "num_tokens": 20457724.0, "reward": 0.875, "reward_std": 0.292504221200943, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 1.4477896690368652, "sampling/importance_sampling_ratio/mean": 1.000186800956726, "sampling/importance_sampling_ratio/min": 0.7323204278945923, "sampling/sampling_logp_difference/max": 0.3700380325317383, "sampling/sampling_logp_difference/mean": 0.010936199687421322, "step": 991 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00011848341091535985, "clip_ratio/low_min": 0.00011848341091535985, "clip_ratio/region_mean": 0.00011848341091535985, "completions/clipped_ratio": 0.0, "completions/max_length": 596.0, "completions/max_terminated_length": 596.0, "completions/mean_length": 389.09375, "completions/mean_terminated_length": 389.09375, "completions/min_length": 128.0, "completions/min_terminated_length": 128.0, "entropy": 0.3102747555822134, "epoch": 0.5310492505353319, "frac_reward_zero_std": 0.5, "grad_norm": 0.008674847893416882, "learning_rate": 1e-05, "loss": 0.0388, "num_tokens": 20473623.0, "reward": 0.875, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 1.4332579374313354, "sampling/importance_sampling_ratio/mean": 1.000070571899414, "sampling/importance_sampling_ratio/min": 0.6004974842071533, "sampling/sampling_logp_difference/max": 0.5099968910217285, "sampling/sampling_logp_difference/mean": 0.011831749230623245, "step": 992 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 677.0, "completions/max_terminated_length": 677.0, "completions/mean_length": 425.71875, "completions/mean_terminated_length": 425.71875, "completions/min_length": 246.0, "completions/min_terminated_length": 246.0, "entropy": 0.22281677462160587, "epoch": 0.5315845824411135, "frac_reward_zero_std": 0.75, "grad_norm": 0.015275092795491219, "learning_rate": 1e-05, "loss": 0.0105, "num_tokens": 20490782.0, "reward": 0.90625, "reward_std": 0.1293872892856598, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.9839650392532349, "sampling/importance_sampling_ratio/mean": 1.0004388093948364, "sampling/importance_sampling_ratio/min": 0.668385922908783, "sampling/sampling_logp_difference/max": 0.6850974559783936, "sampling/sampling_logp_difference/mean": 0.008745191618800163, "step": 993 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 723.0, "completions/mean_length": 414.5, "completions/mean_terminated_length": 390.933349609375, "completions/min_length": 177.0, "completions/min_terminated_length": 177.0, "entropy": 0.2425446305423975, "epoch": 0.5321199143468951, "frac_reward_zero_std": 0.75, "grad_norm": 0.015331036411225796, "learning_rate": 1e-05, "loss": 0.0027, "num_tokens": 20507494.0, "reward": 0.90625, "reward_std": 0.1293872892856598, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.4391626119613647, "sampling/importance_sampling_ratio/mean": 1.0001529455184937, "sampling/importance_sampling_ratio/min": 0.7409034967422485, "sampling/sampling_logp_difference/max": 0.3640613555908203, "sampling/sampling_logp_difference/mean": 0.010235991328954697, "step": 994 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 727.0, "completions/mean_length": 489.0, "completions/mean_terminated_length": 460.137939453125, "completions/min_length": 288.0, "completions/min_terminated_length": 288.0, "entropy": 0.3341273684054613, "epoch": 0.5326552462526767, "frac_reward_zero_std": 0.5, "grad_norm": 0.02347775362432003, "learning_rate": 1e-05, "loss": 0.0056, "num_tokens": 20526894.0, "reward": 0.78125, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.4396318197250366, "sampling/importance_sampling_ratio/mean": 1.000096082687378, "sampling/importance_sampling_ratio/min": 0.6704294681549072, "sampling/sampling_logp_difference/max": 0.39983677864074707, "sampling/sampling_logp_difference/mean": 0.012613601982593536, "step": 995 }, { "clip_ratio/high_max": 7.631257903994992e-05, "clip_ratio/high_mean": 7.631257903994992e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 7.631257903994992e-05, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 757.0, "completions/mean_length": 448.28125, "completions/mean_terminated_length": 402.6071472167969, "completions/min_length": 108.0, "completions/min_terminated_length": 108.0, "entropy": 0.4117588307708502, "epoch": 0.5331905781584583, "frac_reward_zero_std": 0.25, "grad_norm": 0.024844985455274582, "learning_rate": 1e-05, "loss": 0.0555, "num_tokens": 20545055.0, "reward": 0.6875, "reward_std": 0.3745020925998688, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.382009744644165, "sampling/importance_sampling_ratio/mean": 1.0003635883331299, "sampling/importance_sampling_ratio/min": 0.48914584517478943, "sampling/sampling_logp_difference/max": 0.7150945663452148, "sampling/sampling_logp_difference/mean": 0.013542810454964638, "step": 996 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 757.0, "completions/mean_length": 584.0625, "completions/mean_terminated_length": 565.0344848632812, "completions/min_length": 355.0, "completions/min_terminated_length": 355.0, "entropy": 0.24145429581403732, "epoch": 0.5337259100642399, "frac_reward_zero_std": 0.5, "grad_norm": 0.00853966549038887, "learning_rate": 1e-05, "loss": 0.0197, "num_tokens": 20567793.0, "reward": 0.75, "reward_std": 0.2314550280570984, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.3396481275558472, "sampling/importance_sampling_ratio/mean": 1.000004768371582, "sampling/importance_sampling_ratio/min": 0.43636608123779297, "sampling/sampling_logp_difference/max": 0.8292738199234009, "sampling/sampling_logp_difference/mean": 0.008973004296422005, "step": 997 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 722.0, "completions/max_terminated_length": 722.0, "completions/mean_length": 491.15625, "completions/mean_terminated_length": 491.15625, "completions/min_length": 302.0, "completions/min_terminated_length": 302.0, "entropy": 0.20358617417514324, "epoch": 0.5342612419700214, "frac_reward_zero_std": 0.75, "grad_norm": 0.006970930378884077, "learning_rate": 1e-05, "loss": -0.0082, "num_tokens": 20587310.0, "reward": 0.90625, "reward_std": 0.1293872892856598, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.4240021705627441, "sampling/importance_sampling_ratio/mean": 0.9996641278266907, "sampling/importance_sampling_ratio/min": 0.6888296604156494, "sampling/sampling_logp_difference/max": 0.37276124954223633, "sampling/sampling_logp_difference/mean": 0.008362756110727787, "step": 998 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00017769212718121707, "clip_ratio/low_min": 0.00017769212718121707, "clip_ratio/region_mean": 0.00017769212718121707, "completions/clipped_ratio": 0.0, "completions/max_length": 531.0, "completions/max_terminated_length": 531.0, "completions/mean_length": 374.375, "completions/mean_terminated_length": 374.375, "completions/min_length": 255.0, "completions/min_terminated_length": 255.0, "entropy": 0.30256051383912563, "epoch": 0.534796573875803, "frac_reward_zero_std": 0.5, "grad_norm": 0.006298878695815802, "learning_rate": 1e-05, "loss": 0.0083, "num_tokens": 20602714.0, "reward": 0.9375, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.9375, "rewards/accuracy_reward/std": 0.24593468010425568, "sampling/importance_sampling_ratio/max": 1.4680598974227905, "sampling/importance_sampling_ratio/mean": 1.000386357307434, "sampling/importance_sampling_ratio/min": 0.7151246070861816, "sampling/sampling_logp_difference/max": 0.383941650390625, "sampling/sampling_logp_difference/mean": 0.01191803626716137, "step": 999 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 746.0, "completions/mean_length": 566.75, "completions/mean_terminated_length": 545.9310302734375, "completions/min_length": 344.0, "completions/min_terminated_length": 344.0, "entropy": 0.34574132412672043, "epoch": 0.5353319057815846, "frac_reward_zero_std": 0.25, "grad_norm": 0.02217051014304161, "learning_rate": 1e-05, "loss": 0.0523, "num_tokens": 20624474.0, "reward": 0.5625, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.3802006244659424, "sampling/importance_sampling_ratio/mean": 0.9995157718658447, "sampling/importance_sampling_ratio/min": 0.70558100938797, "sampling/sampling_logp_difference/max": 0.34873366355895996, "sampling/sampling_logp_difference/mean": 0.01196676678955555, "step": 1000 }, { "clip_ratio/high_max": 6.742178811691701e-05, "clip_ratio/high_mean": 6.742178811691701e-05, "clip_ratio/low_mean": 6.271951860981062e-05, "clip_ratio/low_min": 6.271951860981062e-05, "clip_ratio/region_mean": 0.00013014130672672763, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 663.0, "completions/mean_length": 461.09375, "completions/mean_terminated_length": 440.63336181640625, "completions/min_length": 275.0, "completions/min_terminated_length": 275.0, "entropy": 0.29909978061914444, "epoch": 0.5358672376873662, "frac_reward_zero_std": 0.5, "grad_norm": 0.006004105322062969, "learning_rate": 1e-05, "loss": 0.0916, "num_tokens": 20643509.0, "reward": 0.9375, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.9375, "rewards/accuracy_reward/std": 0.24593468010425568, "sampling/importance_sampling_ratio/max": 1.5818359851837158, "sampling/importance_sampling_ratio/mean": 0.999842643737793, "sampling/importance_sampling_ratio/min": 0.6266263723373413, "sampling/sampling_logp_difference/max": 0.467404842376709, "sampling/sampling_logp_difference/mean": 0.011380976997315884, "step": 1001 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.003842281643301e-05, "clip_ratio/low_min": 6.003842281643301e-05, "clip_ratio/region_mean": 6.003842281643301e-05, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 730.0, "completions/mean_length": 513.1875, "completions/mean_terminated_length": 476.7857360839844, "completions/min_length": 291.0, "completions/min_terminated_length": 291.0, "entropy": 0.31066639348864555, "epoch": 0.5364025695931478, "frac_reward_zero_std": 0.25, "grad_norm": 0.02892880327999592, "learning_rate": 1e-05, "loss": 0.0193, "num_tokens": 20663891.0, "reward": 0.75, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.4119313955307007, "sampling/importance_sampling_ratio/mean": 0.9999995231628418, "sampling/importance_sampling_ratio/min": 0.7418161630630493, "sampling/sampling_logp_difference/max": 0.3449585437774658, "sampling/sampling_logp_difference/mean": 0.011235960759222507, "step": 1002 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00014886206918163225, "clip_ratio/low_min": 0.00014886206918163225, "clip_ratio/region_mean": 0.00014886206918163225, "completions/clipped_ratio": 0.0, "completions/max_length": 639.0, "completions/max_terminated_length": 639.0, "completions/mean_length": 406.625, "completions/mean_terminated_length": 406.625, "completions/min_length": 312.0, "completions/min_terminated_length": 312.0, "entropy": 0.32252445816993713, "epoch": 0.5369379014989293, "frac_reward_zero_std": 0.75, "grad_norm": 0.008560850284993649, "learning_rate": 1e-05, "loss": 0.007, "num_tokens": 20680823.0, "reward": 0.90625, "reward_std": 0.1293872892856598, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.3867586851119995, "sampling/importance_sampling_ratio/mean": 1.0000890493392944, "sampling/importance_sampling_ratio/min": 0.6831401586532593, "sampling/sampling_logp_difference/max": 0.38105523586273193, "sampling/sampling_logp_difference/mean": 0.012363986112177372, "step": 1003 }, { "clip_ratio/high_max": 5.380973016144708e-05, "clip_ratio/high_mean": 5.380973016144708e-05, "clip_ratio/low_mean": 5.087505269329995e-05, "clip_ratio/low_min": 5.087505269329995e-05, "clip_ratio/region_mean": 0.00010468478285474703, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 729.0, "completions/mean_length": 553.0625, "completions/mean_terminated_length": 530.8275756835938, "completions/min_length": 211.0, "completions/min_terminated_length": 211.0, "entropy": 0.41602638363838196, "epoch": 0.5374732334047109, "frac_reward_zero_std": 0.0, "grad_norm": 0.01616629585623741, "learning_rate": 1e-05, "loss": -0.0111, "num_tokens": 20702265.0, "reward": 0.75, "reward_std": 0.4355512857437134, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.521829605102539, "sampling/importance_sampling_ratio/mean": 0.9995875954627991, "sampling/importance_sampling_ratio/min": 0.6779919266700745, "sampling/sampling_logp_difference/max": 0.41991329193115234, "sampling/sampling_logp_difference/mean": 0.013063188642263412, "step": 1004 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 655.0, "completions/mean_length": 375.8125, "completions/mean_terminated_length": 363.1612854003906, "completions/min_length": 155.0, "completions/min_terminated_length": 155.0, "entropy": 0.2924263346940279, "epoch": 0.5380085653104925, "frac_reward_zero_std": 0.5, "grad_norm": 0.009580021724104881, "learning_rate": 1e-05, "loss": 0.0663, "num_tokens": 20717299.0, "reward": 0.875, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 1.5672733783721924, "sampling/importance_sampling_ratio/mean": 0.9999663233757019, "sampling/importance_sampling_ratio/min": 0.6852931380271912, "sampling/sampling_logp_difference/max": 0.4493374824523926, "sampling/sampling_logp_difference/mean": 0.010947070084512234, "step": 1005 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 716.0, "completions/mean_length": 448.40625, "completions/mean_terminated_length": 438.0967712402344, "completions/min_length": 185.0, "completions/min_terminated_length": 185.0, "entropy": 0.31122429855167866, "epoch": 0.538543897216274, "frac_reward_zero_std": 0.25, "grad_norm": 0.024525612592697144, "learning_rate": 1e-05, "loss": -0.025, "num_tokens": 20735240.0, "reward": 0.625, "reward_std": 0.3924052119255066, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.7791000604629517, "sampling/importance_sampling_ratio/mean": 0.9997047781944275, "sampling/importance_sampling_ratio/min": 0.4711102843284607, "sampling/sampling_logp_difference/max": 0.7526631355285645, "sampling/sampling_logp_difference/mean": 0.011387272737920284, "step": 1006 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00013269779447000474, "clip_ratio/low_min": 0.00013269779447000474, "clip_ratio/region_mean": 0.00013269779447000474, "completions/clipped_ratio": 0.0, "completions/max_length": 708.0, "completions/max_terminated_length": 708.0, "completions/mean_length": 486.5625, "completions/mean_terminated_length": 486.5625, "completions/min_length": 257.0, "completions/min_terminated_length": 257.0, "entropy": 0.2862442471086979, "epoch": 0.5390792291220556, "frac_reward_zero_std": 0.25, "grad_norm": 0.006266626995056868, "learning_rate": 1e-05, "loss": 0.0265, "num_tokens": 20754994.0, "reward": 0.8125, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.5755714178085327, "sampling/importance_sampling_ratio/mean": 0.9999701976776123, "sampling/importance_sampling_ratio/min": 0.7431015968322754, "sampling/sampling_logp_difference/max": 0.454617977142334, "sampling/sampling_logp_difference/mean": 0.010682228952646255, "step": 1007 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 705.0, "completions/max_terminated_length": 705.0, "completions/mean_length": 476.125, "completions/mean_terminated_length": 476.125, "completions/min_length": 315.0, "completions/min_terminated_length": 315.0, "entropy": 0.31628429144620895, "epoch": 0.5396145610278372, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0032, "num_tokens": 20773798.0, "reward": 0.96875, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.96875, "rewards/accuracy_reward/std": 0.1767766922712326, "sampling/importance_sampling_ratio/max": 1.6980595588684082, "sampling/importance_sampling_ratio/mean": 0.9999076724052429, "sampling/importance_sampling_ratio/min": 0.5939722657203674, "sampling/sampling_logp_difference/max": 0.5294861793518066, "sampling/sampling_logp_difference/mean": 0.011187360621988773, "step": 1008 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.319514795904979e-05, "clip_ratio/low_min": 6.319514795904979e-05, "clip_ratio/region_mean": 6.319514795904979e-05, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 731.0, "completions/mean_length": 485.59375, "completions/mean_terminated_length": 466.7666931152344, "completions/min_length": 234.0, "completions/min_terminated_length": 234.0, "entropy": 0.33392519876360893, "epoch": 0.5401498929336188, "frac_reward_zero_std": 0.5, "grad_norm": 0.013179289177060127, "learning_rate": 1e-05, "loss": 0.0438, "num_tokens": 20793513.0, "reward": 0.8125, "reward_std": 0.2587745785713196, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.4700065851211548, "sampling/importance_sampling_ratio/mean": 0.9999333620071411, "sampling/importance_sampling_ratio/min": 0.5124729871749878, "sampling/sampling_logp_difference/max": 0.6685073375701904, "sampling/sampling_logp_difference/mean": 0.011264905333518982, "step": 1009 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.950494985678233e-05, "clip_ratio/low_min": 4.950494985678233e-05, "clip_ratio/region_mean": 4.950494985678233e-05, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 737.0, "completions/mean_length": 504.0625, "completions/mean_terminated_length": 466.357177734375, "completions/min_length": 288.0, "completions/min_terminated_length": 288.0, "entropy": 0.380746778100729, "epoch": 0.5406852248394004, "frac_reward_zero_std": 0.25, "grad_norm": 0.01088571734726429, "learning_rate": 1e-05, "loss": 0.0466, "num_tokens": 20813331.0, "reward": 0.5625, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.389113426208496, "sampling/importance_sampling_ratio/mean": 1.000091314315796, "sampling/importance_sampling_ratio/min": 0.7165794372558594, "sampling/sampling_logp_difference/max": 0.3332662582397461, "sampling/sampling_logp_difference/mean": 0.012350971810519695, "step": 1010 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 676.0, "completions/max_terminated_length": 676.0, "completions/mean_length": 393.9375, "completions/mean_terminated_length": 393.9375, "completions/min_length": 160.0, "completions/min_terminated_length": 160.0, "entropy": 0.2761106453835964, "epoch": 0.541220556745182, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 20829409.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.4311715364456177, "sampling/importance_sampling_ratio/mean": 0.9997698068618774, "sampling/importance_sampling_ratio/min": 0.6138607859611511, "sampling/sampling_logp_difference/max": 0.4879871606826782, "sampling/sampling_logp_difference/mean": 0.010651922784745693, "step": 1011 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.375, "completions/max_length": 768.0, "completions/max_terminated_length": 755.0, "completions/mean_length": 563.9375, "completions/mean_terminated_length": 441.5, "completions/min_length": 223.0, "completions/min_terminated_length": 223.0, "entropy": 0.4022360257804394, "epoch": 0.5417558886509636, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0235, "num_tokens": 20851847.0, "reward": 0.5625, "reward_std": 0.1157275140285492, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.5724354982376099, "sampling/importance_sampling_ratio/mean": 1.0001391172409058, "sampling/importance_sampling_ratio/min": 0.6602086424827576, "sampling/sampling_logp_difference/max": 0.45262575149536133, "sampling/sampling_logp_difference/mean": 0.012675242498517036, "step": 1012 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 548.0, "completions/max_terminated_length": 548.0, "completions/mean_length": 394.4375, "completions/mean_terminated_length": 394.4375, "completions/min_length": 232.0, "completions/min_terminated_length": 232.0, "entropy": 0.2700550053268671, "epoch": 0.5422912205567452, "frac_reward_zero_std": 0.5, "grad_norm": 0.006720743142068386, "learning_rate": 1e-05, "loss": 0.011, "num_tokens": 20868293.0, "reward": 0.90625, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.4353808164596558, "sampling/importance_sampling_ratio/mean": 0.9999120831489563, "sampling/importance_sampling_ratio/min": 0.7047362327575684, "sampling/sampling_logp_difference/max": 0.36143016815185547, "sampling/sampling_logp_difference/mean": 0.010281146503984928, "step": 1013 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 593.0, "completions/max_terminated_length": 593.0, "completions/mean_length": 421.78125, "completions/mean_terminated_length": 421.78125, "completions/min_length": 262.0, "completions/min_terminated_length": 262.0, "entropy": 0.24118858203291893, "epoch": 0.5428265524625268, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 20885838.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.427647352218628, "sampling/importance_sampling_ratio/mean": 1.0003013610839844, "sampling/importance_sampling_ratio/min": 0.7005481123924255, "sampling/sampling_logp_difference/max": 0.35602784156799316, "sampling/sampling_logp_difference/mean": 0.010140156373381615, "step": 1014 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0002263921342091635, "clip_ratio/low_min": 0.0002263921342091635, "clip_ratio/region_mean": 0.0002263921342091635, "completions/clipped_ratio": 0.0, "completions/max_length": 606.0, "completions/max_terminated_length": 606.0, "completions/mean_length": 410.65625, "completions/mean_terminated_length": 410.65625, "completions/min_length": 220.0, "completions/min_terminated_length": 220.0, "entropy": 0.2819908335804939, "epoch": 0.5433618843683083, "frac_reward_zero_std": 0.75, "grad_norm": 0.011591862887144089, "learning_rate": 1e-05, "loss": 0.0156, "num_tokens": 20902467.0, "reward": 0.875, "reward_std": 0.13363061845302582, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 1.436547875404358, "sampling/importance_sampling_ratio/mean": 1.0001517534255981, "sampling/importance_sampling_ratio/min": 0.7365150451660156, "sampling/sampling_logp_difference/max": 0.3622429370880127, "sampling/sampling_logp_difference/mean": 0.010819976218044758, "step": 1015 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 694.0, "completions/mean_length": 474.8125, "completions/mean_terminated_length": 432.9285888671875, "completions/min_length": 242.0, "completions/min_terminated_length": 242.0, "entropy": 0.32844111137092113, "epoch": 0.5438972162740899, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 20921165.0, "reward": 0.75, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.4033128023147583, "sampling/importance_sampling_ratio/mean": 1.0000876188278198, "sampling/importance_sampling_ratio/min": 0.7019321322441101, "sampling/sampling_logp_difference/max": 0.35391855239868164, "sampling/sampling_logp_difference/mean": 0.011300837621092796, "step": 1016 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 619.0, "completions/max_terminated_length": 619.0, "completions/mean_length": 438.25, "completions/mean_terminated_length": 438.25, "completions/min_length": 243.0, "completions/min_terminated_length": 243.0, "entropy": 0.2606982924044132, "epoch": 0.5444325481798715, "frac_reward_zero_std": 0.75, "grad_norm": 0.013174247927963734, "learning_rate": 1e-05, "loss": -0.0082, "num_tokens": 20938957.0, "reward": 0.90625, "reward_std": 0.1293872892856598, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.454656958580017, "sampling/importance_sampling_ratio/mean": 1.0001221895217896, "sampling/importance_sampling_ratio/min": 0.7321741580963135, "sampling/sampling_logp_difference/max": 0.3747701644897461, "sampling/sampling_logp_difference/mean": 0.010059934109449387, "step": 1017 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00010052271682070568, "clip_ratio/low_min": 0.00010052271682070568, "clip_ratio/region_mean": 0.00010052271682070568, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 749.0, "completions/mean_length": 538.96875, "completions/mean_terminated_length": 506.2500305175781, "completions/min_length": 226.0, "completions/min_terminated_length": 226.0, "entropy": 0.32434203661978245, "epoch": 0.5449678800856531, "frac_reward_zero_std": 0.0, "grad_norm": 0.01634792611002922, "learning_rate": 1e-05, "loss": 0.014, "num_tokens": 20960220.0, "reward": 0.375, "reward_std": 0.4671337604522705, "rewards/accuracy_reward/mean": 0.375, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.3839634656906128, "sampling/importance_sampling_ratio/mean": 0.9999974966049194, "sampling/importance_sampling_ratio/min": 0.6750747561454773, "sampling/sampling_logp_difference/max": 0.3929319381713867, "sampling/sampling_logp_difference/mean": 0.011419091373682022, "step": 1018 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.715592124033719e-05, "clip_ratio/low_min": 5.715592124033719e-05, "clip_ratio/region_mean": 5.715592124033719e-05, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 741.0, "completions/mean_length": 509.8125, "completions/mean_terminated_length": 450.23077392578125, "completions/min_length": 331.0, "completions/min_terminated_length": 331.0, "entropy": 0.3778581041842699, "epoch": 0.5455032119914347, "frac_reward_zero_std": 0.25, "grad_norm": 0.024147875607013702, "learning_rate": 1e-05, "loss": 0.0124, "num_tokens": 20980374.0, "reward": 0.625, "reward_std": 0.3104073107242584, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.4404075145721436, "sampling/importance_sampling_ratio/mean": 1.0000821352005005, "sampling/importance_sampling_ratio/min": 0.712488055229187, "sampling/sampling_logp_difference/max": 0.3649260997772217, "sampling/sampling_logp_difference/mean": 0.011622502468526363, "step": 1019 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 744.0, "completions/mean_length": 448.09375, "completions/mean_terminated_length": 426.7666931152344, "completions/min_length": 255.0, "completions/min_terminated_length": 255.0, "entropy": 0.35178061947226524, "epoch": 0.5460385438972163, "frac_reward_zero_std": 0.5, "grad_norm": 0.03291179984807968, "learning_rate": 1e-05, "loss": 0.0386, "num_tokens": 20998073.0, "reward": 0.90625, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.4602396488189697, "sampling/importance_sampling_ratio/mean": 0.9998698234558105, "sampling/importance_sampling_ratio/min": 0.6329456567764282, "sampling/sampling_logp_difference/max": 0.4573707580566406, "sampling/sampling_logp_difference/mean": 0.011893614195287228, "step": 1020 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00010640751861501485, "clip_ratio/low_min": 0.00010640751861501485, "clip_ratio/region_mean": 0.00010640751861501485, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 767.0, "completions/mean_length": 559.125, "completions/mean_terminated_length": 520.4444580078125, "completions/min_length": 295.0, "completions/min_terminated_length": 295.0, "entropy": 0.40700463578104973, "epoch": 0.5465738758029979, "frac_reward_zero_std": 0.25, "grad_norm": 0.01419112365692854, "learning_rate": 1e-05, "loss": 0.0215, "num_tokens": 21019837.0, "reward": 0.6875, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.7007182836532593, "sampling/importance_sampling_ratio/mean": 0.9997385740280151, "sampling/importance_sampling_ratio/min": 0.695031464099884, "sampling/sampling_logp_difference/max": 0.5310506820678711, "sampling/sampling_logp_difference/mean": 0.01345874834805727, "step": 1021 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 7.739938155282289e-05, "clip_ratio/low_min": 7.739938155282289e-05, "clip_ratio/region_mean": 7.739938155282289e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 715.0, "completions/max_terminated_length": 715.0, "completions/mean_length": 426.03125, "completions/mean_terminated_length": 426.03125, "completions/min_length": 281.0, "completions/min_terminated_length": 281.0, "entropy": 0.30674983933568, "epoch": 0.5471092077087795, "frac_reward_zero_std": 0.5, "grad_norm": 0.02698620967566967, "learning_rate": 1e-05, "loss": -0.0005, "num_tokens": 21037142.0, "reward": 0.84375, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.5722301006317139, "sampling/importance_sampling_ratio/mean": 0.9997084736824036, "sampling/importance_sampling_ratio/min": 0.7135961651802063, "sampling/sampling_logp_difference/max": 0.45249509811401367, "sampling/sampling_logp_difference/mean": 0.01148021500557661, "step": 1022 }, { "clip_ratio/high_max": 6.300403038039804e-05, "clip_ratio/high_mean": 6.300403038039804e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 6.300403038039804e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 564.0, "completions/max_terminated_length": 564.0, "completions/mean_length": 425.5625, "completions/mean_terminated_length": 425.5625, "completions/min_length": 241.0, "completions/min_terminated_length": 241.0, "entropy": 0.27811561338603497, "epoch": 0.547644539614561, "frac_reward_zero_std": 0.75, "grad_norm": 0.019728660583496094, "learning_rate": 1e-05, "loss": 0.0063, "num_tokens": 21055040.0, "reward": 0.9375, "reward_std": 0.1157275140285492, "rewards/accuracy_reward/mean": 0.9375, "rewards/accuracy_reward/std": 0.24593468010425568, "sampling/importance_sampling_ratio/max": 1.6341397762298584, "sampling/importance_sampling_ratio/mean": 0.9996365904808044, "sampling/importance_sampling_ratio/min": 0.6480754017829895, "sampling/sampling_logp_difference/max": 0.4911165237426758, "sampling/sampling_logp_difference/mean": 0.010462039150297642, "step": 1023 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 675.0, "completions/mean_length": 491.125, "completions/mean_terminated_length": 482.19354248046875, "completions/min_length": 310.0, "completions/min_terminated_length": 310.0, "entropy": 0.26878377981483936, "epoch": 0.5481798715203426, "frac_reward_zero_std": 0.25, "grad_norm": 0.020172934979200363, "learning_rate": 1e-05, "loss": 0.0082, "num_tokens": 21074644.0, "reward": 0.40625, "reward_std": 0.3471629321575165, "rewards/accuracy_reward/mean": 0.40625, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.2830291986465454, "sampling/importance_sampling_ratio/mean": 1.0000466108322144, "sampling/importance_sampling_ratio/min": 0.643420934677124, "sampling/sampling_logp_difference/max": 0.44095611572265625, "sampling/sampling_logp_difference/mean": 0.009556285105645657, "step": 1024 } ], "logging_steps": 1, "max_steps": 1024, "num_input_tokens_seen": 21074644, "num_train_epochs": 1, "save_steps": 64, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": true, "should_training_stop": true }, "attributes": {} } }, "total_flos": 0.0, "train_batch_size": 4, "trial_name": null, "trial_params": null }