{ "best_global_step": null, "best_metric": null, "best_model_checkpoint": null, "epoch": 0.5481798715203426, "eval_steps": 500, "global_step": 1024, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.5, "completions/max_length": 768.0, "completions/max_terminated_length": 753.0, "completions/mean_length": 664.84375, "completions/mean_terminated_length": 561.6875, "completions/min_length": 322.0, "completions/min_terminated_length": 322.0, "entropy": 0.48598330840468407, "epoch": 0.0005353319057815846, "frac_reward_zero_std": 0.25, "grad_norm": 0.0032464242540299892, "learning_rate": 1e-05, "loss": 0.0473, "num_tokens": 25147.0, "reward": 0.46875, "reward_std": 0.2651650309562683, "rewards/accuracy_reward/mean": 0.46875, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.2930827140808105, "sampling/importance_sampling_ratio/mean": 0.9998305439949036, "sampling/importance_sampling_ratio/min": 0.6979114413261414, "sampling/sampling_logp_difference/max": 0.3596630096435547, "sampling/sampling_logp_difference/mean": 0.012516415677964687, "step": 1 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 743.0, "completions/mean_length": 605.75, "completions/mean_terminated_length": 560.3200073242188, "completions/min_length": 274.0, "completions/min_terminated_length": 274.0, "entropy": 0.4345819912850857, "epoch": 0.0010706638115631692, "frac_reward_zero_std": 0.25, "grad_norm": 0.004117041360586882, "learning_rate": 1e-05, "loss": 0.076, "num_tokens": 48379.0, "reward": 0.59375, "reward_std": 0.3198433816432953, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.8063730001449585, "sampling/importance_sampling_ratio/mean": 1.0002689361572266, "sampling/importance_sampling_ratio/min": 0.5230656266212463, "sampling/sampling_logp_difference/max": 0.6480484008789062, "sampling/sampling_logp_difference/mean": 0.011621036566793919, "step": 2 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.28125, "completions/max_length": 768.0, "completions/max_terminated_length": 698.0, "completions/mean_length": 583.4375, "completions/mean_terminated_length": 511.2174072265625, "completions/min_length": 204.0, "completions/min_terminated_length": 204.0, "entropy": 0.5384182259440422, "epoch": 0.0016059957173447537, "frac_reward_zero_std": 0.0, "grad_norm": 0.01117366086691618, "learning_rate": 1e-05, "loss": 0.1108, "num_tokens": 70521.0, "reward": 0.65625, "reward_std": 0.47137710452079773, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.4263523817062378, "sampling/importance_sampling_ratio/mean": 1.0000293254852295, "sampling/importance_sampling_ratio/min": 0.7268283367156982, "sampling/sampling_logp_difference/max": 0.3551204204559326, "sampling/sampling_logp_difference/mean": 0.013133048079907894, "step": 3 }, { "clip_ratio/high_max": 4.39831092080567e-05, "clip_ratio/high_mean": 4.39831092080567e-05, "clip_ratio/low_mean": 0.00010378849401604384, "clip_ratio/low_min": 0.00010378849401604384, "clip_ratio/region_mean": 0.00014777160322410055, "completions/clipped_ratio": 0.40625, "completions/max_length": 768.0, "completions/max_terminated_length": 734.0, "completions/mean_length": 613.5625, "completions/mean_terminated_length": 507.8947448730469, "completions/min_length": 217.0, "completions/min_terminated_length": 217.0, "entropy": 0.47148050367832184, "epoch": 0.0021413276231263384, "frac_reward_zero_std": 0.5, "grad_norm": 0.013791498728096485, "learning_rate": 1e-05, "loss": -0.0218, "num_tokens": 94467.0, "reward": 0.34375, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.34375, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.3169457912445068, "sampling/importance_sampling_ratio/mean": 1.0000814199447632, "sampling/importance_sampling_ratio/min": 0.7270077466964722, "sampling/sampling_logp_difference/max": 0.3188180923461914, "sampling/sampling_logp_difference/mean": 0.012500625103712082, "step": 4 }, { "clip_ratio/high_max": 4.451566928764805e-05, "clip_ratio/high_mean": 4.451566928764805e-05, "clip_ratio/low_mean": 0.000116039089334663, "clip_ratio/low_min": 0.000116039089334663, "clip_ratio/region_mean": 0.00016055475862231106, "completions/clipped_ratio": 0.34375, "completions/max_length": 768.0, "completions/max_terminated_length": 744.0, "completions/mean_length": 626.59375, "completions/mean_terminated_length": 552.5238037109375, "completions/min_length": 356.0, "completions/min_terminated_length": 356.0, "entropy": 0.47515081241726875, "epoch": 0.0026766595289079227, "frac_reward_zero_std": 0.0, "grad_norm": 0.012919590808451176, "learning_rate": 1e-05, "loss": 0.0341, "num_tokens": 118310.0, "reward": 0.5, "reward_std": 0.5175491571426392, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5080004930496216, "sampling/importance_sampling_ratio/max": 1.4363598823547363, "sampling/importance_sampling_ratio/mean": 1.0000596046447754, "sampling/importance_sampling_ratio/min": 0.7076445817947388, "sampling/sampling_logp_difference/max": 0.36211204528808594, "sampling/sampling_logp_difference/mean": 0.013171524740755558, "step": 5 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0003781330888159573, "clip_ratio/low_min": 0.0003781330888159573, "clip_ratio/region_mean": 0.0003781330888159573, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 763.0, "completions/mean_length": 558.78125, "completions/mean_terminated_length": 520.0370483398438, "completions/min_length": 160.0, "completions/min_terminated_length": 160.0, "entropy": 0.5127999149262905, "epoch": 0.0032119914346895075, "frac_reward_zero_std": 0.5, "grad_norm": 0.011963306926190853, "learning_rate": 1e-05, "loss": 0.0437, "num_tokens": 140095.0, "reward": 0.78125, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.5415970087051392, "sampling/importance_sampling_ratio/mean": 1.0000983476638794, "sampling/importance_sampling_ratio/min": 0.6230328679084778, "sampling/sampling_logp_difference/max": 0.4731559753417969, "sampling/sampling_logp_difference/mean": 0.013560203835368156, "step": 6 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00012046683696098626, "clip_ratio/low_min": 0.00012046683696098626, "clip_ratio/region_mean": 0.00012046683696098626, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 762.0, "completions/mean_length": 565.0, "completions/mean_terminated_length": 544.0, "completions/min_length": 304.0, "completions/min_terminated_length": 304.0, "entropy": 0.4087902195751667, "epoch": 0.003747323340471092, "frac_reward_zero_std": 0.25, "grad_norm": 0.010129212401807308, "learning_rate": 1e-05, "loss": 0.0421, "num_tokens": 161799.0, "reward": 0.71875, "reward_std": 0.3787454068660736, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.552953839302063, "sampling/importance_sampling_ratio/mean": 0.9997699856758118, "sampling/importance_sampling_ratio/min": 0.7039182186126709, "sampling/sampling_logp_difference/max": 0.4401588439941406, "sampling/sampling_logp_difference/mean": 0.011312289163470268, "step": 7 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.375, "completions/max_length": 768.0, "completions/max_terminated_length": 752.0, "completions/mean_length": 670.0, "completions/mean_terminated_length": 611.2000122070312, "completions/min_length": 372.0, "completions/min_terminated_length": 372.0, "entropy": 0.40500492975115776, "epoch": 0.004282655246252677, "frac_reward_zero_std": 0.5, "grad_norm": 0.005402647890150547, "learning_rate": 1e-05, "loss": 0.0368, "num_tokens": 186759.0, "reward": 0.53125, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.4619346857070923, "sampling/importance_sampling_ratio/mean": 0.999567449092865, "sampling/importance_sampling_ratio/min": 0.3118903934955597, "sampling/sampling_logp_difference/max": 1.1651034355163574, "sampling/sampling_logp_difference/mean": 0.010809365659952164, "step": 8 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.375, "completions/max_length": 768.0, "completions/max_terminated_length": 766.0, "completions/mean_length": 647.40625, "completions/mean_terminated_length": 575.0499877929688, "completions/min_length": 280.0, "completions/min_terminated_length": 280.0, "entropy": 0.36652812361717224, "epoch": 0.004817987152034261, "frac_reward_zero_std": 0.5, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0301, "num_tokens": 211412.0, "reward": 0.65625, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.2698006629943848, "sampling/importance_sampling_ratio/mean": 1.0001059770584106, "sampling/importance_sampling_ratio/min": 0.7396385073661804, "sampling/sampling_logp_difference/max": 0.3015937805175781, "sampling/sampling_logp_difference/mean": 0.010424943640828133, "step": 9 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.49688154226169e-05, "clip_ratio/low_min": 6.49688154226169e-05, "clip_ratio/region_mean": 6.49688154226169e-05, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 726.0, "completions/mean_length": 525.96875, "completions/mean_terminated_length": 518.1612548828125, "completions/min_length": 267.0, "completions/min_terminated_length": 267.0, "entropy": 0.36391472816467285, "epoch": 0.0053533190578158455, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0078, "num_tokens": 231275.0, "reward": 0.90625, "reward_std": 0.1293872892856598, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.32644522190094, "sampling/importance_sampling_ratio/mean": 1.0003172159194946, "sampling/importance_sampling_ratio/min": 0.7914782762527466, "sampling/sampling_logp_difference/max": 0.2825026512145996, "sampling/sampling_logp_difference/mean": 0.0099624739959836, "step": 10 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.069010537932627e-05, "clip_ratio/low_min": 4.069010537932627e-05, "clip_ratio/region_mean": 4.069010537932627e-05, "completions/clipped_ratio": 0.65625, "completions/max_length": 768.0, "completions/max_terminated_length": 719.0, "completions/mean_length": 708.84375, "completions/mean_terminated_length": 595.9091186523438, "completions/min_length": 324.0, "completions/min_terminated_length": 324.0, "entropy": 0.6045695021748543, "epoch": 0.005888650963597431, "frac_reward_zero_std": 0.5, "grad_norm": 0.010288149118423462, "learning_rate": 1e-05, "loss": 0.0168, "num_tokens": 257926.0, "reward": 0.1875, "reward_std": 0.249358132481575, "rewards/accuracy_reward/mean": 0.1875, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.4171395301818848, "sampling/importance_sampling_ratio/mean": 0.9999479651451111, "sampling/importance_sampling_ratio/min": 0.7186304330825806, "sampling/sampling_logp_difference/max": 0.34864044189453125, "sampling/sampling_logp_difference/mean": 0.015258465893566608, "step": 11 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 760.0, "completions/mean_length": 551.21875, "completions/mean_terminated_length": 490.5199890136719, "completions/min_length": 295.0, "completions/min_terminated_length": 295.0, "entropy": 0.40489397943019867, "epoch": 0.006423982869379015, "frac_reward_zero_std": 0.25, "grad_norm": 0.012296498753130436, "learning_rate": 1e-05, "loss": 0.0895, "num_tokens": 279069.0, "reward": 0.75, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.4719487428665161, "sampling/importance_sampling_ratio/mean": 0.9999110698699951, "sampling/importance_sampling_ratio/min": 0.704704761505127, "sampling/sampling_logp_difference/max": 0.38658714294433594, "sampling/sampling_logp_difference/mean": 0.011324138380587101, "step": 12 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00010533543172641657, "clip_ratio/low_min": 0.00010533543172641657, "clip_ratio/region_mean": 0.00010533543172641657, "completions/clipped_ratio": 0.28125, "completions/max_length": 768.0, "completions/max_terminated_length": 766.0, "completions/mean_length": 620.625, "completions/mean_terminated_length": 562.95654296875, "completions/min_length": 371.0, "completions/min_terminated_length": 371.0, "entropy": 0.6603880785405636, "epoch": 0.006959314775160599, "frac_reward_zero_std": 0.0, "grad_norm": 0.012373446486890316, "learning_rate": 1e-05, "loss": 0.0984, "num_tokens": 302353.0, "reward": 0.5625, "reward_std": 0.5260357856750488, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.3349218368530273, "sampling/importance_sampling_ratio/mean": 1.0000901222229004, "sampling/importance_sampling_ratio/min": 0.5918812155723572, "sampling/sampling_logp_difference/max": 0.524449348449707, "sampling/sampling_logp_difference/mean": 0.013712276704609394, "step": 13 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 9.522695108898915e-05, "clip_ratio/low_min": 9.522695108898915e-05, "clip_ratio/region_mean": 9.522695108898915e-05, "completions/clipped_ratio": 0.34375, "completions/max_length": 768.0, "completions/max_terminated_length": 741.0, "completions/mean_length": 620.0, "completions/mean_terminated_length": 542.4761962890625, "completions/min_length": 221.0, "completions/min_terminated_length": 221.0, "entropy": 0.39660315588116646, "epoch": 0.007494646680942184, "frac_reward_zero_std": 0.25, "grad_norm": 0.007810923270881176, "learning_rate": 1e-05, "loss": 0.0522, "num_tokens": 325745.0, "reward": 0.625, "reward_std": 0.3104073107242584, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.463096261024475, "sampling/importance_sampling_ratio/mean": 0.9998887181282043, "sampling/importance_sampling_ratio/min": 0.7350841164588928, "sampling/sampling_logp_difference/max": 0.3805549144744873, "sampling/sampling_logp_difference/mean": 0.011892049573361874, "step": 14 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.369415885070339e-05, "clip_ratio/low_min": 5.369415885070339e-05, "clip_ratio/region_mean": 5.369415885070339e-05, "completions/clipped_ratio": 0.40625, "completions/max_length": 768.0, "completions/max_terminated_length": 704.0, "completions/mean_length": 623.4375, "completions/mean_terminated_length": 524.5263061523438, "completions/min_length": 184.0, "completions/min_terminated_length": 184.0, "entropy": 0.38310953974723816, "epoch": 0.008029978586723769, "frac_reward_zero_std": 0.0, "grad_norm": 0.007060438394546509, "learning_rate": 1e-05, "loss": -0.0022, "num_tokens": 349143.0, "reward": 0.40625, "reward_std": 0.4765698313713074, "rewards/accuracy_reward/mean": 0.40625, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.4384175539016724, "sampling/importance_sampling_ratio/mean": 0.9999220371246338, "sampling/importance_sampling_ratio/min": 0.729432225227356, "sampling/sampling_logp_difference/max": 0.3635435104370117, "sampling/sampling_logp_difference/mean": 0.010586755350232124, "step": 15 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.657228055293672e-05, "clip_ratio/low_min": 4.657228055293672e-05, "clip_ratio/region_mean": 4.657228055293672e-05, "completions/clipped_ratio": 0.375, "completions/max_length": 768.0, "completions/max_terminated_length": 720.0, "completions/mean_length": 649.0625, "completions/mean_terminated_length": 577.7000122070312, "completions/min_length": 429.0, "completions/min_terminated_length": 429.0, "entropy": 0.7524018473923206, "epoch": 0.008565310492505354, "frac_reward_zero_std": 0.0, "grad_norm": 0.023527327924966812, "learning_rate": 1e-05, "loss": 0.0716, "num_tokens": 374665.0, "reward": 0.5, "reward_std": 0.4492306709289551, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5080004930496216, "sampling/importance_sampling_ratio/max": 1.5210533142089844, "sampling/importance_sampling_ratio/mean": 1.000063419342041, "sampling/importance_sampling_ratio/min": 0.637096107006073, "sampling/sampling_logp_difference/max": 0.4508347511291504, "sampling/sampling_logp_difference/mean": 0.01580382138490677, "step": 16 }, { "clip_ratio/high_max": 9.996496009989642e-05, "clip_ratio/high_mean": 9.996496009989642e-05, "clip_ratio/low_mean": 0.00015877384066698141, "clip_ratio/low_min": 0.00015877384066698141, "clip_ratio/region_mean": 0.00025873880076687783, "completions/clipped_ratio": 0.34375, "completions/max_length": 768.0, "completions/max_terminated_length": 736.0, "completions/mean_length": 617.75, "completions/mean_terminated_length": 539.047607421875, "completions/min_length": 299.0, "completions/min_terminated_length": 299.0, "entropy": 0.5123349986970425, "epoch": 0.009100642398286937, "frac_reward_zero_std": 0.0, "grad_norm": 0.016295379027724266, "learning_rate": 1e-05, "loss": 0.0207, "num_tokens": 398241.0, "reward": 0.25, "reward_std": 0.4629100561141968, "rewards/accuracy_reward/mean": 0.25, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.3084748983383179, "sampling/importance_sampling_ratio/mean": 0.9996823668479919, "sampling/importance_sampling_ratio/min": 0.7431430816650391, "sampling/sampling_logp_difference/max": 0.29686665534973145, "sampling/sampling_logp_difference/mean": 0.012664510868489742, "step": 17 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0001043877964548301, "clip_ratio/low_min": 0.0001043877964548301, "clip_ratio/region_mean": 0.0001043877964548301, "completions/clipped_ratio": 0.375, "completions/max_length": 768.0, "completions/max_terminated_length": 749.0, "completions/mean_length": 631.15625, "completions/mean_terminated_length": 549.0499877929688, "completions/min_length": 342.0, "completions/min_terminated_length": 342.0, "entropy": 0.42930785194039345, "epoch": 0.009635974304068522, "frac_reward_zero_std": 0.25, "grad_norm": 0.004890018608421087, "learning_rate": 1e-05, "loss": 0.0594, "num_tokens": 422318.0, "reward": 0.65625, "reward_std": 0.3608423173427582, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.3816895484924316, "sampling/importance_sampling_ratio/mean": 1.0002059936523438, "sampling/importance_sampling_ratio/min": 0.7530749440193176, "sampling/sampling_logp_difference/max": 0.3233070373535156, "sampling/sampling_logp_difference/mean": 0.011523136869072914, "step": 18 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.9780009905807674e-05, "clip_ratio/low_min": 5.9780009905807674e-05, "clip_ratio/region_mean": 5.9780009905807674e-05, "completions/clipped_ratio": 0.53125, "completions/max_length": 768.0, "completions/max_terminated_length": 716.0, "completions/mean_length": 647.40625, "completions/mean_terminated_length": 510.7333679199219, "completions/min_length": 260.0, "completions/min_terminated_length": 260.0, "entropy": 0.44347527250647545, "epoch": 0.010171306209850108, "frac_reward_zero_std": 0.25, "grad_norm": 0.010858778841793537, "learning_rate": 1e-05, "loss": 0.0795, "num_tokens": 447203.0, "reward": 0.4375, "reward_std": 0.3514062762260437, "rewards/accuracy_reward/mean": 0.4375, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.3750196695327759, "sampling/importance_sampling_ratio/mean": 0.9996398687362671, "sampling/importance_sampling_ratio/min": 0.7026919722557068, "sampling/sampling_logp_difference/max": 0.35283660888671875, "sampling/sampling_logp_difference/mean": 0.01247062161564827, "step": 19 }, { "clip_ratio/high_max": 6.782419950468466e-05, "clip_ratio/high_mean": 6.782419950468466e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 6.782419950468466e-05, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 707.0, "completions/mean_length": 496.125, "completions/mean_terminated_length": 445.77777099609375, "completions/min_length": 189.0, "completions/min_terminated_length": 189.0, "entropy": 0.49348969385027885, "epoch": 0.010706638115631691, "frac_reward_zero_std": 0.25, "grad_norm": 0.016775470227003098, "learning_rate": 1e-05, "loss": 0.0511, "num_tokens": 466759.0, "reward": 0.65625, "reward_std": 0.3377465009689331, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.519389271736145, "sampling/importance_sampling_ratio/mean": 0.9998260736465454, "sampling/importance_sampling_ratio/min": 0.7127357721328735, "sampling/sampling_logp_difference/max": 0.4183084964752197, "sampling/sampling_logp_difference/mean": 0.01263282261788845, "step": 20 }, { "clip_ratio/high_max": 5.5506217904621735e-05, "clip_ratio/high_mean": 5.5506217904621735e-05, "clip_ratio/low_mean": 4.98802874062676e-05, "clip_ratio/low_min": 4.98802874062676e-05, "clip_ratio/region_mean": 0.00010538650531088933, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 757.0, "completions/mean_length": 526.78125, "completions/mean_terminated_length": 482.1111145019531, "completions/min_length": 197.0, "completions/min_terminated_length": 197.0, "entropy": 0.4062811993062496, "epoch": 0.011241970021413276, "frac_reward_zero_std": 0.25, "grad_norm": 0.006292411591857672, "learning_rate": 1e-05, "loss": 0.0678, "num_tokens": 487648.0, "reward": 0.65625, "reward_std": 0.3198433816432953, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.2623493671417236, "sampling/importance_sampling_ratio/mean": 0.9999346733093262, "sampling/importance_sampling_ratio/min": 0.6255134344100952, "sampling/sampling_logp_difference/max": 0.46918249130249023, "sampling/sampling_logp_difference/mean": 0.011023581959307194, "step": 21 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.1377028537681326e-05, "clip_ratio/low_min": 4.1377028537681326e-05, "clip_ratio/region_mean": 4.1377028537681326e-05, "completions/clipped_ratio": 0.28125, "completions/max_length": 768.0, "completions/max_terminated_length": 756.0, "completions/mean_length": 599.15625, "completions/mean_terminated_length": 533.0869750976562, "completions/min_length": 280.0, "completions/min_terminated_length": 280.0, "entropy": 0.4398523196578026, "epoch": 0.011777301927194861, "frac_reward_zero_std": 0.0, "grad_norm": 0.0055059464648365974, "learning_rate": 1e-05, "loss": -0.0123, "num_tokens": 510197.0, "reward": 0.71875, "reward_std": 0.4534739851951599, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.4150367975234985, "sampling/importance_sampling_ratio/mean": 0.999511182308197, "sampling/importance_sampling_ratio/min": 0.6897582411766052, "sampling/sampling_logp_difference/max": 0.3714141845703125, "sampling/sampling_logp_difference/mean": 0.012775862589478493, "step": 22 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00016042274728533812, "clip_ratio/low_min": 0.00016042274728533812, "clip_ratio/region_mean": 0.00016042274728533812, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 717.0, "completions/mean_length": 589.46875, "completions/mean_terminated_length": 548.2692260742188, "completions/min_length": 303.0, "completions/min_terminated_length": 303.0, "entropy": 0.4353533983230591, "epoch": 0.012312633832976445, "frac_reward_zero_std": 0.25, "grad_norm": 0.020144155248999596, "learning_rate": 1e-05, "loss": 0.0159, "num_tokens": 533172.0, "reward": 0.71875, "reward_std": 0.3198433816432953, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.5432878732681274, "sampling/importance_sampling_ratio/mean": 1.0002663135528564, "sampling/importance_sampling_ratio/min": 0.69236159324646, "sampling/sampling_logp_difference/max": 0.4339151382446289, "sampling/sampling_logp_difference/mean": 0.01157721970230341, "step": 23 }, { "clip_ratio/high_max": 4.7819434257689863e-05, "clip_ratio/high_mean": 4.7819434257689863e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 4.7819434257689863e-05, "completions/clipped_ratio": 0.28125, "completions/max_length": 768.0, "completions/max_terminated_length": 739.0, "completions/mean_length": 628.625, "completions/mean_terminated_length": 574.0869750976562, "completions/min_length": 341.0, "completions/min_terminated_length": 341.0, "entropy": 0.511908870190382, "epoch": 0.01284796573875803, "frac_reward_zero_std": 0.0, "grad_norm": 0.00383503595367074, "learning_rate": 1e-05, "loss": 0.0682, "num_tokens": 557360.0, "reward": 0.65625, "reward_std": 0.3808925747871399, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.4700541496276855, "sampling/importance_sampling_ratio/mean": 0.99983811378479, "sampling/importance_sampling_ratio/min": 0.36265435814857483, "sampling/sampling_logp_difference/max": 1.0143051147460938, "sampling/sampling_logp_difference/mean": 0.013304595835506916, "step": 24 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00023704511841060594, "clip_ratio/low_min": 0.00023704511841060594, "clip_ratio/region_mean": 0.00023704511841060594, "completions/clipped_ratio": 0.3125, "completions/max_length": 768.0, "completions/max_terminated_length": 742.0, "completions/mean_length": 619.96875, "completions/mean_terminated_length": 552.6818237304688, "completions/min_length": 317.0, "completions/min_terminated_length": 317.0, "entropy": 0.5788429118692875, "epoch": 0.013383297644539615, "frac_reward_zero_std": 0.0, "grad_norm": 0.016527721658349037, "learning_rate": 1e-05, "loss": 0.1029, "num_tokens": 580695.0, "reward": 0.65625, "reward_std": 0.4628904461860657, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.442878246307373, "sampling/importance_sampling_ratio/mean": 0.9998840689659119, "sampling/importance_sampling_ratio/min": 0.7391331791877747, "sampling/sampling_logp_difference/max": 0.3666398525238037, "sampling/sampling_logp_difference/mean": 0.014603289775550365, "step": 25 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 8.138021075865254e-05, "clip_ratio/low_min": 8.138021075865254e-05, "clip_ratio/region_mean": 8.138021075865254e-05, "completions/clipped_ratio": 0.53125, "completions/max_length": 768.0, "completions/max_terminated_length": 708.0, "completions/mean_length": 666.21875, "completions/mean_terminated_length": 550.86669921875, "completions/min_length": 226.0, "completions/min_terminated_length": 226.0, "entropy": 0.48395277187228203, "epoch": 0.013918629550321198, "frac_reward_zero_std": 0.25, "grad_norm": 0.004640842787921429, "learning_rate": 1e-05, "loss": 0.0397, "num_tokens": 606054.0, "reward": 0.28125, "reward_std": 0.3471629321575165, "rewards/accuracy_reward/mean": 0.28125, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.3310222625732422, "sampling/importance_sampling_ratio/mean": 1.0000438690185547, "sampling/importance_sampling_ratio/min": 0.2625313699245453, "sampling/sampling_logp_difference/max": 1.3373847007751465, "sampling/sampling_logp_difference/mean": 0.01332576759159565, "step": 26 }, { "clip_ratio/high_max": 5.9073725424241275e-05, "clip_ratio/high_mean": 5.9073725424241275e-05, "clip_ratio/low_mean": 9.77333947957959e-05, "clip_ratio/low_min": 9.77333947957959e-05, "clip_ratio/region_mean": 0.00015680712022003718, "completions/clipped_ratio": 0.5625, "completions/max_length": 768.0, "completions/max_terminated_length": 670.0, "completions/mean_length": 666.40625, "completions/mean_terminated_length": 535.7857666015625, "completions/min_length": 272.0, "completions/min_terminated_length": 272.0, "entropy": 0.5612205415964127, "epoch": 0.014453961456102784, "frac_reward_zero_std": 0.0, "grad_norm": 0.021868634968996048, "learning_rate": 1e-05, "loss": 0.0633, "num_tokens": 631235.0, "reward": 0.34375, "reward_std": 0.4218915104866028, "rewards/accuracy_reward/mean": 0.34375, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.4124186038970947, "sampling/importance_sampling_ratio/mean": 0.9998308420181274, "sampling/importance_sampling_ratio/min": 0.7335000038146973, "sampling/sampling_logp_difference/max": 0.3453035354614258, "sampling/sampling_logp_difference/mean": 0.013546819798648357, "step": 27 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.6921923058107495e-05, "clip_ratio/low_min": 4.6921923058107495e-05, "clip_ratio/region_mean": 4.6921923058107495e-05, "completions/clipped_ratio": 0.375, "completions/max_length": 768.0, "completions/max_terminated_length": 768.0, "completions/mean_length": 681.625, "completions/mean_terminated_length": 629.7999877929688, "completions/min_length": 450.0, "completions/min_terminated_length": 450.0, "entropy": 0.4054417908191681, "epoch": 0.014989293361884369, "frac_reward_zero_std": 0.25, "grad_norm": 0.00730283185839653, "learning_rate": 1e-05, "loss": 0.0291, "num_tokens": 657327.0, "reward": 0.375, "reward_std": 0.3924051821231842, "rewards/accuracy_reward/mean": 0.375, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.3310648202896118, "sampling/importance_sampling_ratio/mean": 0.999858558177948, "sampling/importance_sampling_ratio/min": 0.7653622627258301, "sampling/sampling_logp_difference/max": 0.2859792709350586, "sampling/sampling_logp_difference/mean": 0.011034416034817696, "step": 28 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.615453846985474e-05, "clip_ratio/low_min": 5.615453846985474e-05, "clip_ratio/region_mean": 5.615453846985474e-05, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 745.0, "completions/mean_length": 635.21875, "completions/mean_terminated_length": 590.9583740234375, "completions/min_length": 393.0, "completions/min_terminated_length": 393.0, "entropy": 0.3583945333957672, "epoch": 0.015524625267665952, "frac_reward_zero_std": 0.0, "grad_norm": 0.021860741078853607, "learning_rate": 1e-05, "loss": 0.0738, "num_tokens": 681894.0, "reward": 0.65625, "reward_std": 0.4397946000099182, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0001440048217773, "sampling/importance_sampling_ratio/min": 0.6967951655387878, "sampling/sampling_logp_difference/max": 0.967193603515625, "sampling/sampling_logp_difference/mean": 0.010393482632935047, "step": 29 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.53125, "completions/max_length": 768.0, "completions/max_terminated_length": 755.0, "completions/mean_length": 693.8125, "completions/mean_terminated_length": 609.7333374023438, "completions/min_length": 354.0, "completions/min_terminated_length": 354.0, "entropy": 0.7556599229574203, "epoch": 0.016059957173447537, "frac_reward_zero_std": 0.5, "grad_norm": 0.005575866438448429, "learning_rate": 1e-05, "loss": 0.0234, "num_tokens": 708392.0, "reward": 0.3125, "reward_std": 0.2587745785713196, "rewards/accuracy_reward/mean": 0.3125, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.5003604888916016, "sampling/importance_sampling_ratio/mean": 0.9996361136436462, "sampling/importance_sampling_ratio/min": 0.6749789714813232, "sampling/sampling_logp_difference/max": 0.40570545196533203, "sampling/sampling_logp_difference/mean": 0.01721298322081566, "step": 30 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 9.409141421201639e-05, "clip_ratio/low_min": 9.409141421201639e-05, "clip_ratio/region_mean": 9.409141421201639e-05, "completions/clipped_ratio": 0.3125, "completions/max_length": 768.0, "completions/max_terminated_length": 690.0, "completions/mean_length": 607.5, "completions/mean_terminated_length": 534.5454711914062, "completions/min_length": 351.0, "completions/min_terminated_length": 351.0, "entropy": 0.3454369604587555, "epoch": 0.016595289079229122, "frac_reward_zero_std": 0.25, "grad_norm": 0.01821541041135788, "learning_rate": 1e-05, "loss": 0.0106, "num_tokens": 731512.0, "reward": 0.6875, "reward_std": 0.292504221200943, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.6597185134887695, "sampling/importance_sampling_ratio/mean": 1.0002830028533936, "sampling/importance_sampling_ratio/min": 0.7118902206420898, "sampling/sampling_logp_difference/max": 0.506648063659668, "sampling/sampling_logp_difference/mean": 0.00990574061870575, "step": 31 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 9.516424688627012e-05, "clip_ratio/low_min": 9.516424688627012e-05, "clip_ratio/region_mean": 9.516424688627012e-05, "completions/clipped_ratio": 0.375, "completions/max_length": 768.0, "completions/max_terminated_length": 731.0, "completions/mean_length": 613.40625, "completions/mean_terminated_length": 520.6500244140625, "completions/min_length": 179.0, "completions/min_terminated_length": 179.0, "entropy": 0.3875400647521019, "epoch": 0.017130620985010708, "frac_reward_zero_std": 0.0, "grad_norm": 0.013307609595358372, "learning_rate": 1e-05, "loss": 0.0834, "num_tokens": 754885.0, "reward": 0.46875, "reward_std": 0.4534739851951599, "rewards/accuracy_reward/mean": 0.46875, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0003117322921753, "sampling/importance_sampling_ratio/min": 0.687193751335144, "sampling/sampling_logp_difference/max": 1.1515445709228516, "sampling/sampling_logp_difference/mean": 0.011079014278948307, "step": 32 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.041565939085558e-05, "clip_ratio/low_min": 6.041565939085558e-05, "clip_ratio/region_mean": 6.041565939085558e-05, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 761.0, "completions/mean_length": 566.3125, "completions/mean_terminated_length": 537.5, "completions/min_length": 279.0, "completions/min_terminated_length": 279.0, "entropy": 0.4358655549585819, "epoch": 0.017665952890792293, "frac_reward_zero_std": 0.5, "grad_norm": 0.00740576907992363, "learning_rate": 1e-05, "loss": 0.0131, "num_tokens": 776631.0, "reward": 0.625, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.449811339378357, "sampling/importance_sampling_ratio/mean": 1.0000813007354736, "sampling/importance_sampling_ratio/min": 0.7329732775688171, "sampling/sampling_logp_difference/max": 0.3714334964752197, "sampling/sampling_logp_difference/mean": 0.012371337972581387, "step": 33 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.46875, "completions/max_length": 768.0, "completions/max_terminated_length": 768.0, "completions/mean_length": 663.46875, "completions/mean_terminated_length": 571.2352905273438, "completions/min_length": 199.0, "completions/min_terminated_length": 199.0, "entropy": 0.5534002110362053, "epoch": 0.018201284796573874, "frac_reward_zero_std": 0.5, "grad_norm": 0.008740512654185295, "learning_rate": 1e-05, "loss": 0.0472, "num_tokens": 801750.0, "reward": 0.46875, "reward_std": 0.2630178928375244, "rewards/accuracy_reward/mean": 0.46875, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.5338393449783325, "sampling/importance_sampling_ratio/mean": 1.0004141330718994, "sampling/importance_sampling_ratio/min": 0.5497848391532898, "sampling/sampling_logp_difference/max": 0.5982282161712646, "sampling/sampling_logp_difference/mean": 0.013347679749131203, "step": 34 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.1848008550005034e-05, "clip_ratio/low_min": 4.1848008550005034e-05, "clip_ratio/region_mean": 4.1848008550005034e-05, "completions/clipped_ratio": 0.34375, "completions/max_length": 768.0, "completions/max_terminated_length": 755.0, "completions/mean_length": 625.46875, "completions/mean_terminated_length": 550.8095092773438, "completions/min_length": 296.0, "completions/min_terminated_length": 296.0, "entropy": 0.5512795709073544, "epoch": 0.01873661670235546, "frac_reward_zero_std": 0.25, "grad_norm": 0.007339011412113905, "learning_rate": 1e-05, "loss": 0.0789, "num_tokens": 825661.0, "reward": 0.59375, "reward_std": 0.3608423173427582, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.2961760759353638, "sampling/importance_sampling_ratio/mean": 1.0002658367156982, "sampling/importance_sampling_ratio/min": 0.7105289697647095, "sampling/sampling_logp_difference/max": 0.34174561500549316, "sampling/sampling_logp_difference/mean": 0.013928350992500782, "step": 35 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00014985193411121145, "clip_ratio/low_min": 0.00014985193411121145, "clip_ratio/region_mean": 0.00014985193411121145, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 733.0, "completions/mean_length": 564.9375, "completions/mean_terminated_length": 518.0769653320312, "completions/min_length": 194.0, "completions/min_terminated_length": 194.0, "entropy": 0.3453544117510319, "epoch": 0.019271948608137045, "frac_reward_zero_std": 0.25, "grad_norm": 0.005400785245001316, "learning_rate": 1e-05, "loss": -0.0211, "num_tokens": 847395.0, "reward": 0.59375, "reward_std": 0.3377465009689331, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.2773507833480835, "sampling/importance_sampling_ratio/mean": 1.000115156173706, "sampling/importance_sampling_ratio/min": 0.6307759881019592, "sampling/sampling_logp_difference/max": 0.46080446243286133, "sampling/sampling_logp_difference/mean": 0.009833919815719128, "step": 36 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.5126354962121695e-05, "clip_ratio/low_min": 4.5126354962121695e-05, "clip_ratio/region_mean": 4.5126354962121695e-05, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 737.0, "completions/mean_length": 590.5, "completions/mean_terminated_length": 549.5384521484375, "completions/min_length": 369.0, "completions/min_terminated_length": 369.0, "entropy": 0.4163068048655987, "epoch": 0.01980728051391863, "frac_reward_zero_std": 0.25, "grad_norm": 0.019713765010237694, "learning_rate": 1e-05, "loss": 0.0623, "num_tokens": 870251.0, "reward": 0.71875, "reward_std": 0.2651650309562683, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.416803002357483, "sampling/importance_sampling_ratio/mean": 1.0000214576721191, "sampling/importance_sampling_ratio/min": 0.6959530711174011, "sampling/sampling_logp_difference/max": 0.3624730110168457, "sampling/sampling_logp_difference/mean": 0.011494419537484646, "step": 37 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 9.050188964465633e-05, "clip_ratio/low_min": 9.050188964465633e-05, "clip_ratio/region_mean": 9.050188964465633e-05, "completions/clipped_ratio": 0.3125, "completions/max_length": 768.0, "completions/max_terminated_length": 755.0, "completions/mean_length": 635.71875, "completions/mean_terminated_length": 575.5909423828125, "completions/min_length": 368.0, "completions/min_terminated_length": 368.0, "entropy": 0.41894080862402916, "epoch": 0.020342612419700215, "frac_reward_zero_std": 0.0, "grad_norm": 0.00975870992988348, "learning_rate": 1e-05, "loss": 0.0046, "num_tokens": 894474.0, "reward": 0.46875, "reward_std": 0.4628904461860657, "rewards/accuracy_reward/mean": 0.46875, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.442466139793396, "sampling/importance_sampling_ratio/mean": 1.0000383853912354, "sampling/importance_sampling_ratio/min": 0.6907856464385986, "sampling/sampling_logp_difference/max": 0.36992573738098145, "sampling/sampling_logp_difference/mean": 0.012097334489226341, "step": 38 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.9682035751175135e-05, "clip_ratio/low_min": 4.9682035751175135e-05, "clip_ratio/region_mean": 4.9682035751175135e-05, "completions/clipped_ratio": 0.375, "completions/max_length": 768.0, "completions/max_terminated_length": 768.0, "completions/mean_length": 631.65625, "completions/mean_terminated_length": 549.8500366210938, "completions/min_length": 230.0, "completions/min_terminated_length": 230.0, "entropy": 0.40683867037296295, "epoch": 0.0208779443254818, "frac_reward_zero_std": 0.25, "grad_norm": 0.007603779900819063, "learning_rate": 1e-05, "loss": 0.0653, "num_tokens": 918247.0, "reward": 0.5, "reward_std": 0.3514062464237213, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5080004930496216, "sampling/importance_sampling_ratio/max": 1.395285964012146, "sampling/importance_sampling_ratio/mean": 0.9997595548629761, "sampling/importance_sampling_ratio/min": 0.6868053674697876, "sampling/sampling_logp_difference/max": 0.375704288482666, "sampling/sampling_logp_difference/mean": 0.010730033740401268, "step": 39 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.225752465776168e-05, "clip_ratio/low_min": 5.225752465776168e-05, "clip_ratio/region_mean": 5.225752465776168e-05, "completions/clipped_ratio": 0.34375, "completions/max_length": 768.0, "completions/max_terminated_length": 746.0, "completions/mean_length": 612.875, "completions/mean_terminated_length": 531.6190795898438, "completions/min_length": 380.0, "completions/min_terminated_length": 380.0, "entropy": 0.6648869961500168, "epoch": 0.021413276231263382, "frac_reward_zero_std": 0.5, "grad_norm": 0.0019930468406528234, "learning_rate": 1e-05, "loss": 0.0421, "num_tokens": 941723.0, "reward": 0.5, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5080004930496216, "sampling/importance_sampling_ratio/max": 1.2759915590286255, "sampling/importance_sampling_ratio/mean": 1.0007076263427734, "sampling/importance_sampling_ratio/min": 0.7806393504142761, "sampling/sampling_logp_difference/max": 0.24764204025268555, "sampling/sampling_logp_difference/mean": 0.01612691953778267, "step": 40 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.137690095580183e-05, "clip_ratio/low_min": 5.137690095580183e-05, "clip_ratio/region_mean": 5.137690095580183e-05, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 685.0, "completions/mean_length": 542.25, "completions/mean_terminated_length": 479.03997802734375, "completions/min_length": 282.0, "completions/min_terminated_length": 282.0, "entropy": 0.40131812915205956, "epoch": 0.021948608137044967, "frac_reward_zero_std": 0.25, "grad_norm": 0.0050626760348677635, "learning_rate": 1e-05, "loss": 0.0117, "num_tokens": 962587.0, "reward": 0.5, "reward_std": 0.292504221200943, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5080004930496216, "sampling/importance_sampling_ratio/max": 1.278491497039795, "sampling/importance_sampling_ratio/mean": 0.9998194575309753, "sampling/importance_sampling_ratio/min": 0.7864078879356384, "sampling/sampling_logp_difference/max": 0.2456808090209961, "sampling/sampling_logp_difference/mean": 0.011833418160676956, "step": 41 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.000168136422871612, "clip_ratio/low_min": 0.000168136422871612, "clip_ratio/region_mean": 0.000168136422871612, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 753.0, "completions/mean_length": 565.09375, "completions/mean_terminated_length": 527.5184936523438, "completions/min_length": 215.0, "completions/min_terminated_length": 215.0, "entropy": 0.3964141681790352, "epoch": 0.022483940042826552, "frac_reward_zero_std": 0.0, "grad_norm": 0.017772329971194267, "learning_rate": 1e-05, "loss": 0.0801, "num_tokens": 984094.0, "reward": 0.59375, "reward_std": 0.4628904461860657, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.447232723236084, "sampling/importance_sampling_ratio/mean": 0.9995170831680298, "sampling/importance_sampling_ratio/min": 0.6852384805679321, "sampling/sampling_logp_difference/max": 0.377988338470459, "sampling/sampling_logp_difference/mean": 0.011716882698237896, "step": 42 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 9.637625043978915e-05, "clip_ratio/low_min": 9.637625043978915e-05, "clip_ratio/region_mean": 9.637625043978915e-05, "completions/clipped_ratio": 0.3125, "completions/max_length": 768.0, "completions/max_terminated_length": 758.0, "completions/mean_length": 568.65625, "completions/mean_terminated_length": 478.04547119140625, "completions/min_length": 233.0, "completions/min_terminated_length": 233.0, "entropy": 0.39531830325722694, "epoch": 0.023019271948608137, "frac_reward_zero_std": 0.5, "grad_norm": 0.0030931883957237005, "learning_rate": 1e-05, "loss": 0.0374, "num_tokens": 1006171.0, "reward": 0.5, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5080004930496216, "sampling/importance_sampling_ratio/max": 1.4095613956451416, "sampling/importance_sampling_ratio/mean": 1.0000072717666626, "sampling/importance_sampling_ratio/min": 0.7386751770973206, "sampling/sampling_logp_difference/max": 0.3432786464691162, "sampling/sampling_logp_difference/mean": 0.011342217214405537, "step": 43 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.0220973207615316e-05, "clip_ratio/low_min": 5.0220973207615316e-05, "clip_ratio/region_mean": 5.0220973207615316e-05, "completions/clipped_ratio": 0.28125, "completions/max_length": 768.0, "completions/max_terminated_length": 767.0, "completions/mean_length": 591.84375, "completions/mean_terminated_length": 522.9130249023438, "completions/min_length": 268.0, "completions/min_terminated_length": 268.0, "entropy": 0.39927684888243675, "epoch": 0.023554603854389723, "frac_reward_zero_std": 0.25, "grad_norm": 0.005198259372264147, "learning_rate": 1e-05, "loss": 0.0131, "num_tokens": 1028838.0, "reward": 0.75, "reward_std": 0.292504221200943, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.3477733135223389, "sampling/importance_sampling_ratio/mean": 0.9998369216918945, "sampling/importance_sampling_ratio/min": 0.6835851669311523, "sampling/sampling_logp_difference/max": 0.380403995513916, "sampling/sampling_logp_difference/mean": 0.00997208058834076, "step": 44 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.4658805563813075e-05, "clip_ratio/low_min": 4.4658805563813075e-05, "clip_ratio/region_mean": 4.4658805563813075e-05, "completions/clipped_ratio": 0.28125, "completions/max_length": 768.0, "completions/max_terminated_length": 764.0, "completions/mean_length": 632.125, "completions/mean_terminated_length": 578.95654296875, "completions/min_length": 437.0, "completions/min_terminated_length": 437.0, "entropy": 0.4036395661532879, "epoch": 0.024089935760171308, "frac_reward_zero_std": 0.25, "grad_norm": 0.007570364512503147, "learning_rate": 1e-05, "loss": 0.0508, "num_tokens": 1053194.0, "reward": 0.6875, "reward_std": 0.3514062464237213, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.365864872932434, "sampling/importance_sampling_ratio/mean": 1.0003732442855835, "sampling/importance_sampling_ratio/min": 0.7070538401603699, "sampling/sampling_logp_difference/max": 0.3466484546661377, "sampling/sampling_logp_difference/mean": 0.011278525926172733, "step": 45 }, { "clip_ratio/high_max": 4.98802874062676e-05, "clip_ratio/high_mean": 4.98802874062676e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 4.98802874062676e-05, "completions/clipped_ratio": 0.40625, "completions/max_length": 768.0, "completions/max_terminated_length": 765.0, "completions/mean_length": 636.6875, "completions/mean_terminated_length": 546.8421020507812, "completions/min_length": 301.0, "completions/min_terminated_length": 301.0, "entropy": 0.40303152799606323, "epoch": 0.02462526766595289, "frac_reward_zero_std": 0.25, "grad_norm": 0.0037608416751027107, "learning_rate": 1e-05, "loss": 0.0215, "num_tokens": 1077584.0, "reward": 0.65625, "reward_std": 0.3061639666557312, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.2744766473770142, "sampling/importance_sampling_ratio/mean": 0.999945342540741, "sampling/importance_sampling_ratio/min": 0.7364591360092163, "sampling/sampling_logp_difference/max": 0.30590152740478516, "sampling/sampling_logp_difference/mean": 0.01124804001301527, "step": 46 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 752.0, "completions/mean_length": 557.90625, "completions/mean_terminated_length": 527.8928833007812, "completions/min_length": 315.0, "completions/min_terminated_length": 315.0, "entropy": 0.5076882019639015, "epoch": 0.025160599571734475, "frac_reward_zero_std": 0.0, "grad_norm": 0.010317984037101269, "learning_rate": 1e-05, "loss": 0.048, "num_tokens": 1099197.0, "reward": 0.65625, "reward_std": 0.4355708956718445, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.2960543632507324, "sampling/importance_sampling_ratio/mean": 0.999902069568634, "sampling/importance_sampling_ratio/min": 0.627313494682312, "sampling/sampling_logp_difference/max": 0.4663088321685791, "sampling/sampling_logp_difference/mean": 0.013130159117281437, "step": 47 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 751.0, "completions/mean_length": 497.90625, "completions/mean_terminated_length": 469.96551513671875, "completions/min_length": 329.0, "completions/min_terminated_length": 329.0, "entropy": 0.4591529108583927, "epoch": 0.02569593147751606, "frac_reward_zero_std": 0.5, "grad_norm": 0.00704563595354557, "learning_rate": 1e-05, "loss": 0.0141, "num_tokens": 1118266.0, "reward": 0.78125, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.2926690578460693, "sampling/importance_sampling_ratio/mean": 0.9999036192893982, "sampling/importance_sampling_ratio/min": 0.6051042079925537, "sampling/sampling_logp_difference/max": 0.502354621887207, "sampling/sampling_logp_difference/mean": 0.011426439508795738, "step": 48 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00020978445900254883, "clip_ratio/low_min": 0.00020978445900254883, "clip_ratio/region_mean": 0.00020978445900254883, "completions/clipped_ratio": 0.3125, "completions/max_length": 768.0, "completions/max_terminated_length": 731.0, "completions/mean_length": 565.28125, "completions/mean_terminated_length": 473.1363830566406, "completions/min_length": 257.0, "completions/min_terminated_length": 257.0, "entropy": 0.5920190773904324, "epoch": 0.026231263383297645, "frac_reward_zero_std": 0.0, "grad_norm": 0.025336306542158127, "learning_rate": 1e-05, "loss": 0.0812, "num_tokens": 1140291.0, "reward": 0.59375, "reward_std": 0.4218915104866028, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.2873826026916504, "sampling/importance_sampling_ratio/mean": 0.9996625781059265, "sampling/importance_sampling_ratio/min": 0.7075410485267639, "sampling/sampling_logp_difference/max": 0.3459596633911133, "sampling/sampling_logp_difference/mean": 0.015037329867482185, "step": 49 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 768.0, "completions/mean_length": 567.90625, "completions/mean_terminated_length": 501.2083435058594, "completions/min_length": 245.0, "completions/min_terminated_length": 245.0, "entropy": 0.37092962861061096, "epoch": 0.02676659528907923, "frac_reward_zero_std": 0.75, "grad_norm": 0.006071723531931639, "learning_rate": 1e-05, "loss": 0.0158, "num_tokens": 1162016.0, "reward": 0.5625, "reward_std": 0.1157275140285492, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.3203513622283936, "sampling/importance_sampling_ratio/mean": 1.0002940893173218, "sampling/importance_sampling_ratio/min": 0.7036080360412598, "sampling/sampling_logp_difference/max": 0.3515338897705078, "sampling/sampling_logp_difference/mean": 0.010587909258902073, "step": 50 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 738.0, "completions/mean_length": 604.125, "completions/mean_terminated_length": 573.7777709960938, "completions/min_length": 364.0, "completions/min_terminated_length": 364.0, "entropy": 0.37366531416773796, "epoch": 0.027301927194860815, "frac_reward_zero_std": 0.25, "grad_norm": 0.006548416335135698, "learning_rate": 1e-05, "loss": 0.0512, "num_tokens": 1185244.0, "reward": 0.65625, "reward_std": 0.3471629321575165, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.4837409257888794, "sampling/importance_sampling_ratio/mean": 1.0000320672988892, "sampling/importance_sampling_ratio/min": 0.7552201747894287, "sampling/sampling_logp_difference/max": 0.39456653594970703, "sampling/sampling_logp_difference/mean": 0.0101691335439682, "step": 51 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00010076310354634188, "clip_ratio/low_min": 0.00010076310354634188, "clip_ratio/region_mean": 0.00010076310354634188, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 768.0, "completions/mean_length": 580.0, "completions/mean_terminated_length": 517.3333740234375, "completions/min_length": 247.0, "completions/min_terminated_length": 247.0, "entropy": 0.43295803666114807, "epoch": 0.027837259100642397, "frac_reward_zero_std": 0.25, "grad_norm": 0.011636517941951752, "learning_rate": 1e-05, "loss": 0.0631, "num_tokens": 1207460.0, "reward": 0.4375, "reward_std": 0.2925041913986206, "rewards/accuracy_reward/mean": 0.4375, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.3694878816604614, "sampling/importance_sampling_ratio/mean": 0.9999204277992249, "sampling/importance_sampling_ratio/min": 0.7504498958587646, "sampling/sampling_logp_difference/max": 0.3144369125366211, "sampling/sampling_logp_difference/mean": 0.012383855879306793, "step": 52 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.8823530707741156e-05, "clip_ratio/low_min": 5.8823530707741156e-05, "clip_ratio/region_mean": 5.8823530707741156e-05, "completions/clipped_ratio": 0.28125, "completions/max_length": 768.0, "completions/max_terminated_length": 753.0, "completions/mean_length": 640.71875, "completions/mean_terminated_length": 590.9130249023438, "completions/min_length": 338.0, "completions/min_terminated_length": 338.0, "entropy": 0.42586465552449226, "epoch": 0.028372591006423982, "frac_reward_zero_std": 0.0, "grad_norm": 0.01022303570061922, "learning_rate": 1e-05, "loss": 0.039, "num_tokens": 1231555.0, "reward": 0.6875, "reward_std": 0.4671337604522705, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.3079744577407837, "sampling/importance_sampling_ratio/mean": 1.0001554489135742, "sampling/importance_sampling_ratio/min": 0.6923260688781738, "sampling/sampling_logp_difference/max": 0.36769819259643555, "sampling/sampling_logp_difference/mean": 0.011949202977120876, "step": 53 }, { "clip_ratio/high_max": 5.577866977546364e-05, "clip_ratio/high_mean": 5.577866977546364e-05, "clip_ratio/low_mean": 0.00014533525245497003, "clip_ratio/low_min": 0.00014533525245497003, "clip_ratio/region_mean": 0.00020111392223043367, "completions/clipped_ratio": 0.40625, "completions/max_length": 768.0, "completions/max_terminated_length": 732.0, "completions/mean_length": 623.90625, "completions/mean_terminated_length": 525.3157958984375, "completions/min_length": 348.0, "completions/min_terminated_length": 348.0, "entropy": 0.554719552397728, "epoch": 0.028907922912205567, "frac_reward_zero_std": 0.25, "grad_norm": 0.011497048661112785, "learning_rate": 1e-05, "loss": 0.069, "num_tokens": 1255304.0, "reward": 0.5, "reward_std": 0.3514062464237213, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5080004930496216, "sampling/importance_sampling_ratio/max": 1.4615116119384766, "sampling/importance_sampling_ratio/mean": 0.9998668432235718, "sampling/importance_sampling_ratio/min": 0.6180641651153564, "sampling/sampling_logp_difference/max": 0.48116302490234375, "sampling/sampling_logp_difference/mean": 0.013421766459941864, "step": 54 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.28125, "completions/max_length": 768.0, "completions/max_terminated_length": 759.0, "completions/mean_length": 639.9375, "completions/mean_terminated_length": 589.8261108398438, "completions/min_length": 411.0, "completions/min_terminated_length": 411.0, "entropy": 0.48903779312968254, "epoch": 0.029443254817987152, "frac_reward_zero_std": 0.0, "grad_norm": 0.02093012072145939, "learning_rate": 1e-05, "loss": 0.0902, "num_tokens": 1279526.0, "reward": 0.75, "reward_std": 0.4355512857437134, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.558075189590454, "sampling/importance_sampling_ratio/mean": 1.0000821352005005, "sampling/importance_sampling_ratio/min": 0.7071018815040588, "sampling/sampling_logp_difference/max": 0.4434511661529541, "sampling/sampling_logp_difference/mean": 0.012461931444704533, "step": 55 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00010899157132371329, "clip_ratio/low_min": 0.00010899157132371329, "clip_ratio/region_mean": 0.00010899157132371329, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 757.0, "completions/mean_length": 573.6875, "completions/mean_terminated_length": 519.2799682617188, "completions/min_length": 228.0, "completions/min_terminated_length": 228.0, "entropy": 0.5139472596347332, "epoch": 0.029978586723768737, "frac_reward_zero_std": 0.5, "grad_norm": 0.012574407272040844, "learning_rate": 1e-05, "loss": -0.0136, "num_tokens": 1301644.0, "reward": 0.65625, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.303346872329712, "sampling/importance_sampling_ratio/mean": 1.0001070499420166, "sampling/importance_sampling_ratio/min": 0.7205821871757507, "sampling/sampling_logp_difference/max": 0.3276958465576172, "sampling/sampling_logp_difference/mean": 0.013474510982632637, "step": 56 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.1759834605036303e-05, "clip_ratio/low_min": 5.1759834605036303e-05, "clip_ratio/region_mean": 5.1759834605036303e-05, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 662.0, "completions/mean_length": 491.625, "completions/mean_terminated_length": 427.8461608886719, "completions/min_length": 263.0, "completions/min_terminated_length": 263.0, "entropy": 0.4833686873316765, "epoch": 0.030513918629550323, "frac_reward_zero_std": 0.25, "grad_norm": 0.004445187747478485, "learning_rate": 1e-05, "loss": 0.0981, "num_tokens": 1320616.0, "reward": 0.78125, "reward_std": 0.3061639964580536, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.339613914489746, "sampling/importance_sampling_ratio/mean": 0.9995917677879333, "sampling/importance_sampling_ratio/min": 0.7019649744033813, "sampling/sampling_logp_difference/max": 0.35387176275253296, "sampling/sampling_logp_difference/mean": 0.01272429060190916, "step": 57 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.459507545107044e-05, "clip_ratio/low_min": 4.459507545107044e-05, "clip_ratio/region_mean": 4.459507545107044e-05, "completions/clipped_ratio": 0.40625, "completions/max_length": 768.0, "completions/max_terminated_length": 733.0, "completions/mean_length": 648.4375, "completions/mean_terminated_length": 566.631591796875, "completions/min_length": 399.0, "completions/min_terminated_length": 399.0, "entropy": 0.4782155305147171, "epoch": 0.031049250535331904, "frac_reward_zero_std": 0.5, "grad_norm": 0.011375176720321178, "learning_rate": 1e-05, "loss": 0.0096, "num_tokens": 1345430.0, "reward": 0.5, "reward_std": 0.2314550280570984, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5080004930496216, "sampling/importance_sampling_ratio/max": 1.4411849975585938, "sampling/importance_sampling_ratio/mean": 1.000040054321289, "sampling/importance_sampling_ratio/min": 0.7014603614807129, "sampling/sampling_logp_difference/max": 0.3654656410217285, "sampling/sampling_logp_difference/mean": 0.012318131513893604, "step": 58 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0003555068797140848, "clip_ratio/low_min": 0.0003555068797140848, "clip_ratio/region_mean": 0.0003555068797140848, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 759.0, "completions/mean_length": 510.15625, "completions/mean_terminated_length": 492.9667053222656, "completions/min_length": 297.0, "completions/min_terminated_length": 297.0, "entropy": 0.562933512032032, "epoch": 0.03158458244111349, "frac_reward_zero_std": 0.25, "grad_norm": 0.00894081499427557, "learning_rate": 1e-05, "loss": -0.0186, "num_tokens": 1365619.0, "reward": 0.625, "reward_std": 0.3650856614112854, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.313398838043213, "sampling/importance_sampling_ratio/mean": 0.9997080564498901, "sampling/importance_sampling_ratio/min": 0.5310248136520386, "sampling/sampling_logp_difference/max": 0.6329464912414551, "sampling/sampling_logp_difference/mean": 0.013425687327980995, "step": 59 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.34375, "completions/max_length": 768.0, "completions/max_terminated_length": 642.0, "completions/mean_length": 583.03125, "completions/mean_terminated_length": 486.1428527832031, "completions/min_length": 339.0, "completions/min_terminated_length": 339.0, "entropy": 0.5129562467336655, "epoch": 0.032119914346895075, "frac_reward_zero_std": 0.5, "grad_norm": 0.005703146103769541, "learning_rate": 1e-05, "loss": 0.0481, "num_tokens": 1388652.0, "reward": 0.625, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.3739655017852783, "sampling/importance_sampling_ratio/mean": 0.9998830556869507, "sampling/importance_sampling_ratio/min": 0.45569929480552673, "sampling/sampling_logp_difference/max": 0.7859221696853638, "sampling/sampling_logp_difference/mean": 0.014459557831287384, "step": 60 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00010529793144087307, "clip_ratio/low_min": 0.00010529793144087307, "clip_ratio/region_mean": 0.00010529793144087307, "completions/clipped_ratio": 0.28125, "completions/max_length": 768.0, "completions/max_terminated_length": 752.0, "completions/mean_length": 596.875, "completions/mean_terminated_length": 529.9130249023438, "completions/min_length": 328.0, "completions/min_terminated_length": 328.0, "entropy": 0.45681414380669594, "epoch": 0.032655246252676656, "frac_reward_zero_std": 0.5, "grad_norm": 0.013348175212740898, "learning_rate": 1e-05, "loss": -0.0109, "num_tokens": 1411592.0, "reward": 0.5, "reward_std": 0.2314550280570984, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5080004930496216, "sampling/importance_sampling_ratio/max": 1.4582934379577637, "sampling/importance_sampling_ratio/mean": 0.9998890161514282, "sampling/importance_sampling_ratio/min": 0.7152434587478638, "sampling/sampling_logp_difference/max": 0.37726688385009766, "sampling/sampling_logp_difference/mean": 0.012740189209580421, "step": 61 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 768.0, "completions/mean_length": 509.625, "completions/mean_terminated_length": 450.0000305175781, "completions/min_length": 263.0, "completions/min_terminated_length": 263.0, "entropy": 0.5020339004695415, "epoch": 0.033190578158458245, "frac_reward_zero_std": 0.5, "grad_norm": 0.006378734949976206, "learning_rate": 1e-05, "loss": 0.0418, "num_tokens": 1431348.0, "reward": 0.53125, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.4372516870498657, "sampling/importance_sampling_ratio/mean": 1.000119924545288, "sampling/importance_sampling_ratio/min": 0.509554922580719, "sampling/sampling_logp_difference/max": 0.674217700958252, "sampling/sampling_logp_difference/mean": 0.014347701333463192, "step": 62 }, { "clip_ratio/high_max": 5.548158151214011e-05, "clip_ratio/high_mean": 5.548158151214011e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 5.548158151214011e-05, "completions/clipped_ratio": 0.46875, "completions/max_length": 768.0, "completions/max_terminated_length": 616.0, "completions/mean_length": 619.4375, "completions/mean_terminated_length": 488.3529357910156, "completions/min_length": 359.0, "completions/min_terminated_length": 359.0, "entropy": 0.5028683356940746, "epoch": 0.03372591006423983, "frac_reward_zero_std": 0.75, "grad_norm": 0.008214807137846947, "learning_rate": 1e-05, "loss": 0.047, "num_tokens": 1454962.0, "reward": 0.34375, "reward_std": 0.1293872892856598, "rewards/accuracy_reward/mean": 0.34375, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.3948371410369873, "sampling/importance_sampling_ratio/mean": 0.9998055696487427, "sampling/importance_sampling_ratio/min": 0.6906365752220154, "sampling/sampling_logp_difference/max": 0.3701416254043579, "sampling/sampling_logp_difference/mean": 0.013865773566067219, "step": 63 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00010791813110699877, "clip_ratio/low_min": 0.00010791813110699877, "clip_ratio/region_mean": 0.00010791813110699877, "completions/clipped_ratio": 0.28125, "completions/max_length": 768.0, "completions/max_terminated_length": 749.0, "completions/mean_length": 579.0, "completions/mean_terminated_length": 505.0434875488281, "completions/min_length": 194.0, "completions/min_terminated_length": 194.0, "entropy": 0.5049542114138603, "epoch": 0.034261241970021415, "frac_reward_zero_std": 0.25, "grad_norm": 0.011847138404846191, "learning_rate": 1e-05, "loss": 0.0918, "num_tokens": 1477394.0, "reward": 0.5, "reward_std": 0.3745020925998688, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5080004930496216, "sampling/importance_sampling_ratio/max": 1.4061970710754395, "sampling/importance_sampling_ratio/mean": 1.000219702720642, "sampling/importance_sampling_ratio/min": 0.39653947949409485, "sampling/sampling_logp_difference/max": 0.9249796867370605, "sampling/sampling_logp_difference/mean": 0.013348663225769997, "step": 64 }, { "clip_ratio/high_max": 5.86579080845695e-05, "clip_ratio/high_mean": 5.86579080845695e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 5.86579080845695e-05, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 746.0, "completions/mean_length": 606.375, "completions/mean_terminated_length": 576.4444580078125, "completions/min_length": 351.0, "completions/min_terminated_length": 351.0, "entropy": 0.3726109452545643, "epoch": 0.034796573875803, "frac_reward_zero_std": 0.5, "grad_norm": 0.012685705907642841, "learning_rate": 1e-05, "loss": 0.0339, "num_tokens": 1500550.0, "reward": 0.8125, "reward_std": 0.249358132481575, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.3059941530227661, "sampling/importance_sampling_ratio/mean": 1.0000492334365845, "sampling/importance_sampling_ratio/min": 0.6384655833244324, "sampling/sampling_logp_difference/max": 0.4486875534057617, "sampling/sampling_logp_difference/mean": 0.010820028372108936, "step": 65 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.527345299720764e-05, "clip_ratio/low_min": 4.527345299720764e-05, "clip_ratio/region_mean": 4.527345299720764e-05, "completions/clipped_ratio": 0.28125, "completions/max_length": 768.0, "completions/max_terminated_length": 745.0, "completions/mean_length": 582.0625, "completions/mean_terminated_length": 509.3043518066406, "completions/min_length": 306.0, "completions/min_terminated_length": 306.0, "entropy": 0.3910229839384556, "epoch": 0.035331905781584586, "frac_reward_zero_std": 0.5, "grad_norm": 0.00877825915813446, "learning_rate": 1e-05, "loss": 0.031, "num_tokens": 1522968.0, "reward": 0.6875, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.3906184434890747, "sampling/importance_sampling_ratio/mean": 1.0001715421676636, "sampling/importance_sampling_ratio/min": 0.7516657710075378, "sampling/sampling_logp_difference/max": 0.32974863052368164, "sampling/sampling_logp_difference/mean": 0.010856068693101406, "step": 66 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.145937055000104e-05, "clip_ratio/low_min": 4.145937055000104e-05, "clip_ratio/region_mean": 4.145937055000104e-05, "completions/clipped_ratio": 0.5, "completions/max_length": 768.0, "completions/max_terminated_length": 711.0, "completions/mean_length": 656.03125, "completions/mean_terminated_length": 544.0625, "completions/min_length": 342.0, "completions/min_terminated_length": 342.0, "entropy": 0.4979759305715561, "epoch": 0.03586723768736617, "frac_reward_zero_std": 0.5, "grad_norm": 0.0032111932523548603, "learning_rate": 1e-05, "loss": 0.017, "num_tokens": 1547993.0, "reward": 0.21875, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.21875, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.33022141456604, "sampling/importance_sampling_ratio/mean": 0.99981689453125, "sampling/importance_sampling_ratio/min": 0.5292699337005615, "sampling/sampling_logp_difference/max": 0.6362566947937012, "sampling/sampling_logp_difference/mean": 0.01353519968688488, "step": 67 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 697.0, "completions/mean_length": 469.375, "completions/mean_terminated_length": 459.7419128417969, "completions/min_length": 261.0, "completions/min_terminated_length": 261.0, "entropy": 0.43256181851029396, "epoch": 0.03640256959314775, "frac_reward_zero_std": 0.5, "grad_norm": 0.004933727905154228, "learning_rate": 1e-05, "loss": 0.0256, "num_tokens": 1566565.0, "reward": 0.8125, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.2607918977737427, "sampling/importance_sampling_ratio/mean": 0.9996660351753235, "sampling/importance_sampling_ratio/min": 0.6787375211715698, "sampling/sampling_logp_difference/max": 0.38752079010009766, "sampling/sampling_logp_difference/mean": 0.01155771128833294, "step": 68 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 758.0, "completions/mean_length": 567.375, "completions/mean_terminated_length": 511.1999816894531, "completions/min_length": 269.0, "completions/min_terminated_length": 269.0, "entropy": 0.34118273109197617, "epoch": 0.03693790149892934, "frac_reward_zero_std": 0.25, "grad_norm": 0.012756981886923313, "learning_rate": 1e-05, "loss": 0.0819, "num_tokens": 1588321.0, "reward": 0.75, "reward_std": 0.3514062762260437, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.448341727256775, "sampling/importance_sampling_ratio/mean": 0.9997310638427734, "sampling/importance_sampling_ratio/min": 0.6787981390953064, "sampling/sampling_logp_difference/max": 0.3874315023422241, "sampling/sampling_logp_difference/mean": 0.009688274003565311, "step": 69 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.5580258049303666e-05, "clip_ratio/low_min": 5.5580258049303666e-05, "clip_ratio/region_mean": 5.5580258049303666e-05, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 760.0, "completions/mean_length": 582.4375, "completions/mean_terminated_length": 555.9285888671875, "completions/min_length": 208.0, "completions/min_terminated_length": 208.0, "entropy": 0.35030674561858177, "epoch": 0.03747323340471092, "frac_reward_zero_std": 0.5, "grad_norm": 0.006231345701962709, "learning_rate": 1e-05, "loss": 0.0373, "num_tokens": 1610327.0, "reward": 0.8125, "reward_std": 0.2587745785713196, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.34609055519104, "sampling/importance_sampling_ratio/mean": 0.9999900460243225, "sampling/importance_sampling_ratio/min": 0.585841953754425, "sampling/sampling_logp_difference/max": 0.5347051620483398, "sampling/sampling_logp_difference/mean": 0.010446908883750439, "step": 70 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.541077891597524e-05, "clip_ratio/low_min": 6.541077891597524e-05, "clip_ratio/region_mean": 6.541077891597524e-05, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 605.0, "completions/mean_length": 507.0625, "completions/mean_terminated_length": 446.8461608886719, "completions/min_length": 199.0, "completions/min_terminated_length": 199.0, "entropy": 0.530906118452549, "epoch": 0.03800856531049251, "frac_reward_zero_std": 0.0, "grad_norm": 0.02199765481054783, "learning_rate": 1e-05, "loss": 0.0789, "num_tokens": 1630025.0, "reward": 0.71875, "reward_std": 0.4397946000099182, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.4410661458969116, "sampling/importance_sampling_ratio/mean": 0.9997541308403015, "sampling/importance_sampling_ratio/min": 0.5709772109985352, "sampling/sampling_logp_difference/max": 0.560405969619751, "sampling/sampling_logp_difference/mean": 0.01314842514693737, "step": 71 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 695.0, "completions/mean_length": 527.125, "completions/mean_terminated_length": 471.5384826660156, "completions/min_length": 334.0, "completions/min_terminated_length": 334.0, "entropy": 0.5136992037296295, "epoch": 0.03854389721627409, "frac_reward_zero_std": 0.5, "grad_norm": 0.012632464058697224, "learning_rate": 1e-05, "loss": 0.0355, "num_tokens": 1650381.0, "reward": 0.46875, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.46875, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.2877466678619385, "sampling/importance_sampling_ratio/mean": 0.9998239874839783, "sampling/importance_sampling_ratio/min": 0.7674005627632141, "sampling/sampling_logp_difference/max": 0.26474642753601074, "sampling/sampling_logp_difference/mean": 0.013403840363025665, "step": 72 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00011858078505611047, "clip_ratio/low_min": 0.00011858078505611047, "clip_ratio/region_mean": 0.00011858078505611047, "completions/clipped_ratio": 0.3125, "completions/max_length": 768.0, "completions/max_terminated_length": 683.0, "completions/mean_length": 578.84375, "completions/mean_terminated_length": 492.8636474609375, "completions/min_length": 277.0, "completions/min_terminated_length": 277.0, "entropy": 0.47807905450463295, "epoch": 0.03907922912205567, "frac_reward_zero_std": 0.0, "grad_norm": 0.008951799012720585, "learning_rate": 1e-05, "loss": 0.0555, "num_tokens": 1672192.0, "reward": 0.40625, "reward_std": 0.4218915104866028, "rewards/accuracy_reward/mean": 0.40625, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.3621253967285156, "sampling/importance_sampling_ratio/mean": 1.0000659227371216, "sampling/importance_sampling_ratio/min": 0.4066939353942871, "sampling/sampling_logp_difference/max": 0.8996944427490234, "sampling/sampling_logp_difference/mean": 0.01292353868484497, "step": 73 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.883259629830718e-05, "clip_ratio/low_min": 6.883259629830718e-05, "clip_ratio/region_mean": 6.883259629830718e-05, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 714.0, "completions/mean_length": 511.875, "completions/mean_terminated_length": 475.2857360839844, "completions/min_length": 255.0, "completions/min_terminated_length": 255.0, "entropy": 0.330965094268322, "epoch": 0.03961456102783726, "frac_reward_zero_std": 0.5, "grad_norm": 0.005720760673284531, "learning_rate": 1e-05, "loss": -0.0119, "num_tokens": 1692436.0, "reward": 0.84375, "reward_std": 0.22201895713806152, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.2868467569351196, "sampling/importance_sampling_ratio/mean": 1.000016450881958, "sampling/importance_sampling_ratio/min": 0.5379886627197266, "sampling/sampling_logp_difference/max": 0.6199178695678711, "sampling/sampling_logp_difference/mean": 0.00998774729669094, "step": 74 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 698.0, "completions/mean_length": 538.875, "completions/mean_terminated_length": 474.7200012207031, "completions/min_length": 252.0, "completions/min_terminated_length": 252.0, "entropy": 0.43403178453445435, "epoch": 0.04014989293361884, "frac_reward_zero_std": 0.5, "grad_norm": 0.018599966540932655, "learning_rate": 1e-05, "loss": 0.0573, "num_tokens": 1713048.0, "reward": 0.8125, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.2929112911224365, "sampling/importance_sampling_ratio/mean": 0.9999213218688965, "sampling/importance_sampling_ratio/min": 0.6580591797828674, "sampling/sampling_logp_difference/max": 0.4184603691101074, "sampling/sampling_logp_difference/mean": 0.011468234471976757, "step": 75 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00014653716425527819, "clip_ratio/low_min": 0.00014653716425527819, "clip_ratio/region_mean": 0.00014653716425527819, "completions/clipped_ratio": 0.3125, "completions/max_length": 768.0, "completions/max_terminated_length": 767.0, "completions/mean_length": 605.78125, "completions/mean_terminated_length": 532.0454711914062, "completions/min_length": 337.0, "completions/min_terminated_length": 337.0, "entropy": 0.5242552533745766, "epoch": 0.04068522483940043, "frac_reward_zero_std": 0.0, "grad_norm": 0.019541600719094276, "learning_rate": 1e-05, "loss": 0.0685, "num_tokens": 1736833.0, "reward": 0.5625, "reward_std": 0.5081326961517334, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.443228840827942, "sampling/importance_sampling_ratio/mean": 1.00018310546875, "sampling/importance_sampling_ratio/min": 0.6059591174125671, "sampling/sampling_logp_difference/max": 0.5009427070617676, "sampling/sampling_logp_difference/mean": 0.01432492583990097, "step": 76 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.489679460879415e-05, "clip_ratio/low_min": 5.489679460879415e-05, "clip_ratio/region_mean": 5.489679460879415e-05, "completions/clipped_ratio": 0.28125, "completions/max_length": 768.0, "completions/max_terminated_length": 747.0, "completions/mean_length": 627.125, "completions/mean_terminated_length": 572.0, "completions/min_length": 364.0, "completions/min_terminated_length": 364.0, "entropy": 0.47469621524214745, "epoch": 0.04122055674518201, "frac_reward_zero_std": 0.5, "grad_norm": 0.003933256026357412, "learning_rate": 1e-05, "loss": 0.0347, "num_tokens": 1760869.0, "reward": 0.71875, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.6860499382019043, "sampling/importance_sampling_ratio/mean": 0.9998744130134583, "sampling/importance_sampling_ratio/min": 0.5010150671005249, "sampling/sampling_logp_difference/max": 0.6911191940307617, "sampling/sampling_logp_difference/mean": 0.012595373205840588, "step": 77 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00010221772754448466, "clip_ratio/low_min": 0.00010221772754448466, "clip_ratio/region_mean": 0.00010221772754448466, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 757.0, "completions/mean_length": 509.15625, "completions/mean_terminated_length": 461.22222900390625, "completions/min_length": 215.0, "completions/min_terminated_length": 215.0, "entropy": 0.47905098646879196, "epoch": 0.0417558886509636, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0284, "num_tokens": 1781314.0, "reward": 0.53125, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.3845901489257812, "sampling/importance_sampling_ratio/mean": 0.9996688365936279, "sampling/importance_sampling_ratio/min": 0.6213145852088928, "sampling/sampling_logp_difference/max": 0.4759178161621094, "sampling/sampling_logp_difference/mean": 0.013783096335828304, "step": 78 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.40625, "completions/max_length": 768.0, "completions/max_terminated_length": 759.0, "completions/mean_length": 647.09375, "completions/mean_terminated_length": 564.368408203125, "completions/min_length": 364.0, "completions/min_terminated_length": 364.0, "entropy": 0.4179849214851856, "epoch": 0.04229122055674518, "frac_reward_zero_std": 0.25, "grad_norm": 0.007662573829293251, "learning_rate": 1e-05, "loss": 0.0308, "num_tokens": 1805413.0, "reward": 0.59375, "reward_std": 0.3198433816432953, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.4343494176864624, "sampling/importance_sampling_ratio/mean": 1.0001981258392334, "sampling/importance_sampling_ratio/min": 0.7690429091453552, "sampling/sampling_logp_difference/max": 0.36071133613586426, "sampling/sampling_logp_difference/mean": 0.011692270636558533, "step": 79 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 9.750395838636905e-05, "clip_ratio/low_min": 9.750395838636905e-05, "clip_ratio/region_mean": 9.750395838636905e-05, "completions/clipped_ratio": 0.375, "completions/max_length": 768.0, "completions/max_terminated_length": 598.0, "completions/mean_length": 588.03125, "completions/mean_terminated_length": 480.0500183105469, "completions/min_length": 355.0, "completions/min_terminated_length": 355.0, "entropy": 0.5071725882589817, "epoch": 0.042826552462526764, "frac_reward_zero_std": 0.0, "grad_norm": 0.012563863769173622, "learning_rate": 1e-05, "loss": 0.084, "num_tokens": 1828502.0, "reward": 0.5625, "reward_std": 0.49022960662841797, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.5360150337219238, "sampling/importance_sampling_ratio/mean": 1.000058650970459, "sampling/importance_sampling_ratio/min": 0.47326546907424927, "sampling/sampling_logp_difference/max": 0.7480988502502441, "sampling/sampling_logp_difference/mean": 0.01280238013714552, "step": 80 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 767.0, "completions/mean_length": 567.09375, "completions/mean_terminated_length": 520.7307739257812, "completions/min_length": 289.0, "completions/min_terminated_length": 289.0, "entropy": 0.4767908751964569, "epoch": 0.04336188436830835, "frac_reward_zero_std": 0.25, "grad_norm": 0.007349553983658552, "learning_rate": 1e-05, "loss": 0.0046, "num_tokens": 1850625.0, "reward": 0.78125, "reward_std": 0.3061639964580536, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.4524610042572021, "sampling/importance_sampling_ratio/mean": 1.0003424882888794, "sampling/importance_sampling_ratio/min": 0.54655921459198, "sampling/sampling_logp_difference/max": 0.6041126251220703, "sampling/sampling_logp_difference/mean": 0.012623358517885208, "step": 81 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.28125, "completions/max_length": 768.0, "completions/max_terminated_length": 698.0, "completions/mean_length": 561.9375, "completions/mean_terminated_length": 481.3043518066406, "completions/min_length": 278.0, "completions/min_terminated_length": 278.0, "entropy": 0.48736458644270897, "epoch": 0.043897216274089934, "frac_reward_zero_std": 0.25, "grad_norm": 0.007987036369740963, "learning_rate": 1e-05, "loss": 0.0336, "num_tokens": 1872207.0, "reward": 0.53125, "reward_std": 0.3377464711666107, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.511791706085205, "sampling/importance_sampling_ratio/mean": 0.9997742772102356, "sampling/importance_sampling_ratio/min": 0.5764255523681641, "sampling/sampling_logp_difference/max": 0.5509090423583984, "sampling/sampling_logp_difference/mean": 0.01264932844787836, "step": 82 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00019177383364876732, "clip_ratio/low_min": 0.00019177383364876732, "clip_ratio/region_mean": 0.00019177383364876732, "completions/clipped_ratio": 0.28125, "completions/max_length": 768.0, "completions/max_terminated_length": 755.0, "completions/mean_length": 644.78125, "completions/mean_terminated_length": 596.5652465820312, "completions/min_length": 440.0, "completions/min_terminated_length": 440.0, "entropy": 0.4271051101386547, "epoch": 0.04443254817987152, "frac_reward_zero_std": 0.25, "grad_norm": 0.0072464970871806145, "learning_rate": 1e-05, "loss": 0.0279, "num_tokens": 1896696.0, "reward": 0.40625, "reward_std": 0.3198433816432953, "rewards/accuracy_reward/mean": 0.40625, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.3683228492736816, "sampling/importance_sampling_ratio/mean": 1.0000728368759155, "sampling/importance_sampling_ratio/min": 0.7364410758018494, "sampling/sampling_logp_difference/max": 0.3135857582092285, "sampling/sampling_logp_difference/mean": 0.011938256211578846, "step": 83 }, { "clip_ratio/high_max": 0.00016138809587573633, "clip_ratio/high_mean": 0.00016138809587573633, "clip_ratio/low_mean": 9.469286669627763e-05, "clip_ratio/low_min": 9.469286669627763e-05, "clip_ratio/region_mean": 0.00025608096257201396, "completions/clipped_ratio": 0.375, "completions/max_length": 768.0, "completions/max_terminated_length": 695.0, "completions/mean_length": 646.5625, "completions/mean_terminated_length": 573.7000122070312, "completions/min_length": 356.0, "completions/min_terminated_length": 356.0, "entropy": 0.489109069108963, "epoch": 0.044967880085653104, "frac_reward_zero_std": 0.0, "grad_norm": 0.015514599159359932, "learning_rate": 1e-05, "loss": 0.0385, "num_tokens": 1921354.0, "reward": 0.4375, "reward_std": 0.5081326961517334, "rewards/accuracy_reward/mean": 0.4375, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.5057592391967773, "sampling/importance_sampling_ratio/mean": 1.0001661777496338, "sampling/importance_sampling_ratio/min": 0.47354331612586975, "sampling/sampling_logp_difference/max": 0.7475118637084961, "sampling/sampling_logp_difference/mean": 0.013187098316848278, "step": 84 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 739.0, "completions/mean_length": 546.25, "completions/mean_terminated_length": 495.0769348144531, "completions/min_length": 318.0, "completions/min_terminated_length": 318.0, "entropy": 0.37117818370461464, "epoch": 0.045503211991434686, "frac_reward_zero_std": 0.5, "grad_norm": 0.003233251627534628, "learning_rate": 1e-05, "loss": 0.0603, "num_tokens": 1942650.0, "reward": 0.59375, "reward_std": 0.22201895713806152, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.415218710899353, "sampling/importance_sampling_ratio/mean": 1.0000778436660767, "sampling/importance_sampling_ratio/min": 0.7080608010292053, "sampling/sampling_logp_difference/max": 0.34728407859802246, "sampling/sampling_logp_difference/mean": 0.010211372748017311, "step": 85 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00013728720659855753, "clip_ratio/low_min": 0.00013728720659855753, "clip_ratio/region_mean": 0.00013728720659855753, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 696.0, "completions/mean_length": 522.5625, "completions/mean_terminated_length": 465.923095703125, "completions/min_length": 42.0, "completions/min_terminated_length": 42.0, "entropy": 0.44907190278172493, "epoch": 0.046038543897216275, "frac_reward_zero_std": 0.25, "grad_norm": 0.011993853375315666, "learning_rate": 1e-05, "loss": 0.0042, "num_tokens": 1962612.0, "reward": 0.5625, "reward_std": 0.3745020925998688, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.3522263765335083, "sampling/importance_sampling_ratio/mean": 0.9996841549873352, "sampling/importance_sampling_ratio/min": 0.6549509763717651, "sampling/sampling_logp_difference/max": 0.42319488525390625, "sampling/sampling_logp_difference/mean": 0.012766262516379356, "step": 86 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.7801146138226613e-05, "clip_ratio/low_min": 4.7801146138226613e-05, "clip_ratio/region_mean": 4.7801146138226613e-05, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 758.0, "completions/mean_length": 592.3125, "completions/mean_terminated_length": 551.7692260742188, "completions/min_length": 334.0, "completions/min_terminated_length": 334.0, "entropy": 0.3904706798493862, "epoch": 0.046573875802997856, "frac_reward_zero_std": 0.25, "grad_norm": 0.01267465390264988, "learning_rate": 1e-05, "loss": 0.0293, "num_tokens": 1985590.0, "reward": 0.625, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.4399200677871704, "sampling/importance_sampling_ratio/mean": 0.9999144673347473, "sampling/importance_sampling_ratio/min": 0.6691920757293701, "sampling/sampling_logp_difference/max": 0.4016841650009155, "sampling/sampling_logp_difference/mean": 0.011088584549725056, "step": 87 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.40625, "completions/max_length": 768.0, "completions/max_terminated_length": 733.0, "completions/mean_length": 665.09375, "completions/mean_terminated_length": 594.6842041015625, "completions/min_length": 291.0, "completions/min_terminated_length": 291.0, "entropy": 0.4168216176331043, "epoch": 0.047109207708779445, "frac_reward_zero_std": 0.25, "grad_norm": 0.003892525564879179, "learning_rate": 1e-05, "loss": -0.0202, "num_tokens": 2010857.0, "reward": 0.40625, "reward_std": 0.3198433816432953, "rewards/accuracy_reward/mean": 0.40625, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.4377671480178833, "sampling/importance_sampling_ratio/mean": 1.000050663948059, "sampling/importance_sampling_ratio/min": 0.7132178544998169, "sampling/sampling_logp_difference/max": 0.36309123039245605, "sampling/sampling_logp_difference/mean": 0.011454454623162746, "step": 88 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.34375, "completions/max_length": 768.0, "completions/max_terminated_length": 768.0, "completions/mean_length": 627.46875, "completions/mean_terminated_length": 553.857177734375, "completions/min_length": 377.0, "completions/min_terminated_length": 377.0, "entropy": 0.5533029772341251, "epoch": 0.04764453961456103, "frac_reward_zero_std": 0.25, "grad_norm": 0.011965334415435791, "learning_rate": 1e-05, "loss": 0.0681, "num_tokens": 2034944.0, "reward": 0.5, "reward_std": 0.3745020925998688, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5080004930496216, "sampling/importance_sampling_ratio/max": 1.4319475889205933, "sampling/importance_sampling_ratio/mean": 1.0000640153884888, "sampling/importance_sampling_ratio/min": 0.4541526734828949, "sampling/sampling_logp_difference/max": 0.7893218994140625, "sampling/sampling_logp_difference/mean": 0.01460959017276764, "step": 89 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00014825596372247674, "clip_ratio/low_min": 0.00014825596372247674, "clip_ratio/region_mean": 0.00014825596372247674, "completions/clipped_ratio": 0.28125, "completions/max_length": 768.0, "completions/max_terminated_length": 747.0, "completions/mean_length": 615.65625, "completions/mean_terminated_length": 556.0435180664062, "completions/min_length": 366.0, "completions/min_terminated_length": 366.0, "entropy": 0.44857458397746086, "epoch": 0.048179871520342615, "frac_reward_zero_std": 0.0, "grad_norm": 0.0067571490071713924, "learning_rate": 1e-05, "loss": 0.0206, "num_tokens": 2058973.0, "reward": 0.46875, "reward_std": 0.4355708956718445, "rewards/accuracy_reward/mean": 0.46875, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.3036197423934937, "sampling/importance_sampling_ratio/mean": 1.0000296831130981, "sampling/importance_sampling_ratio/min": 0.5692623853683472, "sampling/sampling_logp_difference/max": 0.5634138584136963, "sampling/sampling_logp_difference/mean": 0.012198539450764656, "step": 90 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 8.138021075865254e-05, "clip_ratio/low_min": 8.138021075865254e-05, "clip_ratio/region_mean": 8.138021075865254e-05, "completions/clipped_ratio": 0.34375, "completions/max_length": 768.0, "completions/max_terminated_length": 763.0, "completions/mean_length": 599.40625, "completions/mean_terminated_length": 511.0952453613281, "completions/min_length": 237.0, "completions/min_terminated_length": 237.0, "entropy": 0.3983592949807644, "epoch": 0.0487152034261242, "frac_reward_zero_std": 0.25, "grad_norm": 0.014063858427107334, "learning_rate": 1e-05, "loss": 0.049, "num_tokens": 2081882.0, "reward": 0.5625, "reward_std": 0.292504221200943, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.2893673181533813, "sampling/importance_sampling_ratio/mean": 0.9998114109039307, "sampling/importance_sampling_ratio/min": 0.6123552322387695, "sampling/sampling_logp_difference/max": 0.49044275283813477, "sampling/sampling_logp_difference/mean": 0.011362754739820957, "step": 91 }, { "clip_ratio/high_max": 5.444250928121619e-05, "clip_ratio/high_mean": 5.444250928121619e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 5.444250928121619e-05, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 713.0, "completions/mean_length": 488.09375, "completions/mean_terminated_length": 469.433349609375, "completions/min_length": 231.0, "completions/min_terminated_length": 231.0, "entropy": 0.44644933193922043, "epoch": 0.04925053533190578, "frac_reward_zero_std": 0.25, "grad_norm": 0.00947182159870863, "learning_rate": 1e-05, "loss": 0.0589, "num_tokens": 2100909.0, "reward": 0.78125, "reward_std": 0.3471629321575165, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.4443336725234985, "sampling/importance_sampling_ratio/mean": 0.9998829364776611, "sampling/importance_sampling_ratio/min": 0.756104052066803, "sampling/sampling_logp_difference/max": 0.3676481246948242, "sampling/sampling_logp_difference/mean": 0.012396569363772869, "step": 92 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.000111259454570245, "clip_ratio/low_min": 0.000111259454570245, "clip_ratio/region_mean": 0.000111259454570245, "completions/clipped_ratio": 0.3125, "completions/max_length": 768.0, "completions/max_terminated_length": 706.0, "completions/mean_length": 620.6875, "completions/mean_terminated_length": 553.727294921875, "completions/min_length": 269.0, "completions/min_terminated_length": 269.0, "entropy": 0.47858962416648865, "epoch": 0.04978586723768737, "frac_reward_zero_std": 0.0, "grad_norm": 0.013834147714078426, "learning_rate": 1e-05, "loss": 0.0884, "num_tokens": 2124595.0, "reward": 0.53125, "reward_std": 0.521792471408844, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.6744105815887451, "sampling/importance_sampling_ratio/mean": 1.0001462697982788, "sampling/importance_sampling_ratio/min": 0.739568293094635, "sampling/sampling_logp_difference/max": 0.5154612064361572, "sampling/sampling_logp_difference/mean": 0.012730774469673634, "step": 93 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.53125, "completions/max_length": 768.0, "completions/max_terminated_length": 751.0, "completions/mean_length": 673.3125, "completions/mean_terminated_length": 566.0, "completions/min_length": 300.0, "completions/min_terminated_length": 300.0, "entropy": 0.5602152310311794, "epoch": 0.05032119914346895, "frac_reward_zero_std": 0.25, "grad_norm": 0.007184832822531462, "learning_rate": 1e-05, "loss": 0.0351, "num_tokens": 2149981.0, "reward": 0.46875, "reward_std": 0.38816186785697937, "rewards/accuracy_reward/mean": 0.46875, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.4355807304382324, "sampling/importance_sampling_ratio/mean": 0.9999159574508667, "sampling/importance_sampling_ratio/min": 0.6217223405838013, "sampling/sampling_logp_difference/max": 0.4752616882324219, "sampling/sampling_logp_difference/mean": 0.012786953710019588, "step": 94 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 714.0, "completions/mean_length": 506.34375, "completions/mean_terminated_length": 488.9000244140625, "completions/min_length": 261.0, "completions/min_terminated_length": 261.0, "entropy": 0.3649308830499649, "epoch": 0.05085653104925054, "frac_reward_zero_std": 0.5, "grad_norm": 0.006838271860033274, "learning_rate": 1e-05, "loss": 0.0272, "num_tokens": 2169376.0, "reward": 0.875, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 1.2903103828430176, "sampling/importance_sampling_ratio/mean": 0.9998345971107483, "sampling/importance_sampling_ratio/min": 0.4351949691772461, "sampling/sampling_logp_difference/max": 0.8319611549377441, "sampling/sampling_logp_difference/mean": 0.01059436984360218, "step": 95 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 740.0, "completions/mean_length": 564.09375, "completions/mean_terminated_length": 517.0384521484375, "completions/min_length": 283.0, "completions/min_terminated_length": 283.0, "entropy": 0.4984537586569786, "epoch": 0.05139186295503212, "frac_reward_zero_std": 0.5, "grad_norm": 0.034659527242183685, "learning_rate": 1e-05, "loss": -0.0084, "num_tokens": 2190947.0, "reward": 0.65625, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.3317999839782715, "sampling/importance_sampling_ratio/mean": 0.9997941851615906, "sampling/importance_sampling_ratio/min": 0.767650306224823, "sampling/sampling_logp_difference/max": 0.2865314483642578, "sampling/sampling_logp_difference/mean": 0.013492072932422161, "step": 96 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0001035625537042506, "clip_ratio/low_min": 0.0001035625537042506, "clip_ratio/region_mean": 0.0001035625537042506, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 757.0, "completions/mean_length": 589.46875, "completions/mean_terminated_length": 539.47998046875, "completions/min_length": 284.0, "completions/min_terminated_length": 284.0, "entropy": 0.32192447781562805, "epoch": 0.05192719486081371, "frac_reward_zero_std": 0.25, "grad_norm": 0.005379266571253538, "learning_rate": 1e-05, "loss": 0.0321, "num_tokens": 2213514.0, "reward": 0.65625, "reward_std": 0.3061639666557312, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.280397891998291, "sampling/importance_sampling_ratio/mean": 0.9998688697814941, "sampling/importance_sampling_ratio/min": 0.6917656064033508, "sampling/sampling_logp_difference/max": 0.36850810050964355, "sampling/sampling_logp_difference/mean": 0.009419996291399002, "step": 97 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 731.0, "completions/mean_length": 513.90625, "completions/mean_terminated_length": 505.70965576171875, "completions/min_length": 317.0, "completions/min_terminated_length": 317.0, "entropy": 0.30539928935468197, "epoch": 0.05246252676659529, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": -0.0027, "num_tokens": 2233727.0, "reward": 0.71875, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.3576973676681519, "sampling/importance_sampling_ratio/mean": 1.0000336170196533, "sampling/importance_sampling_ratio/min": 0.7603753805160522, "sampling/sampling_logp_difference/max": 0.3057901859283447, "sampling/sampling_logp_difference/mean": 0.00919144693762064, "step": 98 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.28125, "completions/max_length": 768.0, "completions/max_terminated_length": 711.0, "completions/mean_length": 575.78125, "completions/mean_terminated_length": 500.5652160644531, "completions/min_length": 215.0, "completions/min_terminated_length": 215.0, "entropy": 0.42042646184563637, "epoch": 0.05299785867237687, "frac_reward_zero_std": 0.25, "grad_norm": 0.009321073070168495, "learning_rate": 1e-05, "loss": 0.0352, "num_tokens": 2255712.0, "reward": 0.71875, "reward_std": 0.3471629321575165, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.2689825296401978, "sampling/importance_sampling_ratio/mean": 0.9999005794525146, "sampling/importance_sampling_ratio/min": 0.6649126410484314, "sampling/sampling_logp_difference/max": 0.4080996513366699, "sampling/sampling_logp_difference/mean": 0.01154857873916626, "step": 99 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 9.067827340913936e-05, "clip_ratio/low_min": 9.067827340913936e-05, "clip_ratio/region_mean": 9.067827340913936e-05, "completions/clipped_ratio": 0.5, "completions/max_length": 768.0, "completions/max_terminated_length": 761.0, "completions/mean_length": 681.0, "completions/mean_terminated_length": 594.0, "completions/min_length": 435.0, "completions/min_terminated_length": 435.0, "entropy": 0.5039323531091213, "epoch": 0.05353319057815846, "frac_reward_zero_std": 0.25, "grad_norm": 0.006315401755273342, "learning_rate": 1e-05, "loss": 0.0556, "num_tokens": 2281672.0, "reward": 0.4375, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.4375, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.447018027305603, "sampling/importance_sampling_ratio/mean": 0.9999662637710571, "sampling/importance_sampling_ratio/min": 0.634667694568634, "sampling/sampling_logp_difference/max": 0.4546537399291992, "sampling/sampling_logp_difference/mean": 0.013043881393969059, "step": 100 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.40625, "completions/max_length": 768.0, "completions/max_terminated_length": 755.0, "completions/mean_length": 646.1875, "completions/mean_terminated_length": 562.8421020507812, "completions/min_length": 279.0, "completions/min_terminated_length": 279.0, "entropy": 0.5601640939712524, "epoch": 0.05406852248394004, "frac_reward_zero_std": 0.5, "grad_norm": 0.006256799213588238, "learning_rate": 1e-05, "loss": 0.0272, "num_tokens": 2306390.0, "reward": 0.5, "reward_std": 0.2314550280570984, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5080004930496216, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9998123049736023, "sampling/importance_sampling_ratio/min": 0.5658286213874817, "sampling/sampling_logp_difference/max": 0.763413667678833, "sampling/sampling_logp_difference/mean": 0.013222371228039265, "step": 101 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.085686618462205e-05, "clip_ratio/low_min": 6.085686618462205e-05, "clip_ratio/region_mean": 6.085686618462205e-05, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 693.0, "completions/mean_length": 509.0, "completions/mean_terminated_length": 482.2069091796875, "completions/min_length": 246.0, "completions/min_terminated_length": 246.0, "entropy": 0.4597771391272545, "epoch": 0.05460385438972163, "frac_reward_zero_std": 0.0, "grad_norm": 0.015706215053796768, "learning_rate": 1e-05, "loss": 0.0256, "num_tokens": 2326190.0, "reward": 0.6875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.4313578605651855, "sampling/importance_sampling_ratio/mean": 0.9998531937599182, "sampling/importance_sampling_ratio/min": 0.5052533149719238, "sampling/sampling_logp_difference/max": 0.6826953887939453, "sampling/sampling_logp_difference/mean": 0.012012065388262272, "step": 102 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 8.138021075865254e-05, "clip_ratio/low_min": 8.138021075865254e-05, "clip_ratio/region_mean": 8.138021075865254e-05, "completions/clipped_ratio": 0.59375, "completions/max_length": 768.0, "completions/max_terminated_length": 765.0, "completions/mean_length": 687.71875, "completions/mean_terminated_length": 570.3846435546875, "completions/min_length": 337.0, "completions/min_terminated_length": 337.0, "entropy": 0.3641216792166233, "epoch": 0.05513918629550321, "frac_reward_zero_std": 0.0, "grad_norm": 0.007397511508315802, "learning_rate": 1e-05, "loss": 0.0905, "num_tokens": 2351853.0, "reward": 0.4375, "reward_std": 0.4492306709289551, "rewards/accuracy_reward/mean": 0.4375, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.27895188331604, "sampling/importance_sampling_ratio/mean": 0.9999485015869141, "sampling/importance_sampling_ratio/min": 0.7077409625053406, "sampling/sampling_logp_difference/max": 0.34567713737487793, "sampling/sampling_logp_difference/mean": 0.010833405889570713, "step": 103 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 739.0, "completions/mean_length": 482.53125, "completions/mean_terminated_length": 473.32257080078125, "completions/min_length": 194.0, "completions/min_terminated_length": 194.0, "entropy": 0.4147394150495529, "epoch": 0.055674518201284794, "frac_reward_zero_std": 0.75, "grad_norm": 0.006192837841808796, "learning_rate": 1e-05, "loss": 0.0371, "num_tokens": 2370654.0, "reward": 0.9375, "reward_std": 0.1157275140285492, "rewards/accuracy_reward/mean": 0.9375, "rewards/accuracy_reward/std": 0.24593468010425568, "sampling/importance_sampling_ratio/max": 1.321974754333496, "sampling/importance_sampling_ratio/mean": 1.0000211000442505, "sampling/importance_sampling_ratio/min": 0.777760922908783, "sampling/sampling_logp_difference/max": 0.2791266441345215, "sampling/sampling_logp_difference/mean": 0.011425119824707508, "step": 104 }, { "clip_ratio/high_max": 9.846078319242224e-05, "clip_ratio/high_mean": 9.846078319242224e-05, "clip_ratio/low_mean": 9.072280590771697e-05, "clip_ratio/low_min": 9.072280590771697e-05, "clip_ratio/region_mean": 0.00018918358910013922, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 753.0, "completions/mean_length": 628.0, "completions/mean_terminated_length": 588.7999877929688, "completions/min_length": 284.0, "completions/min_terminated_length": 284.0, "entropy": 0.5660983696579933, "epoch": 0.05620985010706638, "frac_reward_zero_std": 0.0, "grad_norm": 0.010016806423664093, "learning_rate": 1e-05, "loss": 0.08, "num_tokens": 2394278.0, "reward": 0.71875, "reward_std": 0.4628904461860657, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.2932615280151367, "sampling/importance_sampling_ratio/mean": 1.0002484321594238, "sampling/importance_sampling_ratio/min": 0.6581019163131714, "sampling/sampling_logp_difference/max": 0.4183955192565918, "sampling/sampling_logp_difference/mean": 0.01387240830808878, "step": 105 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 736.0, "completions/mean_length": 572.59375, "completions/mean_terminated_length": 527.5, "completions/min_length": 173.0, "completions/min_terminated_length": 173.0, "entropy": 0.38742442801594734, "epoch": 0.056745182012847964, "frac_reward_zero_std": 0.5, "grad_norm": 0.012721240520477295, "learning_rate": 1e-05, "loss": 0.0182, "num_tokens": 2416249.0, "reward": 0.8125, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.437535047531128, "sampling/importance_sampling_ratio/mean": 1.0001819133758545, "sampling/importance_sampling_ratio/min": 0.655653715133667, "sampling/sampling_logp_difference/max": 0.4221224784851074, "sampling/sampling_logp_difference/mean": 0.01118245255202055, "step": 106 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 9.860571663011797e-05, "clip_ratio/low_min": 9.860571663011797e-05, "clip_ratio/region_mean": 9.860571663011797e-05, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 732.0, "completions/mean_length": 593.3125, "completions/mean_terminated_length": 553.0, "completions/min_length": 249.0, "completions/min_terminated_length": 249.0, "entropy": 0.3981010913848877, "epoch": 0.05728051391862955, "frac_reward_zero_std": 0.25, "grad_norm": 0.005357117857784033, "learning_rate": 1e-05, "loss": 0.0758, "num_tokens": 2438947.0, "reward": 0.6875, "reward_std": 0.3471825420856476, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.3418827056884766, "sampling/importance_sampling_ratio/mean": 1.000197172164917, "sampling/importance_sampling_ratio/min": 0.5009822845458984, "sampling/sampling_logp_difference/max": 0.6911845207214355, "sampling/sampling_logp_difference/mean": 0.011721469461917877, "step": 107 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00019840320237562992, "clip_ratio/low_min": 0.00019840320237562992, "clip_ratio/region_mean": 0.00019840320237562992, "completions/clipped_ratio": 0.46875, "completions/max_length": 768.0, "completions/max_terminated_length": 754.0, "completions/mean_length": 664.71875, "completions/mean_terminated_length": 573.5882568359375, "completions/min_length": 405.0, "completions/min_terminated_length": 405.0, "entropy": 0.43526025488972664, "epoch": 0.057815845824411134, "frac_reward_zero_std": 0.0, "grad_norm": 0.012608899734914303, "learning_rate": 1e-05, "loss": 0.0713, "num_tokens": 2464170.0, "reward": 0.34375, "reward_std": 0.47137710452079773, "rewards/accuracy_reward/mean": 0.34375, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.4328020811080933, "sampling/importance_sampling_ratio/mean": 0.9997937679290771, "sampling/importance_sampling_ratio/min": 0.6727548837661743, "sampling/sampling_logp_difference/max": 0.39637428522109985, "sampling/sampling_logp_difference/mean": 0.01205401960760355, "step": 108 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.46875, "completions/max_length": 768.0, "completions/max_terminated_length": 736.0, "completions/mean_length": 655.25, "completions/mean_terminated_length": 555.7647094726562, "completions/min_length": 51.0, "completions/min_terminated_length": 51.0, "entropy": 0.46148813143372536, "epoch": 0.05835117773019272, "frac_reward_zero_std": 0.25, "grad_norm": 0.009041559882462025, "learning_rate": 1e-05, "loss": 0.007, "num_tokens": 2489610.0, "reward": 0.5625, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.276477336883545, "sampling/importance_sampling_ratio/mean": 0.9997236132621765, "sampling/importance_sampling_ratio/min": 0.6942884922027588, "sampling/sampling_logp_difference/max": 0.3648676872253418, "sampling/sampling_logp_difference/mean": 0.011251892894506454, "step": 109 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0001304120960412547, "clip_ratio/low_min": 0.0001304120960412547, "clip_ratio/region_mean": 0.0001304120960412547, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 732.0, "completions/mean_length": 519.3125, "completions/mean_terminated_length": 493.5862121582031, "completions/min_length": 201.0, "completions/min_terminated_length": 201.0, "entropy": 0.43273311108350754, "epoch": 0.058886509635974305, "frac_reward_zero_std": 0.0, "grad_norm": 0.009418485686182976, "learning_rate": 1e-05, "loss": 0.0315, "num_tokens": 2510188.0, "reward": 0.5625, "reward_std": 0.3945523500442505, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.2939728498458862, "sampling/importance_sampling_ratio/mean": 1.000428318977356, "sampling/importance_sampling_ratio/min": 0.5578873753547668, "sampling/sampling_logp_difference/max": 0.5835981369018555, "sampling/sampling_logp_difference/mean": 0.012478448450565338, "step": 110 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.443299207603559e-05, "clip_ratio/low_min": 6.443299207603559e-05, "clip_ratio/region_mean": 6.443299207603559e-05, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 761.0, "completions/mean_length": 560.28125, "completions/mean_terminated_length": 538.7930908203125, "completions/min_length": 250.0, "completions/min_terminated_length": 250.0, "entropy": 0.3870910219848156, "epoch": 0.059421841541755886, "frac_reward_zero_std": 0.25, "grad_norm": 0.004824988543987274, "learning_rate": 1e-05, "loss": 0.0504, "num_tokens": 2531821.0, "reward": 0.25, "reward_std": 0.3650856614112854, "rewards/accuracy_reward/mean": 0.25, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.4037129878997803, "sampling/importance_sampling_ratio/mean": 1.0002878904342651, "sampling/importance_sampling_ratio/min": 0.6825191378593445, "sampling/sampling_logp_difference/max": 0.38196468353271484, "sampling/sampling_logp_difference/mean": 0.010554767213761806, "step": 111 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.3125, "completions/max_length": 768.0, "completions/max_terminated_length": 720.0, "completions/mean_length": 587.375, "completions/mean_terminated_length": 505.2727355957031, "completions/min_length": 201.0, "completions/min_terminated_length": 201.0, "entropy": 0.6201117932796478, "epoch": 0.059957173447537475, "frac_reward_zero_std": 0.25, "grad_norm": 0.008925995789468288, "learning_rate": 1e-05, "loss": 0.0605, "num_tokens": 2554177.0, "reward": 0.5, "reward_std": 0.3745020925998688, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5080004930496216, "sampling/importance_sampling_ratio/max": 1.4320957660675049, "sampling/importance_sampling_ratio/mean": 0.9996970295906067, "sampling/importance_sampling_ratio/min": 0.69200599193573, "sampling/sampling_logp_difference/max": 0.3681607246398926, "sampling/sampling_logp_difference/mean": 0.014479795470833778, "step": 112 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.186202204436995e-05, "clip_ratio/low_min": 4.186202204436995e-05, "clip_ratio/region_mean": 4.186202204436995e-05, "completions/clipped_ratio": 0.40625, "completions/max_length": 768.0, "completions/max_terminated_length": 739.0, "completions/mean_length": 663.96875, "completions/mean_terminated_length": 592.7894897460938, "completions/min_length": 222.0, "completions/min_terminated_length": 222.0, "entropy": 0.3596794046461582, "epoch": 0.06049250535331906, "frac_reward_zero_std": 0.5, "grad_norm": 0.009059062227606773, "learning_rate": 1e-05, "loss": 0.0189, "num_tokens": 2579048.0, "reward": 0.5625, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.3382359743118286, "sampling/importance_sampling_ratio/mean": 1.0003551244735718, "sampling/importance_sampling_ratio/min": 0.6658063530921936, "sampling/sampling_logp_difference/max": 0.4067564010620117, "sampling/sampling_logp_difference/mean": 0.010846213437616825, "step": 113 }, { "clip_ratio/high_max": 6.130456313258037e-05, "clip_ratio/high_mean": 6.130456313258037e-05, "clip_ratio/low_mean": 0.00011080646800110117, "clip_ratio/low_min": 0.00011080646800110117, "clip_ratio/region_mean": 0.00017211103113368154, "completions/clipped_ratio": 0.3125, "completions/max_length": 768.0, "completions/max_terminated_length": 766.0, "completions/mean_length": 626.15625, "completions/mean_terminated_length": 561.6818237304688, "completions/min_length": 279.0, "completions/min_terminated_length": 279.0, "entropy": 0.3593634217977524, "epoch": 0.061027837259100645, "frac_reward_zero_std": 0.25, "grad_norm": 0.0030713516753166914, "learning_rate": 1e-05, "loss": -0.0141, "num_tokens": 2602733.0, "reward": 0.625, "reward_std": 0.3104073107242584, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.2897512912750244, "sampling/importance_sampling_ratio/mean": 1.0003292560577393, "sampling/importance_sampling_ratio/min": 0.4559897184371948, "sampling/sampling_logp_difference/max": 0.7852849960327148, "sampling/sampling_logp_difference/mean": 0.010105901397764683, "step": 114 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.8543690354563296e-05, "clip_ratio/low_min": 4.8543690354563296e-05, "clip_ratio/region_mean": 4.8543690354563296e-05, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 722.0, "completions/mean_length": 576.25, "completions/mean_terminated_length": 548.857177734375, "completions/min_length": 363.0, "completions/min_terminated_length": 363.0, "entropy": 0.35390249639749527, "epoch": 0.06156316916488223, "frac_reward_zero_std": 0.5, "grad_norm": 0.006338918581604958, "learning_rate": 1e-05, "loss": 0.0285, "num_tokens": 2624925.0, "reward": 0.5625, "reward_std": 0.249358132481575, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.575213074684143, "sampling/importance_sampling_ratio/mean": 1.0000450611114502, "sampling/importance_sampling_ratio/min": 0.6898723244667053, "sampling/sampling_logp_difference/max": 0.4543905258178711, "sampling/sampling_logp_difference/mean": 0.010251270607113838, "step": 115 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 644.0, "completions/mean_length": 552.5, "completions/mean_terminated_length": 512.5925903320312, "completions/min_length": 248.0, "completions/min_terminated_length": 248.0, "entropy": 0.3580743409693241, "epoch": 0.06209850107066381, "frac_reward_zero_std": 0.25, "grad_norm": 0.018461303785443306, "learning_rate": 1e-05, "loss": 0.0289, "num_tokens": 2646693.0, "reward": 0.75, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.462385892868042, "sampling/importance_sampling_ratio/mean": 1.0000616312026978, "sampling/importance_sampling_ratio/min": 0.757928729057312, "sampling/sampling_logp_difference/max": 0.3800692558288574, "sampling/sampling_logp_difference/mean": 0.010540477931499481, "step": 116 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.509034963324666e-05, "clip_ratio/low_min": 5.509034963324666e-05, "clip_ratio/region_mean": 5.509034963324666e-05, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 763.0, "completions/mean_length": 523.03125, "completions/mean_terminated_length": 497.6896667480469, "completions/min_length": 312.0, "completions/min_terminated_length": 312.0, "entropy": 0.38762374222278595, "epoch": 0.0626338329764454, "frac_reward_zero_std": 0.0, "grad_norm": 0.007358966860920191, "learning_rate": 1e-05, "loss": 0.0195, "num_tokens": 2667110.0, "reward": 0.6875, "reward_std": 0.4492306709289551, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.4650076627731323, "sampling/importance_sampling_ratio/mean": 1.0000468492507935, "sampling/importance_sampling_ratio/min": 0.6492045521736145, "sampling/sampling_logp_difference/max": 0.43200743198394775, "sampling/sampling_logp_difference/mean": 0.010364267975091934, "step": 117 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0001240694837179035, "clip_ratio/low_min": 0.0001240694837179035, "clip_ratio/region_mean": 0.0001240694837179035, "completions/clipped_ratio": 0.40625, "completions/max_length": 768.0, "completions/max_terminated_length": 659.0, "completions/mean_length": 609.875, "completions/mean_terminated_length": 501.6842041015625, "completions/min_length": 323.0, "completions/min_terminated_length": 323.0, "entropy": 0.2991199493408203, "epoch": 0.06316916488222699, "frac_reward_zero_std": 0.25, "grad_norm": 0.007873529568314552, "learning_rate": 1e-05, "loss": 0.0348, "num_tokens": 2689874.0, "reward": 0.46875, "reward_std": 0.3061639666557312, "rewards/accuracy_reward/mean": 0.46875, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.2994112968444824, "sampling/importance_sampling_ratio/mean": 1.0000189542770386, "sampling/importance_sampling_ratio/min": 0.7097275853157043, "sampling/sampling_logp_difference/max": 0.34287405014038086, "sampling/sampling_logp_difference/mean": 0.00929878931492567, "step": 118 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.161118340562098e-05, "clip_ratio/low_min": 4.161118340562098e-05, "clip_ratio/region_mean": 4.161118340562098e-05, "completions/clipped_ratio": 0.5625, "completions/max_length": 768.0, "completions/max_terminated_length": 748.0, "completions/mean_length": 721.03125, "completions/mean_terminated_length": 660.6428833007812, "completions/min_length": 458.0, "completions/min_terminated_length": 458.0, "entropy": 0.5329389944672585, "epoch": 0.06370449678800856, "frac_reward_zero_std": 0.25, "grad_norm": 0.01718723401427269, "learning_rate": 1e-05, "loss": 0.0414, "num_tokens": 2717387.0, "reward": 0.28125, "reward_std": 0.3198433816432953, "rewards/accuracy_reward/mean": 0.28125, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.489748239517212, "sampling/importance_sampling_ratio/mean": 1.0000871419906616, "sampling/importance_sampling_ratio/min": 0.7058092355728149, "sampling/sampling_logp_difference/max": 0.39860713481903076, "sampling/sampling_logp_difference/mean": 0.013629668392241001, "step": 119 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 762.0, "completions/mean_length": 585.25, "completions/mean_terminated_length": 534.0800170898438, "completions/min_length": 337.0, "completions/min_terminated_length": 337.0, "entropy": 0.43758948892354965, "epoch": 0.06423982869379015, "frac_reward_zero_std": 0.5, "grad_norm": 0.012252735905349255, "learning_rate": 1e-05, "loss": 0.0065, "num_tokens": 2739611.0, "reward": 0.59375, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.29102623462677, "sampling/importance_sampling_ratio/mean": 1.00021231174469, "sampling/importance_sampling_ratio/min": 0.319496750831604, "sampling/sampling_logp_difference/max": 1.1410081386566162, "sampling/sampling_logp_difference/mean": 0.011854034848511219, "step": 120 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00010165071580559015, "clip_ratio/low_min": 0.00010165071580559015, "clip_ratio/region_mean": 0.00010165071580559015, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 765.0, "completions/mean_length": 586.625, "completions/mean_terminated_length": 526.1666870117188, "completions/min_length": 343.0, "completions/min_terminated_length": 343.0, "entropy": 0.3946254886686802, "epoch": 0.06477516059957174, "frac_reward_zero_std": 0.25, "grad_norm": 0.002445508725941181, "learning_rate": 1e-05, "loss": 0.0439, "num_tokens": 2762151.0, "reward": 0.5, "reward_std": 0.2925041913986206, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5080004930496216, "sampling/importance_sampling_ratio/max": 1.2958875894546509, "sampling/importance_sampling_ratio/mean": 0.9999931454658508, "sampling/importance_sampling_ratio/min": 0.6989948749542236, "sampling/sampling_logp_difference/max": 0.3581118583679199, "sampling/sampling_logp_difference/mean": 0.011642329394817352, "step": 121 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 767.0, "completions/mean_length": 559.0625, "completions/mean_terminated_length": 510.8461608886719, "completions/min_length": 373.0, "completions/min_terminated_length": 373.0, "entropy": 0.41690022498369217, "epoch": 0.06531049250535331, "frac_reward_zero_std": 0.0, "grad_norm": 0.004603108856827021, "learning_rate": 1e-05, "loss": 0.0679, "num_tokens": 2784537.0, "reward": 0.6875, "reward_std": 0.3535533845424652, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.4526787996292114, "sampling/importance_sampling_ratio/mean": 1.000157117843628, "sampling/importance_sampling_ratio/min": 0.7026703953742981, "sampling/sampling_logp_difference/max": 0.3734092712402344, "sampling/sampling_logp_difference/mean": 0.012163394130766392, "step": 122 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.634779543266632e-05, "clip_ratio/low_min": 4.634779543266632e-05, "clip_ratio/region_mean": 4.634779543266632e-05, "completions/clipped_ratio": 0.34375, "completions/max_length": 768.0, "completions/max_terminated_length": 741.0, "completions/mean_length": 597.96875, "completions/mean_terminated_length": 508.90478515625, "completions/min_length": 276.0, "completions/min_terminated_length": 276.0, "entropy": 0.34927118197083473, "epoch": 0.0658458244111349, "frac_reward_zero_std": 0.5, "grad_norm": 0.008883594535291195, "learning_rate": 1e-05, "loss": 0.0435, "num_tokens": 2807608.0, "reward": 0.5, "reward_std": 0.2587745785713196, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5080004930496216, "sampling/importance_sampling_ratio/max": 1.4907944202423096, "sampling/importance_sampling_ratio/mean": 0.9999949932098389, "sampling/importance_sampling_ratio/min": 0.5515795350074768, "sampling/sampling_logp_difference/max": 0.5949692726135254, "sampling/sampling_logp_difference/mean": 0.010144411586225033, "step": 123 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.166748789837584e-05, "clip_ratio/low_min": 6.166748789837584e-05, "clip_ratio/region_mean": 6.166748789837584e-05, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 766.0, "completions/mean_length": 549.375, "completions/mean_terminated_length": 488.1600036621094, "completions/min_length": 178.0, "completions/min_terminated_length": 178.0, "entropy": 0.5086312592029572, "epoch": 0.06638115631691649, "frac_reward_zero_std": 0.25, "grad_norm": 0.02062125876545906, "learning_rate": 1e-05, "loss": 0.129, "num_tokens": 2828620.0, "reward": 0.71875, "reward_std": 0.3471629321575165, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.3955836296081543, "sampling/importance_sampling_ratio/mean": 1.0000624656677246, "sampling/importance_sampling_ratio/min": 0.5925219058990479, "sampling/sampling_logp_difference/max": 0.5233674049377441, "sampling/sampling_logp_difference/mean": 0.013531128875911236, "step": 124 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00019864087880705483, "clip_ratio/low_min": 0.00019864087880705483, "clip_ratio/region_mean": 0.00019864087880705483, "completions/clipped_ratio": 0.28125, "completions/max_length": 768.0, "completions/max_terminated_length": 766.0, "completions/mean_length": 629.15625, "completions/mean_terminated_length": 574.8261108398438, "completions/min_length": 228.0, "completions/min_terminated_length": 228.0, "entropy": 0.454969085752964, "epoch": 0.06691648822269808, "frac_reward_zero_std": 0.0, "grad_norm": 0.01148031372576952, "learning_rate": 1e-05, "loss": 0.0791, "num_tokens": 2852721.0, "reward": 0.4375, "reward_std": 0.4492306709289551, "rewards/accuracy_reward/mean": 0.4375, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.4142060279846191, "sampling/importance_sampling_ratio/mean": 1.0000925064086914, "sampling/importance_sampling_ratio/min": 0.5978776812553406, "sampling/sampling_logp_difference/max": 0.514369010925293, "sampling/sampling_logp_difference/mean": 0.012700111605226994, "step": 125 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 8.633586185169406e-05, "clip_ratio/low_min": 8.633586185169406e-05, "clip_ratio/region_mean": 8.633586185169406e-05, "completions/clipped_ratio": 0.3125, "completions/max_length": 768.0, "completions/max_terminated_length": 739.0, "completions/mean_length": 603.53125, "completions/mean_terminated_length": 528.7727661132812, "completions/min_length": 235.0, "completions/min_terminated_length": 235.0, "entropy": 0.4669857695698738, "epoch": 0.06745182012847965, "frac_reward_zero_std": 0.0, "grad_norm": 0.01519722305238247, "learning_rate": 1e-05, "loss": 0.0196, "num_tokens": 2875546.0, "reward": 0.25, "reward_std": 0.4629100561141968, "rewards/accuracy_reward/mean": 0.25, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.5019454956054688, "sampling/importance_sampling_ratio/mean": 1.0000461339950562, "sampling/importance_sampling_ratio/min": 0.652527928352356, "sampling/sampling_logp_difference/max": 0.42690134048461914, "sampling/sampling_logp_difference/mean": 0.012970094569027424, "step": 126 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00023071422037901357, "clip_ratio/low_min": 0.00023071422037901357, "clip_ratio/region_mean": 0.00023071422037901357, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 700.0, "completions/mean_length": 543.0625, "completions/mean_terminated_length": 480.0799865722656, "completions/min_length": 78.0, "completions/min_terminated_length": 78.0, "entropy": 0.709203228354454, "epoch": 0.06798715203426124, "frac_reward_zero_std": 0.0, "grad_norm": 0.012273093685507774, "learning_rate": 1e-05, "loss": 0.0624, "num_tokens": 2897252.0, "reward": 0.4375, "reward_std": 0.47655022144317627, "rewards/accuracy_reward/mean": 0.4375, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.3052661418914795, "sampling/importance_sampling_ratio/mean": 1.0001472234725952, "sampling/importance_sampling_ratio/min": 0.7167006731033325, "sampling/sampling_logp_difference/max": 0.333096981048584, "sampling/sampling_logp_difference/mean": 0.01649046689271927, "step": 127 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.827505810884759e-05, "clip_ratio/low_min": 5.827505810884759e-05, "clip_ratio/region_mean": 5.827505810884759e-05, "completions/clipped_ratio": 0.3125, "completions/max_length": 768.0, "completions/max_terminated_length": 755.0, "completions/mean_length": 566.46875, "completions/mean_terminated_length": 474.8636474609375, "completions/min_length": 244.0, "completions/min_terminated_length": 244.0, "entropy": 0.5634264834225178, "epoch": 0.06852248394004283, "frac_reward_zero_std": 0.75, "grad_norm": 0.0031958629842847586, "learning_rate": 1e-05, "loss": 0.0132, "num_tokens": 2919315.0, "reward": 0.3125, "reward_std": 0.1157275140285492, "rewards/accuracy_reward/mean": 0.3125, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.5357681512832642, "sampling/importance_sampling_ratio/mean": 1.0004392862319946, "sampling/importance_sampling_ratio/min": 0.524953305721283, "sampling/sampling_logp_difference/max": 0.6444458961486816, "sampling/sampling_logp_difference/mean": 0.014474543742835522, "step": 128 }, { "clip_ratio/high_max": 5.1652892580023035e-05, "clip_ratio/high_mean": 5.1652892580023035e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 5.1652892580023035e-05, "completions/clipped_ratio": 0.28125, "completions/max_length": 768.0, "completions/max_terminated_length": 753.0, "completions/mean_length": 640.65625, "completions/mean_terminated_length": 590.8261108398438, "completions/min_length": 372.0, "completions/min_terminated_length": 372.0, "entropy": 0.617782287299633, "epoch": 0.0690578158458244, "frac_reward_zero_std": 0.25, "grad_norm": 0.016598865389823914, "learning_rate": 1e-05, "loss": 0.0449, "num_tokens": 2944568.0, "reward": 0.625, "reward_std": 0.292504221200943, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.4206124544143677, "sampling/importance_sampling_ratio/mean": 0.9997906684875488, "sampling/importance_sampling_ratio/min": 0.6488592028617859, "sampling/sampling_logp_difference/max": 0.43253952264785767, "sampling/sampling_logp_difference/mean": 0.013920563273131847, "step": 129 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00016284645244013518, "clip_ratio/low_min": 0.00016284645244013518, "clip_ratio/region_mean": 0.00016284645244013518, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 753.0, "completions/mean_length": 539.59375, "completions/mean_terminated_length": 475.6399841308594, "completions/min_length": 304.0, "completions/min_terminated_length": 304.0, "entropy": 0.4281056188046932, "epoch": 0.069593147751606, "frac_reward_zero_std": 0.25, "grad_norm": 0.01196902897208929, "learning_rate": 1e-05, "loss": 0.0088, "num_tokens": 2966171.0, "reward": 0.6875, "reward_std": 0.3514062464237213, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.2752894163131714, "sampling/importance_sampling_ratio/mean": 0.9999509453773499, "sampling/importance_sampling_ratio/min": 0.6416014432907104, "sampling/sampling_logp_difference/max": 0.4437880516052246, "sampling/sampling_logp_difference/mean": 0.01213695015758276, "step": 130 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 682.0, "completions/mean_length": 541.9375, "completions/mean_terminated_length": 500.0740661621094, "completions/min_length": 177.0, "completions/min_terminated_length": 177.0, "entropy": 0.3594672866165638, "epoch": 0.07012847965738758, "frac_reward_zero_std": 0.0, "grad_norm": 0.009060511365532875, "learning_rate": 1e-05, "loss": 0.0066, "num_tokens": 2986961.0, "reward": 0.625, "reward_std": 0.4671337604522705, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.4494447708129883, "sampling/importance_sampling_ratio/mean": 1.0001531839370728, "sampling/importance_sampling_ratio/min": 0.6335175633430481, "sampling/sampling_logp_difference/max": 0.4564676284790039, "sampling/sampling_logp_difference/mean": 0.011128030717372894, "step": 131 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 763.0, "completions/mean_length": 624.3125, "completions/mean_terminated_length": 584.0800170898438, "completions/min_length": 262.0, "completions/min_terminated_length": 262.0, "entropy": 0.3743956610560417, "epoch": 0.07066381156316917, "frac_reward_zero_std": 0.5, "grad_norm": 0.010629815980792046, "learning_rate": 1e-05, "loss": 0.0168, "num_tokens": 3010971.0, "reward": 0.78125, "reward_std": 0.2630178928375244, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.292447805404663, "sampling/importance_sampling_ratio/mean": 0.9998924136161804, "sampling/importance_sampling_ratio/min": 0.622527003288269, "sampling/sampling_logp_difference/max": 0.4739682674407959, "sampling/sampling_logp_difference/mean": 0.010707241483032703, "step": 132 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 760.0, "completions/mean_length": 542.09375, "completions/mean_terminated_length": 500.25927734375, "completions/min_length": 214.0, "completions/min_terminated_length": 214.0, "entropy": 0.3791262358427048, "epoch": 0.07119914346895075, "frac_reward_zero_std": 0.25, "grad_norm": 0.008368417620658875, "learning_rate": 1e-05, "loss": 0.0846, "num_tokens": 3032086.0, "reward": 0.8125, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.4262453317642212, "sampling/importance_sampling_ratio/mean": 1.0005323886871338, "sampling/importance_sampling_ratio/min": 0.694298267364502, "sampling/sampling_logp_difference/max": 0.3648536205291748, "sampling/sampling_logp_difference/mean": 0.011444810777902603, "step": 133 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 8.72600357979536e-05, "clip_ratio/low_min": 8.72600357979536e-05, "clip_ratio/region_mean": 8.72600357979536e-05, "completions/clipped_ratio": 0.28125, "completions/max_length": 768.0, "completions/max_terminated_length": 756.0, "completions/mean_length": 626.0, "completions/mean_terminated_length": 570.434814453125, "completions/min_length": 314.0, "completions/min_terminated_length": 314.0, "entropy": 0.40805576741695404, "epoch": 0.07173447537473233, "frac_reward_zero_std": 0.0, "grad_norm": 0.008140301331877708, "learning_rate": 1e-05, "loss": 0.1021, "num_tokens": 3056126.0, "reward": 0.75, "reward_std": 0.4355512857437134, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.4351403713226318, "sampling/importance_sampling_ratio/mean": 1.000121831893921, "sampling/importance_sampling_ratio/min": 0.7144694924354553, "sampling/sampling_logp_difference/max": 0.36126255989074707, "sampling/sampling_logp_difference/mean": 0.012150084599852562, "step": 134 }, { "clip_ratio/high_max": 5.597850395133719e-05, "clip_ratio/high_mean": 5.597850395133719e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 5.597850395133719e-05, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 712.0, "completions/mean_length": 462.875, "completions/mean_terminated_length": 442.5333557128906, "completions/min_length": 317.0, "completions/min_terminated_length": 317.0, "entropy": 0.4520307444036007, "epoch": 0.07226980728051392, "frac_reward_zero_std": 0.5, "grad_norm": 0.024488739669322968, "learning_rate": 1e-05, "loss": 0.0455, "num_tokens": 3074610.0, "reward": 0.84375, "reward_std": 0.22201895713806152, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.2992031574249268, "sampling/importance_sampling_ratio/mean": 1.0000141859054565, "sampling/importance_sampling_ratio/min": 0.7068355679512024, "sampling/sampling_logp_difference/max": 0.3469572067260742, "sampling/sampling_logp_difference/mean": 0.01334022544324398, "step": 135 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.610413063550368e-05, "clip_ratio/low_min": 5.610413063550368e-05, "clip_ratio/region_mean": 5.610413063550368e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 754.0, "completions/max_terminated_length": 754.0, "completions/mean_length": 527.53125, "completions/mean_terminated_length": 527.53125, "completions/min_length": 192.0, "completions/min_terminated_length": 192.0, "entropy": 0.3511754237115383, "epoch": 0.0728051391862955, "frac_reward_zero_std": 0.5, "grad_norm": 0.004012619145214558, "learning_rate": 1e-05, "loss": 0.0407, "num_tokens": 3095515.0, "reward": 0.75, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.3161683082580566, "sampling/importance_sampling_ratio/mean": 0.9997851848602295, "sampling/importance_sampling_ratio/min": 0.721417248249054, "sampling/sampling_logp_difference/max": 0.3265376091003418, "sampling/sampling_logp_difference/mean": 0.01028917171061039, "step": 136 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.83746116515249e-05, "clip_ratio/low_min": 4.83746116515249e-05, "clip_ratio/region_mean": 4.83746116515249e-05, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 730.0, "completions/mean_length": 566.3125, "completions/mean_terminated_length": 537.5, "completions/min_length": 339.0, "completions/min_terminated_length": 339.0, "entropy": 0.33679792284965515, "epoch": 0.07334047109207709, "frac_reward_zero_std": 0.5, "grad_norm": 0.0023172921501100063, "learning_rate": 1e-05, "loss": 0.0861, "num_tokens": 3117069.0, "reward": 0.8125, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.3584282398223877, "sampling/importance_sampling_ratio/mean": 1.0001167058944702, "sampling/importance_sampling_ratio/min": 0.7186372876167297, "sampling/sampling_logp_difference/max": 0.3303985595703125, "sampling/sampling_logp_difference/mean": 0.010091305710375309, "step": 137 }, { "clip_ratio/high_max": 5.666364450007677e-05, "clip_ratio/high_mean": 5.666364450007677e-05, "clip_ratio/low_mean": 5.509034963324666e-05, "clip_ratio/low_min": 5.509034963324666e-05, "clip_ratio/region_mean": 0.00011175399413332343, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 756.0, "completions/mean_length": 559.03125, "completions/mean_terminated_length": 520.3333129882812, "completions/min_length": 249.0, "completions/min_terminated_length": 249.0, "entropy": 0.3786024637520313, "epoch": 0.07387580299785867, "frac_reward_zero_std": 0.25, "grad_norm": 0.02267376147210598, "learning_rate": 1e-05, "loss": 0.0742, "num_tokens": 3138678.0, "reward": 0.75, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.4510283470153809, "sampling/importance_sampling_ratio/mean": 0.9997961521148682, "sampling/importance_sampling_ratio/min": 0.7228491306304932, "sampling/sampling_logp_difference/max": 0.3722724914550781, "sampling/sampling_logp_difference/mean": 0.011123294942080975, "step": 138 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 756.0, "completions/mean_length": 540.09375, "completions/mean_terminated_length": 507.5357360839844, "completions/min_length": 241.0, "completions/min_terminated_length": 241.0, "entropy": 0.3804616779088974, "epoch": 0.07441113490364026, "frac_reward_zero_std": 0.25, "grad_norm": 0.011127207428216934, "learning_rate": 1e-05, "loss": 0.0509, "num_tokens": 3159161.0, "reward": 0.6875, "reward_std": 0.3471825420856476, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.4004617929458618, "sampling/importance_sampling_ratio/mean": 1.0003966093063354, "sampling/importance_sampling_ratio/min": 0.7051466703414917, "sampling/sampling_logp_difference/max": 0.3493494987487793, "sampling/sampling_logp_difference/mean": 0.011153659783303738, "step": 139 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.263301525497809e-05, "clip_ratio/low_min": 4.263301525497809e-05, "clip_ratio/region_mean": 4.263301525497809e-05, "completions/clipped_ratio": 0.34375, "completions/max_length": 768.0, "completions/max_terminated_length": 759.0, "completions/mean_length": 608.53125, "completions/mean_terminated_length": 525.0, "completions/min_length": 338.0, "completions/min_terminated_length": 338.0, "entropy": 0.4498027302324772, "epoch": 0.07494646680942184, "frac_reward_zero_std": 0.25, "grad_norm": 0.003857298055663705, "learning_rate": 1e-05, "loss": 0.0745, "num_tokens": 3181954.0, "reward": 0.5625, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.3739570379257202, "sampling/importance_sampling_ratio/mean": 0.9998639225959778, "sampling/importance_sampling_ratio/min": 0.3522147238254547, "sampling/sampling_logp_difference/max": 1.0435142517089844, "sampling/sampling_logp_difference/mean": 0.011898073367774487, "step": 140 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00020862984820269048, "clip_ratio/low_min": 0.00020862984820269048, "clip_ratio/region_mean": 0.00020862984820269048, "completions/clipped_ratio": 0.40625, "completions/max_length": 768.0, "completions/max_terminated_length": 735.0, "completions/mean_length": 677.0, "completions/mean_terminated_length": 614.73681640625, "completions/min_length": 495.0, "completions/min_terminated_length": 495.0, "entropy": 0.4274517484009266, "epoch": 0.07548179871520343, "frac_reward_zero_std": 0.0, "grad_norm": 0.010394889861345291, "learning_rate": 1e-05, "loss": 0.0708, "num_tokens": 3207506.0, "reward": 0.5625, "reward_std": 0.49022960662841797, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.375177025794983, "sampling/importance_sampling_ratio/mean": 0.999890148639679, "sampling/importance_sampling_ratio/min": 0.594200611114502, "sampling/sampling_logp_difference/max": 0.520538330078125, "sampling/sampling_logp_difference/mean": 0.012031642720103264, "step": 141 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 603.0, "completions/mean_length": 451.5, "completions/mean_terminated_length": 441.2903137207031, "completions/min_length": 175.0, "completions/min_terminated_length": 175.0, "entropy": 0.3740456774830818, "epoch": 0.07601713062098502, "frac_reward_zero_std": 0.75, "grad_norm": 0.00951254740357399, "learning_rate": 1e-05, "loss": 0.0289, "num_tokens": 3225306.0, "reward": 0.90625, "reward_std": 0.1293872892856598, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.455404281616211, "sampling/importance_sampling_ratio/mean": 0.9996922016143799, "sampling/importance_sampling_ratio/min": 0.5443582534790039, "sampling/sampling_logp_difference/max": 0.6081476211547852, "sampling/sampling_logp_difference/mean": 0.010931653901934624, "step": 142 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00014522213677992113, "clip_ratio/low_min": 0.00014522213677992113, "clip_ratio/region_mean": 0.00014522213677992113, "completions/clipped_ratio": 0.3125, "completions/max_length": 768.0, "completions/max_terminated_length": 753.0, "completions/mean_length": 631.84375, "completions/mean_terminated_length": 569.95458984375, "completions/min_length": 367.0, "completions/min_terminated_length": 367.0, "entropy": 0.48440078645944595, "epoch": 0.07655246252676659, "frac_reward_zero_std": 0.0, "grad_norm": 0.010871878825128078, "learning_rate": 1e-05, "loss": 0.0896, "num_tokens": 3249189.0, "reward": 0.5, "reward_std": 0.4082317352294922, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5080004930496216, "sampling/importance_sampling_ratio/max": 1.4944756031036377, "sampling/importance_sampling_ratio/mean": 0.9997045397758484, "sampling/importance_sampling_ratio/min": 0.04641878977417946, "sampling/sampling_logp_difference/max": 3.0700509548187256, "sampling/sampling_logp_difference/mean": 0.013384725898504257, "step": 143 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.835667616338469e-05, "clip_ratio/low_min": 5.835667616338469e-05, "clip_ratio/region_mean": 5.835667616338469e-05, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 761.0, "completions/mean_length": 521.78125, "completions/mean_terminated_length": 476.1851806640625, "completions/min_length": 249.0, "completions/min_terminated_length": 249.0, "entropy": 0.4121195822954178, "epoch": 0.07708779443254818, "frac_reward_zero_std": 0.25, "grad_norm": 0.01367057953029871, "learning_rate": 1e-05, "loss": -0.013, "num_tokens": 3269502.0, "reward": 0.6875, "reward_std": 0.292504221200943, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.3120893239974976, "sampling/importance_sampling_ratio/mean": 1.0002217292785645, "sampling/importance_sampling_ratio/min": 0.6409497857093811, "sampling/sampling_logp_difference/max": 0.44480419158935547, "sampling/sampling_logp_difference/mean": 0.012407545000314713, "step": 144 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 9.98272662400268e-05, "clip_ratio/low_min": 9.98272662400268e-05, "clip_ratio/region_mean": 9.98272662400268e-05, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 727.0, "completions/mean_length": 566.59375, "completions/mean_terminated_length": 520.1154174804688, "completions/min_length": 217.0, "completions/min_terminated_length": 217.0, "entropy": 0.43163253366947174, "epoch": 0.07762312633832977, "frac_reward_zero_std": 0.25, "grad_norm": 0.010163567960262299, "learning_rate": 1e-05, "loss": 0.0858, "num_tokens": 3291289.0, "reward": 0.78125, "reward_std": 0.3061639666557312, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.2963974475860596, "sampling/importance_sampling_ratio/mean": 1.0003619194030762, "sampling/importance_sampling_ratio/min": 0.6973382830619812, "sampling/sampling_logp_difference/max": 0.36048460006713867, "sampling/sampling_logp_difference/mean": 0.011822182685136795, "step": 145 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 638.0, "completions/mean_length": 438.0, "completions/mean_terminated_length": 427.3548278808594, "completions/min_length": 146.0, "completions/min_terminated_length": 146.0, "entropy": 0.4192674495279789, "epoch": 0.07815845824411134, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": -0.0508, "num_tokens": 3308793.0, "reward": 0.84375, "reward_std": 0.1293872892856598, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.4280418157577515, "sampling/importance_sampling_ratio/mean": 0.9996333122253418, "sampling/importance_sampling_ratio/min": 0.6107430458068848, "sampling/sampling_logp_difference/max": 0.49307894706726074, "sampling/sampling_logp_difference/mean": 0.012461808510124683, "step": 146 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.28125, "completions/max_length": 768.0, "completions/max_terminated_length": 704.0, "completions/mean_length": 567.40625, "completions/mean_terminated_length": 488.9130554199219, "completions/min_length": 289.0, "completions/min_terminated_length": 289.0, "entropy": 0.5919518247246742, "epoch": 0.07869379014989293, "frac_reward_zero_std": 0.25, "grad_norm": 0.009975074790418148, "learning_rate": 1e-05, "loss": 0.0444, "num_tokens": 3330606.0, "reward": 0.625, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.3223353624343872, "sampling/importance_sampling_ratio/mean": 0.9999397993087769, "sampling/importance_sampling_ratio/min": 0.6502242088317871, "sampling/sampling_logp_difference/max": 0.4304380416870117, "sampling/sampling_logp_difference/mean": 0.014194437302649021, "step": 147 }, { "clip_ratio/high_max": 6.088650843594223e-05, "clip_ratio/high_mean": 6.088650843594223e-05, "clip_ratio/low_mean": 5.2764880820177495e-05, "clip_ratio/low_min": 5.2764880820177495e-05, "clip_ratio/region_mean": 0.00011365138925611973, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 644.0, "completions/mean_length": 564.59375, "completions/mean_terminated_length": 496.79168701171875, "completions/min_length": 327.0, "completions/min_terminated_length": 327.0, "entropy": 0.39994125068187714, "epoch": 0.07922912205567452, "frac_reward_zero_std": 0.5, "grad_norm": 0.006876808591187, "learning_rate": 1e-05, "loss": -0.0087, "num_tokens": 3352321.0, "reward": 0.625, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.4018239974975586, "sampling/importance_sampling_ratio/mean": 1.0000483989715576, "sampling/importance_sampling_ratio/min": 0.7175582051277161, "sampling/sampling_logp_difference/max": 0.33777427673339844, "sampling/sampling_logp_difference/mean": 0.011508024297654629, "step": 148 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00015411724598379806, "clip_ratio/low_min": 0.00015411724598379806, "clip_ratio/region_mean": 0.00015411724598379806, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 746.0, "completions/mean_length": 605.71875, "completions/mean_terminated_length": 582.5357666015625, "completions/min_length": 397.0, "completions/min_terminated_length": 397.0, "entropy": 0.33412376418709755, "epoch": 0.07976445396145611, "frac_reward_zero_std": 0.25, "grad_norm": 0.011310392990708351, "learning_rate": 1e-05, "loss": 0.0791, "num_tokens": 3375168.0, "reward": 0.75, "reward_std": 0.3650856614112854, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.3741053342819214, "sampling/importance_sampling_ratio/mean": 1.0000348091125488, "sampling/importance_sampling_ratio/min": 0.6577353477478027, "sampling/sampling_logp_difference/max": 0.4189525842666626, "sampling/sampling_logp_difference/mean": 0.010629684664309025, "step": 149 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00010651181219145656, "clip_ratio/low_min": 0.00010651181219145656, "clip_ratio/region_mean": 0.00010651181219145656, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 727.0, "completions/mean_length": 526.53125, "completions/mean_terminated_length": 510.433349609375, "completions/min_length": 281.0, "completions/min_terminated_length": 281.0, "entropy": 0.3149258513003588, "epoch": 0.08029978586723768, "frac_reward_zero_std": 0.5, "grad_norm": 0.0036895317025482655, "learning_rate": 1e-05, "loss": -0.0039, "num_tokens": 3395641.0, "reward": 0.8125, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.4270235300064087, "sampling/importance_sampling_ratio/mean": 0.9999079704284668, "sampling/importance_sampling_ratio/min": 0.6860512495040894, "sampling/sampling_logp_difference/max": 0.376802921295166, "sampling/sampling_logp_difference/mean": 0.010782305151224136, "step": 150 }, { "clip_ratio/high_max": 4.77281391795259e-05, "clip_ratio/high_mean": 4.77281391795259e-05, "clip_ratio/low_mean": 0.0003206419605703559, "clip_ratio/low_min": 0.0003206419605703559, "clip_ratio/region_mean": 0.0003683700997498818, "completions/clipped_ratio": 0.3125, "completions/max_length": 768.0, "completions/max_terminated_length": 669.0, "completions/mean_length": 615.0, "completions/mean_terminated_length": 545.45458984375, "completions/min_length": 262.0, "completions/min_terminated_length": 262.0, "entropy": 0.4121477007865906, "epoch": 0.08083511777301927, "frac_reward_zero_std": 0.0, "grad_norm": 0.00616479804739356, "learning_rate": 1e-05, "loss": 0.0981, "num_tokens": 3419313.0, "reward": 0.5, "reward_std": 0.4671337604522705, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5080004930496216, "sampling/importance_sampling_ratio/max": 1.6228677034378052, "sampling/importance_sampling_ratio/mean": 1.0000944137573242, "sampling/importance_sampling_ratio/min": 0.7266005873680115, "sampling/sampling_logp_difference/max": 0.4841947555541992, "sampling/sampling_logp_difference/mean": 0.012517292983829975, "step": 151 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00013014055730309337, "clip_ratio/low_min": 0.00013014055730309337, "clip_ratio/region_mean": 0.00013014055730309337, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 764.0, "completions/mean_length": 547.90625, "completions/mean_terminated_length": 516.4642944335938, "completions/min_length": 148.0, "completions/min_terminated_length": 148.0, "entropy": 0.4624267853796482, "epoch": 0.08137044967880086, "frac_reward_zero_std": 0.25, "grad_norm": 0.011894823983311653, "learning_rate": 1e-05, "loss": 0.0366, "num_tokens": 3440686.0, "reward": 0.75, "reward_std": 0.3650856614112854, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.5203043222427368, "sampling/importance_sampling_ratio/mean": 1.0004301071166992, "sampling/importance_sampling_ratio/min": 0.7335498332977295, "sampling/sampling_logp_difference/max": 0.41891050338745117, "sampling/sampling_logp_difference/mean": 0.012454214505851269, "step": 152 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00010090626892633736, "clip_ratio/low_min": 0.00010090626892633736, "clip_ratio/region_mean": 0.00010090626892633736, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 748.0, "completions/mean_length": 571.09375, "completions/mean_terminated_length": 542.9642944335938, "completions/min_length": 129.0, "completions/min_terminated_length": 129.0, "entropy": 0.43677468225359917, "epoch": 0.08190578158458243, "frac_reward_zero_std": 0.25, "grad_norm": 0.020136717706918716, "learning_rate": 1e-05, "loss": 0.065, "num_tokens": 3462585.0, "reward": 0.71875, "reward_std": 0.2651650309562683, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.4207556247711182, "sampling/importance_sampling_ratio/mean": 1.0001400709152222, "sampling/importance_sampling_ratio/min": 0.6170693635940552, "sampling/sampling_logp_difference/max": 0.4827737808227539, "sampling/sampling_logp_difference/mean": 0.012302085757255554, "step": 153 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.747126488131471e-05, "clip_ratio/low_min": 5.747126488131471e-05, "clip_ratio/region_mean": 5.747126488131471e-05, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 757.0, "completions/mean_length": 511.8125, "completions/mean_terminated_length": 475.21429443359375, "completions/min_length": 252.0, "completions/min_terminated_length": 252.0, "entropy": 0.4246552102267742, "epoch": 0.08244111349036402, "frac_reward_zero_std": 0.5, "grad_norm": 0.0030132178217172623, "learning_rate": 1e-05, "loss": 0.0813, "num_tokens": 3483059.0, "reward": 0.8125, "reward_std": 0.249358132481575, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.4191759824752808, "sampling/importance_sampling_ratio/mean": 1.0000072717666626, "sampling/importance_sampling_ratio/min": 0.6935054659843445, "sampling/sampling_logp_difference/max": 0.3659961223602295, "sampling/sampling_logp_difference/mean": 0.012803583405911922, "step": 154 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.28125, "completions/max_length": 768.0, "completions/max_terminated_length": 743.0, "completions/mean_length": 568.84375, "completions/mean_terminated_length": 490.9130554199219, "completions/min_length": 260.0, "completions/min_terminated_length": 260.0, "entropy": 0.3456209786236286, "epoch": 0.08297644539614561, "frac_reward_zero_std": 0.75, "grad_norm": 0.008401411585509777, "learning_rate": 1e-05, "loss": 0.0158, "num_tokens": 3505118.0, "reward": 0.65625, "reward_std": 0.1293872892856598, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.3933793306350708, "sampling/importance_sampling_ratio/mean": 1.000069260597229, "sampling/importance_sampling_ratio/min": 0.4956018328666687, "sampling/sampling_logp_difference/max": 0.7019823789596558, "sampling/sampling_logp_difference/mean": 0.010431949980556965, "step": 155 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00012369978139759041, "clip_ratio/low_min": 0.00012369978139759041, "clip_ratio/region_mean": 0.00012369978139759041, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 747.0, "completions/mean_length": 555.34375, "completions/mean_terminated_length": 515.9629516601562, "completions/min_length": 226.0, "completions/min_terminated_length": 226.0, "entropy": 0.35629772394895554, "epoch": 0.0835117773019272, "frac_reward_zero_std": 0.25, "grad_norm": 0.007527804002165794, "learning_rate": 1e-05, "loss": 0.0314, "num_tokens": 3526785.0, "reward": 0.65625, "reward_std": 0.3471629321575165, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.6204110383987427, "sampling/importance_sampling_ratio/mean": 1.0002415180206299, "sampling/importance_sampling_ratio/min": 0.6880589127540588, "sampling/sampling_logp_difference/max": 0.4826798439025879, "sampling/sampling_logp_difference/mean": 0.011526282876729965, "step": 156 }, { "clip_ratio/high_max": 6.561679765582085e-05, "clip_ratio/high_mean": 6.561679765582085e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 6.561679765582085e-05, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 675.0, "completions/mean_length": 469.46875, "completions/mean_terminated_length": 459.83868408203125, "completions/min_length": 274.0, "completions/min_terminated_length": 274.0, "entropy": 0.3269459344446659, "epoch": 0.08404710920770878, "frac_reward_zero_std": 0.75, "grad_norm": 0.020380010828375816, "learning_rate": 1e-05, "loss": 0.043, "num_tokens": 3545336.0, "reward": 0.96875, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.96875, "rewards/accuracy_reward/std": 0.1767766922712326, "sampling/importance_sampling_ratio/max": 1.3741401433944702, "sampling/importance_sampling_ratio/mean": 0.9999512434005737, "sampling/importance_sampling_ratio/min": 0.7038865685462952, "sampling/sampling_logp_difference/max": 0.3511381149291992, "sampling/sampling_logp_difference/mean": 0.010684875771403313, "step": 157 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.076811041566543e-05, "clip_ratio/low_min": 6.076811041566543e-05, "clip_ratio/region_mean": 6.076811041566543e-05, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 750.0, "completions/mean_length": 556.21875, "completions/mean_terminated_length": 496.91998291015625, "completions/min_length": 331.0, "completions/min_terminated_length": 331.0, "entropy": 0.4623580873012543, "epoch": 0.08458244111349036, "frac_reward_zero_std": 0.5, "grad_norm": 0.009135546162724495, "learning_rate": 1e-05, "loss": 0.0285, "num_tokens": 3567031.0, "reward": 0.65625, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.430266261100769, "sampling/importance_sampling_ratio/mean": 0.9999651312828064, "sampling/importance_sampling_ratio/min": 0.6599296927452087, "sampling/sampling_logp_difference/max": 0.4156219959259033, "sampling/sampling_logp_difference/mean": 0.013079782947897911, "step": 158 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 7.042253855615854e-05, "clip_ratio/low_min": 7.042253855615854e-05, "clip_ratio/region_mean": 7.042253855615854e-05, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 711.0, "completions/mean_length": 515.21875, "completions/mean_terminated_length": 498.36669921875, "completions/min_length": 222.0, "completions/min_terminated_length": 222.0, "entropy": 0.39342348650097847, "epoch": 0.08511777301927195, "frac_reward_zero_std": 0.5, "grad_norm": 0.01613238826394081, "learning_rate": 1e-05, "loss": 0.0178, "num_tokens": 3587214.0, "reward": 0.8125, "reward_std": 0.2587745785713196, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.441656470298767, "sampling/importance_sampling_ratio/mean": 1.000588059425354, "sampling/importance_sampling_ratio/min": 0.6850452423095703, "sampling/sampling_logp_difference/max": 0.37827038764953613, "sampling/sampling_logp_difference/mean": 0.01200867723673582, "step": 159 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00010963959721266292, "clip_ratio/low_min": 0.00010963959721266292, "clip_ratio/region_mean": 0.00010963959721266292, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 761.0, "completions/mean_length": 559.65625, "completions/mean_terminated_length": 521.0740966796875, "completions/min_length": 331.0, "completions/min_terminated_length": 331.0, "entropy": 0.5497498512268066, "epoch": 0.08565310492505353, "frac_reward_zero_std": 0.0, "grad_norm": 0.010425073094666004, "learning_rate": 1e-05, "loss": 0.1146, "num_tokens": 3608723.0, "reward": 0.625, "reward_std": 0.48503684997558594, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.3026947975158691, "sampling/importance_sampling_ratio/mean": 1.0003063678741455, "sampling/importance_sampling_ratio/min": 0.697893500328064, "sampling/sampling_logp_difference/max": 0.35968875885009766, "sampling/sampling_logp_difference/mean": 0.01383618451654911, "step": 160 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00019822225294774398, "clip_ratio/low_min": 0.00019822225294774398, "clip_ratio/region_mean": 0.00019822225294774398, "completions/clipped_ratio": 0.40625, "completions/max_length": 768.0, "completions/max_terminated_length": 757.0, "completions/mean_length": 639.375, "completions/mean_terminated_length": 551.368408203125, "completions/min_length": 316.0, "completions/min_terminated_length": 316.0, "entropy": 0.6377146951854229, "epoch": 0.08618843683083512, "frac_reward_zero_std": 0.0, "grad_norm": 0.021822459995746613, "learning_rate": 1e-05, "loss": 0.1258, "num_tokens": 3632943.0, "reward": 0.375, "reward_std": 0.4492306709289551, "rewards/accuracy_reward/mean": 0.375, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.2977290153503418, "sampling/importance_sampling_ratio/mean": 0.9999541640281677, "sampling/importance_sampling_ratio/min": 0.7110767960548401, "sampling/sampling_logp_difference/max": 0.3409748077392578, "sampling/sampling_logp_difference/mean": 0.01600494794547558, "step": 161 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00011524299407028593, "clip_ratio/low_min": 0.00011524299407028593, "clip_ratio/region_mean": 0.00011524299407028593, "completions/clipped_ratio": 0.375, "completions/max_length": 768.0, "completions/max_terminated_length": 746.0, "completions/mean_length": 595.1875, "completions/mean_terminated_length": 491.5, "completions/min_length": 214.0, "completions/min_terminated_length": 214.0, "entropy": 0.688156247138977, "epoch": 0.0867237687366167, "frac_reward_zero_std": 0.25, "grad_norm": 0.024438703432679176, "learning_rate": 1e-05, "loss": 0.0774, "num_tokens": 3655837.0, "reward": 0.5625, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.3606370687484741, "sampling/importance_sampling_ratio/mean": 1.0005593299865723, "sampling/importance_sampling_ratio/min": 0.5256821513175964, "sampling/sampling_logp_difference/max": 0.6430585384368896, "sampling/sampling_logp_difference/mean": 0.015769150108098984, "step": 162 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.3146257414482534e-05, "clip_ratio/low_min": 5.3146257414482534e-05, "clip_ratio/region_mean": 5.3146257414482534e-05, "completions/clipped_ratio": 0.40625, "completions/max_length": 768.0, "completions/max_terminated_length": 739.0, "completions/mean_length": 620.71875, "completions/mean_terminated_length": 519.9473876953125, "completions/min_length": 334.0, "completions/min_terminated_length": 334.0, "entropy": 0.6735220775008202, "epoch": 0.0872591006423983, "frac_reward_zero_std": 0.75, "grad_norm": 0.0015876619145274162, "learning_rate": 1e-05, "loss": -0.011, "num_tokens": 3680036.0, "reward": 0.21875, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.21875, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.445966124534607, "sampling/importance_sampling_ratio/mean": 0.9998120665550232, "sampling/importance_sampling_ratio/min": 0.7061793208122253, "sampling/sampling_logp_difference/max": 0.36877763271331787, "sampling/sampling_logp_difference/mean": 0.018158145248889923, "step": 163 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.313131598290056e-05, "clip_ratio/low_min": 6.313131598290056e-05, "clip_ratio/region_mean": 6.313131598290056e-05, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 729.0, "completions/mean_length": 552.53125, "completions/mean_terminated_length": 512.629638671875, "completions/min_length": 291.0, "completions/min_terminated_length": 291.0, "entropy": 0.4686658903956413, "epoch": 0.08779443254817987, "frac_reward_zero_std": 0.25, "grad_norm": 0.014513184316456318, "learning_rate": 1e-05, "loss": 0.0026, "num_tokens": 3701349.0, "reward": 0.5625, "reward_std": 0.2925041913986206, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.3749197721481323, "sampling/importance_sampling_ratio/mean": 0.9998638033866882, "sampling/importance_sampling_ratio/min": 0.743603527545929, "sampling/sampling_logp_difference/max": 0.31839537620544434, "sampling/sampling_logp_difference/mean": 0.012418043799698353, "step": 164 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.291108780307695e-05, "clip_ratio/low_min": 4.291108780307695e-05, "clip_ratio/region_mean": 4.291108780307695e-05, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 702.0, "completions/mean_length": 572.625, "completions/mean_terminated_length": 527.5384521484375, "completions/min_length": 325.0, "completions/min_terminated_length": 325.0, "entropy": 0.38484838977456093, "epoch": 0.08832976445396146, "frac_reward_zero_std": 0.5, "grad_norm": 0.008719516918063164, "learning_rate": 1e-05, "loss": 0.0423, "num_tokens": 3722873.0, "reward": 0.53125, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.4222228527069092, "sampling/importance_sampling_ratio/mean": 0.9997617602348328, "sampling/importance_sampling_ratio/min": 0.5860301852226257, "sampling/sampling_logp_difference/max": 0.53438401222229, "sampling/sampling_logp_difference/mean": 0.011152560822665691, "step": 165 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00011565401655388996, "clip_ratio/low_min": 0.00011565401655388996, "clip_ratio/region_mean": 0.00011565401655388996, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 756.0, "completions/mean_length": 555.40625, "completions/mean_terminated_length": 525.0357666015625, "completions/min_length": 307.0, "completions/min_terminated_length": 307.0, "entropy": 0.3937302567064762, "epoch": 0.08886509635974305, "frac_reward_zero_std": 0.25, "grad_norm": 0.01666826754808426, "learning_rate": 1e-05, "loss": 0.0691, "num_tokens": 3744302.0, "reward": 0.8125, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.4650695323944092, "sampling/importance_sampling_ratio/mean": 0.9999182820320129, "sampling/importance_sampling_ratio/min": 0.6995806097984314, "sampling/sampling_logp_difference/max": 0.38190269470214844, "sampling/sampling_logp_difference/mean": 0.011928286403417587, "step": 166 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.9465768825029954e-05, "clip_ratio/low_min": 4.9465768825029954e-05, "clip_ratio/region_mean": 4.9465768825029954e-05, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 748.0, "completions/mean_length": 618.34375, "completions/mean_terminated_length": 590.629638671875, "completions/min_length": 308.0, "completions/min_terminated_length": 308.0, "entropy": 0.42345913872122765, "epoch": 0.08940042826552462, "frac_reward_zero_std": 0.0, "grad_norm": 0.020113669335842133, "learning_rate": 1e-05, "loss": 0.0251, "num_tokens": 3767561.0, "reward": 0.75, "reward_std": 0.4261348247528076, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.5204405784606934, "sampling/importance_sampling_ratio/mean": 0.9999188184738159, "sampling/importance_sampling_ratio/min": 0.6613926887512207, "sampling/sampling_logp_difference/max": 0.41900014877319336, "sampling/sampling_logp_difference/mean": 0.01242354791611433, "step": 167 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.518763646250591e-05, "clip_ratio/low_min": 5.518763646250591e-05, "clip_ratio/region_mean": 5.518763646250591e-05, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 763.0, "completions/mean_length": 587.15625, "completions/mean_terminated_length": 545.423095703125, "completions/min_length": 295.0, "completions/min_terminated_length": 295.0, "entropy": 0.5502735823392868, "epoch": 0.08993576017130621, "frac_reward_zero_std": 0.25, "grad_norm": 0.009137660264968872, "learning_rate": 1e-05, "loss": 0.0746, "num_tokens": 3789854.0, "reward": 0.71875, "reward_std": 0.3608423173427582, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.3804175853729248, "sampling/importance_sampling_ratio/mean": 0.9998465776443481, "sampling/importance_sampling_ratio/min": 0.41815128922462463, "sampling/sampling_logp_difference/max": 0.8719120025634766, "sampling/sampling_logp_difference/mean": 0.013950812630355358, "step": 168 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 727.0, "completions/mean_length": 538.34375, "completions/mean_terminated_length": 461.79168701171875, "completions/min_length": 192.0, "completions/min_terminated_length": 192.0, "entropy": 0.5538909025490284, "epoch": 0.0904710920770878, "frac_reward_zero_std": 0.5, "grad_norm": 0.002491187071427703, "learning_rate": 1e-05, "loss": -0.006, "num_tokens": 3811297.0, "reward": 0.6875, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.396300196647644, "sampling/importance_sampling_ratio/mean": 1.0003321170806885, "sampling/importance_sampling_ratio/min": 0.7034684419631958, "sampling/sampling_logp_difference/max": 0.3517322540283203, "sampling/sampling_logp_difference/mean": 0.01397632621228695, "step": 169 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 648.0, "completions/mean_length": 493.21875, "completions/mean_terminated_length": 453.96429443359375, "completions/min_length": 277.0, "completions/min_terminated_length": 277.0, "entropy": 0.49675028398633003, "epoch": 0.09100642398286937, "frac_reward_zero_std": 0.25, "grad_norm": 0.0036047326866537333, "learning_rate": 1e-05, "loss": 0.0405, "num_tokens": 3830944.0, "reward": 0.71875, "reward_std": 0.3198433816432953, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.3590514659881592, "sampling/importance_sampling_ratio/mean": 1.0000966787338257, "sampling/importance_sampling_ratio/min": 0.6605715155601501, "sampling/sampling_logp_difference/max": 0.41464996337890625, "sampling/sampling_logp_difference/mean": 0.014946396462619305, "step": 170 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.941064591752365e-05, "clip_ratio/low_min": 5.941064591752365e-05, "clip_ratio/region_mean": 5.941064591752365e-05, "completions/clipped_ratio": 0.34375, "completions/max_length": 768.0, "completions/max_terminated_length": 733.0, "completions/mean_length": 617.0, "completions/mean_terminated_length": 537.90478515625, "completions/min_length": 246.0, "completions/min_terminated_length": 246.0, "entropy": 0.33820777013897896, "epoch": 0.09154175588865096, "frac_reward_zero_std": 0.5, "grad_norm": 0.007598424796015024, "learning_rate": 1e-05, "loss": 0.0358, "num_tokens": 3854616.0, "reward": 0.4375, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.4375, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.322767734527588, "sampling/importance_sampling_ratio/mean": 1.0002580881118774, "sampling/importance_sampling_ratio/min": 0.7275824546813965, "sampling/sampling_logp_difference/max": 0.31802797317504883, "sampling/sampling_logp_difference/mean": 0.010352818295359612, "step": 171 }, { "clip_ratio/high_max": 7.931471918709576e-05, "clip_ratio/high_mean": 7.931471918709576e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 7.931471918709576e-05, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 757.0, "completions/mean_length": 482.625, "completions/mean_terminated_length": 473.4193420410156, "completions/min_length": 266.0, "completions/min_terminated_length": 266.0, "entropy": 0.37833021581172943, "epoch": 0.09207708779443255, "frac_reward_zero_std": 0.5, "grad_norm": 0.002545375842601061, "learning_rate": 1e-05, "loss": -0.009, "num_tokens": 3873516.0, "reward": 0.9375, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.9375, "rewards/accuracy_reward/std": 0.24593468010425568, "sampling/importance_sampling_ratio/max": 1.3294203281402588, "sampling/importance_sampling_ratio/mean": 1.0000876188278198, "sampling/importance_sampling_ratio/min": 0.7000898718833923, "sampling/sampling_logp_difference/max": 0.3565465211868286, "sampling/sampling_logp_difference/mean": 0.011979430913925171, "step": 172 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 9.448358468944207e-05, "clip_ratio/low_min": 9.448358468944207e-05, "clip_ratio/region_mean": 9.448358468944207e-05, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 727.0, "completions/mean_length": 543.09375, "completions/mean_terminated_length": 510.96429443359375, "completions/min_length": 320.0, "completions/min_terminated_length": 320.0, "entropy": 0.4378381334245205, "epoch": 0.09261241970021414, "frac_reward_zero_std": 0.75, "grad_norm": 0.0038268915377557278, "learning_rate": 1e-05, "loss": 0.0027, "num_tokens": 3894975.0, "reward": 0.53125, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.4210575819015503, "sampling/importance_sampling_ratio/mean": 1.0002058744430542, "sampling/importance_sampling_ratio/min": 0.6890934705734253, "sampling/sampling_logp_difference/max": 0.3723783493041992, "sampling/sampling_logp_difference/mean": 0.013296155259013176, "step": 173 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 732.0, "completions/mean_length": 470.59375, "completions/mean_terminated_length": 450.7666931152344, "completions/min_length": 198.0, "completions/min_terminated_length": 198.0, "entropy": 0.37185758352279663, "epoch": 0.09314775160599571, "frac_reward_zero_std": 0.25, "grad_norm": 0.023561233654618263, "learning_rate": 1e-05, "loss": 0.04, "num_tokens": 3913578.0, "reward": 0.78125, "reward_std": 0.3608423173427582, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.4191864728927612, "sampling/importance_sampling_ratio/mean": 1.000483751296997, "sampling/importance_sampling_ratio/min": 0.6901246905326843, "sampling/sampling_logp_difference/max": 0.3708829879760742, "sampling/sampling_logp_difference/mean": 0.011572744697332382, "step": 174 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 728.0, "completions/mean_length": 529.375, "completions/mean_terminated_length": 521.6774291992188, "completions/min_length": 306.0, "completions/min_terminated_length": 306.0, "entropy": 0.3523942269384861, "epoch": 0.0936830835117773, "frac_reward_zero_std": 0.5, "grad_norm": 0.0074403719045221806, "learning_rate": 1e-05, "loss": 0.0161, "num_tokens": 3934438.0, "reward": 0.65625, "reward_std": 0.22201895713806152, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.410582423210144, "sampling/importance_sampling_ratio/mean": 0.9998390674591064, "sampling/importance_sampling_ratio/min": 0.6631200313568115, "sampling/sampling_logp_difference/max": 0.4107992649078369, "sampling/sampling_logp_difference/mean": 0.011772280558943748, "step": 175 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 683.0, "completions/mean_length": 536.65625, "completions/mean_terminated_length": 459.54168701171875, "completions/min_length": 342.0, "completions/min_terminated_length": 342.0, "entropy": 0.5548071675002575, "epoch": 0.09421841541755889, "frac_reward_zero_std": 0.5, "grad_norm": 0.00894884392619133, "learning_rate": 1e-05, "loss": 0.0421, "num_tokens": 3955827.0, "reward": 0.3125, "reward_std": 0.249358132481575, "rewards/accuracy_reward/mean": 0.3125, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.5687905550003052, "sampling/importance_sampling_ratio/mean": 1.0000354051589966, "sampling/importance_sampling_ratio/min": 0.697902500629425, "sampling/sampling_logp_difference/max": 0.4503049850463867, "sampling/sampling_logp_difference/mean": 0.014390457421541214, "step": 176 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 8.932847413234413e-05, "clip_ratio/low_min": 8.932847413234413e-05, "clip_ratio/region_mean": 8.932847413234413e-05, "completions/clipped_ratio": 0.4375, "completions/max_length": 768.0, "completions/max_terminated_length": 738.0, "completions/mean_length": 637.34375, "completions/mean_terminated_length": 535.7222290039062, "completions/min_length": 301.0, "completions/min_terminated_length": 301.0, "entropy": 0.42246196419000626, "epoch": 0.09475374732334046, "frac_reward_zero_std": 0.25, "grad_norm": 0.008636223152279854, "learning_rate": 1e-05, "loss": -0.0237, "num_tokens": 3980702.0, "reward": 0.3125, "reward_std": 0.3745020925998688, "rewards/accuracy_reward/mean": 0.3125, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.3391796350479126, "sampling/importance_sampling_ratio/mean": 0.9999925494194031, "sampling/importance_sampling_ratio/min": 0.5327543020248413, "sampling/sampling_logp_difference/max": 0.629694938659668, "sampling/sampling_logp_difference/mean": 0.011958497576415539, "step": 177 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 750.0, "completions/mean_length": 466.6875, "completions/mean_terminated_length": 382.3199768066406, "completions/min_length": 219.0, "completions/min_terminated_length": 219.0, "entropy": 0.6823995895683765, "epoch": 0.09528907922912205, "frac_reward_zero_std": 0.75, "grad_norm": 0.005268024280667305, "learning_rate": 1e-05, "loss": 0.0028, "num_tokens": 3999092.0, "reward": 0.5625, "reward_std": 0.1157275140285492, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.514682650566101, "sampling/importance_sampling_ratio/mean": 1.0001227855682373, "sampling/importance_sampling_ratio/min": 0.7262601852416992, "sampling/sampling_logp_difference/max": 0.4152059555053711, "sampling/sampling_logp_difference/mean": 0.016032084822654724, "step": 178 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.463288445957005e-05, "clip_ratio/low_min": 6.463288445957005e-05, "clip_ratio/region_mean": 6.463288445957005e-05, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 692.0, "completions/mean_length": 490.625, "completions/mean_terminated_length": 461.9310302734375, "completions/min_length": 274.0, "completions/min_terminated_length": 274.0, "entropy": 0.5883337892591953, "epoch": 0.09582441113490364, "frac_reward_zero_std": 0.25, "grad_norm": 0.018840458244085312, "learning_rate": 1e-05, "loss": -0.0069, "num_tokens": 4018368.0, "reward": 0.53125, "reward_std": 0.3471629321575165, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.6545922756195068, "sampling/importance_sampling_ratio/mean": 0.9998740553855896, "sampling/importance_sampling_ratio/min": 0.5516250133514404, "sampling/sampling_logp_difference/max": 0.5948867797851562, "sampling/sampling_logp_difference/mean": 0.014858265407383442, "step": 179 }, { "clip_ratio/high_max": 5.3236795793054625e-05, "clip_ratio/high_mean": 5.3236795793054625e-05, "clip_ratio/low_mean": 5.484861685545184e-05, "clip_ratio/low_min": 5.484861685545184e-05, "clip_ratio/region_mean": 0.00010808541264850646, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 752.0, "completions/mean_length": 601.78125, "completions/mean_terminated_length": 546.375, "completions/min_length": 306.0, "completions/min_terminated_length": 306.0, "entropy": 0.3723117969930172, "epoch": 0.09635974304068523, "frac_reward_zero_std": 0.25, "grad_norm": 0.008200142532587051, "learning_rate": 1e-05, "loss": 0.0146, "num_tokens": 4041609.0, "reward": 0.71875, "reward_std": 0.3471629321575165, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.2989987134933472, "sampling/importance_sampling_ratio/mean": 1.000217080116272, "sampling/importance_sampling_ratio/min": 0.7527726292610168, "sampling/sampling_logp_difference/max": 0.28399205207824707, "sampling/sampling_logp_difference/mean": 0.011099657043814659, "step": 180 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.228201527846977e-05, "clip_ratio/low_min": 6.228201527846977e-05, "clip_ratio/region_mean": 6.228201527846977e-05, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 714.0, "completions/mean_length": 418.21875, "completions/mean_terminated_length": 406.93548583984375, "completions/min_length": 155.0, "completions/min_terminated_length": 155.0, "entropy": 0.4289981424808502, "epoch": 0.0968950749464668, "frac_reward_zero_std": 0.5, "grad_norm": 0.005340190604329109, "learning_rate": 1e-05, "loss": -0.0135, "num_tokens": 4058584.0, "reward": 0.78125, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.5157135725021362, "sampling/importance_sampling_ratio/mean": 0.9994528889656067, "sampling/importance_sampling_ratio/min": 0.7148792743682861, "sampling/sampling_logp_difference/max": 0.41588640213012695, "sampling/sampling_logp_difference/mean": 0.013451691716909409, "step": 181 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 719.0, "completions/mean_length": 585.3125, "completions/mean_terminated_length": 524.4166870117188, "completions/min_length": 229.0, "completions/min_terminated_length": 229.0, "entropy": 0.3565916270017624, "epoch": 0.0974304068522484, "frac_reward_zero_std": 0.25, "grad_norm": 0.008056404069066048, "learning_rate": 1e-05, "loss": 0.0141, "num_tokens": 4080922.0, "reward": 0.65625, "reward_std": 0.38816186785697937, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.3575481176376343, "sampling/importance_sampling_ratio/mean": 1.0001626014709473, "sampling/importance_sampling_ratio/min": 0.6891089081764221, "sampling/sampling_logp_difference/max": 0.37235593795776367, "sampling/sampling_logp_difference/mean": 0.010848519392311573, "step": 182 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00014377634579432197, "clip_ratio/low_min": 0.00014377634579432197, "clip_ratio/region_mean": 0.00014377634579432197, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 757.0, "completions/mean_length": 571.25, "completions/mean_terminated_length": 550.8965454101562, "completions/min_length": 320.0, "completions/min_terminated_length": 320.0, "entropy": 0.33851267769932747, "epoch": 0.09796573875802998, "frac_reward_zero_std": 0.5, "grad_norm": 0.018591763451695442, "learning_rate": 1e-05, "loss": 0.0479, "num_tokens": 4103602.0, "reward": 0.75, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.4362455606460571, "sampling/importance_sampling_ratio/mean": 1.0002412796020508, "sampling/importance_sampling_ratio/min": 0.7441653609275818, "sampling/sampling_logp_difference/max": 0.362032413482666, "sampling/sampling_logp_difference/mean": 0.010752060450613499, "step": 183 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 8.833922038320452e-05, "clip_ratio/low_min": 8.833922038320452e-05, "clip_ratio/region_mean": 8.833922038320452e-05, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 758.0, "completions/mean_length": 572.6875, "completions/mean_terminated_length": 518.0, "completions/min_length": 329.0, "completions/min_terminated_length": 329.0, "entropy": 0.4437847100198269, "epoch": 0.09850107066381156, "frac_reward_zero_std": 0.25, "grad_norm": 0.032799091190099716, "learning_rate": 1e-05, "loss": 0.0756, "num_tokens": 4126248.0, "reward": 0.71875, "reward_std": 0.35564959049224854, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9999086856842041, "sampling/importance_sampling_ratio/min": 0.6396790146827698, "sampling/sampling_logp_difference/max": 0.7101085186004639, "sampling/sampling_logp_difference/mean": 0.012340809218585491, "step": 184 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.278716344037093e-05, "clip_ratio/low_min": 5.278716344037093e-05, "clip_ratio/region_mean": 5.278716344037093e-05, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 752.0, "completions/mean_length": 584.28125, "completions/mean_terminated_length": 550.25927734375, "completions/min_length": 323.0, "completions/min_terminated_length": 323.0, "entropy": 0.39026234298944473, "epoch": 0.09903640256959315, "frac_reward_zero_std": 0.25, "grad_norm": 0.006969504989683628, "learning_rate": 1e-05, "loss": 0.0439, "num_tokens": 4148705.0, "reward": 0.8125, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.47065007686615, "sampling/importance_sampling_ratio/mean": 1.0001626014709473, "sampling/importance_sampling_ratio/min": 0.6491507291793823, "sampling/sampling_logp_difference/max": 0.43209028244018555, "sampling/sampling_logp_difference/mean": 0.012676937505602837, "step": 185 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00020644002506742254, "clip_ratio/low_min": 0.00020644002506742254, "clip_ratio/region_mean": 0.00020644002506742254, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 761.0, "completions/mean_length": 624.90625, "completions/mean_terminated_length": 584.8399658203125, "completions/min_length": 288.0, "completions/min_terminated_length": 288.0, "entropy": 0.4165581353008747, "epoch": 0.09957173447537473, "frac_reward_zero_std": 0.0, "grad_norm": 0.010572649538516998, "learning_rate": 1e-05, "loss": 0.0231, "num_tokens": 4172662.0, "reward": 0.59375, "reward_std": 0.4218915104866028, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.907720923423767, "sampling/importance_sampling_ratio/mean": 0.9999308586120605, "sampling/importance_sampling_ratio/min": 0.6693372130393982, "sampling/sampling_logp_difference/max": 0.645909309387207, "sampling/sampling_logp_difference/mean": 0.011922947131097317, "step": 186 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00020344323638710193, "clip_ratio/low_min": 0.00020344323638710193, "clip_ratio/region_mean": 0.00020344323638710193, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 671.0, "completions/mean_length": 431.28125, "completions/mean_terminated_length": 408.8333435058594, "completions/min_length": 198.0, "completions/min_terminated_length": 198.0, "entropy": 0.4276481121778488, "epoch": 0.10010706638115632, "frac_reward_zero_std": 0.25, "grad_norm": 0.007953577674925327, "learning_rate": 1e-05, "loss": 0.0361, "num_tokens": 4189735.0, "reward": 0.25, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.25, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.3328940868377686, "sampling/importance_sampling_ratio/mean": 1.0002760887145996, "sampling/importance_sampling_ratio/min": 0.6931687593460083, "sampling/sampling_logp_difference/max": 0.3664817810058594, "sampling/sampling_logp_difference/mean": 0.013134675100445747, "step": 187 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00010754277172964066, "clip_ratio/low_min": 0.00010754277172964066, "clip_ratio/region_mean": 0.00010754277172964066, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 690.0, "completions/mean_length": 556.96875, "completions/mean_terminated_length": 517.888916015625, "completions/min_length": 335.0, "completions/min_terminated_length": 335.0, "entropy": 0.4502966143190861, "epoch": 0.1006423982869379, "frac_reward_zero_std": 0.25, "grad_norm": 0.015682239085435867, "learning_rate": 1e-05, "loss": 0.0952, "num_tokens": 4211462.0, "reward": 0.65625, "reward_std": 0.3608423173427582, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.3125203847885132, "sampling/importance_sampling_ratio/mean": 0.9996594190597534, "sampling/importance_sampling_ratio/min": 0.5370357036590576, "sampling/sampling_logp_difference/max": 0.6216907501220703, "sampling/sampling_logp_difference/mean": 0.012791034765541553, "step": 188 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.763719334732741e-05, "clip_ratio/low_min": 4.763719334732741e-05, "clip_ratio/region_mean": 4.763719334732741e-05, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 758.0, "completions/mean_length": 570.28125, "completions/mean_terminated_length": 514.9199829101562, "completions/min_length": 275.0, "completions/min_terminated_length": 275.0, "entropy": 0.46106133610010147, "epoch": 0.10117773019271949, "frac_reward_zero_std": 0.25, "grad_norm": 0.007562241051346064, "learning_rate": 1e-05, "loss": -0.0074, "num_tokens": 4233487.0, "reward": 0.59375, "reward_std": 0.3377465009689331, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.3308215141296387, "sampling/importance_sampling_ratio/mean": 0.9997634291648865, "sampling/importance_sampling_ratio/min": 0.5934041142463684, "sampling/sampling_logp_difference/max": 0.5218796730041504, "sampling/sampling_logp_difference/mean": 0.013349337503314018, "step": 189 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0001560395503474865, "clip_ratio/low_min": 0.0001560395503474865, "clip_ratio/region_mean": 0.0001560395503474865, "completions/clipped_ratio": 0.5, "completions/max_length": 768.0, "completions/max_terminated_length": 713.0, "completions/mean_length": 650.53125, "completions/mean_terminated_length": 533.0625, "completions/min_length": 280.0, "completions/min_terminated_length": 280.0, "entropy": 0.46249398961663246, "epoch": 0.10171306209850108, "frac_reward_zero_std": 0.5, "grad_norm": 0.0019809517543762922, "learning_rate": 1e-05, "loss": 0.0118, "num_tokens": 4258840.0, "reward": 0.0625, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.0625, "rewards/accuracy_reward/std": 0.24593468010425568, "sampling/importance_sampling_ratio/max": 1.507697343826294, "sampling/importance_sampling_ratio/mean": 0.9998849630355835, "sampling/importance_sampling_ratio/min": 0.6196276545524597, "sampling/sampling_logp_difference/max": 0.4786365032196045, "sampling/sampling_logp_difference/mean": 0.013673453591763973, "step": 190 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0001461528445361182, "clip_ratio/low_min": 0.0001461528445361182, "clip_ratio/region_mean": 0.0001461528445361182, "completions/clipped_ratio": 0.28125, "completions/max_length": 768.0, "completions/max_terminated_length": 766.0, "completions/mean_length": 653.9375, "completions/mean_terminated_length": 609.3043823242188, "completions/min_length": 418.0, "completions/min_terminated_length": 418.0, "entropy": 0.33082620427012444, "epoch": 0.10224839400428265, "frac_reward_zero_std": 0.0, "grad_norm": 0.007062896620482206, "learning_rate": 1e-05, "loss": 0.0543, "num_tokens": 4284094.0, "reward": 0.53125, "reward_std": 0.4397946000099182, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.336458683013916, "sampling/importance_sampling_ratio/mean": 0.999947726726532, "sampling/importance_sampling_ratio/min": 0.644375205039978, "sampling/sampling_logp_difference/max": 0.43947410583496094, "sampling/sampling_logp_difference/mean": 0.01081669982522726, "step": 191 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0001768947513482999, "clip_ratio/low_min": 0.0001768947513482999, "clip_ratio/region_mean": 0.0001768947513482999, "completions/clipped_ratio": 0.375, "completions/max_length": 768.0, "completions/max_terminated_length": 713.0, "completions/mean_length": 640.125, "completions/mean_terminated_length": 563.4000244140625, "completions/min_length": 347.0, "completions/min_terminated_length": 347.0, "entropy": 0.42493443191051483, "epoch": 0.10278372591006424, "frac_reward_zero_std": 0.25, "grad_norm": 0.012724976055324078, "learning_rate": 1e-05, "loss": 0.0633, "num_tokens": 4308482.0, "reward": 0.3125, "reward_std": 0.3514062464237213, "rewards/accuracy_reward/mean": 0.3125, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.426673173904419, "sampling/importance_sampling_ratio/mean": 0.9997580051422119, "sampling/importance_sampling_ratio/min": 0.571729302406311, "sampling/sampling_logp_difference/max": 0.5590896606445312, "sampling/sampling_logp_difference/mean": 0.013163600116968155, "step": 192 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.3125, "completions/max_length": 768.0, "completions/max_terminated_length": 675.0, "completions/mean_length": 585.34375, "completions/mean_terminated_length": 502.3182067871094, "completions/min_length": 279.0, "completions/min_terminated_length": 279.0, "entropy": 0.5026186592876911, "epoch": 0.10331905781584583, "frac_reward_zero_std": 0.5, "grad_norm": 0.005640897434204817, "learning_rate": 1e-05, "loss": 0.0759, "num_tokens": 4331141.0, "reward": 0.625, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.4223015308380127, "sampling/importance_sampling_ratio/mean": 0.9998229742050171, "sampling/importance_sampling_ratio/min": 0.6328842639923096, "sampling/sampling_logp_difference/max": 0.45746779441833496, "sampling/sampling_logp_difference/mean": 0.015014126896858215, "step": 193 }, { "clip_ratio/high_max": 4.4642856664722785e-05, "clip_ratio/high_mean": 4.4642856664722785e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 4.4642856664722785e-05, "completions/clipped_ratio": 0.46875, "completions/max_length": 768.0, "completions/max_terminated_length": 725.0, "completions/mean_length": 624.28125, "completions/mean_terminated_length": 497.4705810546875, "completions/min_length": 311.0, "completions/min_terminated_length": 311.0, "entropy": 0.5654965825378895, "epoch": 0.10385438972162742, "frac_reward_zero_std": 0.25, "grad_norm": 0.010605625808238983, "learning_rate": 1e-05, "loss": 0.0616, "num_tokens": 4355278.0, "reward": 0.4375, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.4375, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.486506700515747, "sampling/importance_sampling_ratio/mean": 0.999529242515564, "sampling/importance_sampling_ratio/min": 0.6654889583587646, "sampling/sampling_logp_difference/max": 0.40723323822021484, "sampling/sampling_logp_difference/mean": 0.015287080779671669, "step": 194 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 711.0, "completions/mean_length": 507.34375, "completions/mean_terminated_length": 470.107177734375, "completions/min_length": 291.0, "completions/min_terminated_length": 291.0, "entropy": 0.4125858396291733, "epoch": 0.10438972162740899, "frac_reward_zero_std": 0.5, "grad_norm": 0.007816245779395103, "learning_rate": 1e-05, "loss": 0.0552, "num_tokens": 4375073.0, "reward": 0.8125, "reward_std": 0.249358132481575, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.4488462209701538, "sampling/importance_sampling_ratio/mean": 0.9996964335441589, "sampling/importance_sampling_ratio/min": 0.6287140846252441, "sampling/sampling_logp_difference/max": 0.4640786647796631, "sampling/sampling_logp_difference/mean": 0.012324759736657143, "step": 195 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 651.0, "completions/mean_length": 497.375, "completions/mean_terminated_length": 469.3793029785156, "completions/min_length": 262.0, "completions/min_terminated_length": 262.0, "entropy": 0.3259465880692005, "epoch": 0.10492505353319058, "frac_reward_zero_std": 0.0, "grad_norm": 0.019756974652409554, "learning_rate": 1e-05, "loss": 0.0587, "num_tokens": 4394677.0, "reward": 0.78125, "reward_std": 0.4218915104866028, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.435684084892273, "sampling/importance_sampling_ratio/mean": 1.0003255605697632, "sampling/importance_sampling_ratio/min": 0.6057069301605225, "sampling/sampling_logp_difference/max": 0.5013589859008789, "sampling/sampling_logp_difference/mean": 0.011160098947584629, "step": 196 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.484861685545184e-05, "clip_ratio/low_min": 5.484861685545184e-05, "clip_ratio/region_mean": 5.484861685545184e-05, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 716.0, "completions/mean_length": 523.46875, "completions/mean_terminated_length": 478.1851806640625, "completions/min_length": 283.0, "completions/min_terminated_length": 283.0, "entropy": 0.41297532618045807, "epoch": 0.10546038543897217, "frac_reward_zero_std": 0.5, "grad_norm": 0.003697456093505025, "learning_rate": 1e-05, "loss": 0.0223, "num_tokens": 4415124.0, "reward": 0.75, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.3766028881072998, "sampling/importance_sampling_ratio/mean": 1.0000152587890625, "sampling/importance_sampling_ratio/min": 0.6974722743034363, "sampling/sampling_logp_difference/max": 0.3602924346923828, "sampling/sampling_logp_difference/mean": 0.012368491850793362, "step": 197 }, { "clip_ratio/high_max": 5.451373726828024e-05, "clip_ratio/high_mean": 5.451373726828024e-05, "clip_ratio/low_mean": 0.00015712289678049274, "clip_ratio/low_min": 0.00015712289678049274, "clip_ratio/region_mean": 0.0002116366413247306, "completions/clipped_ratio": 0.3125, "completions/max_length": 768.0, "completions/max_terminated_length": 704.0, "completions/mean_length": 593.09375, "completions/mean_terminated_length": 513.5909423828125, "completions/min_length": 351.0, "completions/min_terminated_length": 351.0, "entropy": 0.3787360694259405, "epoch": 0.10599571734475374, "frac_reward_zero_std": 0.0, "grad_norm": 0.0069842226803302765, "learning_rate": 1e-05, "loss": 0.116, "num_tokens": 4438199.0, "reward": 0.46875, "reward_std": 0.4397946000099182, "rewards/accuracy_reward/mean": 0.46875, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.346086025238037, "sampling/importance_sampling_ratio/mean": 0.9994926452636719, "sampling/importance_sampling_ratio/min": 0.6092809438705444, "sampling/sampling_logp_difference/max": 0.49547576904296875, "sampling/sampling_logp_difference/mean": 0.012435591779649258, "step": 198 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 733.0, "completions/mean_length": 552.0625, "completions/mean_terminated_length": 502.23077392578125, "completions/min_length": 232.0, "completions/min_terminated_length": 232.0, "entropy": 0.3516416922211647, "epoch": 0.10653104925053533, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0351, "num_tokens": 4459329.0, "reward": 0.65625, "reward_std": 0.1293872892856598, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.3627244234085083, "sampling/importance_sampling_ratio/mean": 1.0005079507827759, "sampling/importance_sampling_ratio/min": 0.6949403285980225, "sampling/sampling_logp_difference/max": 0.36392927169799805, "sampling/sampling_logp_difference/mean": 0.011472605168819427, "step": 199 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.0444875998655334e-05, "clip_ratio/low_min": 6.0444875998655334e-05, "clip_ratio/region_mean": 6.0444875998655334e-05, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 742.0, "completions/mean_length": 489.0625, "completions/mean_terminated_length": 470.4667053222656, "completions/min_length": 201.0, "completions/min_terminated_length": 201.0, "entropy": 0.43827785551548004, "epoch": 0.10706638115631692, "frac_reward_zero_std": 0.25, "grad_norm": 0.006218034774065018, "learning_rate": 1e-05, "loss": 0.1175, "num_tokens": 4478507.0, "reward": 0.75, "reward_std": 0.3514062464237213, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.473978877067566, "sampling/importance_sampling_ratio/mean": 0.9999464750289917, "sampling/importance_sampling_ratio/min": 0.6813673973083496, "sampling/sampling_logp_difference/max": 0.38796544075012207, "sampling/sampling_logp_difference/mean": 0.013890287838876247, "step": 200 }, { "clip_ratio/high_max": 6.459948053816333e-05, "clip_ratio/high_mean": 6.459948053816333e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 6.459948053816333e-05, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 742.0, "completions/mean_length": 472.90625, "completions/mean_terminated_length": 463.3870849609375, "completions/min_length": 192.0, "completions/min_terminated_length": 192.0, "entropy": 0.36613602563738823, "epoch": 0.1076017130620985, "frac_reward_zero_std": 0.75, "grad_norm": 0.002796619199216366, "learning_rate": 1e-05, "loss": 0.0344, "num_tokens": 4496936.0, "reward": 0.96875, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.96875, "rewards/accuracy_reward/std": 0.1767766922712326, "sampling/importance_sampling_ratio/max": 1.561207890510559, "sampling/importance_sampling_ratio/mean": 1.0001063346862793, "sampling/importance_sampling_ratio/min": 0.6822316646575928, "sampling/sampling_logp_difference/max": 0.44545984268188477, "sampling/sampling_logp_difference/mean": 0.011922389268875122, "step": 201 }, { "clip_ratio/high_max": 4.932912270305678e-05, "clip_ratio/high_mean": 4.932912270305678e-05, "clip_ratio/low_mean": 5.267593587632291e-05, "clip_ratio/low_min": 5.267593587632291e-05, "clip_ratio/region_mean": 0.00010200505857937969, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 768.0, "completions/mean_length": 574.40625, "completions/mean_terminated_length": 538.5555419921875, "completions/min_length": 273.0, "completions/min_terminated_length": 273.0, "entropy": 0.323601009324193, "epoch": 0.10813704496788008, "frac_reward_zero_std": 0.5, "grad_norm": 0.011047563515603542, "learning_rate": 1e-05, "loss": 0.0461, "num_tokens": 4518797.0, "reward": 0.875, "reward_std": 0.2314550280570984, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 1.422509789466858, "sampling/importance_sampling_ratio/mean": 0.9997566938400269, "sampling/importance_sampling_ratio/min": 0.5621358752250671, "sampling/sampling_logp_difference/max": 0.5760116577148438, "sampling/sampling_logp_difference/mean": 0.010617411695420742, "step": 202 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 7.418397581204772e-05, "clip_ratio/low_min": 7.418397581204772e-05, "clip_ratio/region_mean": 7.418397581204772e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 708.0, "completions/max_terminated_length": 708.0, "completions/mean_length": 488.59375, "completions/mean_terminated_length": 488.59375, "completions/min_length": 92.0, "completions/min_terminated_length": 92.0, "entropy": 0.3624422922730446, "epoch": 0.10867237687366167, "frac_reward_zero_std": 0.25, "grad_norm": 0.012993577867746353, "learning_rate": 1e-05, "loss": 0.0867, "num_tokens": 4537864.0, "reward": 0.75, "reward_std": 0.3745020925998688, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.447041630744934, "sampling/importance_sampling_ratio/mean": 0.9999805688858032, "sampling/importance_sampling_ratio/min": 0.7270780205726624, "sampling/sampling_logp_difference/max": 0.3695211410522461, "sampling/sampling_logp_difference/mean": 0.01146488357335329, "step": 203 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 9.194557060254738e-05, "clip_ratio/low_min": 9.194557060254738e-05, "clip_ratio/region_mean": 9.194557060254738e-05, "completions/clipped_ratio": 0.34375, "completions/max_length": 768.0, "completions/max_terminated_length": 757.0, "completions/mean_length": 642.96875, "completions/mean_terminated_length": 577.4761962890625, "completions/min_length": 360.0, "completions/min_terminated_length": 360.0, "entropy": 0.4151980020105839, "epoch": 0.10920770877944326, "frac_reward_zero_std": 0.0, "grad_norm": 0.005442460533231497, "learning_rate": 1e-05, "loss": 0.0616, "num_tokens": 4562223.0, "reward": 0.40625, "reward_std": 0.4534739851951599, "rewards/accuracy_reward/mean": 0.40625, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.459665298461914, "sampling/importance_sampling_ratio/mean": 1.0001134872436523, "sampling/importance_sampling_ratio/min": 0.7412106394767761, "sampling/sampling_logp_difference/max": 0.3782072067260742, "sampling/sampling_logp_difference/mean": 0.012796086259186268, "step": 204 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 8.480325777782127e-05, "clip_ratio/low_min": 8.480325777782127e-05, "clip_ratio/region_mean": 8.480325777782127e-05, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 738.0, "completions/mean_length": 459.90625, "completions/mean_terminated_length": 428.03448486328125, "completions/min_length": 259.0, "completions/min_terminated_length": 259.0, "entropy": 0.5418744124472141, "epoch": 0.10974304068522484, "frac_reward_zero_std": 0.25, "grad_norm": 0.009512354619801044, "learning_rate": 1e-05, "loss": -0.0017, "num_tokens": 4580988.0, "reward": 0.6875, "reward_std": 0.3924051821231842, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.3982853889465332, "sampling/importance_sampling_ratio/mean": 1.0001652240753174, "sampling/importance_sampling_ratio/min": 0.6685997247695923, "sampling/sampling_logp_difference/max": 0.4025697708129883, "sampling/sampling_logp_difference/mean": 0.01568397879600525, "step": 205 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 726.0, "completions/mean_length": 508.5, "completions/mean_terminated_length": 500.1290283203125, "completions/min_length": 246.0, "completions/min_terminated_length": 246.0, "entropy": 0.35495569556951523, "epoch": 0.11027837259100642, "frac_reward_zero_std": 0.5, "grad_norm": 0.005454814527183771, "learning_rate": 1e-05, "loss": 0.0625, "num_tokens": 4601236.0, "reward": 0.84375, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.3583024740219116, "sampling/importance_sampling_ratio/mean": 1.00021493434906, "sampling/importance_sampling_ratio/min": 0.41868987679481506, "sampling/sampling_logp_difference/max": 0.8706247806549072, "sampling/sampling_logp_difference/mean": 0.01119942031800747, "step": 206 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00011782906949520111, "clip_ratio/low_min": 0.00011782906949520111, "clip_ratio/region_mean": 0.00011782906949520111, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 752.0, "completions/mean_length": 536.0, "completions/mean_terminated_length": 471.03997802734375, "completions/min_length": 288.0, "completions/min_terminated_length": 288.0, "entropy": 0.46669817715883255, "epoch": 0.11081370449678801, "frac_reward_zero_std": 0.0, "grad_norm": 0.01467866264283657, "learning_rate": 1e-05, "loss": 0.1617, "num_tokens": 4621980.0, "reward": 0.53125, "reward_std": 0.4218915104866028, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.2943097352981567, "sampling/importance_sampling_ratio/mean": 0.9996358156204224, "sampling/importance_sampling_ratio/min": 0.6145366430282593, "sampling/sampling_logp_difference/max": 0.48688673973083496, "sampling/sampling_logp_difference/mean": 0.013230583630502224, "step": 207 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.5110067731002346e-05, "clip_ratio/low_min": 4.5110067731002346e-05, "clip_ratio/region_mean": 4.5110067731002346e-05, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 756.0, "completions/mean_length": 606.8125, "completions/mean_terminated_length": 576.9629516601562, "completions/min_length": 340.0, "completions/min_terminated_length": 340.0, "entropy": 0.4399042911827564, "epoch": 0.11134903640256959, "frac_reward_zero_std": 0.5, "grad_norm": 0.015372531488537788, "learning_rate": 1e-05, "loss": 0.0271, "num_tokens": 4645102.0, "reward": 0.75, "reward_std": 0.2587745785713196, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0002062320709229, "sampling/importance_sampling_ratio/min": 0.6881018280982971, "sampling/sampling_logp_difference/max": 0.806495189666748, "sampling/sampling_logp_difference/mean": 0.01302797719836235, "step": 208 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 673.0, "completions/mean_length": 489.0, "completions/mean_terminated_length": 460.137939453125, "completions/min_length": 269.0, "completions/min_terminated_length": 269.0, "entropy": 0.44632333144545555, "epoch": 0.11188436830835118, "frac_reward_zero_std": 0.25, "grad_norm": 0.005208560265600681, "learning_rate": 1e-05, "loss": 0.0688, "num_tokens": 4664198.0, "reward": 0.53125, "reward_std": 0.3471629321575165, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.4133529663085938, "sampling/importance_sampling_ratio/mean": 1.0002318620681763, "sampling/importance_sampling_ratio/min": 0.6086702942848206, "sampling/sampling_logp_difference/max": 0.4964785575866699, "sampling/sampling_logp_difference/mean": 0.013368175365030766, "step": 209 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0001271186483791098, "clip_ratio/low_min": 0.0001271186483791098, "clip_ratio/region_mean": 0.0001271186483791098, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 748.0, "completions/mean_length": 480.25, "completions/mean_terminated_length": 413.8461608886719, "completions/min_length": 138.0, "completions/min_terminated_length": 138.0, "entropy": 0.4682172201573849, "epoch": 0.11241970021413276, "frac_reward_zero_std": 0.25, "grad_norm": 0.015574317425489426, "learning_rate": 1e-05, "loss": 0.058, "num_tokens": 4683006.0, "reward": 0.625, "reward_std": 0.3104073107242584, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.48087477684021, "sampling/importance_sampling_ratio/mean": 1.000162124633789, "sampling/importance_sampling_ratio/min": 0.6997669339179993, "sampling/sampling_logp_difference/max": 0.39263296127319336, "sampling/sampling_logp_difference/mean": 0.01364071387797594, "step": 210 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 735.0, "completions/mean_length": 544.9375, "completions/mean_terminated_length": 493.4615478515625, "completions/min_length": 354.0, "completions/min_terminated_length": 354.0, "entropy": 0.3690958619117737, "epoch": 0.11295503211991435, "frac_reward_zero_std": 0.75, "grad_norm": 0.010793657973408699, "learning_rate": 1e-05, "loss": 0.0356, "num_tokens": 4703948.0, "reward": 0.65625, "reward_std": 0.1293872892856598, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.4460468292236328, "sampling/importance_sampling_ratio/mean": 1.000065565109253, "sampling/importance_sampling_ratio/min": 0.7057710289955139, "sampling/sampling_logp_difference/max": 0.3688335418701172, "sampling/sampling_logp_difference/mean": 0.011855688877403736, "step": 211 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 708.0, "completions/mean_length": 487.625, "completions/mean_terminated_length": 468.933349609375, "completions/min_length": 315.0, "completions/min_terminated_length": 315.0, "entropy": 0.39685725048184395, "epoch": 0.11349036402569593, "frac_reward_zero_std": 0.25, "grad_norm": 0.01544248778373003, "learning_rate": 1e-05, "loss": 0.0475, "num_tokens": 4723304.0, "reward": 0.65625, "reward_std": 0.3608423173427582, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.290382742881775, "sampling/importance_sampling_ratio/mean": 0.9997295141220093, "sampling/importance_sampling_ratio/min": 0.6577038168907166, "sampling/sampling_logp_difference/max": 0.41900062561035156, "sampling/sampling_logp_difference/mean": 0.012623356655240059, "step": 212 }, { "clip_ratio/high_max": 4.5306271204026416e-05, "clip_ratio/high_mean": 4.5306271204026416e-05, "clip_ratio/low_mean": 0.00014086080045672134, "clip_ratio/low_min": 0.00014086080045672134, "clip_ratio/region_mean": 0.00018616707166074775, "completions/clipped_ratio": 0.40625, "completions/max_length": 768.0, "completions/max_terminated_length": 752.0, "completions/mean_length": 641.3125, "completions/mean_terminated_length": 554.631591796875, "completions/min_length": 394.0, "completions/min_terminated_length": 394.0, "entropy": 0.528442993760109, "epoch": 0.11402569593147752, "frac_reward_zero_std": 0.25, "grad_norm": 0.005189127754420042, "learning_rate": 1e-05, "loss": 0.0175, "num_tokens": 4747714.0, "reward": 0.53125, "reward_std": 0.2651650309562683, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.4936127662658691, "sampling/importance_sampling_ratio/mean": 0.9998847246170044, "sampling/importance_sampling_ratio/min": 0.6634731888771057, "sampling/sampling_logp_difference/max": 0.4102668762207031, "sampling/sampling_logp_difference/mean": 0.014573306776583195, "step": 213 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00010960105282720178, "clip_ratio/low_min": 0.00010960105282720178, "clip_ratio/region_mean": 0.00010960105282720178, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 679.0, "completions/mean_length": 561.40625, "completions/mean_terminated_length": 513.7307739257812, "completions/min_length": 316.0, "completions/min_terminated_length": 316.0, "entropy": 0.4899198245257139, "epoch": 0.1145610278372591, "frac_reward_zero_std": 0.25, "grad_norm": 0.017145702615380287, "learning_rate": 1e-05, "loss": 0.0085, "num_tokens": 4769695.0, "reward": 0.625, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.40604829788208, "sampling/importance_sampling_ratio/mean": 0.9998758435249329, "sampling/importance_sampling_ratio/min": 0.6533907055854797, "sampling/sampling_logp_difference/max": 0.4255800247192383, "sampling/sampling_logp_difference/mean": 0.014461631886661053, "step": 214 }, { "clip_ratio/high_max": 5.921364208916202e-05, "clip_ratio/high_mean": 5.921364208916202e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 5.921364208916202e-05, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 761.0, "completions/mean_length": 494.28125, "completions/mean_terminated_length": 476.0333557128906, "completions/min_length": 233.0, "completions/min_terminated_length": 233.0, "entropy": 0.33969808742403984, "epoch": 0.11509635974304068, "frac_reward_zero_std": 0.5, "grad_norm": 0.007109393365681171, "learning_rate": 1e-05, "loss": -0.0182, "num_tokens": 4789488.0, "reward": 0.90625, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.5345942974090576, "sampling/importance_sampling_ratio/mean": 1.0005249977111816, "sampling/importance_sampling_ratio/min": 0.5697072744369507, "sampling/sampling_logp_difference/max": 0.5626325607299805, "sampling/sampling_logp_difference/mean": 0.0109400674700737, "step": 215 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 765.0, "completions/mean_length": 496.65625, "completions/mean_terminated_length": 468.5862121582031, "completions/min_length": 228.0, "completions/min_terminated_length": 228.0, "entropy": 0.4533826895058155, "epoch": 0.11563169164882227, "frac_reward_zero_std": 0.5, "grad_norm": 0.0041758473962545395, "learning_rate": 1e-05, "loss": 0.0326, "num_tokens": 4809077.0, "reward": 0.75, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.7522910833358765, "sampling/importance_sampling_ratio/mean": 1.0000641345977783, "sampling/importance_sampling_ratio/min": 0.49681904911994934, "sampling/sampling_logp_difference/max": 0.6995294094085693, "sampling/sampling_logp_difference/mean": 0.013441476039588451, "step": 216 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00011785111200879328, "clip_ratio/low_min": 0.00011785111200879328, "clip_ratio/region_mean": 0.00011785111200879328, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 705.0, "completions/mean_length": 537.71875, "completions/mean_terminated_length": 504.8214416503906, "completions/min_length": 284.0, "completions/min_terminated_length": 284.0, "entropy": 0.43806789815425873, "epoch": 0.11616702355460386, "frac_reward_zero_std": 0.5, "grad_norm": 0.008020127192139626, "learning_rate": 1e-05, "loss": 0.0196, "num_tokens": 4829884.0, "reward": 0.71875, "reward_std": 0.2630178928375244, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.5796546936035156, "sampling/importance_sampling_ratio/mean": 0.9999369978904724, "sampling/importance_sampling_ratio/min": 0.6530105471611023, "sampling/sampling_logp_difference/max": 0.45720624923706055, "sampling/sampling_logp_difference/mean": 0.012817000970244408, "step": 217 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0002680727484403178, "clip_ratio/low_min": 0.0002680727484403178, "clip_ratio/region_mean": 0.0002680727484403178, "completions/clipped_ratio": 0.5, "completions/max_length": 768.0, "completions/max_terminated_length": 697.0, "completions/mean_length": 665.96875, "completions/mean_terminated_length": 563.9375, "completions/min_length": 452.0, "completions/min_terminated_length": 452.0, "entropy": 0.5910607874393463, "epoch": 0.11670235546038545, "frac_reward_zero_std": 0.25, "grad_norm": 0.009125817567110062, "learning_rate": 1e-05, "loss": 0.0537, "num_tokens": 4855643.0, "reward": 0.3125, "reward_std": 0.3514062464237213, "rewards/accuracy_reward/mean": 0.3125, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.4317469596862793, "sampling/importance_sampling_ratio/mean": 0.9997588992118835, "sampling/importance_sampling_ratio/min": 0.4819125831127167, "sampling/sampling_logp_difference/max": 0.7299926280975342, "sampling/sampling_logp_difference/mean": 0.016217265278100967, "step": 218 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00032712527900002897, "clip_ratio/low_min": 0.00032712527900002897, "clip_ratio/region_mean": 0.00032712527900002897, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 702.0, "completions/mean_length": 572.15625, "completions/mean_terminated_length": 506.875, "completions/min_length": 327.0, "completions/min_terminated_length": 327.0, "entropy": 0.48503001406788826, "epoch": 0.11723768736616702, "frac_reward_zero_std": 0.25, "grad_norm": 0.011963547207415104, "learning_rate": 1e-05, "loss": 0.0439, "num_tokens": 4878128.0, "reward": 0.46875, "reward_std": 0.3471629321575165, "rewards/accuracy_reward/mean": 0.46875, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.5091300010681152, "sampling/importance_sampling_ratio/mean": 0.999998927116394, "sampling/importance_sampling_ratio/min": 0.6804026365280151, "sampling/sampling_logp_difference/max": 0.4115333557128906, "sampling/sampling_logp_difference/mean": 0.014249210245907307, "step": 219 }, { "clip_ratio/high_max": 6.670223956461996e-05, "clip_ratio/high_mean": 6.670223956461996e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 6.670223956461996e-05, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 717.0, "completions/mean_length": 504.15625, "completions/mean_terminated_length": 466.46429443359375, "completions/min_length": 232.0, "completions/min_terminated_length": 232.0, "entropy": 0.5912829264998436, "epoch": 0.11777301927194861, "frac_reward_zero_std": 0.5, "grad_norm": 0.01798270083963871, "learning_rate": 1e-05, "loss": 0.042, "num_tokens": 4898125.0, "reward": 0.78125, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.4862390756607056, "sampling/importance_sampling_ratio/mean": 1.0002179145812988, "sampling/importance_sampling_ratio/min": 0.6412653923034668, "sampling/sampling_logp_difference/max": 0.44431185722351074, "sampling/sampling_logp_difference/mean": 0.013367424719035625, "step": 220 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.689576573786326e-05, "clip_ratio/low_min": 5.689576573786326e-05, "clip_ratio/region_mean": 5.689576573786326e-05, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 757.0, "completions/mean_length": 538.5, "completions/mean_terminated_length": 523.2000122070312, "completions/min_length": 309.0, "completions/min_terminated_length": 309.0, "entropy": 0.39681804180145264, "epoch": 0.1183083511777302, "frac_reward_zero_std": 0.25, "grad_norm": 0.004230021964758635, "learning_rate": 1e-05, "loss": 0.0221, "num_tokens": 4918917.0, "reward": 0.875, "reward_std": 0.292504221200943, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 1.4435296058654785, "sampling/importance_sampling_ratio/mean": 0.9999265074729919, "sampling/importance_sampling_ratio/min": 0.6420924663543701, "sampling/sampling_logp_difference/max": 0.4430229663848877, "sampling/sampling_logp_difference/mean": 0.012738961726427078, "step": 221 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00011799021012848243, "clip_ratio/low_min": 0.00011799021012848243, "clip_ratio/region_mean": 0.00011799021012848243, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 742.0, "completions/mean_length": 506.8125, "completions/mean_terminated_length": 469.5000305175781, "completions/min_length": 303.0, "completions/min_terminated_length": 303.0, "entropy": 0.391423586755991, "epoch": 0.11884368308351177, "frac_reward_zero_std": 0.25, "grad_norm": 0.0145324831828475, "learning_rate": 1e-05, "loss": 0.0036, "num_tokens": 4938575.0, "reward": 0.59375, "reward_std": 0.3787454068660736, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.4798321723937988, "sampling/importance_sampling_ratio/mean": 1.0004429817199707, "sampling/importance_sampling_ratio/min": 0.6881264448165894, "sampling/sampling_logp_difference/max": 0.39192867279052734, "sampling/sampling_logp_difference/mean": 0.01311441045254469, "step": 222 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00010432070484966971, "clip_ratio/low_min": 0.00010432070484966971, "clip_ratio/region_mean": 0.00010432070484966971, "completions/clipped_ratio": 0.34375, "completions/max_length": 768.0, "completions/max_terminated_length": 693.0, "completions/mean_length": 548.84375, "completions/mean_terminated_length": 434.0476379394531, "completions/min_length": 251.0, "completions/min_terminated_length": 251.0, "entropy": 0.3972848244011402, "epoch": 0.11937901498929336, "frac_reward_zero_std": 0.5, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0416, "num_tokens": 4959962.0, "reward": 0.46875, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.46875, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.345275640487671, "sampling/importance_sampling_ratio/mean": 1.0000486373901367, "sampling/importance_sampling_ratio/min": 0.6178843975067139, "sampling/sampling_logp_difference/max": 0.48145389556884766, "sampling/sampling_logp_difference/mean": 0.012422376312315464, "step": 223 }, { "clip_ratio/high_max": 4.792944673681632e-05, "clip_ratio/high_mean": 4.792944673681632e-05, "clip_ratio/low_mean": 5.4537522373721004e-05, "clip_ratio/low_min": 5.4537522373721004e-05, "clip_ratio/region_mean": 0.00010246696911053732, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 738.0, "completions/mean_length": 565.96875, "completions/mean_terminated_length": 519.34619140625, "completions/min_length": 308.0, "completions/min_terminated_length": 308.0, "entropy": 0.42224637046456337, "epoch": 0.11991434689507495, "frac_reward_zero_std": 0.0, "grad_norm": 0.007893293164670467, "learning_rate": 1e-05, "loss": 0.097, "num_tokens": 4981217.0, "reward": 0.75, "reward_std": 0.3945523500442505, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.8203226327896118, "sampling/importance_sampling_ratio/mean": 1.0004703998565674, "sampling/importance_sampling_ratio/min": 0.5529597997665405, "sampling/sampling_logp_difference/max": 0.5990138053894043, "sampling/sampling_logp_difference/mean": 0.012788786552846432, "step": 224 }, { "clip_ratio/high_max": 6.648935959674418e-05, "clip_ratio/high_mean": 6.648935959674418e-05, "clip_ratio/low_mean": 0.0001456488753319718, "clip_ratio/low_min": 0.0001456488753319718, "clip_ratio/region_mean": 0.00021213823492871597, "completions/clipped_ratio": 0.375, "completions/max_length": 768.0, "completions/max_terminated_length": 680.0, "completions/mean_length": 610.59375, "completions/mean_terminated_length": 516.1500244140625, "completions/min_length": 210.0, "completions/min_terminated_length": 210.0, "entropy": 0.5052645802497864, "epoch": 0.12044967880085652, "frac_reward_zero_std": 0.25, "grad_norm": 0.00404767319560051, "learning_rate": 1e-05, "loss": 0.0522, "num_tokens": 5004828.0, "reward": 0.4375, "reward_std": 0.292504221200943, "rewards/accuracy_reward/mean": 0.4375, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.5891460180282593, "sampling/importance_sampling_ratio/mean": 0.9998494982719421, "sampling/importance_sampling_ratio/min": 0.6003414392471313, "sampling/sampling_logp_difference/max": 0.5102567672729492, "sampling/sampling_logp_difference/mean": 0.014668923802673817, "step": 225 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.8112505939789116e-05, "clip_ratio/low_min": 5.8112505939789116e-05, "clip_ratio/region_mean": 5.8112505939789116e-05, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 752.0, "completions/mean_length": 471.5, "completions/mean_terminated_length": 451.7333679199219, "completions/min_length": 127.0, "completions/min_terminated_length": 127.0, "entropy": 0.502563439309597, "epoch": 0.12098501070663811, "frac_reward_zero_std": 0.25, "grad_norm": 0.02190265990793705, "learning_rate": 1e-05, "loss": -0.002, "num_tokens": 5023972.0, "reward": 0.625, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.3722007274627686, "sampling/importance_sampling_ratio/mean": 0.9995844960212708, "sampling/importance_sampling_ratio/min": 0.5808719992637634, "sampling/sampling_logp_difference/max": 0.5432248115539551, "sampling/sampling_logp_difference/mean": 0.013688228093087673, "step": 226 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00019691576017066836, "clip_ratio/low_min": 0.00019691576017066836, "clip_ratio/region_mean": 0.00019691576017066836, "completions/clipped_ratio": 0.4375, "completions/max_length": 768.0, "completions/max_terminated_length": 766.0, "completions/mean_length": 644.46875, "completions/mean_terminated_length": 548.388916015625, "completions/min_length": 257.0, "completions/min_terminated_length": 257.0, "entropy": 0.7140659429132938, "epoch": 0.1215203426124197, "frac_reward_zero_std": 0.5, "grad_norm": 0.01852392591536045, "learning_rate": 1e-05, "loss": 0.0198, "num_tokens": 5049363.0, "reward": 0.34375, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.34375, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.6167372465133667, "sampling/importance_sampling_ratio/mean": 1.0000755786895752, "sampling/importance_sampling_ratio/min": 0.6656412482261658, "sampling/sampling_logp_difference/max": 0.480410099029541, "sampling/sampling_logp_difference/mean": 0.017896713688969612, "step": 227 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 713.0, "completions/mean_length": 562.59375, "completions/mean_terminated_length": 524.5555419921875, "completions/min_length": 368.0, "completions/min_terminated_length": 368.0, "entropy": 0.3955683335661888, "epoch": 0.12205567451820129, "frac_reward_zero_std": 0.5, "grad_norm": 0.004563800059258938, "learning_rate": 1e-05, "loss": 0.0843, "num_tokens": 5071078.0, "reward": 0.65625, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.3121706247329712, "sampling/importance_sampling_ratio/mean": 1.0000855922698975, "sampling/importance_sampling_ratio/min": 0.3818032443523407, "sampling/sampling_logp_difference/max": 0.9628498554229736, "sampling/sampling_logp_difference/mean": 0.01162648480385542, "step": 228 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 759.0, "completions/mean_length": 584.78125, "completions/mean_terminated_length": 550.8518676757812, "completions/min_length": 327.0, "completions/min_terminated_length": 327.0, "entropy": 0.3032823149114847, "epoch": 0.12259100642398287, "frac_reward_zero_std": 0.25, "grad_norm": 0.004989946726709604, "learning_rate": 1e-05, "loss": -0.0156, "num_tokens": 5093527.0, "reward": 0.65625, "reward_std": 0.3198433816432953, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.8463772535324097, "sampling/importance_sampling_ratio/mean": 1.0001384019851685, "sampling/importance_sampling_ratio/min": 0.5978705883026123, "sampling/sampling_logp_difference/max": 0.6132254600524902, "sampling/sampling_logp_difference/mean": 0.01033024676144123, "step": 229 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.303580448729917e-05, "clip_ratio/low_min": 6.303580448729917e-05, "clip_ratio/region_mean": 6.303580448729917e-05, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 711.0, "completions/mean_length": 464.75, "completions/mean_terminated_length": 444.5333557128906, "completions/min_length": 280.0, "completions/min_terminated_length": 280.0, "entropy": 0.3863627575337887, "epoch": 0.12312633832976445, "frac_reward_zero_std": 0.25, "grad_norm": 0.005575043149292469, "learning_rate": 1e-05, "loss": 0.0657, "num_tokens": 5112143.0, "reward": 0.78125, "reward_std": 0.3377465009689331, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.5059571266174316, "sampling/importance_sampling_ratio/mean": 0.9998601078987122, "sampling/importance_sampling_ratio/min": 0.7042276263237, "sampling/sampling_logp_difference/max": 0.40942859649658203, "sampling/sampling_logp_difference/mean": 0.013208088465034962, "step": 230 }, { "clip_ratio/high_max": 4.976114723831415e-05, "clip_ratio/high_mean": 4.976114723831415e-05, "clip_ratio/low_mean": 4.9407113692723215e-05, "clip_ratio/low_min": 4.9407113692723215e-05, "clip_ratio/region_mean": 9.916826093103737e-05, "completions/clipped_ratio": 0.3125, "completions/max_length": 768.0, "completions/max_terminated_length": 711.0, "completions/mean_length": 540.0625, "completions/mean_terminated_length": 436.4545593261719, "completions/min_length": 203.0, "completions/min_terminated_length": 203.0, "entropy": 0.5061399415135384, "epoch": 0.12366167023554604, "frac_reward_zero_std": 0.5, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": -0.0552, "num_tokens": 5133089.0, "reward": 0.625, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.3599090576171875, "sampling/importance_sampling_ratio/mean": 1.0001215934753418, "sampling/importance_sampling_ratio/min": 0.29600051045417786, "sampling/sampling_logp_difference/max": 1.2173941135406494, "sampling/sampling_logp_difference/mean": 0.014033789746463299, "step": 231 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00010252309220959432, "clip_ratio/low_min": 0.00010252309220959432, "clip_ratio/region_mean": 0.00010252309220959432, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 761.0, "completions/mean_length": 533.46875, "completions/mean_terminated_length": 509.2069091796875, "completions/min_length": 193.0, "completions/min_terminated_length": 193.0, "entropy": 0.391541488468647, "epoch": 0.12419700214132762, "frac_reward_zero_std": 0.25, "grad_norm": 0.021787628531455994, "learning_rate": 1e-05, "loss": 0.0325, "num_tokens": 5153784.0, "reward": 0.78125, "reward_std": 0.3377464711666107, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.3615431785583496, "sampling/importance_sampling_ratio/mean": 0.9998968243598938, "sampling/importance_sampling_ratio/min": 0.5969482064247131, "sampling/sampling_logp_difference/max": 0.5159249305725098, "sampling/sampling_logp_difference/mean": 0.012499446049332619, "step": 232 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00020247400607331656, "clip_ratio/low_min": 0.00020247400607331656, "clip_ratio/region_mean": 0.00020247400607331656, "completions/clipped_ratio": 0.5, "completions/max_length": 768.0, "completions/max_terminated_length": 658.0, "completions/mean_length": 625.28125, "completions/mean_terminated_length": 482.5625, "completions/min_length": 213.0, "completions/min_terminated_length": 213.0, "entropy": 0.4108714498579502, "epoch": 0.1247323340471092, "frac_reward_zero_std": 0.0, "grad_norm": 0.01006588339805603, "learning_rate": 1e-05, "loss": 0.0793, "num_tokens": 5177801.0, "reward": 0.5, "reward_std": 0.47655022144317627, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5080004930496216, "sampling/importance_sampling_ratio/max": 1.3702772855758667, "sampling/importance_sampling_ratio/mean": 0.9998976588249207, "sampling/importance_sampling_ratio/min": 0.48900964856147766, "sampling/sampling_logp_difference/max": 0.7153730392456055, "sampling/sampling_logp_difference/mean": 0.011515702120959759, "step": 233 }, { "clip_ratio/high_max": 9.656446491135284e-05, "clip_ratio/high_mean": 9.656446491135284e-05, "clip_ratio/low_mean": 9.113035412156023e-05, "clip_ratio/low_min": 9.113035412156023e-05, "clip_ratio/region_mean": 0.00018769481903291307, "completions/clipped_ratio": 0.34375, "completions/max_length": 768.0, "completions/max_terminated_length": 703.0, "completions/mean_length": 600.125, "completions/mean_terminated_length": 512.1904907226562, "completions/min_length": 280.0, "completions/min_terminated_length": 280.0, "entropy": 0.46367712691426277, "epoch": 0.1252676659528908, "frac_reward_zero_std": 0.0, "grad_norm": 0.006998525932431221, "learning_rate": 1e-05, "loss": 0.0301, "num_tokens": 5200581.0, "reward": 0.5625, "reward_std": 0.4082317352294922, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.6151835918426514, "sampling/importance_sampling_ratio/mean": 0.999909520149231, "sampling/importance_sampling_ratio/min": 0.7212191224098206, "sampling/sampling_logp_difference/max": 0.4794485569000244, "sampling/sampling_logp_difference/mean": 0.01389430183917284, "step": 234 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 732.0, "completions/mean_length": 460.5, "completions/mean_terminated_length": 440.0000305175781, "completions/min_length": 208.0, "completions/min_terminated_length": 208.0, "entropy": 0.45329560339450836, "epoch": 0.12580299785867238, "frac_reward_zero_std": 0.5, "grad_norm": 0.003511700313538313, "learning_rate": 1e-05, "loss": 0.0792, "num_tokens": 5219517.0, "reward": 0.6875, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.8255608081817627, "sampling/importance_sampling_ratio/mean": 0.999989926815033, "sampling/importance_sampling_ratio/min": 0.7056424021720886, "sampling/sampling_logp_difference/max": 0.6018872261047363, "sampling/sampling_logp_difference/mean": 0.01322560291737318, "step": 235 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 747.0, "completions/mean_length": 538.09375, "completions/mean_terminated_length": 485.0384826660156, "completions/min_length": 158.0, "completions/min_terminated_length": 158.0, "entropy": 0.4739189185202122, "epoch": 0.12633832976445397, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": -0.0018, "num_tokens": 5241744.0, "reward": 0.6875, "reward_std": 0.1157275140285492, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.3244550228118896, "sampling/importance_sampling_ratio/mean": 0.9999309182167053, "sampling/importance_sampling_ratio/min": 0.44490817189216614, "sampling/sampling_logp_difference/max": 0.8098874092102051, "sampling/sampling_logp_difference/mean": 0.013555406592786312, "step": 236 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.805843855137937e-05, "clip_ratio/low_min": 4.805843855137937e-05, "clip_ratio/region_mean": 4.805843855137937e-05, "completions/clipped_ratio": 0.28125, "completions/max_length": 768.0, "completions/max_terminated_length": 655.0, "completions/mean_length": 522.96875, "completions/mean_terminated_length": 427.08697509765625, "completions/min_length": 223.0, "completions/min_terminated_length": 223.0, "entropy": 0.4349859729409218, "epoch": 0.12687366167023553, "frac_reward_zero_std": 0.25, "grad_norm": 0.014106686227023602, "learning_rate": 1e-05, "loss": 0.1133, "num_tokens": 5262583.0, "reward": 0.6875, "reward_std": 0.292504221200943, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.8777523040771484, "sampling/importance_sampling_ratio/mean": 1.000289797782898, "sampling/importance_sampling_ratio/min": 0.663354218006134, "sampling/sampling_logp_difference/max": 0.6300754547119141, "sampling/sampling_logp_difference/mean": 0.013021737337112427, "step": 237 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 761.0, "completions/mean_length": 566.34375, "completions/mean_terminated_length": 537.5357666015625, "completions/min_length": 81.0, "completions/min_terminated_length": 81.0, "entropy": 0.3597859516739845, "epoch": 0.12740899357601712, "frac_reward_zero_std": 0.5, "grad_norm": 0.010164322331547737, "learning_rate": 1e-05, "loss": 0.0136, "num_tokens": 5284354.0, "reward": 0.8125, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.2968369722366333, "sampling/importance_sampling_ratio/mean": 0.9998154640197754, "sampling/importance_sampling_ratio/min": 0.6164509057998657, "sampling/sampling_logp_difference/max": 0.4837765693664551, "sampling/sampling_logp_difference/mean": 0.010971475392580032, "step": 238 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.3125, "completions/max_length": 768.0, "completions/max_terminated_length": 683.0, "completions/mean_length": 570.6875, "completions/mean_terminated_length": 481.0, "completions/min_length": 281.0, "completions/min_terminated_length": 281.0, "entropy": 0.42755215615034103, "epoch": 0.1279443254817987, "frac_reward_zero_std": 0.5, "grad_norm": 0.00346189271658659, "learning_rate": 1e-05, "loss": -0.0083, "num_tokens": 5306896.0, "reward": 0.65625, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.620949625968933, "sampling/importance_sampling_ratio/mean": 1.0003163814544678, "sampling/importance_sampling_ratio/min": 0.5884921550750732, "sampling/sampling_logp_difference/max": 0.5301916599273682, "sampling/sampling_logp_difference/mean": 0.013137415051460266, "step": 239 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.300403038039804e-05, "clip_ratio/low_min": 6.300403038039804e-05, "clip_ratio/region_mean": 6.300403038039804e-05, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 705.0, "completions/mean_length": 486.90625, "completions/mean_terminated_length": 434.85186767578125, "completions/min_length": 164.0, "completions/min_terminated_length": 164.0, "entropy": 0.4919372908771038, "epoch": 0.1284796573875803, "frac_reward_zero_std": 0.5, "grad_norm": 0.013900874182581902, "learning_rate": 1e-05, "loss": 0.0791, "num_tokens": 5326453.0, "reward": 0.625, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.276257872581482, "sampling/importance_sampling_ratio/mean": 1.0000147819519043, "sampling/importance_sampling_ratio/min": 0.6976582407951355, "sampling/sampling_logp_difference/max": 0.36002588272094727, "sampling/sampling_logp_difference/mean": 0.01409463956952095, "step": 240 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 9.580544792697765e-05, "clip_ratio/low_min": 9.580544792697765e-05, "clip_ratio/region_mean": 9.580544792697765e-05, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 767.0, "completions/mean_length": 543.0, "completions/mean_terminated_length": 510.857177734375, "completions/min_length": 292.0, "completions/min_terminated_length": 292.0, "entropy": 0.442779716104269, "epoch": 0.1290149892933619, "frac_reward_zero_std": 0.0, "grad_norm": 0.01613631099462509, "learning_rate": 1e-05, "loss": 0.068, "num_tokens": 5347565.0, "reward": 0.6875, "reward_std": 0.4355512857437134, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.4268854856491089, "sampling/importance_sampling_ratio/mean": 1.0003812313079834, "sampling/importance_sampling_ratio/min": 0.6826814413070679, "sampling/sampling_logp_difference/max": 0.3817269802093506, "sampling/sampling_logp_difference/mean": 0.013200968503952026, "step": 241 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 697.0, "completions/mean_length": 568.21875, "completions/mean_terminated_length": 512.2799682617188, "completions/min_length": 325.0, "completions/min_terminated_length": 325.0, "entropy": 0.36066998913884163, "epoch": 0.12955032119914348, "frac_reward_zero_std": 0.75, "grad_norm": 0.02156771533191204, "learning_rate": 1e-05, "loss": 0.0185, "num_tokens": 5369700.0, "reward": 0.71875, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.430492639541626, "sampling/importance_sampling_ratio/mean": 0.9997615218162537, "sampling/importance_sampling_ratio/min": 0.641115128993988, "sampling/sampling_logp_difference/max": 0.4445462226867676, "sampling/sampling_logp_difference/mean": 0.011267188936471939, "step": 242 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 750.0, "completions/mean_length": 489.125, "completions/mean_terminated_length": 470.5333557128906, "completions/min_length": 269.0, "completions/min_terminated_length": 269.0, "entropy": 0.35818955674767494, "epoch": 0.13008565310492506, "frac_reward_zero_std": 0.25, "grad_norm": 0.01729537546634674, "learning_rate": 1e-05, "loss": 0.0497, "num_tokens": 5388928.0, "reward": 0.71875, "reward_std": 0.3787454068660736, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.608560562133789, "sampling/importance_sampling_ratio/mean": 1.000006079673767, "sampling/importance_sampling_ratio/min": 0.6876170635223389, "sampling/sampling_logp_difference/max": 0.4753396511077881, "sampling/sampling_logp_difference/mean": 0.011617233045399189, "step": 243 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00011165935939061455, "clip_ratio/low_min": 0.00011165935939061455, "clip_ratio/region_mean": 0.00011165935939061455, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 761.0, "completions/mean_length": 503.4375, "completions/mean_terminated_length": 476.0689697265625, "completions/min_length": 277.0, "completions/min_terminated_length": 277.0, "entropy": 0.4210197776556015, "epoch": 0.13062098501070663, "frac_reward_zero_std": 0.5, "grad_norm": 0.010469350032508373, "learning_rate": 1e-05, "loss": 0.0743, "num_tokens": 5408782.0, "reward": 0.71875, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.4662446975708008, "sampling/importance_sampling_ratio/mean": 1.0000981092453003, "sampling/importance_sampling_ratio/min": 0.7117865085601807, "sampling/sampling_logp_difference/max": 0.382704496383667, "sampling/sampling_logp_difference/mean": 0.01223764382302761, "step": 244 }, { "clip_ratio/high_max": 6.517205474665388e-05, "clip_ratio/high_mean": 6.517205474665388e-05, "clip_ratio/low_mean": 4.8543690354563296e-05, "clip_ratio/low_min": 4.8543690354563296e-05, "clip_ratio/region_mean": 0.00011371574510121718, "completions/clipped_ratio": 0.28125, "completions/max_length": 768.0, "completions/max_terminated_length": 723.0, "completions/mean_length": 613.71875, "completions/mean_terminated_length": 553.3478393554688, "completions/min_length": 408.0, "completions/min_terminated_length": 408.0, "entropy": 0.47519996762275696, "epoch": 0.1311563169164882, "frac_reward_zero_std": 0.25, "grad_norm": 0.005689447280019522, "learning_rate": 1e-05, "loss": 0.0899, "num_tokens": 5432653.0, "reward": 0.59375, "reward_std": 0.3471629321575165, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.4368473291397095, "sampling/importance_sampling_ratio/mean": 1.0000447034835815, "sampling/importance_sampling_ratio/min": 0.6039672493934631, "sampling/sampling_logp_difference/max": 0.5042352676391602, "sampling/sampling_logp_difference/mean": 0.013447093777358532, "step": 245 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 732.0, "completions/mean_length": 510.125, "completions/mean_terminated_length": 492.933349609375, "completions/min_length": 238.0, "completions/min_terminated_length": 238.0, "entropy": 0.379620973020792, "epoch": 0.1316916488222698, "frac_reward_zero_std": 0.25, "grad_norm": 0.015214172191917896, "learning_rate": 1e-05, "loss": 0.0442, "num_tokens": 5452585.0, "reward": 0.78125, "reward_std": 0.3377464711666107, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.4035483598709106, "sampling/importance_sampling_ratio/mean": 0.9997231364250183, "sampling/importance_sampling_ratio/min": 0.701497495174408, "sampling/sampling_logp_difference/max": 0.3545379638671875, "sampling/sampling_logp_difference/mean": 0.011838559992611408, "step": 246 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 756.0, "completions/mean_length": 493.15625, "completions/mean_terminated_length": 464.72412109375, "completions/min_length": 282.0, "completions/min_terminated_length": 282.0, "entropy": 0.47054021805524826, "epoch": 0.1322269807280514, "frac_reward_zero_std": 0.25, "grad_norm": 0.013052414171397686, "learning_rate": 1e-05, "loss": 0.0219, "num_tokens": 5472206.0, "reward": 0.78125, "reward_std": 0.3377464711666107, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.5128657817840576, "sampling/importance_sampling_ratio/mean": 0.9999228119850159, "sampling/importance_sampling_ratio/min": 0.6195247173309326, "sampling/sampling_logp_difference/max": 0.4788026809692383, "sampling/sampling_logp_difference/mean": 0.013454243540763855, "step": 247 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 620.0, "completions/mean_length": 466.65625, "completions/mean_terminated_length": 446.5666809082031, "completions/min_length": 228.0, "completions/min_terminated_length": 228.0, "entropy": 0.3530896082520485, "epoch": 0.13276231263383298, "frac_reward_zero_std": 0.25, "grad_norm": 0.004537623841315508, "learning_rate": 1e-05, "loss": 0.094, "num_tokens": 5490395.0, "reward": 0.875, "reward_std": 0.292504221200943, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 1.4455835819244385, "sampling/importance_sampling_ratio/mean": 1.000109076499939, "sampling/importance_sampling_ratio/min": 0.6275488138198853, "sampling/sampling_logp_difference/max": 0.46593379974365234, "sampling/sampling_logp_difference/mean": 0.011388519778847694, "step": 248 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00011779661872424185, "clip_ratio/low_min": 0.00011779661872424185, "clip_ratio/region_mean": 0.00011779661872424185, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 715.0, "completions/mean_length": 511.34375, "completions/mean_terminated_length": 484.7930908203125, "completions/min_length": 265.0, "completions/min_terminated_length": 265.0, "entropy": 0.4604758098721504, "epoch": 0.13329764453961457, "frac_reward_zero_std": 0.25, "grad_norm": 0.018156999722123146, "learning_rate": 1e-05, "loss": 0.0354, "num_tokens": 5510750.0, "reward": 0.46875, "reward_std": 0.35564959049224854, "rewards/accuracy_reward/mean": 0.46875, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.4280118942260742, "sampling/importance_sampling_ratio/mean": 1.0001708269119263, "sampling/importance_sampling_ratio/min": 0.6212390065193176, "sampling/sampling_logp_difference/max": 0.47603940963745117, "sampling/sampling_logp_difference/mean": 0.011983473785221577, "step": 249 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 622.0, "completions/mean_length": 464.625, "completions/mean_terminated_length": 394.6153869628906, "completions/min_length": 191.0, "completions/min_terminated_length": 191.0, "entropy": 0.577805370092392, "epoch": 0.13383297644539616, "frac_reward_zero_std": 0.5, "grad_norm": 0.009487269446253777, "learning_rate": 1e-05, "loss": 0.0476, "num_tokens": 5529762.0, "reward": 0.59375, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.3565547466278076, "sampling/importance_sampling_ratio/mean": 0.9998091459274292, "sampling/importance_sampling_ratio/min": 0.549450695514679, "sampling/sampling_logp_difference/max": 0.5988361835479736, "sampling/sampling_logp_difference/mean": 0.01527494564652443, "step": 250 }, { "clip_ratio/high_max": 7.936507608974352e-05, "clip_ratio/high_mean": 7.936507608974352e-05, "clip_ratio/low_mean": 5.755064557888545e-05, "clip_ratio/low_min": 5.755064557888545e-05, "clip_ratio/region_mean": 0.00013691572166862898, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 760.0, "completions/mean_length": 466.75, "completions/mean_terminated_length": 435.5862121582031, "completions/min_length": 257.0, "completions/min_terminated_length": 257.0, "entropy": 0.36938830465078354, "epoch": 0.13436830835117772, "frac_reward_zero_std": 0.0, "grad_norm": 0.015231940895318985, "learning_rate": 1e-05, "loss": 0.0357, "num_tokens": 5548282.0, "reward": 0.53125, "reward_std": 0.3808925747871399, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.4448702335357666, "sampling/importance_sampling_ratio/mean": 0.9998409152030945, "sampling/importance_sampling_ratio/min": 0.5664653182029724, "sampling/sampling_logp_difference/max": 0.5683393478393555, "sampling/sampling_logp_difference/mean": 0.012200850062072277, "step": 251 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.40625, "completions/max_length": 768.0, "completions/max_terminated_length": 766.0, "completions/mean_length": 632.78125, "completions/mean_terminated_length": 540.26318359375, "completions/min_length": 274.0, "completions/min_terminated_length": 274.0, "entropy": 0.5005229525268078, "epoch": 0.1349036402569593, "frac_reward_zero_std": 0.25, "grad_norm": 0.008063266053795815, "learning_rate": 1e-05, "loss": -0.0078, "num_tokens": 5573043.0, "reward": 0.34375, "reward_std": 0.3608423173427582, "rewards/accuracy_reward/mean": 0.34375, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.525425910949707, "sampling/importance_sampling_ratio/mean": 1.0000256299972534, "sampling/importance_sampling_ratio/min": 0.6791707277297974, "sampling/sampling_logp_difference/max": 0.4222736358642578, "sampling/sampling_logp_difference/mean": 0.014093023724853992, "step": 252 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 600.0, "completions/max_terminated_length": 600.0, "completions/mean_length": 410.25, "completions/mean_terminated_length": 410.25, "completions/min_length": 196.0, "completions/min_terminated_length": 196.0, "entropy": 0.2839566823095083, "epoch": 0.1354389721627409, "frac_reward_zero_std": 0.5, "grad_norm": 0.0196959488093853, "learning_rate": 1e-05, "loss": -0.0341, "num_tokens": 5589523.0, "reward": 0.90625, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.3948116302490234, "sampling/importance_sampling_ratio/mean": 0.9999096393585205, "sampling/importance_sampling_ratio/min": 0.5720772743225098, "sampling/sampling_logp_difference/max": 0.5584812164306641, "sampling/sampling_logp_difference/mean": 0.009745274670422077, "step": 253 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 7.212925265775993e-05, "clip_ratio/low_min": 7.212925265775993e-05, "clip_ratio/region_mean": 7.212925265775993e-05, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 731.0, "completions/mean_length": 534.6875, "completions/mean_terminated_length": 491.4814758300781, "completions/min_length": 226.0, "completions/min_terminated_length": 226.0, "entropy": 0.4521285966038704, "epoch": 0.13597430406852248, "frac_reward_zero_std": 0.5, "grad_norm": 0.01752840168774128, "learning_rate": 1e-05, "loss": 0.016, "num_tokens": 5610433.0, "reward": 0.75, "reward_std": 0.26726123690605164, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.389081597328186, "sampling/importance_sampling_ratio/mean": 0.9998871684074402, "sampling/importance_sampling_ratio/min": 0.7004028558731079, "sampling/sampling_logp_difference/max": 0.35609960556030273, "sampling/sampling_logp_difference/mean": 0.013969801366329193, "step": 254 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 766.0, "completions/mean_length": 460.15625, "completions/mean_terminated_length": 439.63336181640625, "completions/min_length": 224.0, "completions/min_terminated_length": 224.0, "entropy": 0.3674822635948658, "epoch": 0.13650963597430407, "frac_reward_zero_std": 0.25, "grad_norm": 0.011356225237250328, "learning_rate": 1e-05, "loss": 0.0501, "num_tokens": 5628414.0, "reward": 0.65625, "reward_std": 0.3471629321575165, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.422326922416687, "sampling/importance_sampling_ratio/mean": 1.0003477334976196, "sampling/importance_sampling_ratio/min": 0.6975710988044739, "sampling/sampling_logp_difference/max": 0.3601508140563965, "sampling/sampling_logp_difference/mean": 0.011541708372533321, "step": 255 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.041565939085558e-05, "clip_ratio/low_min": 6.041565939085558e-05, "clip_ratio/region_mean": 6.041565939085558e-05, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 748.0, "completions/mean_length": 487.15625, "completions/mean_terminated_length": 435.1481628417969, "completions/min_length": 250.0, "completions/min_terminated_length": 250.0, "entropy": 0.4013260640203953, "epoch": 0.13704496788008566, "frac_reward_zero_std": 0.0, "grad_norm": 0.013696649111807346, "learning_rate": 1e-05, "loss": 0.0458, "num_tokens": 5647571.0, "reward": 0.46875, "reward_std": 0.4218915104866028, "rewards/accuracy_reward/mean": 0.46875, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.4437862634658813, "sampling/importance_sampling_ratio/mean": 1.000393271446228, "sampling/importance_sampling_ratio/min": 0.7036470770835876, "sampling/sampling_logp_difference/max": 0.36726903915405273, "sampling/sampling_logp_difference/mean": 0.012749886140227318, "step": 256 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 647.0, "completions/max_terminated_length": 647.0, "completions/mean_length": 445.03125, "completions/mean_terminated_length": 445.03125, "completions/min_length": 277.0, "completions/min_terminated_length": 277.0, "entropy": 0.34894824028015137, "epoch": 0.13758029978586725, "frac_reward_zero_std": 0.5, "grad_norm": 0.002513580024242401, "learning_rate": 1e-05, "loss": 0.0504, "num_tokens": 5665012.0, "reward": 0.8125, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9998238682746887, "sampling/importance_sampling_ratio/min": 0.635778546333313, "sampling/sampling_logp_difference/max": 0.6938328742980957, "sampling/sampling_logp_difference/mean": 0.012405021116137505, "step": 257 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 7.01065655448474e-05, "clip_ratio/low_min": 7.01065655448474e-05, "clip_ratio/region_mean": 7.01065655448474e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 672.0, "completions/max_terminated_length": 672.0, "completions/mean_length": 451.0, "completions/mean_terminated_length": 451.0, "completions/min_length": 285.0, "completions/min_terminated_length": 285.0, "entropy": 0.3489055410027504, "epoch": 0.1381156316916488, "frac_reward_zero_std": 0.5, "grad_norm": 0.006209916900843382, "learning_rate": 1e-05, "loss": 0.0214, "num_tokens": 5682900.0, "reward": 0.75, "reward_std": 0.2314550280570984, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.4003722667694092, "sampling/importance_sampling_ratio/mean": 0.99953693151474, "sampling/importance_sampling_ratio/min": 0.6066949367523193, "sampling/sampling_logp_difference/max": 0.4997291564941406, "sampling/sampling_logp_difference/mean": 0.012071177363395691, "step": 258 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.002000762033276e-05, "clip_ratio/low_min": 5.002000762033276e-05, "clip_ratio/region_mean": 5.002000762033276e-05, "completions/clipped_ratio": 0.28125, "completions/max_length": 768.0, "completions/max_terminated_length": 764.0, "completions/mean_length": 618.65625, "completions/mean_terminated_length": 560.2174072265625, "completions/min_length": 239.0, "completions/min_terminated_length": 239.0, "entropy": 0.5505796335637569, "epoch": 0.1386509635974304, "frac_reward_zero_std": 0.5, "grad_norm": 0.004257765598595142, "learning_rate": 1e-05, "loss": 0.0181, "num_tokens": 5706433.0, "reward": 0.25, "reward_std": 0.2587745785713196, "rewards/accuracy_reward/mean": 0.25, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.7159582376480103, "sampling/importance_sampling_ratio/mean": 1.000075340270996, "sampling/importance_sampling_ratio/min": 0.5809783935546875, "sampling/sampling_logp_difference/max": 0.5430417060852051, "sampling/sampling_logp_difference/mean": 0.013373611494898796, "step": 259 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 7.467144314432517e-05, "clip_ratio/low_min": 7.467144314432517e-05, "clip_ratio/region_mean": 7.467144314432517e-05, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 734.0, "completions/mean_length": 479.5, "completions/mean_terminated_length": 460.2666931152344, "completions/min_length": 182.0, "completions/min_terminated_length": 182.0, "entropy": 0.508996956050396, "epoch": 0.139186295503212, "frac_reward_zero_std": 0.25, "grad_norm": 0.013239891268312931, "learning_rate": 1e-05, "loss": 0.0312, "num_tokens": 5725161.0, "reward": 0.65625, "reward_std": 0.3471629321575165, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.2882016897201538, "sampling/importance_sampling_ratio/mean": 0.9997792840003967, "sampling/importance_sampling_ratio/min": 0.5010967254638672, "sampling/sampling_logp_difference/max": 0.6909561157226562, "sampling/sampling_logp_difference/mean": 0.01288861408829689, "step": 260 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 9.622786456020549e-05, "clip_ratio/low_min": 9.622786456020549e-05, "clip_ratio/region_mean": 9.622786456020549e-05, "completions/clipped_ratio": 0.375, "completions/max_length": 768.0, "completions/max_terminated_length": 754.0, "completions/mean_length": 636.1875, "completions/mean_terminated_length": 557.1000366210938, "completions/min_length": 382.0, "completions/min_terminated_length": 382.0, "entropy": 0.5438135378062725, "epoch": 0.13972162740899358, "frac_reward_zero_std": 0.5, "grad_norm": 0.007354440167546272, "learning_rate": 1e-05, "loss": 0.0466, "num_tokens": 5749543.0, "reward": 0.375, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.375, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.8053772449493408, "sampling/importance_sampling_ratio/mean": 1.0000406503677368, "sampling/importance_sampling_ratio/min": 0.6841415166854858, "sampling/sampling_logp_difference/max": 0.5907695293426514, "sampling/sampling_logp_difference/mean": 0.01508672907948494, "step": 261 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00017080266115954146, "clip_ratio/low_min": 0.00017080266115954146, "clip_ratio/region_mean": 0.00017080266115954146, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 693.0, "completions/mean_length": 423.375, "completions/mean_terminated_length": 374.14288330078125, "completions/min_length": 171.0, "completions/min_terminated_length": 171.0, "entropy": 0.4612344726920128, "epoch": 0.14025695931477516, "frac_reward_zero_std": 0.0, "grad_norm": 0.011318651027977467, "learning_rate": 1e-05, "loss": 0.041, "num_tokens": 5766467.0, "reward": 0.625, "reward_std": 0.49871626496315, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.363800287246704, "sampling/importance_sampling_ratio/mean": 1.0003305673599243, "sampling/importance_sampling_ratio/min": 0.5737609267234802, "sampling/sampling_logp_difference/max": 0.5555424690246582, "sampling/sampling_logp_difference/mean": 0.012841030955314636, "step": 262 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.000243418642639881, "clip_ratio/low_min": 0.000243418642639881, "clip_ratio/region_mean": 0.000243418642639881, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 760.0, "completions/mean_length": 629.1875, "completions/mean_terminated_length": 582.9166870117188, "completions/min_length": 365.0, "completions/min_terminated_length": 365.0, "entropy": 0.5080892257392406, "epoch": 0.14079229122055675, "frac_reward_zero_std": 0.25, "grad_norm": 0.011443656869232655, "learning_rate": 1e-05, "loss": 0.0449, "num_tokens": 5790153.0, "reward": 0.46875, "reward_std": 0.3471629321575165, "rewards/accuracy_reward/mean": 0.46875, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.5307055711746216, "sampling/importance_sampling_ratio/mean": 1.000052809715271, "sampling/importance_sampling_ratio/min": 0.3311429023742676, "sampling/sampling_logp_difference/max": 1.1052052974700928, "sampling/sampling_logp_difference/mean": 0.014086173847317696, "step": 263 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.771006544819102e-05, "clip_ratio/low_min": 5.771006544819102e-05, "clip_ratio/region_mean": 5.771006544819102e-05, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 702.0, "completions/mean_length": 508.25, "completions/mean_terminated_length": 448.3077087402344, "completions/min_length": 246.0, "completions/min_terminated_length": 246.0, "entropy": 0.3992535434663296, "epoch": 0.14132762312633834, "frac_reward_zero_std": 0.75, "grad_norm": 0.025639759376645088, "learning_rate": 1e-05, "loss": 0.01, "num_tokens": 5810369.0, "reward": 0.28125, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.28125, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.3812230825424194, "sampling/importance_sampling_ratio/mean": 1.0002771615982056, "sampling/importance_sampling_ratio/min": 0.6831875443458557, "sampling/sampling_logp_difference/max": 0.3809858560562134, "sampling/sampling_logp_difference/mean": 0.012271767482161522, "step": 264 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.275100167840719e-05, "clip_ratio/low_min": 6.275100167840719e-05, "clip_ratio/region_mean": 6.275100167840719e-05, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 739.0, "completions/mean_length": 567.59375, "completions/mean_terminated_length": 538.9642944335938, "completions/min_length": 336.0, "completions/min_terminated_length": 336.0, "entropy": 0.4798845537006855, "epoch": 0.1418629550321199, "frac_reward_zero_std": 0.5, "grad_norm": 0.006482654716819525, "learning_rate": 1e-05, "loss": 0.0399, "num_tokens": 5832204.0, "reward": 0.71875, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.3956878185272217, "sampling/importance_sampling_ratio/mean": 1.0003331899642944, "sampling/importance_sampling_ratio/min": 0.6024718880653381, "sampling/sampling_logp_difference/max": 0.5067142248153687, "sampling/sampling_logp_difference/mean": 0.013263930566608906, "step": 265 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0001232134090969339, "clip_ratio/low_min": 0.0001232134090969339, "clip_ratio/region_mean": 0.0001232134090969339, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 712.0, "completions/mean_length": 577.28125, "completions/mean_terminated_length": 550.0357666015625, "completions/min_length": 266.0, "completions/min_terminated_length": 266.0, "entropy": 0.4998796731233597, "epoch": 0.1423982869379015, "frac_reward_zero_std": 0.5, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0423, "num_tokens": 5854829.0, "reward": 0.78125, "reward_std": 0.2630178928375244, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.3215910196304321, "sampling/importance_sampling_ratio/mean": 1.000644564628601, "sampling/importance_sampling_ratio/min": 0.6367124915122986, "sampling/sampling_logp_difference/max": 0.45143699645996094, "sampling/sampling_logp_difference/mean": 0.01436928752809763, "step": 266 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 709.0, "completions/mean_length": 445.59375, "completions/mean_terminated_length": 435.19354248046875, "completions/min_length": 212.0, "completions/min_terminated_length": 212.0, "entropy": 0.3613501340150833, "epoch": 0.14293361884368308, "frac_reward_zero_std": 0.5, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0861, "num_tokens": 5872408.0, "reward": 0.9375, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.9375, "rewards/accuracy_reward/std": 0.24593468010425568, "sampling/importance_sampling_ratio/max": 1.4321389198303223, "sampling/importance_sampling_ratio/mean": 0.9999266266822815, "sampling/importance_sampling_ratio/min": 0.7264004945755005, "sampling/sampling_logp_difference/max": 0.35916900634765625, "sampling/sampling_logp_difference/mean": 0.011632355861365795, "step": 267 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 673.0, "completions/mean_length": 519.40625, "completions/mean_terminated_length": 502.8333740234375, "completions/min_length": 348.0, "completions/min_terminated_length": 348.0, "entropy": 0.32485054805874825, "epoch": 0.14346895074946467, "frac_reward_zero_std": 0.25, "grad_norm": 0.006728060077875853, "learning_rate": 1e-05, "loss": 0.0159, "num_tokens": 5892653.0, "reward": 0.84375, "reward_std": 0.3061639964580536, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.497738242149353, "sampling/importance_sampling_ratio/mean": 1.0001980066299438, "sampling/importance_sampling_ratio/min": 0.684363067150116, "sampling/sampling_logp_difference/max": 0.40395617485046387, "sampling/sampling_logp_difference/mean": 0.010887635871767998, "step": 268 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00013224303984316066, "clip_ratio/low_min": 0.00013224303984316066, "clip_ratio/region_mean": 0.00013224303984316066, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 729.0, "completions/mean_length": 468.09375, "completions/mean_terminated_length": 448.10003662109375, "completions/min_length": 279.0, "completions/min_terminated_length": 279.0, "entropy": 0.40673037990927696, "epoch": 0.14400428265524626, "frac_reward_zero_std": 0.25, "grad_norm": 0.010233355686068535, "learning_rate": 1e-05, "loss": -0.0387, "num_tokens": 5910984.0, "reward": 0.5625, "reward_std": 0.3471825420856476, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.5287798643112183, "sampling/importance_sampling_ratio/mean": 0.999657928943634, "sampling/importance_sampling_ratio/min": 0.7221192121505737, "sampling/sampling_logp_difference/max": 0.4244699478149414, "sampling/sampling_logp_difference/mean": 0.012970454059541225, "step": 269 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00020064205455128103, "clip_ratio/low_min": 0.00020064205455128103, "clip_ratio/region_mean": 0.00020064205455128103, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 522.0, "completions/mean_length": 445.59375, "completions/mean_terminated_length": 385.8888854980469, "completions/min_length": 201.0, "completions/min_terminated_length": 201.0, "entropy": 0.46112507954239845, "epoch": 0.14453961456102785, "frac_reward_zero_std": 0.25, "grad_norm": 0.019001614302396774, "learning_rate": 1e-05, "loss": 0.0536, "num_tokens": 5928531.0, "reward": 0.5625, "reward_std": 0.3514062762260437, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.360998272895813, "sampling/importance_sampling_ratio/mean": 0.9998810887336731, "sampling/importance_sampling_ratio/min": 0.5097512602806091, "sampling/sampling_logp_difference/max": 0.6738324165344238, "sampling/sampling_logp_difference/mean": 0.01386426854878664, "step": 270 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.9860391300171614e-05, "clip_ratio/low_min": 4.9860391300171614e-05, "clip_ratio/region_mean": 4.9860391300171614e-05, "completions/clipped_ratio": 0.3125, "completions/max_length": 768.0, "completions/max_terminated_length": 706.0, "completions/mean_length": 608.34375, "completions/mean_terminated_length": 535.7727661132812, "completions/min_length": 277.0, "completions/min_terminated_length": 277.0, "entropy": 0.44793014228343964, "epoch": 0.14507494646680943, "frac_reward_zero_std": 0.25, "grad_norm": 0.0058585829101502895, "learning_rate": 1e-05, "loss": 0.0425, "num_tokens": 5951374.0, "reward": 0.625, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.7821135520935059, "sampling/importance_sampling_ratio/mean": 1.0003591775894165, "sampling/importance_sampling_ratio/min": 0.7277841567993164, "sampling/sampling_logp_difference/max": 0.5778000354766846, "sampling/sampling_logp_difference/mean": 0.012861361727118492, "step": 271 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.380806735251099e-05, "clip_ratio/low_min": 6.380806735251099e-05, "clip_ratio/region_mean": 6.380806735251099e-05, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 659.0, "completions/mean_length": 478.78125, "completions/mean_terminated_length": 459.5000305175781, "completions/min_length": 298.0, "completions/min_terminated_length": 298.0, "entropy": 0.39119698852300644, "epoch": 0.145610278372591, "frac_reward_zero_std": 0.0, "grad_norm": 0.02509799413383007, "learning_rate": 1e-05, "loss": 0.0942, "num_tokens": 5970415.0, "reward": 0.71875, "reward_std": 0.3808925747871399, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.4179117679595947, "sampling/importance_sampling_ratio/mean": 0.9994146823883057, "sampling/importance_sampling_ratio/min": 0.6299112439155579, "sampling/sampling_logp_difference/max": 0.4621763229370117, "sampling/sampling_logp_difference/mean": 0.012396533973515034, "step": 272 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 760.0, "completions/mean_length": 484.125, "completions/mean_terminated_length": 404.6399841308594, "completions/min_length": 188.0, "completions/min_terminated_length": 188.0, "entropy": 0.48611050844192505, "epoch": 0.14614561027837258, "frac_reward_zero_std": 0.25, "grad_norm": 0.020668551325798035, "learning_rate": 1e-05, "loss": 0.0815, "num_tokens": 5989203.0, "reward": 0.40625, "reward_std": 0.3471629321575165, "rewards/accuracy_reward/mean": 0.40625, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.5681416988372803, "sampling/importance_sampling_ratio/mean": 1.0000240802764893, "sampling/importance_sampling_ratio/min": 0.7110249400138855, "sampling/sampling_logp_difference/max": 0.4498913288116455, "sampling/sampling_logp_difference/mean": 0.014063272625207901, "step": 273 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 733.0, "completions/mean_length": 518.78125, "completions/mean_terminated_length": 493.0, "completions/min_length": 267.0, "completions/min_terminated_length": 267.0, "entropy": 0.33634426072239876, "epoch": 0.14668094218415417, "frac_reward_zero_std": 0.25, "grad_norm": 0.01658659800887108, "learning_rate": 1e-05, "loss": -0.0045, "num_tokens": 6009292.0, "reward": 0.53125, "reward_std": 0.3198433816432953, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.4125926494598389, "sampling/importance_sampling_ratio/mean": 1.0000663995742798, "sampling/importance_sampling_ratio/min": 0.5959081649780273, "sampling/sampling_logp_difference/max": 0.5176687240600586, "sampling/sampling_logp_difference/mean": 0.011553918942809105, "step": 274 }, { "clip_ratio/high_max": 6.281406967900693e-05, "clip_ratio/high_mean": 6.281406967900693e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 6.281406967900693e-05, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 684.0, "completions/mean_length": 444.28125, "completions/mean_terminated_length": 410.7930908203125, "completions/min_length": 201.0, "completions/min_terminated_length": 201.0, "entropy": 0.4818809889256954, "epoch": 0.14721627408993576, "frac_reward_zero_std": 0.25, "grad_norm": 0.023966826498508453, "learning_rate": 1e-05, "loss": -0.0106, "num_tokens": 6027237.0, "reward": 0.71875, "reward_std": 0.3471629321575165, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.4246875047683716, "sampling/importance_sampling_ratio/mean": 1.0001494884490967, "sampling/importance_sampling_ratio/min": 0.6532939672470093, "sampling/sampling_logp_difference/max": 0.42572808265686035, "sampling/sampling_logp_difference/mean": 0.015202755108475685, "step": 275 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 723.0, "completions/mean_length": 482.40625, "completions/mean_terminated_length": 463.36669921875, "completions/min_length": 272.0, "completions/min_terminated_length": 272.0, "entropy": 0.4784157834947109, "epoch": 0.14775160599571735, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": -0.0169, "num_tokens": 6046514.0, "reward": 0.875, "reward_std": 0.13363061845302582, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 1.4301837682724, "sampling/importance_sampling_ratio/mean": 1.0002397298812866, "sampling/importance_sampling_ratio/min": 0.6200202107429504, "sampling/sampling_logp_difference/max": 0.47800326347351074, "sampling/sampling_logp_difference/mean": 0.013694281689822674, "step": 276 }, { "clip_ratio/high_max": 5.287647945806384e-05, "clip_ratio/high_mean": 5.287647945806384e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 5.287647945806384e-05, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 708.0, "completions/mean_length": 504.875, "completions/mean_terminated_length": 477.6551818847656, "completions/min_length": 270.0, "completions/min_terminated_length": 270.0, "entropy": 0.3827531524002552, "epoch": 0.14828693790149894, "frac_reward_zero_std": 0.25, "grad_norm": 0.012952609919011593, "learning_rate": 1e-05, "loss": -0.0061, "num_tokens": 6066726.0, "reward": 0.6875, "reward_std": 0.3924052119255066, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.362608551979065, "sampling/importance_sampling_ratio/mean": 1.0001440048217773, "sampling/importance_sampling_ratio/min": 0.6704936027526855, "sampling/sampling_logp_difference/max": 0.39974117279052734, "sampling/sampling_logp_difference/mean": 0.011616443283855915, "step": 277 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 718.0, "completions/mean_length": 463.6875, "completions/mean_terminated_length": 420.21429443359375, "completions/min_length": 255.0, "completions/min_terminated_length": 255.0, "entropy": 0.37863264977931976, "epoch": 0.14882226980728053, "frac_reward_zero_std": 0.25, "grad_norm": 0.01699407957494259, "learning_rate": 1e-05, "loss": 0.0916, "num_tokens": 6085332.0, "reward": 0.59375, "reward_std": 0.3608423173427582, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.336287498474121, "sampling/importance_sampling_ratio/mean": 0.9995310306549072, "sampling/importance_sampling_ratio/min": 0.6001459360122681, "sampling/sampling_logp_difference/max": 0.510582447052002, "sampling/sampling_logp_difference/mean": 0.012289334088563919, "step": 278 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00038535775820491835, "clip_ratio/low_min": 0.00038535775820491835, "clip_ratio/region_mean": 0.00038535775820491835, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 752.0, "completions/mean_length": 561.4375, "completions/mean_terminated_length": 503.5999755859375, "completions/min_length": 319.0, "completions/min_terminated_length": 319.0, "entropy": 0.5481689944863319, "epoch": 0.1493576017130621, "frac_reward_zero_std": 0.25, "grad_norm": 0.019931795075535774, "learning_rate": 1e-05, "loss": 0.102, "num_tokens": 6107858.0, "reward": 0.5625, "reward_std": 0.3514062762260437, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.4254674911499023, "sampling/importance_sampling_ratio/mean": 1.000180721282959, "sampling/importance_sampling_ratio/min": 0.6351988911628723, "sampling/sampling_logp_difference/max": 0.45381712913513184, "sampling/sampling_logp_difference/mean": 0.01599624752998352, "step": 279 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 8.821453957352787e-05, "clip_ratio/low_min": 8.821453957352787e-05, "clip_ratio/region_mean": 8.821453957352787e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 598.0, "completions/max_terminated_length": 598.0, "completions/mean_length": 414.5, "completions/mean_terminated_length": 414.5, "completions/min_length": 200.0, "completions/min_terminated_length": 200.0, "entropy": 0.31437014415860176, "epoch": 0.14989293361884368, "frac_reward_zero_std": 0.5, "grad_norm": 0.014123039320111275, "learning_rate": 1e-05, "loss": -0.059, "num_tokens": 6124210.0, "reward": 0.90625, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.435381531715393, "sampling/importance_sampling_ratio/mean": 1.0001773834228516, "sampling/importance_sampling_ratio/min": 0.6299055218696594, "sampling/sampling_logp_difference/max": 0.4621853828430176, "sampling/sampling_logp_difference/mean": 0.01068188063800335, "step": 280 }, { "clip_ratio/high_max": 7.110352453310043e-05, "clip_ratio/high_mean": 7.110352453310043e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 7.110352453310043e-05, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 746.0, "completions/mean_length": 525.15625, "completions/mean_terminated_length": 508.9667053222656, "completions/min_length": 353.0, "completions/min_terminated_length": 353.0, "entropy": 0.4922599531710148, "epoch": 0.15042826552462527, "frac_reward_zero_std": 0.25, "grad_norm": 0.025635600090026855, "learning_rate": 1e-05, "loss": 0.0608, "num_tokens": 6144887.0, "reward": 0.8125, "reward_std": 0.3104073107242584, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.5917540788650513, "sampling/importance_sampling_ratio/mean": 1.0004243850708008, "sampling/importance_sampling_ratio/min": 0.609991729259491, "sampling/sampling_logp_difference/max": 0.4943099021911621, "sampling/sampling_logp_difference/mean": 0.013856773264706135, "step": 281 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.2410901844268665e-05, "clip_ratio/low_min": 5.2410901844268665e-05, "clip_ratio/region_mean": 5.2410901844268665e-05, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 716.0, "completions/mean_length": 526.4375, "completions/mean_terminated_length": 481.7037048339844, "completions/min_length": 169.0, "completions/min_terminated_length": 169.0, "entropy": 0.3575453422963619, "epoch": 0.15096359743040685, "frac_reward_zero_std": 0.5, "grad_norm": 0.008215086534619331, "learning_rate": 1e-05, "loss": 0.0063, "num_tokens": 6165165.0, "reward": 0.71875, "reward_std": 0.2630178928375244, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.4350372552871704, "sampling/importance_sampling_ratio/mean": 1.0001972913742065, "sampling/importance_sampling_ratio/min": 0.626156210899353, "sampling/sampling_logp_difference/max": 0.4681553840637207, "sampling/sampling_logp_difference/mean": 0.011345704086124897, "step": 282 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00010989011207129806, "clip_ratio/low_min": 0.00010989011207129806, "clip_ratio/region_mean": 0.00010989011207129806, "completions/clipped_ratio": 0.40625, "completions/max_length": 768.0, "completions/max_terminated_length": 698.0, "completions/mean_length": 613.9375, "completions/mean_terminated_length": 508.52630615234375, "completions/min_length": 330.0, "completions/min_terminated_length": 330.0, "entropy": 0.5087274610996246, "epoch": 0.15149892933618844, "frac_reward_zero_std": 0.25, "grad_norm": 0.017519496381282806, "learning_rate": 1e-05, "loss": 0.0681, "num_tokens": 6188891.0, "reward": 0.46875, "reward_std": 0.3198433816432953, "rewards/accuracy_reward/mean": 0.46875, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.4468004703521729, "sampling/importance_sampling_ratio/mean": 0.9999304413795471, "sampling/importance_sampling_ratio/min": 0.6442550420761108, "sampling/sampling_logp_difference/max": 0.4396606683731079, "sampling/sampling_logp_difference/mean": 0.013716845773160458, "step": 283 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 686.0, "completions/mean_length": 489.78125, "completions/mean_terminated_length": 480.8064270019531, "completions/min_length": 251.0, "completions/min_terminated_length": 251.0, "entropy": 0.3560822978615761, "epoch": 0.15203426124197003, "frac_reward_zero_std": 0.5, "grad_norm": 0.023355163633823395, "learning_rate": 1e-05, "loss": -0.0782, "num_tokens": 6208372.0, "reward": 0.84375, "reward_std": 0.22201895713806152, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.2881642580032349, "sampling/importance_sampling_ratio/mean": 0.9998629093170166, "sampling/importance_sampling_ratio/min": 0.6603497862815857, "sampling/sampling_logp_difference/max": 0.41498565673828125, "sampling/sampling_logp_difference/mean": 0.01101180911064148, "step": 284 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 736.0, "completions/mean_length": 517.46875, "completions/mean_terminated_length": 481.6785888671875, "completions/min_length": 231.0, "completions/min_terminated_length": 231.0, "entropy": 0.43186208605766296, "epoch": 0.15256959314775162, "frac_reward_zero_std": 0.5, "grad_norm": 0.01367235742509365, "learning_rate": 1e-05, "loss": -0.0141, "num_tokens": 6228891.0, "reward": 0.5625, "reward_std": 0.249358132481575, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.514527440071106, "sampling/importance_sampling_ratio/mean": 1.0001575946807861, "sampling/importance_sampling_ratio/min": 0.6505802273750305, "sampling/sampling_logp_difference/max": 0.42989063262939453, "sampling/sampling_logp_difference/mean": 0.01313034538179636, "step": 285 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 702.0, "completions/mean_length": 443.3125, "completions/mean_terminated_length": 432.83868408203125, "completions/min_length": 162.0, "completions/min_terminated_length": 162.0, "entropy": 0.479502160102129, "epoch": 0.15310492505353318, "frac_reward_zero_std": 0.25, "grad_norm": 0.007370191626250744, "learning_rate": 1e-05, "loss": 0.0603, "num_tokens": 6246645.0, "reward": 0.875, "reward_std": 0.292504221200943, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 1.4618957042694092, "sampling/importance_sampling_ratio/mean": 0.9996649622917175, "sampling/importance_sampling_ratio/min": 0.44690823554992676, "sampling/sampling_logp_difference/max": 0.8054020404815674, "sampling/sampling_logp_difference/mean": 0.014546182006597519, "step": 286 }, { "clip_ratio/high_max": 6.361323175951838e-05, "clip_ratio/high_mean": 6.361323175951838e-05, "clip_ratio/low_mean": 0.00018266693950863555, "clip_ratio/low_min": 0.00018266693950863555, "clip_ratio/region_mean": 0.00024628017126815394, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 709.0, "completions/mean_length": 548.0625, "completions/mean_terminated_length": 507.3333435058594, "completions/min_length": 331.0, "completions/min_terminated_length": 331.0, "entropy": 0.5106589309871197, "epoch": 0.15364025695931477, "frac_reward_zero_std": 0.25, "grad_norm": 0.009625596925616264, "learning_rate": 1e-05, "loss": 0.017, "num_tokens": 6267807.0, "reward": 0.40625, "reward_std": 0.3061639666557312, "rewards/accuracy_reward/mean": 0.40625, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.4405272006988525, "sampling/importance_sampling_ratio/mean": 0.9999322891235352, "sampling/importance_sampling_ratio/min": 0.6387335062026978, "sampling/sampling_logp_difference/max": 0.44826793670654297, "sampling/sampling_logp_difference/mean": 0.014671443961560726, "step": 287 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00018225424355478026, "clip_ratio/low_min": 0.00018225424355478026, "clip_ratio/region_mean": 0.00018225424355478026, "completions/clipped_ratio": 0.34375, "completions/max_length": 768.0, "completions/max_terminated_length": 765.0, "completions/mean_length": 643.71875, "completions/mean_terminated_length": 578.6190795898438, "completions/min_length": 384.0, "completions/min_terminated_length": 384.0, "entropy": 0.4791031889617443, "epoch": 0.15417558886509636, "frac_reward_zero_std": 0.25, "grad_norm": 0.005762732587754726, "learning_rate": 1e-05, "loss": 0.0713, "num_tokens": 6292846.0, "reward": 0.5625, "reward_std": 0.292504221200943, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.5718718767166138, "sampling/importance_sampling_ratio/mean": 1.0001534223556519, "sampling/importance_sampling_ratio/min": 0.7144895792007446, "sampling/sampling_logp_difference/max": 0.4522671699523926, "sampling/sampling_logp_difference/mean": 0.013097360730171204, "step": 288 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 764.0, "completions/mean_length": 547.25, "completions/mean_terminated_length": 496.3077087402344, "completions/min_length": 314.0, "completions/min_terminated_length": 314.0, "entropy": 0.3792828842997551, "epoch": 0.15471092077087795, "frac_reward_zero_std": 0.0, "grad_norm": 0.02202201448380947, "learning_rate": 1e-05, "loss": 0.009, "num_tokens": 6314030.0, "reward": 0.625, "reward_std": 0.4355512857437134, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.3409948348999023, "sampling/importance_sampling_ratio/mean": 0.999615490436554, "sampling/importance_sampling_ratio/min": 0.6302874088287354, "sampling/sampling_logp_difference/max": 0.4615793228149414, "sampling/sampling_logp_difference/mean": 0.011571809649467468, "step": 289 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00012770119064953178, "clip_ratio/low_min": 0.00012770119064953178, "clip_ratio/region_mean": 0.00012770119064953178, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 743.0, "completions/mean_length": 557.40625, "completions/mean_terminated_length": 487.2083435058594, "completions/min_length": 222.0, "completions/min_terminated_length": 222.0, "entropy": 0.43026646226644516, "epoch": 0.15524625267665954, "frac_reward_zero_std": 0.25, "grad_norm": 0.008927677758038044, "learning_rate": 1e-05, "loss": 0.0973, "num_tokens": 6335739.0, "reward": 0.46875, "reward_std": 0.3198433816432953, "rewards/accuracy_reward/mean": 0.46875, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.3829481601715088, "sampling/importance_sampling_ratio/mean": 0.9999781250953674, "sampling/importance_sampling_ratio/min": 0.6179466843605042, "sampling/sampling_logp_difference/max": 0.4813530445098877, "sampling/sampling_logp_difference/mean": 0.012422534637153149, "step": 290 }, { "clip_ratio/high_max": 5.6407941883662716e-05, "clip_ratio/high_mean": 5.6407941883662716e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 5.6407941883662716e-05, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 748.0, "completions/mean_length": 527.53125, "completions/mean_terminated_length": 511.5000305175781, "completions/min_length": 228.0, "completions/min_terminated_length": 228.0, "entropy": 0.27904924005270004, "epoch": 0.15578158458244112, "frac_reward_zero_std": 0.5, "grad_norm": 0.009527213871479034, "learning_rate": 1e-05, "loss": 0.0204, "num_tokens": 6356084.0, "reward": 0.90625, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.3129595518112183, "sampling/importance_sampling_ratio/mean": 0.999772310256958, "sampling/importance_sampling_ratio/min": 0.6971725821495056, "sampling/sampling_logp_difference/max": 0.36072230339050293, "sampling/sampling_logp_difference/mean": 0.01001154538244009, "step": 291 }, { "clip_ratio/high_max": 6.297229265328497e-05, "clip_ratio/high_mean": 6.297229265328497e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 6.297229265328497e-05, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 759.0, "completions/mean_length": 512.34375, "completions/mean_terminated_length": 504.0967712402344, "completions/min_length": 281.0, "completions/min_terminated_length": 281.0, "entropy": 0.42052092403173447, "epoch": 0.15631691648822268, "frac_reward_zero_std": 0.25, "grad_norm": 0.019503682851791382, "learning_rate": 1e-05, "loss": 0.0766, "num_tokens": 6376359.0, "reward": 0.84375, "reward_std": 0.3198433816432953, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.3344718217849731, "sampling/importance_sampling_ratio/mean": 1.0001314878463745, "sampling/importance_sampling_ratio/min": 0.7134794592857361, "sampling/sampling_logp_difference/max": 0.3376016616821289, "sampling/sampling_logp_difference/mean": 0.012851390056312084, "step": 292 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 726.0, "completions/mean_length": 546.3125, "completions/mean_terminated_length": 531.5333862304688, "completions/min_length": 280.0, "completions/min_terminated_length": 280.0, "entropy": 0.6059800870716572, "epoch": 0.15685224839400427, "frac_reward_zero_std": 0.25, "grad_norm": 0.008632062003016472, "learning_rate": 1e-05, "loss": -0.0295, "num_tokens": 6398113.0, "reward": 0.40625, "reward_std": 0.3377465009689331, "rewards/accuracy_reward/mean": 0.40625, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9996324777603149, "sampling/importance_sampling_ratio/min": 0.6368066668510437, "sampling/sampling_logp_difference/max": 0.7614378929138184, "sampling/sampling_logp_difference/mean": 0.013707511126995087, "step": 293 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 686.0, "completions/mean_length": 492.90625, "completions/mean_terminated_length": 441.96295166015625, "completions/min_length": 267.0, "completions/min_terminated_length": 267.0, "entropy": 0.47353770211338997, "epoch": 0.15738758029978586, "frac_reward_zero_std": 0.75, "grad_norm": 0.016395388171076775, "learning_rate": 1e-05, "loss": 0.0199, "num_tokens": 6417526.0, "reward": 0.59375, "reward_std": 0.1293872892856598, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.4660577774047852, "sampling/importance_sampling_ratio/mean": 0.9998359084129333, "sampling/importance_sampling_ratio/min": 0.37846291065216064, "sampling/sampling_logp_difference/max": 0.9716372489929199, "sampling/sampling_logp_difference/mean": 0.014490845613181591, "step": 294 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00014403185195988044, "clip_ratio/low_min": 0.00014403185195988044, "clip_ratio/region_mean": 0.00014403185195988044, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 728.0, "completions/mean_length": 432.6875, "completions/mean_terminated_length": 410.3333435058594, "completions/min_length": 245.0, "completions/min_terminated_length": 245.0, "entropy": 0.4177595153450966, "epoch": 0.15792291220556745, "frac_reward_zero_std": 0.25, "grad_norm": 0.02344527281820774, "learning_rate": 1e-05, "loss": 0.021, "num_tokens": 6435132.0, "reward": 0.6875, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.3946726322174072, "sampling/importance_sampling_ratio/mean": 0.999739944934845, "sampling/importance_sampling_ratio/min": 0.6931111216545105, "sampling/sampling_logp_difference/max": 0.3665649890899658, "sampling/sampling_logp_difference/mean": 0.013593433424830437, "step": 295 }, { "clip_ratio/high_max": 5.482456253957935e-05, "clip_ratio/high_mean": 5.482456253957935e-05, "clip_ratio/low_mean": 0.0001080386973626446, "clip_ratio/low_min": 0.0001080386973626446, "clip_ratio/region_mean": 0.00016286325990222394, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 713.0, "completions/mean_length": 482.71875, "completions/mean_terminated_length": 473.51611328125, "completions/min_length": 170.0, "completions/min_terminated_length": 170.0, "entropy": 0.4640315771102905, "epoch": 0.15845824411134904, "frac_reward_zero_std": 0.0, "grad_norm": 0.01397132221609354, "learning_rate": 1e-05, "loss": -0.0418, "num_tokens": 6453883.0, "reward": 0.5, "reward_std": 0.4355512857437134, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5080004930496216, "sampling/importance_sampling_ratio/max": 1.367895483970642, "sampling/importance_sampling_ratio/mean": 0.9997069835662842, "sampling/importance_sampling_ratio/min": 0.7424412369728088, "sampling/sampling_logp_difference/max": 0.31327342987060547, "sampling/sampling_logp_difference/mean": 0.012333623133599758, "step": 296 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 763.0, "completions/mean_length": 526.40625, "completions/mean_terminated_length": 470.65386962890625, "completions/min_length": 186.0, "completions/min_terminated_length": 186.0, "entropy": 0.5003989450633526, "epoch": 0.15899357601713063, "frac_reward_zero_std": 0.25, "grad_norm": 0.008706404827535152, "learning_rate": 1e-05, "loss": 0.0654, "num_tokens": 6474472.0, "reward": 0.5, "reward_std": 0.3514062464237213, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5080004930496216, "sampling/importance_sampling_ratio/max": 1.3327733278274536, "sampling/importance_sampling_ratio/mean": 1.0000064373016357, "sampling/importance_sampling_ratio/min": 0.7041017413139343, "sampling/sampling_logp_difference/max": 0.350832462310791, "sampling/sampling_logp_difference/mean": 0.014013871550559998, "step": 297 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 766.0, "completions/max_terminated_length": 766.0, "completions/mean_length": 422.6875, "completions/mean_terminated_length": 422.6875, "completions/min_length": 251.0, "completions/min_terminated_length": 251.0, "entropy": 0.34741342440247536, "epoch": 0.15952890792291222, "frac_reward_zero_std": 0.75, "grad_norm": 0.01617433875799179, "learning_rate": 1e-05, "loss": -0.0165, "num_tokens": 6491294.0, "reward": 0.96875, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.96875, "rewards/accuracy_reward/std": 0.1767766922712326, "sampling/importance_sampling_ratio/max": 1.7172808647155762, "sampling/importance_sampling_ratio/mean": 1.0002919435501099, "sampling/importance_sampling_ratio/min": 0.6275865435600281, "sampling/sampling_logp_difference/max": 0.5407421588897705, "sampling/sampling_logp_difference/mean": 0.011559183709323406, "step": 298 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 723.0, "completions/mean_length": 553.90625, "completions/mean_terminated_length": 482.54168701171875, "completions/min_length": 313.0, "completions/min_terminated_length": 313.0, "entropy": 0.5882219597697258, "epoch": 0.16006423982869378, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 6512995.0, "reward": 0.5, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5080004930496216, "sampling/importance_sampling_ratio/max": 1.7449169158935547, "sampling/importance_sampling_ratio/mean": 0.9997621774673462, "sampling/importance_sampling_ratio/min": 0.6054978370666504, "sampling/sampling_logp_difference/max": 0.5567069053649902, "sampling/sampling_logp_difference/mean": 0.013056526891887188, "step": 299 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 755.0, "completions/mean_length": 587.875, "completions/mean_terminated_length": 537.4400024414062, "completions/min_length": 328.0, "completions/min_terminated_length": 328.0, "entropy": 0.3660525269806385, "epoch": 0.16059957173447537, "frac_reward_zero_std": 0.0, "grad_norm": 0.008583523333072662, "learning_rate": 1e-05, "loss": 0.1076, "num_tokens": 6535807.0, "reward": 0.71875, "reward_std": 0.4534739851951599, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.3342469930648804, "sampling/importance_sampling_ratio/mean": 1.0001211166381836, "sampling/importance_sampling_ratio/min": 0.619476318359375, "sampling/sampling_logp_difference/max": 0.4788808822631836, "sampling/sampling_logp_difference/mean": 0.011281672865152359, "step": 300 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 7.720815483480692e-05, "clip_ratio/low_min": 7.720815483480692e-05, "clip_ratio/region_mean": 7.720815483480692e-05, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 748.0, "completions/mean_length": 476.46875, "completions/mean_terminated_length": 434.8214416503906, "completions/min_length": 199.0, "completions/min_terminated_length": 199.0, "entropy": 0.37877320498228073, "epoch": 0.16113490364025695, "frac_reward_zero_std": 0.25, "grad_norm": 0.01476945262402296, "learning_rate": 1e-05, "loss": 0.0181, "num_tokens": 6554766.0, "reward": 0.71875, "reward_std": 0.35564959049224854, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.467990517616272, "sampling/importance_sampling_ratio/mean": 1.000206708908081, "sampling/importance_sampling_ratio/min": 0.5913010239601135, "sampling/sampling_logp_difference/max": 0.5254299640655518, "sampling/sampling_logp_difference/mean": 0.012068652547895908, "step": 301 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 730.0, "completions/mean_length": 539.59375, "completions/mean_terminated_length": 524.36669921875, "completions/min_length": 361.0, "completions/min_terminated_length": 361.0, "entropy": 0.4354598969221115, "epoch": 0.16167023554603854, "frac_reward_zero_std": 0.25, "grad_norm": 0.011717413552105427, "learning_rate": 1e-05, "loss": 0.0041, "num_tokens": 6575857.0, "reward": 0.78125, "reward_std": 0.3608423173427582, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.3261480331420898, "sampling/importance_sampling_ratio/mean": 0.9998117089271545, "sampling/importance_sampling_ratio/min": 0.6434427499771118, "sampling/sampling_logp_difference/max": 0.44092226028442383, "sampling/sampling_logp_difference/mean": 0.01168701983988285, "step": 302 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00011061946861445904, "clip_ratio/low_min": 0.00011061946861445904, "clip_ratio/region_mean": 0.00011061946861445904, "completions/clipped_ratio": 0.28125, "completions/max_length": 768.0, "completions/max_terminated_length": 733.0, "completions/mean_length": 577.0, "completions/mean_terminated_length": 502.2608642578125, "completions/min_length": 280.0, "completions/min_terminated_length": 280.0, "entropy": 0.4856537878513336, "epoch": 0.16220556745182013, "frac_reward_zero_std": 0.25, "grad_norm": 0.012045544572174549, "learning_rate": 1e-05, "loss": 0.0295, "num_tokens": 6597857.0, "reward": 0.5625, "reward_std": 0.3514062464237213, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.597886085510254, "sampling/importance_sampling_ratio/mean": 1.0002676248550415, "sampling/importance_sampling_ratio/min": 0.6344450116157532, "sampling/sampling_logp_difference/max": 0.46868157386779785, "sampling/sampling_logp_difference/mean": 0.014197330921888351, "step": 303 }, { "clip_ratio/high_max": 4.763719334732741e-05, "clip_ratio/high_mean": 4.763719334732741e-05, "clip_ratio/low_mean": 5.755064557888545e-05, "clip_ratio/low_min": 5.755064557888545e-05, "clip_ratio/region_mean": 0.00010518783892621286, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 666.0, "completions/mean_length": 502.75, "completions/mean_terminated_length": 428.47998046875, "completions/min_length": 145.0, "completions/min_terminated_length": 145.0, "entropy": 0.5767744444310665, "epoch": 0.16274089935760172, "frac_reward_zero_std": 0.5, "grad_norm": 0.011516324244439602, "learning_rate": 1e-05, "loss": 0.0064, "num_tokens": 6617729.0, "reward": 0.6875, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.509402871131897, "sampling/importance_sampling_ratio/mean": 0.999971330165863, "sampling/importance_sampling_ratio/min": 0.7389633059501648, "sampling/sampling_logp_difference/max": 0.4117140769958496, "sampling/sampling_logp_difference/mean": 0.014930041506886482, "step": 304 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.978096330887638e-05, "clip_ratio/low_min": 4.978096330887638e-05, "clip_ratio/region_mean": 4.978096330887638e-05, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 740.0, "completions/mean_length": 478.28125, "completions/mean_terminated_length": 424.629638671875, "completions/min_length": 163.0, "completions/min_terminated_length": 163.0, "entropy": 0.5792366713285446, "epoch": 0.1632762312633833, "frac_reward_zero_std": 0.25, "grad_norm": 0.008899957872927189, "learning_rate": 1e-05, "loss": -0.0167, "num_tokens": 6636690.0, "reward": 0.40625, "reward_std": 0.378745436668396, "rewards/accuracy_reward/mean": 0.40625, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.5502060651779175, "sampling/importance_sampling_ratio/mean": 1.000200867652893, "sampling/importance_sampling_ratio/min": 0.6895433664321899, "sampling/sampling_logp_difference/max": 0.4383878707885742, "sampling/sampling_logp_difference/mean": 0.016736779361963272, "step": 305 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0002110216701112222, "clip_ratio/low_min": 0.0002110216701112222, "clip_ratio/region_mean": 0.0002110216701112222, "completions/clipped_ratio": 0.28125, "completions/max_length": 768.0, "completions/max_terminated_length": 718.0, "completions/mean_length": 565.375, "completions/mean_terminated_length": 486.08697509765625, "completions/min_length": 282.0, "completions/min_terminated_length": 282.0, "entropy": 0.48659036308526993, "epoch": 0.16381156316916487, "frac_reward_zero_std": 0.25, "grad_norm": 0.02768571302294731, "learning_rate": 1e-05, "loss": 0.0458, "num_tokens": 6658542.0, "reward": 0.59375, "reward_std": 0.3198433816432953, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.319368839263916, "sampling/importance_sampling_ratio/mean": 1.000253438949585, "sampling/importance_sampling_ratio/min": 0.6450673937797546, "sampling/sampling_logp_difference/max": 0.4384005069732666, "sampling/sampling_logp_difference/mean": 0.013724671676754951, "step": 306 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.8962006985675544e-05, "clip_ratio/low_min": 4.8962006985675544e-05, "clip_ratio/region_mean": 4.8962006985675544e-05, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 738.0, "completions/mean_length": 533.90625, "completions/mean_terminated_length": 509.6896667480469, "completions/min_length": 302.0, "completions/min_terminated_length": 302.0, "entropy": 0.27842517755925655, "epoch": 0.16434689507494646, "frac_reward_zero_std": 0.25, "grad_norm": 0.012529374100267887, "learning_rate": 1e-05, "loss": -0.0076, "num_tokens": 6680067.0, "reward": 0.78125, "reward_std": 0.3377464711666107, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.3033475875854492, "sampling/importance_sampling_ratio/mean": 0.9997432231903076, "sampling/importance_sampling_ratio/min": 0.6831303238868713, "sampling/sampling_logp_difference/max": 0.3810696601867676, "sampling/sampling_logp_difference/mean": 0.009572218172252178, "step": 307 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.53125, "completions/max_length": 768.0, "completions/max_terminated_length": 674.0, "completions/mean_length": 648.8125, "completions/mean_terminated_length": 513.7333374023438, "completions/min_length": 253.0, "completions/min_terminated_length": 253.0, "entropy": 0.48289478570222855, "epoch": 0.16488222698072805, "frac_reward_zero_std": 0.5, "grad_norm": 0.00882739294320345, "learning_rate": 1e-05, "loss": 0.0201, "num_tokens": 6705421.0, "reward": 0.28125, "reward_std": 0.2630178928375244, "rewards/accuracy_reward/mean": 0.28125, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.4294395446777344, "sampling/importance_sampling_ratio/mean": 0.9995339512825012, "sampling/importance_sampling_ratio/min": 0.5330876111984253, "sampling/sampling_logp_difference/max": 0.6290695667266846, "sampling/sampling_logp_difference/mean": 0.012598089873790741, "step": 308 }, { "clip_ratio/high_max": 8.549931953893974e-05, "clip_ratio/high_mean": 8.549931953893974e-05, "clip_ratio/low_mean": 0.0001661965434323065, "clip_ratio/low_min": 0.0001661965434323065, "clip_ratio/region_mean": 0.00025169586297124624, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 747.0, "completions/mean_length": 486.59375, "completions/mean_terminated_length": 467.8333435058594, "completions/min_length": 197.0, "completions/min_terminated_length": 197.0, "entropy": 0.38671203702688217, "epoch": 0.16541755888650964, "frac_reward_zero_std": 0.25, "grad_norm": 0.013369970954954624, "learning_rate": 1e-05, "loss": 0.074, "num_tokens": 6724352.0, "reward": 0.84375, "reward_std": 0.3061639666557312, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.4490798711776733, "sampling/importance_sampling_ratio/mean": 1.0000407695770264, "sampling/importance_sampling_ratio/min": 0.6715885400772095, "sampling/sampling_logp_difference/max": 0.39810943603515625, "sampling/sampling_logp_difference/mean": 0.011065604165196419, "step": 309 }, { "clip_ratio/high_max": 4.9270791350863874e-05, "clip_ratio/high_mean": 4.9270791350863874e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 4.9270791350863874e-05, "completions/clipped_ratio": 0.34375, "completions/max_length": 768.0, "completions/max_terminated_length": 749.0, "completions/mean_length": 592.40625, "completions/mean_terminated_length": 500.4285888671875, "completions/min_length": 241.0, "completions/min_terminated_length": 241.0, "entropy": 0.5058122612535954, "epoch": 0.16595289079229122, "frac_reward_zero_std": 0.25, "grad_norm": 0.021446535363793373, "learning_rate": 1e-05, "loss": 0.0343, "num_tokens": 6747189.0, "reward": 0.625, "reward_std": 0.2925041913986206, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.4081288576126099, "sampling/importance_sampling_ratio/mean": 0.9999823570251465, "sampling/importance_sampling_ratio/min": 0.5873938798904419, "sampling/sampling_logp_difference/max": 0.5320596694946289, "sampling/sampling_logp_difference/mean": 0.012491296976804733, "step": 310 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 764.0, "completions/mean_length": 525.34375, "completions/mean_terminated_length": 469.3461608886719, "completions/min_length": 244.0, "completions/min_terminated_length": 244.0, "entropy": 0.38173941895365715, "epoch": 0.1664882226980728, "frac_reward_zero_std": 0.25, "grad_norm": 0.008090285584330559, "learning_rate": 1e-05, "loss": 0.0175, "num_tokens": 6768536.0, "reward": 0.5625, "reward_std": 0.3471825420856476, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.4165804386138916, "sampling/importance_sampling_ratio/mean": 1.0000367164611816, "sampling/importance_sampling_ratio/min": 0.5375379323959351, "sampling/sampling_logp_difference/max": 0.6207559108734131, "sampling/sampling_logp_difference/mean": 0.011909333989024162, "step": 311 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 8.865247946232557e-05, "clip_ratio/low_min": 8.865247946232557e-05, "clip_ratio/region_mean": 8.865247946232557e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 761.0, "completions/max_terminated_length": 761.0, "completions/mean_length": 407.5, "completions/mean_terminated_length": 407.5, "completions/min_length": 218.0, "completions/min_terminated_length": 218.0, "entropy": 0.4366965591907501, "epoch": 0.1670235546038544, "frac_reward_zero_std": 0.25, "grad_norm": 0.017805999144911766, "learning_rate": 1e-05, "loss": -0.0471, "num_tokens": 6784560.0, "reward": 0.8125, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.3484771251678467, "sampling/importance_sampling_ratio/mean": 1.0002574920654297, "sampling/importance_sampling_ratio/min": 0.7097929120063782, "sampling/sampling_logp_difference/max": 0.34278202056884766, "sampling/sampling_logp_difference/mean": 0.012549689039587975, "step": 312 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 758.0, "completions/mean_length": 474.25, "completions/mean_terminated_length": 454.66668701171875, "completions/min_length": 272.0, "completions/min_terminated_length": 272.0, "entropy": 0.3349683992564678, "epoch": 0.16755888650963596, "frac_reward_zero_std": 0.5, "grad_norm": 0.010974748060107231, "learning_rate": 1e-05, "loss": 0.0115, "num_tokens": 6803768.0, "reward": 0.84375, "reward_std": 0.22201895713806152, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.2873549461364746, "sampling/importance_sampling_ratio/mean": 0.9998072385787964, "sampling/importance_sampling_ratio/min": 0.7049721479415894, "sampling/sampling_logp_difference/max": 0.3495969772338867, "sampling/sampling_logp_difference/mean": 0.010414966382086277, "step": 313 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 736.0, "completions/max_terminated_length": 736.0, "completions/mean_length": 406.3125, "completions/mean_terminated_length": 406.3125, "completions/min_length": 227.0, "completions/min_terminated_length": 227.0, "entropy": 0.36525509133934975, "epoch": 0.16809421841541755, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 6820194.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.4298120737075806, "sampling/importance_sampling_ratio/mean": 1.0004063844680786, "sampling/importance_sampling_ratio/min": 0.7078427076339722, "sampling/sampling_logp_difference/max": 0.3575429916381836, "sampling/sampling_logp_difference/mean": 0.011604657396674156, "step": 314 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 758.0, "completions/mean_length": 463.625, "completions/mean_terminated_length": 453.8064270019531, "completions/min_length": 289.0, "completions/min_terminated_length": 289.0, "entropy": 0.3767676129937172, "epoch": 0.16862955032119914, "frac_reward_zero_std": 0.5, "grad_norm": 0.012036073952913284, "learning_rate": 1e-05, "loss": 0.0586, "num_tokens": 6838502.0, "reward": 0.84375, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.000014305114746, "sampling/importance_sampling_ratio/min": 0.6360156536102295, "sampling/sampling_logp_difference/max": 0.7063698768615723, "sampling/sampling_logp_difference/mean": 0.011565844528377056, "step": 315 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 666.0, "completions/mean_length": 564.125, "completions/mean_terminated_length": 496.16668701171875, "completions/min_length": 314.0, "completions/min_terminated_length": 314.0, "entropy": 0.6370006389915943, "epoch": 0.16916488222698073, "frac_reward_zero_std": 0.75, "grad_norm": 0.002451046369969845, "learning_rate": 1e-05, "loss": -0.0041, "num_tokens": 6860202.0, "reward": 0.46875, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.46875, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.4470359086990356, "sampling/importance_sampling_ratio/mean": 1.000141978263855, "sampling/importance_sampling_ratio/min": 0.43400317430496216, "sampling/sampling_logp_difference/max": 0.8347034454345703, "sampling/sampling_logp_difference/mean": 0.016150325536727905, "step": 316 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 682.0, "completions/mean_length": 508.40625, "completions/mean_terminated_length": 491.10003662109375, "completions/min_length": 266.0, "completions/min_terminated_length": 266.0, "entropy": 0.43147315084934235, "epoch": 0.16970021413276232, "frac_reward_zero_std": 0.0, "grad_norm": 0.026230446994304657, "learning_rate": 1e-05, "loss": 0.0165, "num_tokens": 6880095.0, "reward": 0.6875, "reward_std": 0.44403791427612305, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.3381121158599854, "sampling/importance_sampling_ratio/mean": 1.0000815391540527, "sampling/importance_sampling_ratio/min": 0.6977789998054504, "sampling/sampling_logp_difference/max": 0.35985279083251953, "sampling/sampling_logp_difference/mean": 0.01168067567050457, "step": 317 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 8.394895849050954e-05, "clip_ratio/low_min": 8.394895849050954e-05, "clip_ratio/region_mean": 8.394895849050954e-05, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 655.0, "completions/mean_length": 453.0625, "completions/mean_terminated_length": 442.9031982421875, "completions/min_length": 155.0, "completions/min_terminated_length": 155.0, "entropy": 0.493406031280756, "epoch": 0.1702355460385439, "frac_reward_zero_std": 0.5, "grad_norm": 0.026222607120871544, "learning_rate": 1e-05, "loss": -0.0027, "num_tokens": 6898313.0, "reward": 0.6875, "reward_std": 0.2587745785713196, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.5384159088134766, "sampling/importance_sampling_ratio/mean": 1.0001434087753296, "sampling/importance_sampling_ratio/min": 0.7131491899490356, "sampling/sampling_logp_difference/max": 0.430753231048584, "sampling/sampling_logp_difference/mean": 0.012421787716448307, "step": 318 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.625, "completions/max_length": 768.0, "completions/max_terminated_length": 717.0, "completions/mean_length": 699.15625, "completions/mean_terminated_length": 584.4166870117188, "completions/min_length": 429.0, "completions/min_terminated_length": 429.0, "entropy": 0.5640444606542587, "epoch": 0.1707708779443255, "frac_reward_zero_std": 0.75, "grad_norm": 0.0030002526473253965, "learning_rate": 1e-05, "loss": 0.0239, "num_tokens": 6925246.0, "reward": 0.1875, "reward_std": 0.1157275140285492, "rewards/accuracy_reward/mean": 0.1875, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.5129494667053223, "sampling/importance_sampling_ratio/mean": 0.9998442530632019, "sampling/importance_sampling_ratio/min": 0.6620175242424011, "sampling/sampling_logp_difference/max": 0.4140610694885254, "sampling/sampling_logp_difference/mean": 0.01600034348666668, "step": 319 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 718.0, "completions/mean_length": 561.375, "completions/mean_terminated_length": 531.857177734375, "completions/min_length": 338.0, "completions/min_terminated_length": 338.0, "entropy": 0.300706734880805, "epoch": 0.17130620985010706, "frac_reward_zero_std": 0.75, "grad_norm": 0.004973437171429396, "learning_rate": 1e-05, "loss": 0.0244, "num_tokens": 6947458.0, "reward": 0.90625, "reward_std": 0.1293872892856598, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.4563747644424438, "sampling/importance_sampling_ratio/mean": 0.9997153282165527, "sampling/importance_sampling_ratio/min": 0.6179333329200745, "sampling/sampling_logp_difference/max": 0.48137474060058594, "sampling/sampling_logp_difference/mean": 0.010129036381840706, "step": 320 }, { "clip_ratio/high_max": 6.436663534259424e-05, "clip_ratio/high_mean": 6.436663534259424e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 6.436663534259424e-05, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 751.0, "completions/mean_length": 539.75, "completions/mean_terminated_length": 507.14288330078125, "completions/min_length": 318.0, "completions/min_terminated_length": 318.0, "entropy": 0.3287808820605278, "epoch": 0.17184154175588864, "frac_reward_zero_std": 0.0, "grad_norm": 0.008796916343271732, "learning_rate": 1e-05, "loss": 0.0392, "num_tokens": 6968218.0, "reward": 0.5625, "reward_std": 0.4671337604522705, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.3601312637329102, "sampling/importance_sampling_ratio/mean": 1.0003015995025635, "sampling/importance_sampling_ratio/min": 0.5666290521621704, "sampling/sampling_logp_difference/max": 0.5680503845214844, "sampling/sampling_logp_difference/mean": 0.01035685557872057, "step": 321 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 762.0, "completions/mean_length": 590.96875, "completions/mean_terminated_length": 531.9583740234375, "completions/min_length": 332.0, "completions/min_terminated_length": 332.0, "entropy": 0.5468134395778179, "epoch": 0.17237687366167023, "frac_reward_zero_std": 0.5, "grad_norm": 0.0027159431483596563, "learning_rate": 1e-05, "loss": 0.0476, "num_tokens": 6991185.0, "reward": 0.53125, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.4517290592193604, "sampling/importance_sampling_ratio/mean": 0.9999667406082153, "sampling/importance_sampling_ratio/min": 0.5871145725250244, "sampling/sampling_logp_difference/max": 0.5325352549552917, "sampling/sampling_logp_difference/mean": 0.014989539049565792, "step": 322 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 612.0, "completions/mean_length": 479.625, "completions/mean_terminated_length": 438.4285888671875, "completions/min_length": 207.0, "completions/min_terminated_length": 207.0, "entropy": 0.42262522876262665, "epoch": 0.17291220556745182, "frac_reward_zero_std": 0.5, "grad_norm": 0.001966841984540224, "learning_rate": 1e-05, "loss": 0.0262, "num_tokens": 7010477.0, "reward": 0.6875, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.287473440170288, "sampling/importance_sampling_ratio/mean": 0.9999221563339233, "sampling/importance_sampling_ratio/min": 0.6411926746368408, "sampling/sampling_logp_difference/max": 0.4444253444671631, "sampling/sampling_logp_difference/mean": 0.013051128946244717, "step": 323 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 725.0, "completions/mean_length": 490.125, "completions/mean_terminated_length": 461.3793029785156, "completions/min_length": 82.0, "completions/min_terminated_length": 82.0, "entropy": 0.5104870870709419, "epoch": 0.1734475374732334, "frac_reward_zero_std": 0.75, "grad_norm": 0.003290761960670352, "learning_rate": 1e-05, "loss": 0.0344, "num_tokens": 7029929.0, "reward": 0.71875, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.9737179279327393, "sampling/importance_sampling_ratio/mean": 0.9998589158058167, "sampling/importance_sampling_ratio/min": 0.6362985372543335, "sampling/sampling_logp_difference/max": 0.6799190044403076, "sampling/sampling_logp_difference/mean": 0.01370849646627903, "step": 324 }, { "clip_ratio/high_max": 5.776340185548179e-05, "clip_ratio/high_mean": 5.776340185548179e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 5.776340185548179e-05, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 738.0, "completions/mean_length": 502.21875, "completions/mean_terminated_length": 484.5000305175781, "completions/min_length": 104.0, "completions/min_terminated_length": 104.0, "entropy": 0.36331427469849586, "epoch": 0.173982869379015, "frac_reward_zero_std": 0.25, "grad_norm": 0.0124629195779562, "learning_rate": 1e-05, "loss": -0.0282, "num_tokens": 7049960.0, "reward": 0.78125, "reward_std": 0.3471629321575165, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.3294010162353516, "sampling/importance_sampling_ratio/mean": 0.9997987151145935, "sampling/importance_sampling_ratio/min": 0.5725442171096802, "sampling/sampling_logp_difference/max": 0.5576653480529785, "sampling/sampling_logp_difference/mean": 0.011005447246134281, "step": 325 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 717.0, "completions/mean_length": 506.6875, "completions/mean_terminated_length": 469.357177734375, "completions/min_length": 279.0, "completions/min_terminated_length": 279.0, "entropy": 0.3884149491786957, "epoch": 0.1745182012847966, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": -0.0168, "num_tokens": 7070006.0, "reward": 0.71875, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.3052456378936768, "sampling/importance_sampling_ratio/mean": 0.9998556971549988, "sampling/importance_sampling_ratio/min": 0.6920611262321472, "sampling/sampling_logp_difference/max": 0.36808109283447266, "sampling/sampling_logp_difference/mean": 0.011671111918985844, "step": 326 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 747.0, "completions/mean_length": 524.65625, "completions/mean_terminated_length": 499.4827575683594, "completions/min_length": 260.0, "completions/min_terminated_length": 260.0, "entropy": 0.5351394787430763, "epoch": 0.17505353319057815, "frac_reward_zero_std": 0.5, "grad_norm": 0.015136069618165493, "learning_rate": 1e-05, "loss": 0.0329, "num_tokens": 7090403.0, "reward": 0.75, "reward_std": 0.2587745785713196, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.3184289932250977, "sampling/importance_sampling_ratio/mean": 0.999699592590332, "sampling/importance_sampling_ratio/min": 0.6287856101989746, "sampling/sampling_logp_difference/max": 0.46396493911743164, "sampling/sampling_logp_difference/mean": 0.013748998753726482, "step": 327 }, { "clip_ratio/high_max": 7.557436765637249e-05, "clip_ratio/high_mean": 7.557436765637249e-05, "clip_ratio/low_mean": 5.258729652268812e-05, "clip_ratio/low_min": 5.258729652268812e-05, "clip_ratio/region_mean": 0.0001281616641790606, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 768.0, "completions/mean_length": 526.375, "completions/mean_terminated_length": 510.2666931152344, "completions/min_length": 198.0, "completions/min_terminated_length": 198.0, "entropy": 0.4058164283633232, "epoch": 0.17558886509635974, "frac_reward_zero_std": 0.5, "grad_norm": 0.004701528698205948, "learning_rate": 1e-05, "loss": 0.0075, "num_tokens": 7111367.0, "reward": 0.8125, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.4546986818313599, "sampling/importance_sampling_ratio/mean": 0.9999401569366455, "sampling/importance_sampling_ratio/min": 0.6548423171043396, "sampling/sampling_logp_difference/max": 0.42336082458496094, "sampling/sampling_logp_difference/mean": 0.01178866159170866, "step": 328 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.623032120638527e-05, "clip_ratio/low_min": 5.623032120638527e-05, "clip_ratio/region_mean": 5.623032120638527e-05, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 668.0, "completions/mean_length": 496.09375, "completions/mean_terminated_length": 457.2500305175781, "completions/min_length": 283.0, "completions/min_terminated_length": 283.0, "entropy": 0.43937700241804123, "epoch": 0.17612419700214133, "frac_reward_zero_std": 0.0, "grad_norm": 0.010809614323079586, "learning_rate": 1e-05, "loss": 0.0993, "num_tokens": 7131090.0, "reward": 0.59375, "reward_std": 0.4944729208946228, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.4216543436050415, "sampling/importance_sampling_ratio/mean": 1.0000747442245483, "sampling/importance_sampling_ratio/min": 0.753114640712738, "sampling/sampling_logp_difference/max": 0.3518211841583252, "sampling/sampling_logp_difference/mean": 0.012104019522666931, "step": 329 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.774637272930704e-05, "clip_ratio/low_min": 4.774637272930704e-05, "clip_ratio/region_mean": 4.774637272930704e-05, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 765.0, "completions/mean_length": 547.1875, "completions/mean_terminated_length": 524.3448486328125, "completions/min_length": 198.0, "completions/min_terminated_length": 198.0, "entropy": 0.5076432041823864, "epoch": 0.1766595289079229, "frac_reward_zero_std": 0.25, "grad_norm": 0.014773234724998474, "learning_rate": 1e-05, "loss": -0.0199, "num_tokens": 7152880.0, "reward": 0.78125, "reward_std": 0.3377465009689331, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.3977811336517334, "sampling/importance_sampling_ratio/mean": 0.9996352195739746, "sampling/importance_sampling_ratio/min": 0.5488889813423157, "sampling/sampling_logp_difference/max": 0.5998589992523193, "sampling/sampling_logp_difference/mean": 0.014740390703082085, "step": 330 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.582411879207939e-05, "clip_ratio/low_min": 6.582411879207939e-05, "clip_ratio/region_mean": 6.582411879207939e-05, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 709.0, "completions/mean_length": 448.6875, "completions/mean_terminated_length": 438.3870849609375, "completions/min_length": 213.0, "completions/min_terminated_length": 213.0, "entropy": 0.49838024750351906, "epoch": 0.1771948608137045, "frac_reward_zero_std": 0.25, "grad_norm": 0.005636438727378845, "learning_rate": 1e-05, "loss": -0.0361, "num_tokens": 7170654.0, "reward": 0.71875, "reward_std": 0.2651650309562683, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.6480814218521118, "sampling/importance_sampling_ratio/mean": 1.0003423690795898, "sampling/importance_sampling_ratio/min": 0.3878595232963562, "sampling/sampling_logp_difference/max": 0.9471120834350586, "sampling/sampling_logp_difference/mean": 0.014354714192450047, "step": 331 }, { "clip_ratio/high_max": 0.00011916110815946013, "clip_ratio/high_mean": 0.00011916110815946013, "clip_ratio/low_mean": 7.090187136782333e-05, "clip_ratio/low_min": 7.090187136782333e-05, "clip_ratio/region_mean": 0.00019006297952728346, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 717.0, "completions/mean_length": 508.125, "completions/mean_terminated_length": 499.7419128417969, "completions/min_length": 283.0, "completions/min_terminated_length": 283.0, "entropy": 0.5146004147827625, "epoch": 0.1777301927194861, "frac_reward_zero_std": 0.0, "grad_norm": 0.015472235158085823, "learning_rate": 1e-05, "loss": 0.0445, "num_tokens": 7190322.0, "reward": 0.6875, "reward_std": 0.4671337604522705, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.4304981231689453, "sampling/importance_sampling_ratio/mean": 1.0004292726516724, "sampling/importance_sampling_ratio/min": 0.5301676392555237, "sampling/sampling_logp_difference/max": 0.6345620155334473, "sampling/sampling_logp_difference/mean": 0.013377378694713116, "step": 332 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.28125, "completions/max_length": 768.0, "completions/max_terminated_length": 765.0, "completions/mean_length": 593.53125, "completions/mean_terminated_length": 525.2608642578125, "completions/min_length": 320.0, "completions/min_terminated_length": 320.0, "entropy": 0.4686245284974575, "epoch": 0.17826552462526768, "frac_reward_zero_std": 0.0, "grad_norm": 0.012375380843877792, "learning_rate": 1e-05, "loss": -0.0692, "num_tokens": 7212899.0, "reward": 0.4375, "reward_std": 0.4492306709289551, "rewards/accuracy_reward/mean": 0.4375, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.428321123123169, "sampling/importance_sampling_ratio/mean": 1.0000466108322144, "sampling/importance_sampling_ratio/min": 0.6934236288070679, "sampling/sampling_logp_difference/max": 0.36611413955688477, "sampling/sampling_logp_difference/mean": 0.012941376306116581, "step": 333 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00011114602239103988, "clip_ratio/low_min": 0.00011114602239103988, "clip_ratio/region_mean": 0.00011114602239103988, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 709.0, "completions/mean_length": 528.28125, "completions/mean_terminated_length": 494.0357360839844, "completions/min_length": 326.0, "completions/min_terminated_length": 326.0, "entropy": 0.8232426717877388, "epoch": 0.17880085653104924, "frac_reward_zero_std": 0.0, "grad_norm": 0.020998461171984673, "learning_rate": 1e-05, "loss": 0.0593, "num_tokens": 7233724.0, "reward": 0.6875, "reward_std": 0.4082317352294922, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.5058468580245972, "sampling/importance_sampling_ratio/mean": 1.0002719163894653, "sampling/importance_sampling_ratio/min": 0.7372462749481201, "sampling/sampling_logp_difference/max": 0.40935540199279785, "sampling/sampling_logp_difference/mean": 0.01719389110803604, "step": 334 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00016946071264101192, "clip_ratio/low_min": 0.00016946071264101192, "clip_ratio/region_mean": 0.00016946071264101192, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 756.0, "completions/mean_length": 575.71875, "completions/mean_terminated_length": 521.8800048828125, "completions/min_length": 348.0, "completions/min_terminated_length": 348.0, "entropy": 0.4106746129691601, "epoch": 0.17933618843683083, "frac_reward_zero_std": 0.25, "grad_norm": 0.01226749550551176, "learning_rate": 1e-05, "loss": 0.0232, "num_tokens": 7256003.0, "reward": 0.71875, "reward_std": 0.3471629321575165, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.431685447692871, "sampling/importance_sampling_ratio/mean": 0.999666690826416, "sampling/importance_sampling_ratio/min": 0.6186524033546448, "sampling/sampling_logp_difference/max": 0.4802117347717285, "sampling/sampling_logp_difference/mean": 0.012043134309351444, "step": 335 }, { "clip_ratio/high_max": 0.00012898632849100977, "clip_ratio/high_mean": 0.00012898632849100977, "clip_ratio/low_mean": 5.43714668310713e-05, "clip_ratio/low_min": 5.43714668310713e-05, "clip_ratio/region_mean": 0.00018335779532208107, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 739.0, "completions/mean_length": 537.71875, "completions/mean_terminated_length": 522.36669921875, "completions/min_length": 198.0, "completions/min_terminated_length": 198.0, "entropy": 0.31302709877491, "epoch": 0.17987152034261242, "frac_reward_zero_std": 0.5, "grad_norm": 0.008577987551689148, "learning_rate": 1e-05, "loss": 0.0453, "num_tokens": 7277146.0, "reward": 0.84375, "reward_std": 0.22201895713806152, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.3239972591400146, "sampling/importance_sampling_ratio/mean": 1.000171184539795, "sampling/importance_sampling_ratio/min": 0.7074315547943115, "sampling/sampling_logp_difference/max": 0.3461143970489502, "sampling/sampling_logp_difference/mean": 0.009894965216517448, "step": 336 }, { "clip_ratio/high_max": 9.130658509093337e-05, "clip_ratio/high_mean": 9.130658509093337e-05, "clip_ratio/low_mean": 0.00020746150767081417, "clip_ratio/low_min": 0.00020746150767081417, "clip_ratio/region_mean": 0.00029876809276174754, "completions/clipped_ratio": 0.375, "completions/max_length": 768.0, "completions/max_terminated_length": 753.0, "completions/mean_length": 606.71875, "completions/mean_terminated_length": 509.95001220703125, "completions/min_length": 316.0, "completions/min_terminated_length": 316.0, "entropy": 0.39334094151854515, "epoch": 0.180406852248394, "frac_reward_zero_std": 0.25, "grad_norm": 0.00736922025680542, "learning_rate": 1e-05, "loss": 0.0068, "num_tokens": 7301329.0, "reward": 0.28125, "reward_std": 0.3471629321575165, "rewards/accuracy_reward/mean": 0.28125, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.4514373540878296, "sampling/importance_sampling_ratio/mean": 0.9997961521148682, "sampling/importance_sampling_ratio/min": 0.5209622979164124, "sampling/sampling_logp_difference/max": 0.6520776748657227, "sampling/sampling_logp_difference/mean": 0.011272238567471504, "step": 337 }, { "clip_ratio/high_max": 0.000170951709151268, "clip_ratio/high_mean": 0.000170951709151268, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.000170951709151268, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 758.0, "completions/mean_length": 416.8125, "completions/mean_terminated_length": 405.4838562011719, "completions/min_length": 208.0, "completions/min_terminated_length": 208.0, "entropy": 0.38587110862135887, "epoch": 0.1809421841541756, "frac_reward_zero_std": 0.5, "grad_norm": 0.014478067867457867, "learning_rate": 1e-05, "loss": -0.0152, "num_tokens": 7318211.0, "reward": 0.90625, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.4607062339782715, "sampling/importance_sampling_ratio/mean": 0.9998483061790466, "sampling/importance_sampling_ratio/min": 0.3506035804748535, "sampling/sampling_logp_difference/max": 1.0480990409851074, "sampling/sampling_logp_difference/mean": 0.011547443456947803, "step": 338 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 677.0, "completions/mean_length": 445.46875, "completions/mean_terminated_length": 435.06451416015625, "completions/min_length": 243.0, "completions/min_terminated_length": 243.0, "entropy": 0.35046200081706047, "epoch": 0.18147751605995718, "frac_reward_zero_std": 0.75, "grad_norm": 0.004316780716180801, "learning_rate": 1e-05, "loss": 0.0528, "num_tokens": 7336042.0, "reward": 0.9375, "reward_std": 0.1157275140285492, "rewards/accuracy_reward/mean": 0.9375, "rewards/accuracy_reward/std": 0.24593468010425568, "sampling/importance_sampling_ratio/max": 1.3374011516571045, "sampling/importance_sampling_ratio/mean": 0.9998047947883606, "sampling/importance_sampling_ratio/min": 0.6119147539138794, "sampling/sampling_logp_difference/max": 0.4911623001098633, "sampling/sampling_logp_difference/mean": 0.010776880197227001, "step": 339 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00014489953173324466, "clip_ratio/low_min": 0.00014489953173324466, "clip_ratio/region_mean": 0.00014489953173324466, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 752.0, "completions/mean_length": 534.125, "completions/mean_terminated_length": 490.8148193359375, "completions/min_length": 294.0, "completions/min_terminated_length": 294.0, "entropy": 0.3672318812459707, "epoch": 0.18201284796573874, "frac_reward_zero_std": 0.75, "grad_norm": 0.012201718054711819, "learning_rate": 1e-05, "loss": 0.0498, "num_tokens": 7356454.0, "reward": 0.875, "reward_std": 0.13363061845302582, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 1.293308138847351, "sampling/importance_sampling_ratio/mean": 0.9997571706771851, "sampling/importance_sampling_ratio/min": 0.695732057094574, "sampling/sampling_logp_difference/max": 0.362790584564209, "sampling/sampling_logp_difference/mean": 0.010314418002963066, "step": 340 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0001675299572525546, "clip_ratio/low_min": 0.0001675299572525546, "clip_ratio/region_mean": 0.0001675299572525546, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 710.0, "completions/mean_length": 529.1875, "completions/mean_terminated_length": 474.0769348144531, "completions/min_length": 246.0, "completions/min_terminated_length": 246.0, "entropy": 0.34470563009381294, "epoch": 0.18254817987152033, "frac_reward_zero_std": 0.5, "grad_norm": 0.01242291834205389, "learning_rate": 1e-05, "loss": 0.0236, "num_tokens": 7377572.0, "reward": 0.6875, "reward_std": 0.2587745785713196, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.7681902647018433, "sampling/importance_sampling_ratio/mean": 1.0000383853912354, "sampling/importance_sampling_ratio/min": 0.7239490151405334, "sampling/sampling_logp_difference/max": 0.5699565410614014, "sampling/sampling_logp_difference/mean": 0.011041436344385147, "step": 341 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 738.0, "completions/max_terminated_length": 738.0, "completions/mean_length": 475.09375, "completions/mean_terminated_length": 475.09375, "completions/min_length": 299.0, "completions/min_terminated_length": 299.0, "entropy": 0.32042665779590607, "epoch": 0.18308351177730192, "frac_reward_zero_std": 0.5, "grad_norm": 0.023323800414800644, "learning_rate": 1e-05, "loss": -0.0088, "num_tokens": 7396639.0, "reward": 0.9375, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.9375, "rewards/accuracy_reward/std": 0.24593468010425568, "sampling/importance_sampling_ratio/max": 1.3497538566589355, "sampling/importance_sampling_ratio/mean": 1.0001760721206665, "sampling/importance_sampling_ratio/min": 0.7312887907028198, "sampling/sampling_logp_difference/max": 0.31294679641723633, "sampling/sampling_logp_difference/mean": 0.011018609628081322, "step": 342 }, { "clip_ratio/high_max": 6.076811041566543e-05, "clip_ratio/high_mean": 6.076811041566543e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 6.076811041566543e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 673.0, "completions/max_terminated_length": 673.0, "completions/mean_length": 433.0625, "completions/mean_terminated_length": 433.0625, "completions/min_length": 208.0, "completions/min_terminated_length": 208.0, "entropy": 0.44570066407322884, "epoch": 0.1836188436830835, "frac_reward_zero_std": 0.5, "grad_norm": 0.026565412059426308, "learning_rate": 1e-05, "loss": 0.0513, "num_tokens": 7413953.0, "reward": 0.875, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 1.3588019609451294, "sampling/importance_sampling_ratio/mean": 1.0004217624664307, "sampling/importance_sampling_ratio/min": 0.33628877997398376, "sampling/sampling_logp_difference/max": 1.089785099029541, "sampling/sampling_logp_difference/mean": 0.01277589425444603, "step": 343 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 686.0, "completions/mean_length": 562.03125, "completions/mean_terminated_length": 493.375, "completions/min_length": 339.0, "completions/min_terminated_length": 339.0, "entropy": 0.5396241508424282, "epoch": 0.1841541755888651, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0097, "num_tokens": 7436346.0, "reward": 0.4375, "reward_std": 0.1157275140285492, "rewards/accuracy_reward/mean": 0.4375, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.8712308406829834, "sampling/importance_sampling_ratio/mean": 1.000185251235962, "sampling/importance_sampling_ratio/min": 0.5464312434196472, "sampling/sampling_logp_difference/max": 0.6265964508056641, "sampling/sampling_logp_difference/mean": 0.01332900021225214, "step": 344 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.28125, "completions/max_length": 768.0, "completions/max_terminated_length": 758.0, "completions/mean_length": 602.46875, "completions/mean_terminated_length": 537.6956787109375, "completions/min_length": 317.0, "completions/min_terminated_length": 317.0, "entropy": 0.4268195964396, "epoch": 0.1846895074946467, "frac_reward_zero_std": 0.25, "grad_norm": 0.011327230371534824, "learning_rate": 1e-05, "loss": 0.055, "num_tokens": 7459385.0, "reward": 0.6875, "reward_std": 0.38298875093460083, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.4862500429153442, "sampling/importance_sampling_ratio/mean": 1.0000451803207397, "sampling/importance_sampling_ratio/min": 0.7432152628898621, "sampling/sampling_logp_difference/max": 0.3962562084197998, "sampling/sampling_logp_difference/mean": 0.01288197934627533, "step": 345 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 682.0, "completions/mean_length": 511.03125, "completions/mean_terminated_length": 451.73077392578125, "completions/min_length": 238.0, "completions/min_terminated_length": 238.0, "entropy": 0.41892142966389656, "epoch": 0.18522483940042828, "frac_reward_zero_std": 0.5, "grad_norm": 0.008036869578063488, "learning_rate": 1e-05, "loss": 0.0459, "num_tokens": 7479498.0, "reward": 0.65625, "reward_std": 0.22201895713806152, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.2727625370025635, "sampling/importance_sampling_ratio/mean": 1.0000818967819214, "sampling/importance_sampling_ratio/min": 0.6897485256195068, "sampling/sampling_logp_difference/max": 0.3714282512664795, "sampling/sampling_logp_difference/mean": 0.012019198387861252, "step": 346 }, { "clip_ratio/high_max": 4.4770775275537744e-05, "clip_ratio/high_mean": 4.4770775275537744e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 4.4770775275537744e-05, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 746.0, "completions/mean_length": 587.21875, "completions/mean_terminated_length": 536.5999755859375, "completions/min_length": 143.0, "completions/min_terminated_length": 143.0, "entropy": 0.5027334354817867, "epoch": 0.18576017130620984, "frac_reward_zero_std": 0.25, "grad_norm": 0.009729636833071709, "learning_rate": 1e-05, "loss": -0.0369, "num_tokens": 7502217.0, "reward": 0.5, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5080004930496216, "sampling/importance_sampling_ratio/max": 1.3486765623092651, "sampling/importance_sampling_ratio/mean": 0.9997532367706299, "sampling/importance_sampling_ratio/min": 0.6723280549049377, "sampling/sampling_logp_difference/max": 0.39700889587402344, "sampling/sampling_logp_difference/mean": 0.013065842911601067, "step": 347 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 700.0, "completions/mean_length": 423.5, "completions/mean_terminated_length": 412.3870849609375, "completions/min_length": 215.0, "completions/min_terminated_length": 215.0, "entropy": 0.37799080833792686, "epoch": 0.18629550321199143, "frac_reward_zero_std": 0.25, "grad_norm": 0.004395653028041124, "learning_rate": 1e-05, "loss": 0.0827, "num_tokens": 7519377.0, "reward": 0.90625, "reward_std": 0.2651650309562683, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.3875583410263062, "sampling/importance_sampling_ratio/mean": 1.0003392696380615, "sampling/importance_sampling_ratio/min": 0.6281449198722839, "sampling/sampling_logp_difference/max": 0.4649844169616699, "sampling/sampling_logp_difference/mean": 0.01179863978177309, "step": 348 }, { "clip_ratio/high_max": 0.0001152073746197857, "clip_ratio/high_mean": 0.0001152073746197857, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0001152073746197857, "completions/clipped_ratio": 0.3125, "completions/max_length": 768.0, "completions/max_terminated_length": 526.0, "completions/mean_length": 476.84375, "completions/mean_terminated_length": 344.5, "completions/min_length": 209.0, "completions/min_terminated_length": 209.0, "entropy": 0.4453147351741791, "epoch": 0.18683083511777301, "frac_reward_zero_std": 0.5, "grad_norm": 0.006316605024039745, "learning_rate": 1e-05, "loss": 0.0121, "num_tokens": 7538028.0, "reward": 0.65625, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.3541252613067627, "sampling/importance_sampling_ratio/mean": 0.9995315670967102, "sampling/importance_sampling_ratio/min": 0.30046942830085754, "sampling/sampling_logp_difference/max": 1.202409267425537, "sampling/sampling_logp_difference/mean": 0.013666864484548569, "step": 349 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 598.0, "completions/mean_length": 512.59375, "completions/mean_terminated_length": 476.107177734375, "completions/min_length": 336.0, "completions/min_terminated_length": 336.0, "entropy": 0.4042432829737663, "epoch": 0.1873661670235546, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 7558263.0, "reward": 0.75, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.3103748559951782, "sampling/importance_sampling_ratio/mean": 0.999916672706604, "sampling/importance_sampling_ratio/min": 0.6854010820388794, "sampling/sampling_logp_difference/max": 0.37775111198425293, "sampling/sampling_logp_difference/mean": 0.011970157735049725, "step": 350 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 762.0, "completions/mean_length": 464.0625, "completions/mean_terminated_length": 407.77777099609375, "completions/min_length": 279.0, "completions/min_terminated_length": 279.0, "entropy": 0.40735117346048355, "epoch": 0.1879014989293362, "frac_reward_zero_std": 0.5, "grad_norm": 0.00286686304025352, "learning_rate": 1e-05, "loss": 0.0318, "num_tokens": 7576881.0, "reward": 0.5, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5080004930496216, "sampling/importance_sampling_ratio/max": 1.4542512893676758, "sampling/importance_sampling_ratio/mean": 0.9998769760131836, "sampling/importance_sampling_ratio/min": 0.6443073153495789, "sampling/sampling_logp_difference/max": 0.43957948684692383, "sampling/sampling_logp_difference/mean": 0.012052088975906372, "step": 351 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.278716344037093e-05, "clip_ratio/low_min": 5.278716344037093e-05, "clip_ratio/region_mean": 5.278716344037093e-05, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 623.0, "completions/mean_length": 515.625, "completions/mean_terminated_length": 479.5714416503906, "completions/min_length": 341.0, "completions/min_terminated_length": 341.0, "entropy": 0.3545593172311783, "epoch": 0.18843683083511778, "frac_reward_zero_std": 0.0, "grad_norm": 0.011616687290370464, "learning_rate": 1e-05, "loss": 0.0368, "num_tokens": 7597133.0, "reward": 0.625, "reward_std": 0.5081326961517334, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.3291075229644775, "sampling/importance_sampling_ratio/mean": 0.9997459650039673, "sampling/importance_sampling_ratio/min": 0.10021040588617325, "sampling/sampling_logp_difference/max": 2.300483226776123, "sampling/sampling_logp_difference/mean": 0.011193163692951202, "step": 352 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.301445369492285e-05, "clip_ratio/low_min": 4.301445369492285e-05, "clip_ratio/region_mean": 4.301445369492285e-05, "completions/clipped_ratio": 0.34375, "completions/max_length": 768.0, "completions/max_terminated_length": 638.0, "completions/mean_length": 554.25, "completions/mean_terminated_length": 442.2857360839844, "completions/min_length": 166.0, "completions/min_terminated_length": 166.0, "entropy": 0.6081601269543171, "epoch": 0.18897216274089937, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0228, "num_tokens": 7618645.0, "reward": 0.5625, "reward_std": 0.1157275140285492, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.8886375427246094, "sampling/importance_sampling_ratio/mean": 0.9999396204948425, "sampling/importance_sampling_ratio/min": 0.6797575950622559, "sampling/sampling_logp_difference/max": 0.6358556747436523, "sampling/sampling_logp_difference/mean": 0.017190592363476753, "step": 353 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 695.0, "completions/mean_length": 529.9375, "completions/mean_terminated_length": 522.258056640625, "completions/min_length": 376.0, "completions/min_terminated_length": 376.0, "entropy": 0.41264915466308594, "epoch": 0.18950749464668093, "frac_reward_zero_std": 0.0, "grad_norm": 0.005830404348671436, "learning_rate": 1e-05, "loss": 0.0366, "num_tokens": 7639059.0, "reward": 0.84375, "reward_std": 0.3808925747871399, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.6129010915756226, "sampling/importance_sampling_ratio/mean": 0.9999990463256836, "sampling/importance_sampling_ratio/min": 0.704312801361084, "sampling/sampling_logp_difference/max": 0.47803449630737305, "sampling/sampling_logp_difference/mean": 0.011893495917320251, "step": 354 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00010156382631976157, "clip_ratio/low_min": 0.00010156382631976157, "clip_ratio/region_mean": 0.00010156382631976157, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 763.0, "completions/mean_length": 622.84375, "completions/mean_terminated_length": 589.34619140625, "completions/min_length": 249.0, "completions/min_terminated_length": 249.0, "entropy": 0.33784014731645584, "epoch": 0.19004282655246252, "frac_reward_zero_std": 0.5, "grad_norm": 0.0122241685166955, "learning_rate": 1e-05, "loss": -0.025, "num_tokens": 7662854.0, "reward": 0.375, "reward_std": 0.2314550280570984, "rewards/accuracy_reward/mean": 0.375, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.5569756031036377, "sampling/importance_sampling_ratio/mean": 1.000101923942566, "sampling/importance_sampling_ratio/min": 0.6211375594139099, "sampling/sampling_logp_difference/max": 0.47620272636413574, "sampling/sampling_logp_difference/mean": 0.010734748095273972, "step": 355 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.28125, "completions/max_length": 768.0, "completions/max_terminated_length": 718.0, "completions/mean_length": 611.75, "completions/mean_terminated_length": 550.6087036132812, "completions/min_length": 275.0, "completions/min_terminated_length": 275.0, "entropy": 0.4817916601896286, "epoch": 0.1905781584582441, "frac_reward_zero_std": 0.5, "grad_norm": 0.005454639904201031, "learning_rate": 1e-05, "loss": 0.0828, "num_tokens": 7687350.0, "reward": 0.65625, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0002686977386475, "sampling/importance_sampling_ratio/min": 0.6121079325675964, "sampling/sampling_logp_difference/max": 0.8192362785339355, "sampling/sampling_logp_difference/mean": 0.013691947795450687, "step": 356 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 711.0, "completions/mean_length": 481.21875, "completions/mean_terminated_length": 451.5517272949219, "completions/min_length": 191.0, "completions/min_terminated_length": 191.0, "entropy": 0.32394311763346195, "epoch": 0.1911134903640257, "frac_reward_zero_std": 0.5, "grad_norm": 0.015326720662415028, "learning_rate": 1e-05, "loss": 0.0694, "num_tokens": 7705861.0, "reward": 0.78125, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.3399609327316284, "sampling/importance_sampling_ratio/mean": 0.9996441006660461, "sampling/importance_sampling_ratio/min": 0.41950923204421997, "sampling/sampling_logp_difference/max": 0.8686697483062744, "sampling/sampling_logp_difference/mean": 0.010584529489278793, "step": 357 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.978096330887638e-05, "clip_ratio/low_min": 4.978096330887638e-05, "clip_ratio/region_mean": 4.978096330887638e-05, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 767.0, "completions/mean_length": 570.4375, "completions/mean_terminated_length": 533.8518676757812, "completions/min_length": 271.0, "completions/min_terminated_length": 271.0, "entropy": 0.348613727837801, "epoch": 0.19164882226980728, "frac_reward_zero_std": 0.75, "grad_norm": 0.00653418805450201, "learning_rate": 1e-05, "loss": -0.0025, "num_tokens": 7727643.0, "reward": 0.59375, "reward_std": 0.1293872892856598, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.364234447479248, "sampling/importance_sampling_ratio/mean": 0.999725878238678, "sampling/importance_sampling_ratio/min": 0.6682811379432678, "sampling/sampling_logp_difference/max": 0.4030463695526123, "sampling/sampling_logp_difference/mean": 0.010410048067569733, "step": 358 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 625.0, "completions/mean_length": 413.59375, "completions/mean_terminated_length": 402.1612854003906, "completions/min_length": 297.0, "completions/min_terminated_length": 297.0, "entropy": 0.4846441000699997, "epoch": 0.19218415417558887, "frac_reward_zero_std": 0.75, "grad_norm": 0.014523014426231384, "learning_rate": 1e-05, "loss": 0.0435, "num_tokens": 7744902.0, "reward": 0.84375, "reward_std": 0.1293872892856598, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.3490887880325317, "sampling/importance_sampling_ratio/mean": 0.9997272491455078, "sampling/importance_sampling_ratio/min": 0.6908149719238281, "sampling/sampling_logp_difference/max": 0.36988329887390137, "sampling/sampling_logp_difference/mean": 0.014691982418298721, "step": 359 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 698.0, "completions/mean_length": 512.09375, "completions/mean_terminated_length": 485.6206970214844, "completions/min_length": 324.0, "completions/min_terminated_length": 324.0, "entropy": 0.37720469012856483, "epoch": 0.19271948608137046, "frac_reward_zero_std": 0.5, "grad_norm": 0.008529460057616234, "learning_rate": 1e-05, "loss": 0.0046, "num_tokens": 7764865.0, "reward": 0.8125, "reward_std": 0.249358132481575, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.376272201538086, "sampling/importance_sampling_ratio/mean": 0.9998212456703186, "sampling/importance_sampling_ratio/min": 0.7713286876678467, "sampling/sampling_logp_difference/max": 0.3193786144256592, "sampling/sampling_logp_difference/mean": 0.011417688801884651, "step": 360 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.948535206494853e-05, "clip_ratio/low_min": 4.948535206494853e-05, "clip_ratio/region_mean": 4.948535206494853e-05, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 720.0, "completions/mean_length": 542.0625, "completions/mean_terminated_length": 489.923095703125, "completions/min_length": 181.0, "completions/min_terminated_length": 181.0, "entropy": 0.4329718202352524, "epoch": 0.19325481798715202, "frac_reward_zero_std": 0.25, "grad_norm": 0.023381182923913002, "learning_rate": 1e-05, "loss": 0.1034, "num_tokens": 7786211.0, "reward": 0.53125, "reward_std": 0.2651650309562683, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.4117883443832397, "sampling/importance_sampling_ratio/mean": 0.9998693466186523, "sampling/importance_sampling_ratio/min": 0.6523434519767761, "sampling/sampling_logp_difference/max": 0.4271841049194336, "sampling/sampling_logp_difference/mean": 0.013179978355765343, "step": 361 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.446622137445956e-05, "clip_ratio/low_min": 6.446622137445956e-05, "clip_ratio/region_mean": 6.446622137445956e-05, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 702.0, "completions/mean_length": 537.5, "completions/mean_terminated_length": 460.66668701171875, "completions/min_length": 223.0, "completions/min_terminated_length": 223.0, "entropy": 0.592044323682785, "epoch": 0.1937901498929336, "frac_reward_zero_std": 0.25, "grad_norm": 0.007650850340723991, "learning_rate": 1e-05, "loss": 0.1212, "num_tokens": 7807651.0, "reward": 0.6875, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0002923011779785, "sampling/importance_sampling_ratio/min": 0.682411789894104, "sampling/sampling_logp_difference/max": 0.8413052558898926, "sampling/sampling_logp_difference/mean": 0.014110127463936806, "step": 362 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 9.919505100697279e-05, "clip_ratio/low_min": 9.919505100697279e-05, "clip_ratio/region_mean": 9.919505100697279e-05, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 765.0, "completions/mean_length": 586.40625, "completions/mean_terminated_length": 535.5599975585938, "completions/min_length": 303.0, "completions/min_terminated_length": 303.0, "entropy": 0.47988008335232735, "epoch": 0.1943254817987152, "frac_reward_zero_std": 0.0, "grad_norm": 0.018716705963015556, "learning_rate": 1e-05, "loss": 0.0611, "num_tokens": 7830504.0, "reward": 0.46875, "reward_std": 0.4628904461860657, "rewards/accuracy_reward/mean": 0.46875, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.5832464694976807, "sampling/importance_sampling_ratio/mean": 0.9997760057449341, "sampling/importance_sampling_ratio/min": 0.595163881778717, "sampling/sampling_logp_difference/max": 0.518918514251709, "sampling/sampling_logp_difference/mean": 0.013171184808015823, "step": 363 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.58534411538858e-05, "clip_ratio/low_min": 5.58534411538858e-05, "clip_ratio/region_mean": 5.58534411538858e-05, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 759.0, "completions/mean_length": 532.53125, "completions/mean_terminated_length": 498.89288330078125, "completions/min_length": 183.0, "completions/min_terminated_length": 183.0, "entropy": 0.3653341978788376, "epoch": 0.1948608137044968, "frac_reward_zero_std": 0.0, "grad_norm": 0.011009089648723602, "learning_rate": 1e-05, "loss": 0.0511, "num_tokens": 7851737.0, "reward": 0.5625, "reward_std": 0.4261348247528076, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.3536845445632935, "sampling/importance_sampling_ratio/mean": 0.9997643828392029, "sampling/importance_sampling_ratio/min": 0.7109105587005615, "sampling/sampling_logp_difference/max": 0.34120869636535645, "sampling/sampling_logp_difference/mean": 0.011702951975166798, "step": 364 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 7.014590664766729e-05, "clip_ratio/low_min": 7.014590664766729e-05, "clip_ratio/region_mean": 7.014590664766729e-05, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 767.0, "completions/mean_length": 531.4375, "completions/mean_terminated_length": 523.8064575195312, "completions/min_length": 290.0, "completions/min_terminated_length": 290.0, "entropy": 0.3485301695764065, "epoch": 0.19539614561027838, "frac_reward_zero_std": 0.5, "grad_norm": 0.009981510229408741, "learning_rate": 1e-05, "loss": -0.0505, "num_tokens": 7872471.0, "reward": 0.5625, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.4363633394241333, "sampling/importance_sampling_ratio/mean": 1.00010347366333, "sampling/importance_sampling_ratio/min": 0.7413499355316162, "sampling/sampling_logp_difference/max": 0.36211442947387695, "sampling/sampling_logp_difference/mean": 0.010392917320132256, "step": 365 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 725.0, "completions/mean_length": 444.5625, "completions/mean_terminated_length": 434.1290283203125, "completions/min_length": 203.0, "completions/min_terminated_length": 203.0, "entropy": 0.48997560515999794, "epoch": 0.19593147751605997, "frac_reward_zero_std": 0.5, "grad_norm": 0.008400565013289452, "learning_rate": 1e-05, "loss": 0.0457, "num_tokens": 7890321.0, "reward": 0.78125, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.5290173292160034, "sampling/importance_sampling_ratio/mean": 1.00031578540802, "sampling/importance_sampling_ratio/min": 0.7160792946815491, "sampling/sampling_logp_difference/max": 0.4246252775192261, "sampling/sampling_logp_difference/mean": 0.013884284533560276, "step": 366 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0001172342490463052, "clip_ratio/low_min": 0.0001172342490463052, "clip_ratio/region_mean": 0.0001172342490463052, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 653.0, "completions/mean_length": 497.875, "completions/mean_terminated_length": 435.5384826660156, "completions/min_length": 221.0, "completions/min_terminated_length": 221.0, "entropy": 0.5482137557119131, "epoch": 0.19646680942184155, "frac_reward_zero_std": 0.0, "grad_norm": 0.005809069145470858, "learning_rate": 1e-05, "loss": 0.0779, "num_tokens": 7910421.0, "reward": 0.59375, "reward_std": 0.3987956643104553, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.635168194770813, "sampling/importance_sampling_ratio/mean": 1.000160813331604, "sampling/importance_sampling_ratio/min": 0.588154137134552, "sampling/sampling_logp_difference/max": 0.5307661890983582, "sampling/sampling_logp_difference/mean": 0.014795972965657711, "step": 367 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 7.716049731243402e-05, "clip_ratio/low_min": 7.716049731243402e-05, "clip_ratio/region_mean": 7.716049731243402e-05, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 712.0, "completions/mean_length": 452.625, "completions/mean_terminated_length": 442.45159912109375, "completions/min_length": 307.0, "completions/min_terminated_length": 307.0, "entropy": 0.42601926252245903, "epoch": 0.19700214132762311, "frac_reward_zero_std": 0.5, "grad_norm": 0.02773178741335869, "learning_rate": 1e-05, "loss": -0.0051, "num_tokens": 7928681.0, "reward": 0.75, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.3831844329833984, "sampling/importance_sampling_ratio/mean": 0.9999629259109497, "sampling/importance_sampling_ratio/min": 0.46010899543762207, "sampling/sampling_logp_difference/max": 0.7762918472290039, "sampling/sampling_logp_difference/mean": 0.01247339602559805, "step": 368 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 652.0, "completions/mean_length": 434.0625, "completions/mean_terminated_length": 423.2903137207031, "completions/min_length": 201.0, "completions/min_terminated_length": 201.0, "entropy": 0.39646005257964134, "epoch": 0.1975374732334047, "frac_reward_zero_std": 0.5, "grad_norm": 0.004643204156309366, "learning_rate": 1e-05, "loss": -0.0065, "num_tokens": 7945955.0, "reward": 0.9375, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.9375, "rewards/accuracy_reward/std": 0.24593468010425568, "sampling/importance_sampling_ratio/max": 1.3710025548934937, "sampling/importance_sampling_ratio/mean": 1.000269889831543, "sampling/importance_sampling_ratio/min": 0.7469781041145325, "sampling/sampling_logp_difference/max": 0.31554222106933594, "sampling/sampling_logp_difference/mean": 0.012254216708242893, "step": 369 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00010352126264479011, "clip_ratio/low_min": 0.00010352126264479011, "clip_ratio/region_mean": 0.00010352126264479011, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 756.0, "completions/mean_length": 484.40625, "completions/mean_terminated_length": 475.258056640625, "completions/min_length": 184.0, "completions/min_terminated_length": 184.0, "entropy": 0.4069571755826473, "epoch": 0.1980728051391863, "frac_reward_zero_std": 0.0, "grad_norm": 0.010718288831412792, "learning_rate": 1e-05, "loss": 0.0475, "num_tokens": 7965232.0, "reward": 0.5625, "reward_std": 0.49022960662841797, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.3774203062057495, "sampling/importance_sampling_ratio/mean": 0.9999377131462097, "sampling/importance_sampling_ratio/min": 0.3378658592700958, "sampling/sampling_logp_difference/max": 1.085106372833252, "sampling/sampling_logp_difference/mean": 0.013025488704442978, "step": 370 }, { "clip_ratio/high_max": 5.8247904235031456e-05, "clip_ratio/high_mean": 5.8247904235031456e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 5.8247904235031456e-05, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 743.0, "completions/mean_length": 507.46875, "completions/mean_terminated_length": 480.5172424316406, "completions/min_length": 150.0, "completions/min_terminated_length": 150.0, "entropy": 0.48590512201189995, "epoch": 0.19860813704496788, "frac_reward_zero_std": 0.0, "grad_norm": 0.006943477783352137, "learning_rate": 1e-05, "loss": -0.0175, "num_tokens": 7985663.0, "reward": 0.625, "reward_std": 0.48503684997558594, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.3872870206832886, "sampling/importance_sampling_ratio/mean": 1.0000364780426025, "sampling/importance_sampling_ratio/min": 0.6696125268936157, "sampling/sampling_logp_difference/max": 0.40105605125427246, "sampling/sampling_logp_difference/mean": 0.01292111724615097, "step": 371 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0001097573185688816, "clip_ratio/low_min": 0.0001097573185688816, "clip_ratio/region_mean": 0.0001097573185688816, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 742.0, "completions/mean_length": 521.46875, "completions/mean_terminated_length": 513.51611328125, "completions/min_length": 314.0, "completions/min_terminated_length": 314.0, "entropy": 0.3042990453541279, "epoch": 0.19914346895074947, "frac_reward_zero_std": 0.0, "grad_norm": 0.009631731547415257, "learning_rate": 1e-05, "loss": 0.049, "num_tokens": 8005766.0, "reward": 0.65625, "reward_std": 0.4944729208946228, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.4142330884933472, "sampling/importance_sampling_ratio/mean": 1.0002249479293823, "sampling/importance_sampling_ratio/min": 0.6511433720588684, "sampling/sampling_logp_difference/max": 0.42902541160583496, "sampling/sampling_logp_difference/mean": 0.010262439027428627, "step": 372 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0002464971912559122, "clip_ratio/low_min": 0.0002464971912559122, "clip_ratio/region_mean": 0.0002464971912559122, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 710.0, "completions/mean_length": 543.90625, "completions/mean_terminated_length": 502.40740966796875, "completions/min_length": 266.0, "completions/min_terminated_length": 266.0, "entropy": 0.38599660992622375, "epoch": 0.19967880085653106, "frac_reward_zero_std": 0.25, "grad_norm": 0.014723707921802998, "learning_rate": 1e-05, "loss": -0.0421, "num_tokens": 8027483.0, "reward": 0.6875, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.6832029819488525, "sampling/importance_sampling_ratio/mean": 1.0002552270889282, "sampling/importance_sampling_ratio/min": 0.6958727836608887, "sampling/sampling_logp_difference/max": 0.5206985473632812, "sampling/sampling_logp_difference/mean": 0.01208306010812521, "step": 373 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.43714668310713e-05, "clip_ratio/low_min": 5.43714668310713e-05, "clip_ratio/region_mean": 5.43714668310713e-05, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 717.0, "completions/mean_length": 483.96875, "completions/mean_terminated_length": 465.0333557128906, "completions/min_length": 270.0, "completions/min_terminated_length": 270.0, "entropy": 0.3956117704510689, "epoch": 0.20021413276231265, "frac_reward_zero_std": 0.75, "grad_norm": 0.004858255386352539, "learning_rate": 1e-05, "loss": -0.0041, "num_tokens": 8046394.0, "reward": 0.78125, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.5563654899597168, "sampling/importance_sampling_ratio/mean": 1.0002444982528687, "sampling/importance_sampling_ratio/min": 0.11141053587198257, "sampling/sampling_logp_difference/max": 2.194533348083496, "sampling/sampling_logp_difference/mean": 0.01166938804090023, "step": 374 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 672.0, "completions/mean_length": 526.875, "completions/mean_terminated_length": 459.3599853515625, "completions/min_length": 274.0, "completions/min_terminated_length": 274.0, "entropy": 0.5160121358931065, "epoch": 0.2007494646680942, "frac_reward_zero_std": 0.5, "grad_norm": 0.0033759973011910915, "learning_rate": 1e-05, "loss": -0.0044, "num_tokens": 8067190.0, "reward": 0.6875, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.4209799766540527, "sampling/importance_sampling_ratio/mean": 1.0001325607299805, "sampling/importance_sampling_ratio/min": 0.6531583070755005, "sampling/sampling_logp_difference/max": 0.4259357452392578, "sampling/sampling_logp_difference/mean": 0.015096231363713741, "step": 375 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.477651211549528e-05, "clip_ratio/low_min": 5.477651211549528e-05, "clip_ratio/region_mean": 5.477651211549528e-05, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 734.0, "completions/mean_length": 530.53125, "completions/mean_terminated_length": 522.8709716796875, "completions/min_length": 306.0, "completions/min_terminated_length": 306.0, "entropy": 0.40019306913018227, "epoch": 0.2012847965738758, "frac_reward_zero_std": 0.25, "grad_norm": 0.02143034338951111, "learning_rate": 1e-05, "loss": -0.0122, "num_tokens": 8088055.0, "reward": 0.875, "reward_std": 0.292504221200943, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 1.4617897272109985, "sampling/importance_sampling_ratio/mean": 1.0000956058502197, "sampling/importance_sampling_ratio/min": 0.7446685433387756, "sampling/sampling_logp_difference/max": 0.37966156005859375, "sampling/sampling_logp_difference/mean": 0.01237609051167965, "step": 376 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 692.0, "completions/mean_length": 570.65625, "completions/mean_terminated_length": 534.1111450195312, "completions/min_length": 310.0, "completions/min_terminated_length": 310.0, "entropy": 0.39847778528928757, "epoch": 0.20182012847965738, "frac_reward_zero_std": 0.25, "grad_norm": 0.00591615354642272, "learning_rate": 1e-05, "loss": 0.0247, "num_tokens": 8110692.0, "reward": 0.65625, "reward_std": 0.3061639964580536, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.358012318611145, "sampling/importance_sampling_ratio/mean": 0.9998806118965149, "sampling/importance_sampling_ratio/min": 0.7165917158126831, "sampling/sampling_logp_difference/max": 0.3332490921020508, "sampling/sampling_logp_difference/mean": 0.012140526436269283, "step": 377 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 757.0, "completions/mean_length": 535.65625, "completions/mean_terminated_length": 511.6206970214844, "completions/min_length": 267.0, "completions/min_terminated_length": 267.0, "entropy": 0.3456450626254082, "epoch": 0.20235546038543897, "frac_reward_zero_std": 0.25, "grad_norm": 0.008054766803979874, "learning_rate": 1e-05, "loss": 0.0065, "num_tokens": 8131313.0, "reward": 0.75, "reward_std": 0.3514062464237213, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.320526361465454, "sampling/importance_sampling_ratio/mean": 0.9997600317001343, "sampling/importance_sampling_ratio/min": 0.5832955241203308, "sampling/sampling_logp_difference/max": 0.5390613079071045, "sampling/sampling_logp_difference/mean": 0.011443529278039932, "step": 378 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 709.0, "completions/mean_length": 542.6875, "completions/mean_terminated_length": 490.69232177734375, "completions/min_length": 171.0, "completions/min_terminated_length": 171.0, "entropy": 0.4418714940547943, "epoch": 0.20289079229122056, "frac_reward_zero_std": 0.25, "grad_norm": 0.006573774851858616, "learning_rate": 1e-05, "loss": 0.0737, "num_tokens": 8152487.0, "reward": 0.6875, "reward_std": 0.292504221200943, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.425748348236084, "sampling/importance_sampling_ratio/mean": 0.9998875260353088, "sampling/importance_sampling_ratio/min": 0.6758825182914734, "sampling/sampling_logp_difference/max": 0.3917360305786133, "sampling/sampling_logp_difference/mean": 0.013822292909026146, "step": 379 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.9073725424241275e-05, "clip_ratio/low_min": 5.9073725424241275e-05, "clip_ratio/region_mean": 5.9073725424241275e-05, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 703.0, "completions/mean_length": 507.6875, "completions/mean_terminated_length": 447.6153869628906, "completions/min_length": 222.0, "completions/min_terminated_length": 222.0, "entropy": 0.45735814049839973, "epoch": 0.20342612419700215, "frac_reward_zero_std": 0.0, "grad_norm": 0.027993470430374146, "learning_rate": 1e-05, "loss": 0.0774, "num_tokens": 8172277.0, "reward": 0.6875, "reward_std": 0.47655022144317627, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.36721670627594, "sampling/importance_sampling_ratio/mean": 0.9996927976608276, "sampling/importance_sampling_ratio/min": 0.7227435111999512, "sampling/sampling_logp_difference/max": 0.3247009515762329, "sampling/sampling_logp_difference/mean": 0.012482824735343456, "step": 380 }, { "clip_ratio/high_max": 6.853070226497948e-05, "clip_ratio/high_mean": 6.853070226497948e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 6.853070226497948e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 758.0, "completions/max_terminated_length": 758.0, "completions/mean_length": 444.875, "completions/mean_terminated_length": 444.875, "completions/min_length": 219.0, "completions/min_terminated_length": 219.0, "entropy": 0.3879472576081753, "epoch": 0.20396145610278374, "frac_reward_zero_std": 0.25, "grad_norm": 0.004540951922535896, "learning_rate": 1e-05, "loss": -0.0301, "num_tokens": 8190201.0, "reward": 0.90625, "reward_std": 0.2651650309562683, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0005786418914795, "sampling/importance_sampling_ratio/min": 0.6149527430534363, "sampling/sampling_logp_difference/max": 0.7482748031616211, "sampling/sampling_logp_difference/mean": 0.011853198520839214, "step": 381 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 697.0, "completions/mean_length": 475.09375, "completions/mean_terminated_length": 455.5666809082031, "completions/min_length": 253.0, "completions/min_terminated_length": 253.0, "entropy": 0.3254448063671589, "epoch": 0.2044967880085653, "frac_reward_zero_std": 0.5, "grad_norm": 0.008202667348086834, "learning_rate": 1e-05, "loss": 0.035, "num_tokens": 8208916.0, "reward": 0.90625, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.3376712799072266, "sampling/importance_sampling_ratio/mean": 0.9995779991149902, "sampling/importance_sampling_ratio/min": 0.744231641292572, "sampling/sampling_logp_difference/max": 0.29540300369262695, "sampling/sampling_logp_difference/mean": 0.0104160001501441, "step": 382 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 678.0, "completions/mean_length": 515.0625, "completions/mean_terminated_length": 456.69232177734375, "completions/min_length": 226.0, "completions/min_terminated_length": 226.0, "entropy": 0.3882271386682987, "epoch": 0.2050321199143469, "frac_reward_zero_std": 0.25, "grad_norm": 0.026752524077892303, "learning_rate": 1e-05, "loss": 0.0897, "num_tokens": 8228910.0, "reward": 0.78125, "reward_std": 0.3377465009689331, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.364371657371521, "sampling/importance_sampling_ratio/mean": 0.9997440576553345, "sampling/importance_sampling_ratio/min": 0.6323453187942505, "sampling/sampling_logp_difference/max": 0.45831966400146484, "sampling/sampling_logp_difference/mean": 0.012213868089020252, "step": 383 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00015898891433607787, "clip_ratio/low_min": 0.00015898891433607787, "clip_ratio/region_mean": 0.00015898891433607787, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 720.0, "completions/mean_length": 549.4375, "completions/mean_terminated_length": 488.239990234375, "completions/min_length": 334.0, "completions/min_terminated_length": 334.0, "entropy": 0.46953893080353737, "epoch": 0.20556745182012848, "frac_reward_zero_std": 0.5, "grad_norm": 0.006220671813935041, "learning_rate": 1e-05, "loss": -0.0059, "num_tokens": 8250524.0, "reward": 0.59375, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.3521066904067993, "sampling/importance_sampling_ratio/mean": 0.9997193813323975, "sampling/importance_sampling_ratio/min": 0.6970633268356323, "sampling/sampling_logp_difference/max": 0.36087894439697266, "sampling/sampling_logp_difference/mean": 0.014271720312535763, "step": 384 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 9.784736175788566e-05, "clip_ratio/low_min": 9.784736175788566e-05, "clip_ratio/region_mean": 9.784736175788566e-05, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 764.0, "completions/mean_length": 594.21875, "completions/mean_terminated_length": 554.1154174804688, "completions/min_length": 336.0, "completions/min_terminated_length": 336.0, "entropy": 0.3626043200492859, "epoch": 0.20610278372591007, "frac_reward_zero_std": 0.25, "grad_norm": 0.005539539735764265, "learning_rate": 1e-05, "loss": 0.0681, "num_tokens": 8273387.0, "reward": 0.5625, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.3743711709976196, "sampling/importance_sampling_ratio/mean": 1.0000483989715576, "sampling/importance_sampling_ratio/min": 0.713853120803833, "sampling/sampling_logp_difference/max": 0.3370780944824219, "sampling/sampling_logp_difference/mean": 0.01183868758380413, "step": 385 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00013117517664795741, "clip_ratio/low_min": 0.00013117517664795741, "clip_ratio/region_mean": 0.00013117517664795741, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 723.0, "completions/mean_length": 493.78125, "completions/mean_terminated_length": 484.9354553222656, "completions/min_length": 248.0, "completions/min_terminated_length": 248.0, "entropy": 0.367743544280529, "epoch": 0.20663811563169165, "frac_reward_zero_std": 0.25, "grad_norm": 0.015349820256233215, "learning_rate": 1e-05, "loss": -0.0061, "num_tokens": 8292796.0, "reward": 0.59375, "reward_std": 0.3787454068660736, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.6753965616226196, "sampling/importance_sampling_ratio/mean": 0.9997833967208862, "sampling/importance_sampling_ratio/min": 0.7073305249214172, "sampling/sampling_logp_difference/max": 0.516049861907959, "sampling/sampling_logp_difference/mean": 0.011631221510469913, "step": 386 }, { "clip_ratio/high_max": 5.792400406789966e-05, "clip_ratio/high_mean": 5.792400406789966e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 5.792400406789966e-05, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 762.0, "completions/mean_length": 559.3125, "completions/mean_terminated_length": 500.8799743652344, "completions/min_length": 179.0, "completions/min_terminated_length": 179.0, "entropy": 0.3384836260229349, "epoch": 0.20717344753747324, "frac_reward_zero_std": 0.5, "grad_norm": 0.016733651980757713, "learning_rate": 1e-05, "loss": 0.0209, "num_tokens": 8314382.0, "reward": 0.40625, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.40625, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.7089895009994507, "sampling/importance_sampling_ratio/mean": 1.0001564025878906, "sampling/importance_sampling_ratio/min": 0.2909180521965027, "sampling/sampling_logp_difference/max": 1.2347136735916138, "sampling/sampling_logp_difference/mean": 0.010874203406274319, "step": 387 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 761.0, "completions/mean_length": 553.59375, "completions/mean_terminated_length": 504.11541748046875, "completions/min_length": 222.0, "completions/min_terminated_length": 222.0, "entropy": 0.3786693848669529, "epoch": 0.20770877944325483, "frac_reward_zero_std": 0.5, "grad_norm": 0.004186462145298719, "learning_rate": 1e-05, "loss": 0.0287, "num_tokens": 8335961.0, "reward": 0.65625, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.6064395904541016, "sampling/importance_sampling_ratio/mean": 1.0000622272491455, "sampling/importance_sampling_ratio/min": 0.6084471344947815, "sampling/sampling_logp_difference/max": 0.4968452453613281, "sampling/sampling_logp_difference/mean": 0.011787356808781624, "step": 388 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.708097912953235e-05, "clip_ratio/low_min": 4.708097912953235e-05, "clip_ratio/region_mean": 4.708097912953235e-05, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 732.0, "completions/mean_length": 517.6875, "completions/mean_terminated_length": 481.9285888671875, "completions/min_length": 229.0, "completions/min_terminated_length": 229.0, "entropy": 0.3944498412311077, "epoch": 0.2082441113490364, "frac_reward_zero_std": 0.0, "grad_norm": 0.007791217882186174, "learning_rate": 1e-05, "loss": 0.0828, "num_tokens": 8355847.0, "reward": 0.59375, "reward_std": 0.4218915104866028, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.2994695901870728, "sampling/importance_sampling_ratio/mean": 0.9997397065162659, "sampling/importance_sampling_ratio/min": 0.7719744443893433, "sampling/sampling_logp_difference/max": 0.26195621490478516, "sampling/sampling_logp_difference/mean": 0.011835463345050812, "step": 389 }, { "clip_ratio/high_max": 7.318500865949318e-05, "clip_ratio/high_mean": 7.318500865949318e-05, "clip_ratio/low_mean": 5.441880784928799e-05, "clip_ratio/low_min": 5.441880784928799e-05, "clip_ratio/region_mean": 0.00012760381650878116, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 720.0, "completions/mean_length": 499.84375, "completions/mean_terminated_length": 424.7599792480469, "completions/min_length": 146.0, "completions/min_terminated_length": 146.0, "entropy": 0.35400139540433884, "epoch": 0.20877944325481798, "frac_reward_zero_std": 0.25, "grad_norm": 0.005581251811236143, "learning_rate": 1e-05, "loss": 0.0019, "num_tokens": 8375642.0, "reward": 0.5625, "reward_std": 0.292504221200943, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.4042198657989502, "sampling/importance_sampling_ratio/mean": 1.0000568628311157, "sampling/importance_sampling_ratio/min": 0.6956577897071838, "sampling/sampling_logp_difference/max": 0.3628973960876465, "sampling/sampling_logp_difference/mean": 0.012154696509242058, "step": 390 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 651.0, "completions/mean_length": 431.71875, "completions/mean_terminated_length": 420.8709411621094, "completions/min_length": 191.0, "completions/min_terminated_length": 191.0, "entropy": 0.3674179017543793, "epoch": 0.20931477516059957, "frac_reward_zero_std": 0.5, "grad_norm": 0.007779018487781286, "learning_rate": 1e-05, "loss": 0.0409, "num_tokens": 8392905.0, "reward": 0.5, "reward_std": 0.2314550280570984, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5080004930496216, "sampling/importance_sampling_ratio/max": 1.2777687311172485, "sampling/importance_sampling_ratio/mean": 0.9992403984069824, "sampling/importance_sampling_ratio/min": 0.5686952471733093, "sampling/sampling_logp_difference/max": 0.5644105672836304, "sampling/sampling_logp_difference/mean": 0.01222455594688654, "step": 391 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 729.0, "completions/mean_length": 440.09375, "completions/mean_terminated_length": 429.51611328125, "completions/min_length": 219.0, "completions/min_terminated_length": 219.0, "entropy": 0.4279603026807308, "epoch": 0.20985010706638116, "frac_reward_zero_std": 0.75, "grad_norm": 0.012629266828298569, "learning_rate": 1e-05, "loss": 0.0188, "num_tokens": 8410556.0, "reward": 0.875, "reward_std": 0.13363061845302582, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 1.569334864616394, "sampling/importance_sampling_ratio/mean": 0.9999269843101501, "sampling/importance_sampling_ratio/min": 0.5857587456703186, "sampling/sampling_logp_difference/max": 0.5348472595214844, "sampling/sampling_logp_difference/mean": 0.013155401684343815, "step": 392 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 753.0, "completions/mean_length": 566.34375, "completions/mean_terminated_length": 499.125, "completions/min_length": 266.0, "completions/min_terminated_length": 266.0, "entropy": 0.5298543311655521, "epoch": 0.21038543897216275, "frac_reward_zero_std": 0.5, "grad_norm": 0.005383187904953957, "learning_rate": 1e-05, "loss": 0.0662, "num_tokens": 8432511.0, "reward": 0.625, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.8232184648513794, "sampling/importance_sampling_ratio/mean": 0.9999746680259705, "sampling/importance_sampling_ratio/min": 0.47589820623397827, "sampling/sampling_logp_difference/max": 0.742551326751709, "sampling/sampling_logp_difference/mean": 0.015069807879626751, "step": 393 }, { "clip_ratio/high_max": 7.163323607528582e-05, "clip_ratio/high_mean": 7.163323607528582e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 7.163323607528582e-05, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 740.0, "completions/mean_length": 510.53125, "completions/mean_terminated_length": 483.89654541015625, "completions/min_length": 293.0, "completions/min_terminated_length": 293.0, "entropy": 0.5083822384476662, "epoch": 0.21092077087794434, "frac_reward_zero_std": 0.0, "grad_norm": 0.011491530574858189, "learning_rate": 1e-05, "loss": 0.0172, "num_tokens": 8452752.0, "reward": 0.65625, "reward_std": 0.4628904461860657, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.39710533618927, "sampling/importance_sampling_ratio/mean": 1.00003182888031, "sampling/importance_sampling_ratio/min": 0.6281613707542419, "sampling/sampling_logp_difference/max": 0.46495819091796875, "sampling/sampling_logp_difference/mean": 0.01529776118695736, "step": 394 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 762.0, "completions/mean_length": 532.5625, "completions/mean_terminated_length": 498.9285888671875, "completions/min_length": 286.0, "completions/min_terminated_length": 286.0, "entropy": 0.3021841011941433, "epoch": 0.2114561027837259, "frac_reward_zero_std": 0.5, "grad_norm": 0.0022210676688700914, "learning_rate": 1e-05, "loss": 0.0027, "num_tokens": 8474066.0, "reward": 0.90625, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.4248532056808472, "sampling/importance_sampling_ratio/mean": 1.000141978263855, "sampling/importance_sampling_ratio/min": 0.6691626310348511, "sampling/sampling_logp_difference/max": 0.40172815322875977, "sampling/sampling_logp_difference/mean": 0.01092034112662077, "step": 395 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.994963769102469e-05, "clip_ratio/low_min": 6.994963769102469e-05, "clip_ratio/region_mean": 6.994963769102469e-05, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 722.0, "completions/mean_length": 537.15625, "completions/mean_terminated_length": 513.27587890625, "completions/min_length": 341.0, "completions/min_terminated_length": 341.0, "entropy": 0.28629919327795506, "epoch": 0.21199143468950749, "frac_reward_zero_std": 0.0, "grad_norm": 0.008062201552093029, "learning_rate": 1e-05, "loss": 0.0084, "num_tokens": 8495055.0, "reward": 0.65625, "reward_std": 0.4218915104866028, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.4054651260375977, "sampling/importance_sampling_ratio/mean": 0.9998922944068909, "sampling/importance_sampling_ratio/min": 0.697835385799408, "sampling/sampling_logp_difference/max": 0.3597719669342041, "sampling/sampling_logp_difference/mean": 0.01047180313616991, "step": 396 }, { "clip_ratio/high_max": 6.33874224149622e-05, "clip_ratio/high_mean": 6.33874224149622e-05, "clip_ratio/low_mean": 6.091617979109287e-05, "clip_ratio/low_min": 6.091617979109287e-05, "clip_ratio/region_mean": 0.00012430360220605507, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 701.0, "completions/mean_length": 437.625, "completions/mean_terminated_length": 426.9677429199219, "completions/min_length": 168.0, "completions/min_terminated_length": 168.0, "entropy": 0.4095449633896351, "epoch": 0.21252676659528907, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0227, "num_tokens": 8513019.0, "reward": 0.65625, "reward_std": 0.1293872892856598, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.421972632408142, "sampling/importance_sampling_ratio/mean": 0.9997876286506653, "sampling/importance_sampling_ratio/min": 0.6967024803161621, "sampling/sampling_logp_difference/max": 0.36139678955078125, "sampling/sampling_logp_difference/mean": 0.012393293902277946, "step": 397 }, { "clip_ratio/high_max": 5.1292572607053444e-05, "clip_ratio/high_mean": 5.1292572607053444e-05, "clip_ratio/low_mean": 0.00012902263915748335, "clip_ratio/low_min": 0.00012902263915748335, "clip_ratio/region_mean": 0.0001803152117645368, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 762.0, "completions/mean_length": 544.46875, "completions/mean_terminated_length": 469.9583435058594, "completions/min_length": 192.0, "completions/min_terminated_length": 192.0, "entropy": 0.4786128140985966, "epoch": 0.21306209850107066, "frac_reward_zero_std": 0.0, "grad_norm": 0.005949270911514759, "learning_rate": 1e-05, "loss": 0.1088, "num_tokens": 8535058.0, "reward": 0.59375, "reward_std": 0.4218915104866028, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.7323617935180664, "sampling/importance_sampling_ratio/mean": 1.0003728866577148, "sampling/importance_sampling_ratio/min": 0.7132600545883179, "sampling/sampling_logp_difference/max": 0.5494856834411621, "sampling/sampling_logp_difference/mean": 0.014721707440912724, "step": 398 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 729.0, "completions/mean_length": 531.15625, "completions/mean_terminated_length": 497.3214416503906, "completions/min_length": 254.0, "completions/min_terminated_length": 254.0, "entropy": 0.4673791192471981, "epoch": 0.21359743040685225, "frac_reward_zero_std": 0.5, "grad_norm": 0.011197878047823906, "learning_rate": 1e-05, "loss": 0.0846, "num_tokens": 8555911.0, "reward": 0.59375, "reward_std": 0.22201895713806152, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.3088711500167847, "sampling/importance_sampling_ratio/mean": 0.9998037815093994, "sampling/importance_sampling_ratio/min": 0.7026304006576538, "sampling/sampling_logp_difference/max": 0.3529243469238281, "sampling/sampling_logp_difference/mean": 0.013329234905540943, "step": 399 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 722.0, "completions/max_terminated_length": 722.0, "completions/mean_length": 493.53125, "completions/mean_terminated_length": 493.53125, "completions/min_length": 281.0, "completions/min_terminated_length": 281.0, "entropy": 0.31775000505149364, "epoch": 0.21413276231263384, "frac_reward_zero_std": 0.25, "grad_norm": 0.011138384230434895, "learning_rate": 1e-05, "loss": 0.0623, "num_tokens": 8575296.0, "reward": 0.59375, "reward_std": 0.3608423173427582, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.3432323932647705, "sampling/importance_sampling_ratio/mean": 0.9998267889022827, "sampling/importance_sampling_ratio/min": 0.6971604228019714, "sampling/sampling_logp_difference/max": 0.36073970794677734, "sampling/sampling_logp_difference/mean": 0.010663994587957859, "step": 400 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.390254591475241e-05, "clip_ratio/low_min": 5.390254591475241e-05, "clip_ratio/region_mean": 5.390254591475241e-05, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 734.0, "completions/mean_length": 539.09375, "completions/mean_terminated_length": 531.7096557617188, "completions/min_length": 298.0, "completions/min_terminated_length": 298.0, "entropy": 0.2961991000920534, "epoch": 0.21466809421841543, "frac_reward_zero_std": 0.25, "grad_norm": 0.00423061940819025, "learning_rate": 1e-05, "loss": 0.0515, "num_tokens": 8596011.0, "reward": 0.875, "reward_std": 0.2925041913986206, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 1.3731248378753662, "sampling/importance_sampling_ratio/mean": 0.9998635649681091, "sampling/importance_sampling_ratio/min": 0.6961231827735901, "sampling/sampling_logp_difference/max": 0.3622286319732666, "sampling/sampling_logp_difference/mean": 0.010276572778820992, "step": 401 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 656.0, "completions/mean_length": 422.84375, "completions/mean_terminated_length": 411.70965576171875, "completions/min_length": 231.0, "completions/min_terminated_length": 231.0, "entropy": 0.40191569551825523, "epoch": 0.215203426124197, "frac_reward_zero_std": 0.5, "grad_norm": 0.01045410055667162, "learning_rate": 1e-05, "loss": 0.004, "num_tokens": 8612758.0, "reward": 0.5625, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.458146095275879, "sampling/importance_sampling_ratio/mean": 1.000295877456665, "sampling/importance_sampling_ratio/min": 0.7797122597694397, "sampling/sampling_logp_difference/max": 0.3771657943725586, "sampling/sampling_logp_difference/mean": 0.01333489827811718, "step": 402 }, { "clip_ratio/high_max": 8.344459638465196e-05, "clip_ratio/high_mean": 8.344459638465196e-05, "clip_ratio/low_mean": 5.926979793002829e-05, "clip_ratio/low_min": 5.926979793002829e-05, "clip_ratio/region_mean": 0.00014271439431468025, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 719.0, "completions/mean_length": 538.03125, "completions/mean_terminated_length": 484.9615478515625, "completions/min_length": 232.0, "completions/min_terminated_length": 232.0, "entropy": 0.34591904282569885, "epoch": 0.21573875802997858, "frac_reward_zero_std": 0.25, "grad_norm": 0.011563985608518124, "learning_rate": 1e-05, "loss": -0.0105, "num_tokens": 8633623.0, "reward": 0.6875, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.3086165189743042, "sampling/importance_sampling_ratio/mean": 0.9997710585594177, "sampling/importance_sampling_ratio/min": 0.6319659352302551, "sampling/sampling_logp_difference/max": 0.4589197635650635, "sampling/sampling_logp_difference/mean": 0.011281891725957394, "step": 403 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00012117157530155964, "clip_ratio/low_min": 0.00012117157530155964, "clip_ratio/region_mean": 0.00012117157530155964, "completions/clipped_ratio": 0.0, "completions/max_length": 724.0, "completions/max_terminated_length": 724.0, "completions/mean_length": 470.75, "completions/mean_terminated_length": 470.75, "completions/min_length": 179.0, "completions/min_terminated_length": 179.0, "entropy": 0.4018666222691536, "epoch": 0.21627408993576017, "frac_reward_zero_std": 0.5, "grad_norm": 0.010249024257063866, "learning_rate": 1e-05, "loss": 0.0037, "num_tokens": 8652103.0, "reward": 0.8125, "reward_std": 0.249358132481575, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.3936488628387451, "sampling/importance_sampling_ratio/mean": 0.9999995827674866, "sampling/importance_sampling_ratio/min": 0.6596998572349548, "sampling/sampling_logp_difference/max": 0.4159703254699707, "sampling/sampling_logp_difference/mean": 0.011900828219950199, "step": 404 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 596.0, "completions/mean_length": 474.90625, "completions/mean_terminated_length": 465.45159912109375, "completions/min_length": 312.0, "completions/min_terminated_length": 312.0, "entropy": 0.3899528533220291, "epoch": 0.21680942184154176, "frac_reward_zero_std": 0.5, "grad_norm": 0.006876398343592882, "learning_rate": 1e-05, "loss": 0.0244, "num_tokens": 8671068.0, "reward": 0.90625, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.4994722604751587, "sampling/importance_sampling_ratio/mean": 1.0000793933868408, "sampling/importance_sampling_ratio/min": 0.5886525511741638, "sampling/sampling_logp_difference/max": 0.5299191474914551, "sampling/sampling_logp_difference/mean": 0.012460133992135525, "step": 405 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.0386471886886284e-05, "clip_ratio/low_min": 6.0386471886886284e-05, "clip_ratio/region_mean": 6.0386471886886284e-05, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 694.0, "completions/mean_length": 434.53125, "completions/mean_terminated_length": 423.774169921875, "completions/min_length": 200.0, "completions/min_terminated_length": 200.0, "entropy": 0.43231403082609177, "epoch": 0.21734475374732334, "frac_reward_zero_std": 0.25, "grad_norm": 0.012683926150202751, "learning_rate": 1e-05, "loss": -0.0279, "num_tokens": 8688589.0, "reward": 0.53125, "reward_std": 0.378745436668396, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.3972766399383545, "sampling/importance_sampling_ratio/mean": 0.9998230338096619, "sampling/importance_sampling_ratio/min": 0.7335515022277832, "sampling/sampling_logp_difference/max": 0.33452510833740234, "sampling/sampling_logp_difference/mean": 0.012979553081095219, "step": 406 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 765.0, "completions/mean_length": 536.46875, "completions/mean_terminated_length": 521.0333862304688, "completions/min_length": 297.0, "completions/min_terminated_length": 297.0, "entropy": 0.36300141736865044, "epoch": 0.21788008565310493, "frac_reward_zero_std": 0.0, "grad_norm": 0.011247067712247372, "learning_rate": 1e-05, "loss": 0.0251, "num_tokens": 8709852.0, "reward": 0.59375, "reward_std": 0.3987956643104553, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.4313021898269653, "sampling/importance_sampling_ratio/mean": 1.0001040697097778, "sampling/importance_sampling_ratio/min": 0.6243442296981812, "sampling/sampling_logp_difference/max": 0.4710533618927002, "sampling/sampling_logp_difference/mean": 0.010918519459664822, "step": 407 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0002676450021681376, "clip_ratio/low_min": 0.0002676450021681376, "clip_ratio/region_mean": 0.0002676450021681376, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 764.0, "completions/mean_length": 558.375, "completions/mean_terminated_length": 510.0000305175781, "completions/min_length": 265.0, "completions/min_terminated_length": 265.0, "entropy": 0.34893396869301796, "epoch": 0.21841541755888652, "frac_reward_zero_std": 0.5, "grad_norm": 0.01758604310452938, "learning_rate": 1e-05, "loss": 0.0475, "num_tokens": 8731064.0, "reward": 0.71875, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.296985387802124, "sampling/importance_sampling_ratio/mean": 0.9999855160713196, "sampling/importance_sampling_ratio/min": 0.6867310404777527, "sampling/sampling_logp_difference/max": 0.3758125305175781, "sampling/sampling_logp_difference/mean": 0.010774402879178524, "step": 408 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00021035455574747175, "clip_ratio/low_min": 0.00021035455574747175, "clip_ratio/region_mean": 0.00021035455574747175, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 758.0, "completions/mean_length": 492.59375, "completions/mean_terminated_length": 415.47998046875, "completions/min_length": 215.0, "completions/min_terminated_length": 215.0, "entropy": 0.6372664421796799, "epoch": 0.21895074946466808, "frac_reward_zero_std": 0.25, "grad_norm": 0.025644909590482712, "learning_rate": 1e-05, "loss": 0.0392, "num_tokens": 8750483.0, "reward": 0.375, "reward_std": 0.3745020925998688, "rewards/accuracy_reward/mean": 0.375, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.5307456254959106, "sampling/importance_sampling_ratio/mean": 0.9998341798782349, "sampling/importance_sampling_ratio/min": 0.6213971972465515, "sampling/sampling_logp_difference/max": 0.4757847785949707, "sampling/sampling_logp_difference/mean": 0.017566129565238953, "step": 409 }, { "clip_ratio/high_max": 5.830223744851537e-05, "clip_ratio/high_mean": 5.830223744851537e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 5.830223744851537e-05, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 767.0, "completions/mean_length": 527.75, "completions/mean_terminated_length": 511.7333679199219, "completions/min_length": 294.0, "completions/min_terminated_length": 294.0, "entropy": 0.4586259126663208, "epoch": 0.21948608137044967, "frac_reward_zero_std": 0.0, "grad_norm": 0.015534854494035244, "learning_rate": 1e-05, "loss": -0.017, "num_tokens": 8771459.0, "reward": 0.625, "reward_std": 0.4629100561141968, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.3773834705352783, "sampling/importance_sampling_ratio/mean": 0.9997857809066772, "sampling/importance_sampling_ratio/min": 0.6078636646270752, "sampling/sampling_logp_difference/max": 0.4978046417236328, "sampling/sampling_logp_difference/mean": 0.01344833429902792, "step": 410 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.5938992116134614e-05, "clip_ratio/low_min": 4.5938992116134614e-05, "clip_ratio/region_mean": 4.5938992116134614e-05, "completions/clipped_ratio": 0.375, "completions/max_length": 768.0, "completions/max_terminated_length": 711.0, "completions/mean_length": 607.65625, "completions/mean_terminated_length": 511.45001220703125, "completions/min_length": 138.0, "completions/min_terminated_length": 138.0, "entropy": 0.7120710089802742, "epoch": 0.22002141327623126, "frac_reward_zero_std": 0.5, "grad_norm": 0.014514628797769547, "learning_rate": 1e-05, "loss": 0.0509, "num_tokens": 8795328.0, "reward": 0.375, "reward_std": 0.2314550280570984, "rewards/accuracy_reward/mean": 0.375, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0008165836334229, "sampling/importance_sampling_ratio/min": 0.7169780731201172, "sampling/sampling_logp_difference/max": 0.7082998752593994, "sampling/sampling_logp_difference/mean": 0.017513783648610115, "step": 411 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 756.0, "completions/mean_length": 597.875, "completions/mean_terminated_length": 566.370361328125, "completions/min_length": 348.0, "completions/min_terminated_length": 348.0, "entropy": 0.47999877482652664, "epoch": 0.22055674518201285, "frac_reward_zero_std": 0.75, "grad_norm": 0.013548356480896473, "learning_rate": 1e-05, "loss": 0.0417, "num_tokens": 8818004.0, "reward": 0.59375, "reward_std": 0.1293872892856598, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.6668263673782349, "sampling/importance_sampling_ratio/mean": 0.9998438954353333, "sampling/importance_sampling_ratio/min": 0.6898931264877319, "sampling/sampling_logp_difference/max": 0.5109214782714844, "sampling/sampling_logp_difference/mean": 0.014204300940036774, "step": 412 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0001049721431627404, "clip_ratio/low_min": 0.0001049721431627404, "clip_ratio/region_mean": 0.0001049721431627404, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 720.0, "completions/mean_length": 552.375, "completions/mean_terminated_length": 502.61541748046875, "completions/min_length": 246.0, "completions/min_terminated_length": 246.0, "entropy": 0.545205969363451, "epoch": 0.22109207708779444, "frac_reward_zero_std": 0.25, "grad_norm": 0.01512906327843666, "learning_rate": 1e-05, "loss": 0.0318, "num_tokens": 8840008.0, "reward": 0.625, "reward_std": 0.292504221200943, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.3915132284164429, "sampling/importance_sampling_ratio/mean": 0.999944269657135, "sampling/importance_sampling_ratio/min": 0.777584433555603, "sampling/sampling_logp_difference/max": 0.33039188385009766, "sampling/sampling_logp_difference/mean": 0.012664245441555977, "step": 413 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 608.0, "completions/mean_length": 515.875, "completions/mean_terminated_length": 431.8333435058594, "completions/min_length": 291.0, "completions/min_terminated_length": 291.0, "entropy": 0.3869321160018444, "epoch": 0.22162740899357602, "frac_reward_zero_std": 0.25, "grad_norm": 0.0113857202231884, "learning_rate": 1e-05, "loss": 0.0105, "num_tokens": 8860116.0, "reward": 0.46875, "reward_std": 0.3198433816432953, "rewards/accuracy_reward/mean": 0.46875, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.4158447980880737, "sampling/importance_sampling_ratio/mean": 1.0004041194915771, "sampling/importance_sampling_ratio/min": 0.6664666533470154, "sampling/sampling_logp_difference/max": 0.405765175819397, "sampling/sampling_logp_difference/mean": 0.012336570769548416, "step": 414 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00019005159265361726, "clip_ratio/low_min": 0.00019005159265361726, "clip_ratio/region_mean": 0.00019005159265361726, "completions/clipped_ratio": 0.34375, "completions/max_length": 768.0, "completions/max_terminated_length": 705.0, "completions/mean_length": 643.03125, "completions/mean_terminated_length": 577.5714111328125, "completions/min_length": 427.0, "completions/min_terminated_length": 427.0, "entropy": 0.4384213499724865, "epoch": 0.2221627408993576, "frac_reward_zero_std": 0.0, "grad_norm": 0.006695670075714588, "learning_rate": 1e-05, "loss": 0.0061, "num_tokens": 8884813.0, "reward": 0.3125, "reward_std": 0.4082317352294922, "rewards/accuracy_reward/mean": 0.3125, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.500514030456543, "sampling/importance_sampling_ratio/mean": 0.9998024106025696, "sampling/importance_sampling_ratio/min": 0.6033892631530762, "sampling/sampling_logp_difference/max": 0.505192756652832, "sampling/sampling_logp_difference/mean": 0.012900110334157944, "step": 415 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00010464841761859134, "clip_ratio/low_min": 0.00010464841761859134, "clip_ratio/region_mean": 0.00010464841761859134, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 693.0, "completions/mean_length": 557.1875, "completions/mean_terminated_length": 508.5384826660156, "completions/min_length": 238.0, "completions/min_terminated_length": 238.0, "entropy": 0.5661557614803314, "epoch": 0.22269807280513917, "frac_reward_zero_std": 0.5, "grad_norm": 0.008390724658966064, "learning_rate": 1e-05, "loss": -0.0036, "num_tokens": 8906075.0, "reward": 0.71875, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.381516933441162, "sampling/importance_sampling_ratio/mean": 0.9999334216117859, "sampling/importance_sampling_ratio/min": 0.6935417056083679, "sampling/sampling_logp_difference/max": 0.36594390869140625, "sampling/sampling_logp_difference/mean": 0.014079151675105095, "step": 416 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.592826684936881e-05, "clip_ratio/low_min": 6.592826684936881e-05, "clip_ratio/region_mean": 6.592826684936881e-05, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 751.0, "completions/mean_length": 525.78125, "completions/mean_terminated_length": 469.8846435546875, "completions/min_length": 135.0, "completions/min_terminated_length": 135.0, "entropy": 0.5091826841235161, "epoch": 0.22323340471092076, "frac_reward_zero_std": 0.5, "grad_norm": 0.006459562107920647, "learning_rate": 1e-05, "loss": 0.0079, "num_tokens": 8926860.0, "reward": 0.65625, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.4588712453842163, "sampling/importance_sampling_ratio/mean": 1.0000801086425781, "sampling/importance_sampling_ratio/min": 0.5136346817016602, "sampling/sampling_logp_difference/max": 0.6662429571151733, "sampling/sampling_logp_difference/mean": 0.01519847009330988, "step": 417 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00013404825585894287, "clip_ratio/low_min": 0.00013404825585894287, "clip_ratio/region_mean": 0.00013404825585894287, "completions/clipped_ratio": 0.34375, "completions/max_length": 768.0, "completions/max_terminated_length": 622.0, "completions/mean_length": 562.625, "completions/mean_terminated_length": 455.0476379394531, "completions/min_length": 345.0, "completions/min_terminated_length": 345.0, "entropy": 0.577263955026865, "epoch": 0.22376873661670235, "frac_reward_zero_std": 0.5, "grad_norm": 0.006603385787457228, "learning_rate": 1e-05, "loss": 0.0551, "num_tokens": 8949544.0, "reward": 0.46875, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.46875, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.3305896520614624, "sampling/importance_sampling_ratio/mean": 0.9996999502182007, "sampling/importance_sampling_ratio/min": 0.6139088869094849, "sampling/sampling_logp_difference/max": 0.4879087209701538, "sampling/sampling_logp_difference/mean": 0.016703732311725616, "step": 418 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00021168820967432112, "clip_ratio/low_min": 0.00021168820967432112, "clip_ratio/region_mean": 0.00021168820967432112, "completions/clipped_ratio": 0.34375, "completions/max_length": 768.0, "completions/max_terminated_length": 694.0, "completions/mean_length": 593.21875, "completions/mean_terminated_length": 501.66668701171875, "completions/min_length": 288.0, "completions/min_terminated_length": 288.0, "entropy": 0.45473482087254524, "epoch": 0.22430406852248394, "frac_reward_zero_std": 0.25, "grad_norm": 0.021779675036668777, "learning_rate": 1e-05, "loss": 0.0528, "num_tokens": 8972319.0, "reward": 0.59375, "reward_std": 0.3471629321575165, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.4410138130187988, "sampling/importance_sampling_ratio/mean": 0.9995977878570557, "sampling/importance_sampling_ratio/min": 0.5860862135887146, "sampling/sampling_logp_difference/max": 0.5342884063720703, "sampling/sampling_logp_difference/mean": 0.013226356357336044, "step": 419 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 763.0, "completions/mean_length": 577.1875, "completions/mean_terminated_length": 533.1538696289062, "completions/min_length": 317.0, "completions/min_terminated_length": 317.0, "entropy": 0.35899868980050087, "epoch": 0.22483940042826553, "frac_reward_zero_std": 0.25, "grad_norm": 0.006980070844292641, "learning_rate": 1e-05, "loss": 0.102, "num_tokens": 8994421.0, "reward": 0.625, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.3777568340301514, "sampling/importance_sampling_ratio/mean": 0.9996013641357422, "sampling/importance_sampling_ratio/min": 0.6605517864227295, "sampling/sampling_logp_difference/max": 0.41467976570129395, "sampling/sampling_logp_difference/mean": 0.011273551732301712, "step": 420 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0001349163503618911, "clip_ratio/low_min": 0.0001349163503618911, "clip_ratio/region_mean": 0.0001349163503618911, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 742.0, "completions/mean_length": 579.34375, "completions/mean_terminated_length": 544.4074096679688, "completions/min_length": 317.0, "completions/min_terminated_length": 317.0, "entropy": 0.4214400425553322, "epoch": 0.22537473233404712, "frac_reward_zero_std": 0.0, "grad_norm": 0.01015987154096365, "learning_rate": 1e-05, "loss": 0.0362, "num_tokens": 9016784.0, "reward": 0.53125, "reward_std": 0.4765698313713074, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.442196249961853, "sampling/importance_sampling_ratio/mean": 1.0000865459442139, "sampling/importance_sampling_ratio/min": 0.6789780259132385, "sampling/sampling_logp_difference/max": 0.38716650009155273, "sampling/sampling_logp_difference/mean": 0.012763852253556252, "step": 421 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0001649361729505472, "clip_ratio/low_min": 0.0001649361729505472, "clip_ratio/region_mean": 0.0001649361729505472, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 765.0, "completions/mean_length": 540.9375, "completions/mean_terminated_length": 517.4483032226562, "completions/min_length": 306.0, "completions/min_terminated_length": 306.0, "entropy": 0.46569474786520004, "epoch": 0.2259100642398287, "frac_reward_zero_std": 0.25, "grad_norm": 0.013653812929987907, "learning_rate": 1e-05, "loss": -0.0051, "num_tokens": 9038358.0, "reward": 0.59375, "reward_std": 0.3061639964580536, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.4148699045181274, "sampling/importance_sampling_ratio/mean": 0.999466061592102, "sampling/importance_sampling_ratio/min": 0.6482210755348206, "sampling/sampling_logp_difference/max": 0.43352341651916504, "sampling/sampling_logp_difference/mean": 0.013205149210989475, "step": 422 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 766.0, "completions/mean_length": 446.875, "completions/mean_terminated_length": 425.4666748046875, "completions/min_length": 163.0, "completions/min_terminated_length": 163.0, "entropy": 0.4123445525765419, "epoch": 0.22644539614561027, "frac_reward_zero_std": 0.75, "grad_norm": 0.0027087004855275154, "learning_rate": 1e-05, "loss": -0.0216, "num_tokens": 9056074.0, "reward": 0.71875, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.374659538269043, "sampling/importance_sampling_ratio/mean": 0.9997585415840149, "sampling/importance_sampling_ratio/min": 0.7262780666351318, "sampling/sampling_logp_difference/max": 0.3198223114013672, "sampling/sampling_logp_difference/mean": 0.012693490833044052, "step": 423 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.067961294320412e-05, "clip_ratio/low_min": 6.067961294320412e-05, "clip_ratio/region_mean": 6.067961294320412e-05, "completions/clipped_ratio": 0.3125, "completions/max_length": 768.0, "completions/max_terminated_length": 757.0, "completions/mean_length": 578.9375, "completions/mean_terminated_length": 493.0, "completions/min_length": 193.0, "completions/min_terminated_length": 193.0, "entropy": 0.44280479848384857, "epoch": 0.22698072805139186, "frac_reward_zero_std": 0.5, "grad_norm": 0.010412955656647682, "learning_rate": 1e-05, "loss": 0.0155, "num_tokens": 9078776.0, "reward": 0.65625, "reward_std": 0.22201895713806152, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.4163012504577637, "sampling/importance_sampling_ratio/mean": 1.0002821683883667, "sampling/importance_sampling_ratio/min": 0.3965229392051697, "sampling/sampling_logp_difference/max": 0.9250214099884033, "sampling/sampling_logp_difference/mean": 0.012683121487498283, "step": 424 }, { "clip_ratio/high_max": 6.403688166756183e-05, "clip_ratio/high_mean": 6.403688166756183e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 6.403688166756183e-05, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 744.0, "completions/mean_length": 550.84375, "completions/mean_terminated_length": 519.8214721679688, "completions/min_length": 269.0, "completions/min_terminated_length": 269.0, "entropy": 0.3804459273815155, "epoch": 0.22751605995717344, "frac_reward_zero_std": 0.75, "grad_norm": 0.0024838228709995747, "learning_rate": 1e-05, "loss": 0.0207, "num_tokens": 9099835.0, "reward": 0.71875, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.4718800783157349, "sampling/importance_sampling_ratio/mean": 1.0002789497375488, "sampling/importance_sampling_ratio/min": 0.6511872410774231, "sampling/sampling_logp_difference/max": 0.42895805835723877, "sampling/sampling_logp_difference/mean": 0.012384253554046154, "step": 425 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.924170545767993e-05, "clip_ratio/low_min": 5.924170545767993e-05, "clip_ratio/region_mean": 5.924170545767993e-05, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 726.0, "completions/mean_length": 535.75, "completions/mean_terminated_length": 502.5714416503906, "completions/min_length": 252.0, "completions/min_terminated_length": 252.0, "entropy": 0.29265643656253815, "epoch": 0.22805139186295503, "frac_reward_zero_std": 0.5, "grad_norm": 0.012128359638154507, "learning_rate": 1e-05, "loss": -0.0059, "num_tokens": 9120731.0, "reward": 0.75, "reward_std": 0.26726123690605164, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.426264762878418, "sampling/importance_sampling_ratio/mean": 0.9998127222061157, "sampling/importance_sampling_ratio/min": 0.695228636264801, "sampling/sampling_logp_difference/max": 0.36351442337036133, "sampling/sampling_logp_difference/mean": 0.010166767984628677, "step": 426 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.175889575388283e-05, "clip_ratio/low_min": 6.175889575388283e-05, "clip_ratio/region_mean": 6.175889575388283e-05, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 759.0, "completions/mean_length": 533.40625, "completions/mean_terminated_length": 479.2692565917969, "completions/min_length": 269.0, "completions/min_terminated_length": 269.0, "entropy": 0.41359881311655045, "epoch": 0.22858672376873662, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0279, "num_tokens": 9141696.0, "reward": 0.65625, "reward_std": 0.1293872892856598, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.388490915298462, "sampling/importance_sampling_ratio/mean": 0.9996965527534485, "sampling/importance_sampling_ratio/min": 0.6807646155357361, "sampling/sampling_logp_difference/max": 0.3845386505126953, "sampling/sampling_logp_difference/mean": 0.012603675946593285, "step": 427 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.418292130343616e-05, "clip_ratio/low_min": 5.418292130343616e-05, "clip_ratio/region_mean": 5.418292130343616e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 744.0, "completions/max_terminated_length": 744.0, "completions/mean_length": 461.34375, "completions/mean_terminated_length": 461.34375, "completions/min_length": 221.0, "completions/min_terminated_length": 221.0, "entropy": 0.3670461028814316, "epoch": 0.2291220556745182, "frac_reward_zero_std": 0.25, "grad_norm": 0.008405312895774841, "learning_rate": 1e-05, "loss": -0.0129, "num_tokens": 9160003.0, "reward": 0.6875, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.3555933237075806, "sampling/importance_sampling_ratio/mean": 1.0002845525741577, "sampling/importance_sampling_ratio/min": 0.6224300861358643, "sampling/sampling_logp_difference/max": 0.4741239547729492, "sampling/sampling_logp_difference/mean": 0.012463947758078575, "step": 428 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 689.0, "completions/mean_length": 556.90625, "completions/mean_terminated_length": 508.19232177734375, "completions/min_length": 302.0, "completions/min_terminated_length": 302.0, "entropy": 0.35653635300695896, "epoch": 0.2296573875802998, "frac_reward_zero_std": 0.25, "grad_norm": 0.031224975362420082, "learning_rate": 1e-05, "loss": 0.0507, "num_tokens": 9181712.0, "reward": 0.5625, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.421380877494812, "sampling/importance_sampling_ratio/mean": 1.000276803970337, "sampling/importance_sampling_ratio/min": 0.6906217932701111, "sampling/sampling_logp_difference/max": 0.3701629638671875, "sampling/sampling_logp_difference/mean": 0.011170031502842903, "step": 429 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 635.0, "completions/mean_length": 421.78125, "completions/mean_terminated_length": 410.6128845214844, "completions/min_length": 191.0, "completions/min_terminated_length": 191.0, "entropy": 0.38883281126618385, "epoch": 0.23019271948608136, "frac_reward_zero_std": 0.5, "grad_norm": 0.0073440903797745705, "learning_rate": 1e-05, "loss": 0.0618, "num_tokens": 9198737.0, "reward": 0.8125, "reward_std": 0.2587745785713196, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.3637741804122925, "sampling/importance_sampling_ratio/mean": 1.000226616859436, "sampling/importance_sampling_ratio/min": 0.659216582775116, "sampling/sampling_logp_difference/max": 0.4167032241821289, "sampling/sampling_logp_difference/mean": 0.01179863978177309, "step": 430 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 694.0, "completions/mean_length": 475.125, "completions/mean_terminated_length": 433.2857360839844, "completions/min_length": 189.0, "completions/min_terminated_length": 189.0, "entropy": 0.44823867455124855, "epoch": 0.23072805139186295, "frac_reward_zero_std": 0.0, "grad_norm": 0.009986692108213902, "learning_rate": 1e-05, "loss": 0.0481, "num_tokens": 9217573.0, "reward": 0.6875, "reward_std": 0.4492306709289551, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.271128535270691, "sampling/importance_sampling_ratio/mean": 1.0000205039978027, "sampling/importance_sampling_ratio/min": 0.6460689306259155, "sampling/sampling_logp_difference/max": 0.43684911727905273, "sampling/sampling_logp_difference/mean": 0.01257572416216135, "step": 431 }, { "clip_ratio/high_max": 0.00011637931675068103, "clip_ratio/high_mean": 0.00011637931675068103, "clip_ratio/low_mean": 6.053268589312211e-05, "clip_ratio/low_min": 6.053268589312211e-05, "clip_ratio/region_mean": 0.00017691200264380313, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 759.0, "completions/mean_length": 529.46875, "completions/mean_terminated_length": 513.5667114257812, "completions/min_length": 316.0, "completions/min_terminated_length": 316.0, "entropy": 0.42199476435780525, "epoch": 0.23126338329764454, "frac_reward_zero_std": 0.5, "grad_norm": 0.004911072552204132, "learning_rate": 1e-05, "loss": -0.0107, "num_tokens": 9238444.0, "reward": 0.8125, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.4605900049209595, "sampling/importance_sampling_ratio/mean": 1.0000569820404053, "sampling/importance_sampling_ratio/min": 0.6634466052055359, "sampling/sampling_logp_difference/max": 0.4103069305419922, "sampling/sampling_logp_difference/mean": 0.012371168471872807, "step": 432 }, { "clip_ratio/high_max": 5.587840860243887e-05, "clip_ratio/high_mean": 5.587840860243887e-05, "clip_ratio/low_mean": 4.570384044200182e-05, "clip_ratio/low_min": 4.570384044200182e-05, "clip_ratio/region_mean": 0.00010158224904444069, "completions/clipped_ratio": 0.3125, "completions/max_length": 768.0, "completions/max_terminated_length": 737.0, "completions/mean_length": 625.03125, "completions/mean_terminated_length": 560.0454711914062, "completions/min_length": 323.0, "completions/min_terminated_length": 323.0, "entropy": 0.29868466779589653, "epoch": 0.23179871520342613, "frac_reward_zero_std": 0.25, "grad_norm": 0.008090803399682045, "learning_rate": 1e-05, "loss": 0.0503, "num_tokens": 9262277.0, "reward": 0.5625, "reward_std": 0.3514062464237213, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.2700191736221313, "sampling/importance_sampling_ratio/mean": 0.9997373223304749, "sampling/importance_sampling_ratio/min": 0.7011478543281555, "sampling/sampling_logp_difference/max": 0.35503649711608887, "sampling/sampling_logp_difference/mean": 0.009685932658612728, "step": 433 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.34375, "completions/max_length": 768.0, "completions/max_terminated_length": 763.0, "completions/mean_length": 612.3125, "completions/mean_terminated_length": 530.7619018554688, "completions/min_length": 205.0, "completions/min_terminated_length": 205.0, "entropy": 0.5361443646252155, "epoch": 0.23233404710920771, "frac_reward_zero_std": 0.75, "grad_norm": 0.006447171326726675, "learning_rate": 1e-05, "loss": 0.0005, "num_tokens": 9285511.0, "reward": 0.40625, "reward_std": 0.1293872892856598, "rewards/accuracy_reward/mean": 0.40625, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.6125704050064087, "sampling/importance_sampling_ratio/mean": 0.9995403289794922, "sampling/importance_sampling_ratio/min": 0.45170289278030396, "sampling/sampling_logp_difference/max": 0.7947306632995605, "sampling/sampling_logp_difference/mean": 0.014863980002701283, "step": 434 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 659.0, "completions/max_terminated_length": 659.0, "completions/mean_length": 437.78125, "completions/mean_terminated_length": 437.78125, "completions/min_length": 272.0, "completions/min_terminated_length": 272.0, "entropy": 0.4513319730758667, "epoch": 0.2328693790149893, "frac_reward_zero_std": 0.75, "grad_norm": 0.011153290048241615, "learning_rate": 1e-05, "loss": -0.0011, "num_tokens": 9303280.0, "reward": 0.90625, "reward_std": 0.1293872892856598, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.4422086477279663, "sampling/importance_sampling_ratio/mean": 1.0000663995742798, "sampling/importance_sampling_ratio/min": 0.6965842247009277, "sampling/sampling_logp_difference/max": 0.36617565155029297, "sampling/sampling_logp_difference/mean": 0.012972470372915268, "step": 435 }, { "clip_ratio/high_max": 5.9580554079730064e-05, "clip_ratio/high_mean": 5.9580554079730064e-05, "clip_ratio/low_mean": 0.0001631690247450024, "clip_ratio/low_min": 0.0001631690247450024, "clip_ratio/region_mean": 0.00022274957882473245, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 714.0, "completions/mean_length": 554.59375, "completions/mean_terminated_length": 515.0740966796875, "completions/min_length": 322.0, "completions/min_terminated_length": 322.0, "entropy": 0.4475087486207485, "epoch": 0.2334047109207709, "frac_reward_zero_std": 0.0, "grad_norm": 0.008825229480862617, "learning_rate": 1e-05, "loss": 0.1122, "num_tokens": 9324755.0, "reward": 0.53125, "reward_std": 0.4218915104866028, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.4316606521606445, "sampling/importance_sampling_ratio/mean": 0.9998408555984497, "sampling/importance_sampling_ratio/min": 0.6648994088172913, "sampling/sampling_logp_difference/max": 0.40811944007873535, "sampling/sampling_logp_difference/mean": 0.012117262929677963, "step": 436 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 756.0, "completions/mean_length": 527.34375, "completions/mean_terminated_length": 482.77777099609375, "completions/min_length": 202.0, "completions/min_terminated_length": 202.0, "entropy": 0.2966017406433821, "epoch": 0.23394004282655245, "frac_reward_zero_std": 0.0, "grad_norm": 0.010183276608586311, "learning_rate": 1e-05, "loss": 0.0377, "num_tokens": 9344862.0, "reward": 0.6875, "reward_std": 0.47655022144317627, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.282387375831604, "sampling/importance_sampling_ratio/mean": 1.000028371810913, "sampling/importance_sampling_ratio/min": 0.6990740299224854, "sampling/sampling_logp_difference/max": 0.3579986095428467, "sampling/sampling_logp_difference/mean": 0.009799963794648647, "step": 437 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 602.0, "completions/mean_length": 522.25, "completions/mean_terminated_length": 440.3333435058594, "completions/min_length": 298.0, "completions/min_terminated_length": 298.0, "entropy": 0.3842923603951931, "epoch": 0.23447537473233404, "frac_reward_zero_std": 0.5, "grad_norm": 0.0069622802548110485, "learning_rate": 1e-05, "loss": 0.0322, "num_tokens": 9365838.0, "reward": 0.53125, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.5250685214996338, "sampling/importance_sampling_ratio/mean": 1.0001485347747803, "sampling/importance_sampling_ratio/min": 0.6450940370559692, "sampling/sampling_logp_difference/max": 0.43835926055908203, "sampling/sampling_logp_difference/mean": 0.011143209412693977, "step": 438 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.325911090243608e-05, "clip_ratio/low_min": 6.325911090243608e-05, "clip_ratio/region_mean": 6.325911090243608e-05, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 629.0, "completions/mean_length": 459.875, "completions/mean_terminated_length": 449.9354553222656, "completions/min_length": 206.0, "completions/min_terminated_length": 206.0, "entropy": 0.35073431953787804, "epoch": 0.23501070663811563, "frac_reward_zero_std": 0.5, "grad_norm": 0.012068981304764748, "learning_rate": 1e-05, "loss": 0.0094, "num_tokens": 9384074.0, "reward": 0.78125, "reward_std": 0.2630178928375244, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.4477179050445557, "sampling/importance_sampling_ratio/mean": 0.9997803568840027, "sampling/importance_sampling_ratio/min": 0.5760422945022583, "sampling/sampling_logp_difference/max": 0.5515742301940918, "sampling/sampling_logp_difference/mean": 0.011955702677369118, "step": 439 }, { "clip_ratio/high_max": 5.910165418754332e-05, "clip_ratio/high_mean": 5.910165418754332e-05, "clip_ratio/low_mean": 0.00018559396994533017, "clip_ratio/low_min": 0.00018559396994533017, "clip_ratio/region_mean": 0.0002446956241328735, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 716.0, "completions/mean_length": 519.8125, "completions/mean_terminated_length": 473.85186767578125, "completions/min_length": 264.0, "completions/min_terminated_length": 264.0, "entropy": 0.403911329805851, "epoch": 0.23554603854389722, "frac_reward_zero_std": 0.25, "grad_norm": 0.012297307141125202, "learning_rate": 1e-05, "loss": 0.0837, "num_tokens": 9404580.0, "reward": 0.59375, "reward_std": 0.3966485261917114, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.2885267734527588, "sampling/importance_sampling_ratio/mean": 0.9999293684959412, "sampling/importance_sampling_ratio/min": 0.7447843551635742, "sampling/sampling_logp_difference/max": 0.2946605682373047, "sampling/sampling_logp_difference/mean": 0.012314294464886189, "step": 440 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 745.0, "completions/mean_length": 441.90625, "completions/mean_terminated_length": 420.16668701171875, "completions/min_length": 219.0, "completions/min_terminated_length": 219.0, "entropy": 0.43974559381604195, "epoch": 0.2360813704496788, "frac_reward_zero_std": 0.5, "grad_norm": 0.0028224484995007515, "learning_rate": 1e-05, "loss": 0.0734, "num_tokens": 9422121.0, "reward": 0.65625, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.3143641948699951, "sampling/importance_sampling_ratio/mean": 0.9996448755264282, "sampling/importance_sampling_ratio/min": 0.6420266628265381, "sampling/sampling_logp_difference/max": 0.44312548637390137, "sampling/sampling_logp_difference/mean": 0.014007256366312504, "step": 441 }, { "clip_ratio/high_max": 6.027000927133486e-05, "clip_ratio/high_mean": 6.027000927133486e-05, "clip_ratio/low_mean": 6.027000927133486e-05, "clip_ratio/low_min": 6.027000927133486e-05, "clip_ratio/region_mean": 0.00012054001854266971, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 762.0, "completions/mean_length": 535.21875, "completions/mean_terminated_length": 527.7096557617188, "completions/min_length": 273.0, "completions/min_terminated_length": 273.0, "entropy": 0.3419300578534603, "epoch": 0.2366167023554604, "frac_reward_zero_std": 0.25, "grad_norm": 0.006149868480861187, "learning_rate": 1e-05, "loss": 0.0248, "num_tokens": 9443168.0, "reward": 0.78125, "reward_std": 0.3061639964580536, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.4472057819366455, "sampling/importance_sampling_ratio/mean": 0.9997043013572693, "sampling/importance_sampling_ratio/min": 0.6846362352371216, "sampling/sampling_logp_difference/max": 0.37886762619018555, "sampling/sampling_logp_difference/mean": 0.011754102073609829, "step": 442 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.718206739402376e-05, "clip_ratio/low_min": 5.718206739402376e-05, "clip_ratio/region_mean": 5.718206739402376e-05, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 697.0, "completions/mean_length": 532.875, "completions/mean_terminated_length": 499.2857360839844, "completions/min_length": 277.0, "completions/min_terminated_length": 277.0, "entropy": 0.4245343003422022, "epoch": 0.23715203426124196, "frac_reward_zero_std": 0.5, "grad_norm": 0.029914360493421555, "learning_rate": 1e-05, "loss": 0.0002, "num_tokens": 9463580.0, "reward": 0.8125, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.3767448663711548, "sampling/importance_sampling_ratio/mean": 1.0006240606307983, "sampling/importance_sampling_ratio/min": 0.6944502592086792, "sampling/sampling_logp_difference/max": 0.36463475227355957, "sampling/sampling_logp_difference/mean": 0.01291370764374733, "step": 443 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 648.0, "completions/mean_length": 436.125, "completions/mean_terminated_length": 425.4193420410156, "completions/min_length": 226.0, "completions/min_terminated_length": 226.0, "entropy": 0.3653532285243273, "epoch": 0.23768736616702354, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.013, "num_tokens": 9481056.0, "reward": 0.875, "reward_std": 0.13363061845302582, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 1.454010009765625, "sampling/importance_sampling_ratio/mean": 0.9999028444290161, "sampling/importance_sampling_ratio/min": 0.5122338533401489, "sampling/sampling_logp_difference/max": 0.6689740419387817, "sampling/sampling_logp_difference/mean": 0.011901103891432285, "step": 444 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.8548008382786065e-05, "clip_ratio/low_min": 5.8548008382786065e-05, "clip_ratio/region_mean": 5.8548008382786065e-05, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 756.0, "completions/mean_length": 572.0625, "completions/mean_terminated_length": 551.7930908203125, "completions/min_length": 296.0, "completions/min_terminated_length": 296.0, "entropy": 0.30460595712065697, "epoch": 0.23822269807280513, "frac_reward_zero_std": 0.25, "grad_norm": 0.011782202869653702, "learning_rate": 1e-05, "loss": 0.0696, "num_tokens": 9502930.0, "reward": 0.84375, "reward_std": 0.3198433816432953, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.3174147605895996, "sampling/importance_sampling_ratio/mean": 0.9998862147331238, "sampling/importance_sampling_ratio/min": 0.4322245717048645, "sampling/sampling_logp_difference/max": 0.8388099670410156, "sampling/sampling_logp_difference/mean": 0.01028999499976635, "step": 445 }, { "clip_ratio/high_max": 5.460900138132274e-05, "clip_ratio/high_mean": 5.460900138132274e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 5.460900138132274e-05, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 718.0, "completions/mean_length": 520.9375, "completions/mean_terminated_length": 504.4667053222656, "completions/min_length": 294.0, "completions/min_terminated_length": 294.0, "entropy": 0.31242641247808933, "epoch": 0.23875802997858672, "frac_reward_zero_std": 0.25, "grad_norm": 0.007522552274167538, "learning_rate": 1e-05, "loss": 0.0914, "num_tokens": 9523456.0, "reward": 0.875, "reward_std": 0.292504221200943, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 1.4370394945144653, "sampling/importance_sampling_ratio/mean": 0.9999423623085022, "sampling/importance_sampling_ratio/min": 0.6511749029159546, "sampling/sampling_logp_difference/max": 0.42897701263427734, "sampling/sampling_logp_difference/mean": 0.010918579064309597, "step": 446 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00011043973427149467, "clip_ratio/low_min": 0.00011043973427149467, "clip_ratio/region_mean": 0.00011043973427149467, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 738.0, "completions/mean_length": 569.28125, "completions/mean_terminated_length": 562.8709716796875, "completions/min_length": 357.0, "completions/min_terminated_length": 357.0, "entropy": 0.33172520995140076, "epoch": 0.2392933618843683, "frac_reward_zero_std": 0.25, "grad_norm": 0.0036930718924850225, "learning_rate": 1e-05, "loss": 0.0097, "num_tokens": 9545521.0, "reward": 0.6875, "reward_std": 0.292504221200943, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.4489433765411377, "sampling/importance_sampling_ratio/mean": 1.0001510381698608, "sampling/importance_sampling_ratio/min": 0.6575350761413574, "sampling/sampling_logp_difference/max": 0.41925716400146484, "sampling/sampling_logp_difference/mean": 0.011404524557292461, "step": 447 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 724.0, "completions/mean_length": 486.84375, "completions/mean_terminated_length": 477.774169921875, "completions/min_length": 293.0, "completions/min_terminated_length": 293.0, "entropy": 0.33927514031529427, "epoch": 0.2398286937901499, "frac_reward_zero_std": 0.5, "grad_norm": 0.005942976102232933, "learning_rate": 1e-05, "loss": 0.0153, "num_tokens": 9565124.0, "reward": 0.8125, "reward_std": 0.249358132481575, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.4997282028198242, "sampling/importance_sampling_ratio/mean": 0.9999534487724304, "sampling/importance_sampling_ratio/min": 0.6569938063621521, "sampling/sampling_logp_difference/max": 0.42008066177368164, "sampling/sampling_logp_difference/mean": 0.011414112523198128, "step": 448 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.256518124951981e-05, "clip_ratio/low_min": 5.256518124951981e-05, "clip_ratio/region_mean": 5.256518124951981e-05, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 735.0, "completions/mean_length": 550.125, "completions/mean_terminated_length": 509.77777099609375, "completions/min_length": 241.0, "completions/min_terminated_length": 241.0, "entropy": 0.4012523554265499, "epoch": 0.2403640256959315, "frac_reward_zero_std": 0.25, "grad_norm": 0.020486872643232346, "learning_rate": 1e-05, "loss": 0.0501, "num_tokens": 9586368.0, "reward": 0.71875, "reward_std": 0.3608423173427582, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.4344967603683472, "sampling/importance_sampling_ratio/mean": 1.00015127658844, "sampling/importance_sampling_ratio/min": 0.5887377858161926, "sampling/sampling_logp_difference/max": 0.5297744274139404, "sampling/sampling_logp_difference/mean": 0.012876774184405804, "step": 449 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 748.0, "completions/mean_length": 509.75, "completions/mean_terminated_length": 472.857177734375, "completions/min_length": 302.0, "completions/min_terminated_length": 302.0, "entropy": 0.3603517934679985, "epoch": 0.24089935760171305, "frac_reward_zero_std": 0.75, "grad_norm": 0.016826961189508438, "learning_rate": 1e-05, "loss": 0.0223, "num_tokens": 9606056.0, "reward": 0.9375, "reward_std": 0.1157275140285492, "rewards/accuracy_reward/mean": 0.9375, "rewards/accuracy_reward/std": 0.24593468010425568, "sampling/importance_sampling_ratio/max": 1.350253701210022, "sampling/importance_sampling_ratio/mean": 0.9999425411224365, "sampling/importance_sampling_ratio/min": 0.6953261494636536, "sampling/sampling_logp_difference/max": 0.3633742332458496, "sampling/sampling_logp_difference/mean": 0.011786971241235733, "step": 450 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 7.094211468938738e-05, "clip_ratio/low_min": 7.094211468938738e-05, "clip_ratio/region_mean": 7.094211468938738e-05, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 677.0, "completions/mean_length": 507.84375, "completions/mean_terminated_length": 499.45159912109375, "completions/min_length": 353.0, "completions/min_terminated_length": 353.0, "entropy": 0.3076478410512209, "epoch": 0.24143468950749464, "frac_reward_zero_std": 0.5, "grad_norm": 0.0046203541569411755, "learning_rate": 1e-05, "loss": -0.0137, "num_tokens": 9625915.0, "reward": 0.84375, "reward_std": 0.22201895713806152, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.342712163925171, "sampling/importance_sampling_ratio/mean": 0.9998106956481934, "sampling/importance_sampling_ratio/min": 0.687568187713623, "sampling/sampling_logp_difference/max": 0.37459421157836914, "sampling/sampling_logp_difference/mean": 0.010666599497199059, "step": 451 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.188119004946202e-05, "clip_ratio/low_min": 6.188119004946202e-05, "clip_ratio/region_mean": 6.188119004946202e-05, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 762.0, "completions/mean_length": 421.4375, "completions/mean_terminated_length": 398.3333435058594, "completions/min_length": 223.0, "completions/min_terminated_length": 223.0, "entropy": 0.3537568338215351, "epoch": 0.24197002141327623, "frac_reward_zero_std": 0.5, "grad_norm": 0.011190691962838173, "learning_rate": 1e-05, "loss": -0.0793, "num_tokens": 9642721.0, "reward": 0.6875, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.3687958717346191, "sampling/importance_sampling_ratio/mean": 0.9997307062149048, "sampling/importance_sampling_ratio/min": 0.7300587296485901, "sampling/sampling_logp_difference/max": 0.31463027000427246, "sampling/sampling_logp_difference/mean": 0.011748920194804668, "step": 452 }, { "clip_ratio/high_max": 5.193186370888725e-05, "clip_ratio/high_mean": 5.193186370888725e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 5.193186370888725e-05, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 711.0, "completions/mean_length": 530.03125, "completions/mean_terminated_length": 485.96295166015625, "completions/min_length": 261.0, "completions/min_terminated_length": 261.0, "entropy": 0.4937206022441387, "epoch": 0.24250535331905781, "frac_reward_zero_std": 0.0, "grad_norm": 0.04192846640944481, "learning_rate": 1e-05, "loss": 0.0909, "num_tokens": 9663274.0, "reward": 0.71875, "reward_std": 0.4218915104866028, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.475556492805481, "sampling/importance_sampling_ratio/mean": 0.9997404217720032, "sampling/importance_sampling_ratio/min": 0.6958999037742615, "sampling/sampling_logp_difference/max": 0.3890352249145508, "sampling/sampling_logp_difference/mean": 0.0142365712672472, "step": 453 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.160670454846695e-05, "clip_ratio/low_min": 6.160670454846695e-05, "clip_ratio/region_mean": 6.160670454846695e-05, "completions/clipped_ratio": 0.3125, "completions/max_length": 768.0, "completions/max_terminated_length": 749.0, "completions/mean_length": 636.46875, "completions/mean_terminated_length": 576.6818237304688, "completions/min_length": 330.0, "completions/min_terminated_length": 330.0, "entropy": 0.3610347770154476, "epoch": 0.2430406852248394, "frac_reward_zero_std": 0.25, "grad_norm": 0.014986919239163399, "learning_rate": 1e-05, "loss": 0.0566, "num_tokens": 9687753.0, "reward": 0.46875, "reward_std": 0.378745436668396, "rewards/accuracy_reward/mean": 0.46875, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.6268322467803955, "sampling/importance_sampling_ratio/mean": 1.000038504600525, "sampling/importance_sampling_ratio/min": 0.7378782033920288, "sampling/sampling_logp_difference/max": 0.4866347312927246, "sampling/sampling_logp_difference/mean": 0.011606104671955109, "step": 454 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 8.884150884114206e-05, "clip_ratio/low_min": 8.884150884114206e-05, "clip_ratio/region_mean": 8.884150884114206e-05, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 743.0, "completions/mean_length": 564.46875, "completions/mean_terminated_length": 526.7777709960938, "completions/min_length": 267.0, "completions/min_terminated_length": 267.0, "entropy": 0.44936030358076096, "epoch": 0.243576017130621, "frac_reward_zero_std": 0.25, "grad_norm": 0.009507955051958561, "learning_rate": 1e-05, "loss": 0.0201, "num_tokens": 9710056.0, "reward": 0.6875, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.4472140073776245, "sampling/importance_sampling_ratio/mean": 0.9998016953468323, "sampling/importance_sampling_ratio/min": 0.6794170141220093, "sampling/sampling_logp_difference/max": 0.3865201473236084, "sampling/sampling_logp_difference/mean": 0.014226510189473629, "step": 455 }, { "clip_ratio/high_max": 6.157635652925819e-05, "clip_ratio/high_mean": 6.157635652925819e-05, "clip_ratio/low_mean": 0.0001727002309053205, "clip_ratio/low_min": 0.0001727002309053205, "clip_ratio/region_mean": 0.0002342765874345787, "completions/clipped_ratio": 0.28125, "completions/max_length": 768.0, "completions/max_terminated_length": 763.0, "completions/mean_length": 587.75, "completions/mean_terminated_length": 517.2174072265625, "completions/min_length": 289.0, "completions/min_terminated_length": 289.0, "entropy": 0.4764968156814575, "epoch": 0.24411134903640258, "frac_reward_zero_std": 0.0, "grad_norm": 0.013710142113268375, "learning_rate": 1e-05, "loss": 0.0979, "num_tokens": 9732720.0, "reward": 0.5625, "reward_std": 0.3945523500442505, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9998855590820312, "sampling/importance_sampling_ratio/min": 0.6379536986351013, "sampling/sampling_logp_difference/max": 1.1404428482055664, "sampling/sampling_logp_difference/mean": 0.014291264116764069, "step": 456 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 657.0, "completions/mean_length": 465.8125, "completions/mean_terminated_length": 456.06451416015625, "completions/min_length": 325.0, "completions/min_terminated_length": 325.0, "entropy": 0.33709372393786907, "epoch": 0.24464668094218414, "frac_reward_zero_std": 0.25, "grad_norm": 0.006640094332396984, "learning_rate": 1e-05, "loss": 0.0199, "num_tokens": 9751570.0, "reward": 0.71875, "reward_std": 0.3198433816432953, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.4198437929153442, "sampling/importance_sampling_ratio/mean": 0.9998038411140442, "sampling/importance_sampling_ratio/min": 0.7205827832221985, "sampling/sampling_logp_difference/max": 0.35054683685302734, "sampling/sampling_logp_difference/mean": 0.011600518599152565, "step": 457 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.747126488131471e-05, "clip_ratio/low_min": 5.747126488131471e-05, "clip_ratio/region_mean": 5.747126488131471e-05, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 718.0, "completions/mean_length": 470.03125, "completions/mean_terminated_length": 401.2692565917969, "completions/min_length": 214.0, "completions/min_terminated_length": 214.0, "entropy": 0.3748924769461155, "epoch": 0.24518201284796573, "frac_reward_zero_std": 0.75, "grad_norm": 0.019087394699454308, "learning_rate": 1e-05, "loss": -0.0011, "num_tokens": 9770083.0, "reward": 0.8125, "reward_std": 0.1157275140285492, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.4516117572784424, "sampling/importance_sampling_ratio/mean": 1.000480055809021, "sampling/importance_sampling_ratio/min": 0.7660260200500488, "sampling/sampling_logp_difference/max": 0.37267446517944336, "sampling/sampling_logp_difference/mean": 0.012263812124729156, "step": 458 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 8.229097875300795e-05, "clip_ratio/low_min": 8.229097875300795e-05, "clip_ratio/region_mean": 8.229097875300795e-05, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 628.0, "completions/mean_length": 453.46875, "completions/mean_terminated_length": 443.32257080078125, "completions/min_length": 204.0, "completions/min_terminated_length": 204.0, "entropy": 0.3883685953915119, "epoch": 0.24571734475374732, "frac_reward_zero_std": 0.5, "grad_norm": 0.02708948403596878, "learning_rate": 1e-05, "loss": 0.0567, "num_tokens": 9788242.0, "reward": 0.84375, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.4311306476593018, "sampling/importance_sampling_ratio/mean": 0.9997379183769226, "sampling/importance_sampling_ratio/min": 0.6925398707389832, "sampling/sampling_logp_difference/max": 0.367389440536499, "sampling/sampling_logp_difference/mean": 0.011722617782652378, "step": 459 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00011693237320287153, "clip_ratio/low_min": 0.00011693237320287153, "clip_ratio/region_mean": 0.00011693237320287153, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 702.0, "completions/mean_length": 484.53125, "completions/mean_terminated_length": 465.63336181640625, "completions/min_length": 313.0, "completions/min_terminated_length": 313.0, "entropy": 0.3766840063035488, "epoch": 0.2462526766595289, "frac_reward_zero_std": 0.5, "grad_norm": 0.0047309487126767635, "learning_rate": 1e-05, "loss": 0.08, "num_tokens": 9806971.0, "reward": 0.875, "reward_std": 0.2314550280570984, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 1.5795408487319946, "sampling/importance_sampling_ratio/mean": 1.0000979900360107, "sampling/importance_sampling_ratio/min": 0.716194748878479, "sampling/sampling_logp_difference/max": 0.4571342468261719, "sampling/sampling_logp_difference/mean": 0.011670341715216637, "step": 460 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.55308761249762e-05, "clip_ratio/low_min": 5.55308761249762e-05, "clip_ratio/region_mean": 5.55308761249762e-05, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 672.0, "completions/mean_length": 498.59375, "completions/mean_terminated_length": 480.63336181640625, "completions/min_length": 276.0, "completions/min_terminated_length": 276.0, "entropy": 0.34036383405327797, "epoch": 0.2467880085653105, "frac_reward_zero_std": 0.5, "grad_norm": 0.013544617220759392, "learning_rate": 1e-05, "loss": 0.0187, "num_tokens": 9826606.0, "reward": 0.75, "reward_std": 0.2587745785713196, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.3156120777130127, "sampling/importance_sampling_ratio/mean": 0.99959397315979, "sampling/importance_sampling_ratio/min": 0.7834674715995789, "sampling/sampling_logp_difference/max": 0.27430200576782227, "sampling/sampling_logp_difference/mean": 0.011032242327928543, "step": 461 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 745.0, "completions/mean_length": 510.09375, "completions/mean_terminated_length": 492.9000244140625, "completions/min_length": 307.0, "completions/min_terminated_length": 307.0, "entropy": 0.3001006357371807, "epoch": 0.24732334047109208, "frac_reward_zero_std": 0.5, "grad_norm": 0.007335676345974207, "learning_rate": 1e-05, "loss": 0.049, "num_tokens": 9846777.0, "reward": 0.875, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 1.365645170211792, "sampling/importance_sampling_ratio/mean": 0.9996696710586548, "sampling/importance_sampling_ratio/min": 0.6994103193283081, "sampling/sampling_logp_difference/max": 0.35751771926879883, "sampling/sampling_logp_difference/mean": 0.01061292365193367, "step": 462 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.7843590184347704e-05, "clip_ratio/low_min": 5.7843590184347704e-05, "clip_ratio/region_mean": 5.7843590184347704e-05, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 765.0, "completions/mean_length": 574.71875, "completions/mean_terminated_length": 538.9259033203125, "completions/min_length": 330.0, "completions/min_terminated_length": 330.0, "entropy": 0.4043949991464615, "epoch": 0.24785867237687367, "frac_reward_zero_std": 0.25, "grad_norm": 0.007336011156439781, "learning_rate": 1e-05, "loss": 0.0149, "num_tokens": 9869168.0, "reward": 0.75, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.3946592807769775, "sampling/importance_sampling_ratio/mean": 0.9998060464859009, "sampling/importance_sampling_ratio/min": 0.7594910860061646, "sampling/sampling_logp_difference/max": 0.33265018463134766, "sampling/sampling_logp_difference/mean": 0.0124510508030653, "step": 463 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 9.449973003938794e-05, "clip_ratio/low_min": 9.449973003938794e-05, "clip_ratio/region_mean": 9.449973003938794e-05, "completions/clipped_ratio": 0.5, "completions/max_length": 768.0, "completions/max_terminated_length": 761.0, "completions/mean_length": 640.71875, "completions/mean_terminated_length": 513.4375, "completions/min_length": 267.0, "completions/min_terminated_length": 267.0, "entropy": 0.3673878461122513, "epoch": 0.24839400428265523, "frac_reward_zero_std": 0.5, "grad_norm": 0.002930504735559225, "learning_rate": 1e-05, "loss": 0.0505, "num_tokens": 9894095.0, "reward": 0.4375, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.4375, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.408188819885254, "sampling/importance_sampling_ratio/mean": 1.0000945329666138, "sampling/importance_sampling_ratio/min": 0.6061339974403381, "sampling/sampling_logp_difference/max": 0.5006542205810547, "sampling/sampling_logp_difference/mean": 0.011721408925950527, "step": 464 }, { "clip_ratio/high_max": 5.408913784776814e-05, "clip_ratio/high_mean": 5.408913784776814e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 5.408913784776814e-05, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 686.0, "completions/mean_length": 473.59375, "completions/mean_terminated_length": 464.0967712402344, "completions/min_length": 214.0, "completions/min_terminated_length": 214.0, "entropy": 0.305716123431921, "epoch": 0.24892933618843682, "frac_reward_zero_std": 0.25, "grad_norm": 0.01655631512403488, "learning_rate": 1e-05, "loss": -0.0151, "num_tokens": 9912890.0, "reward": 0.71875, "reward_std": 0.35564959049224854, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.4440534114837646, "sampling/importance_sampling_ratio/mean": 0.9998975992202759, "sampling/importance_sampling_ratio/min": 0.45198091864585876, "sampling/sampling_logp_difference/max": 0.7941153049468994, "sampling/sampling_logp_difference/mean": 0.010904813185334206, "step": 465 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0001181104889838025, "clip_ratio/low_min": 0.0001181104889838025, "clip_ratio/region_mean": 0.0001181104889838025, "completions/clipped_ratio": 0.28125, "completions/max_length": 768.0, "completions/max_terminated_length": 728.0, "completions/mean_length": 562.28125, "completions/mean_terminated_length": 481.7826232910156, "completions/min_length": 277.0, "completions/min_terminated_length": 277.0, "entropy": 0.5426614657044411, "epoch": 0.2494646680942184, "frac_reward_zero_std": 0.25, "grad_norm": 0.0032838040497153997, "learning_rate": 1e-05, "loss": 0.0348, "num_tokens": 9934723.0, "reward": 0.15625, "reward_std": 0.3198433816432953, "rewards/accuracy_reward/mean": 0.15625, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.4727178812026978, "sampling/importance_sampling_ratio/mean": 1.00029456615448, "sampling/importance_sampling_ratio/min": 0.677629292011261, "sampling/sampling_logp_difference/max": 0.38915491104125977, "sampling/sampling_logp_difference/mean": 0.015486430376768112, "step": 466 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 732.0, "completions/mean_length": 563.625, "completions/mean_terminated_length": 534.4285888671875, "completions/min_length": 62.0, "completions/min_terminated_length": 62.0, "entropy": 0.3231046833097935, "epoch": 0.25, "frac_reward_zero_std": 0.0, "grad_norm": 0.017085576429963112, "learning_rate": 1e-05, "loss": 0.0034, "num_tokens": 9956231.0, "reward": 0.6875, "reward_std": 0.4808131456375122, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.3188414573669434, "sampling/importance_sampling_ratio/mean": 1.0000824928283691, "sampling/importance_sampling_ratio/min": 0.7050997614860535, "sampling/sampling_logp_difference/max": 0.34941601753234863, "sampling/sampling_logp_difference/mean": 0.011022663675248623, "step": 467 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0001145213027484715, "clip_ratio/low_min": 0.0001145213027484715, "clip_ratio/region_mean": 0.0001145213027484715, "completions/clipped_ratio": 0.34375, "completions/max_length": 768.0, "completions/max_terminated_length": 683.0, "completions/mean_length": 584.28125, "completions/mean_terminated_length": 488.0476379394531, "completions/min_length": 252.0, "completions/min_terminated_length": 252.0, "entropy": 0.44339311495423317, "epoch": 0.2505353319057816, "frac_reward_zero_std": 0.5, "grad_norm": 0.017508333548903465, "learning_rate": 1e-05, "loss": 0.0851, "num_tokens": 9978784.0, "reward": 0.28125, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.28125, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.358830213546753, "sampling/importance_sampling_ratio/mean": 1.0002084970474243, "sampling/importance_sampling_ratio/min": 0.7079991698265076, "sampling/sampling_logp_difference/max": 0.34531235694885254, "sampling/sampling_logp_difference/mean": 0.012901067733764648, "step": 468 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00015160703333094716, "clip_ratio/low_min": 0.00015160703333094716, "clip_ratio/region_mean": 0.00015160703333094716, "completions/clipped_ratio": 0.0, "completions/max_length": 715.0, "completions/max_terminated_length": 715.0, "completions/mean_length": 433.3125, "completions/mean_terminated_length": 433.3125, "completions/min_length": 259.0, "completions/min_terminated_length": 259.0, "entropy": 0.45748865604400635, "epoch": 0.2510706638115632, "frac_reward_zero_std": 0.75, "grad_norm": 0.02379726991057396, "learning_rate": 1e-05, "loss": 0.0276, "num_tokens": 9995986.0, "reward": 0.6875, "reward_std": 0.1157275140285492, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.419661045074463, "sampling/importance_sampling_ratio/mean": 1.0002655982971191, "sampling/importance_sampling_ratio/min": 0.6812078952789307, "sampling/sampling_logp_difference/max": 0.38388776779174805, "sampling/sampling_logp_difference/mean": 0.013905711472034454, "step": 469 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00025374008691869676, "clip_ratio/low_min": 0.00025374008691869676, "clip_ratio/region_mean": 0.00025374008691869676, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 722.0, "completions/mean_length": 561.1875, "completions/mean_terminated_length": 492.25, "completions/min_length": 178.0, "completions/min_terminated_length": 178.0, "entropy": 0.42329367622733116, "epoch": 0.25160599571734477, "frac_reward_zero_std": 0.5, "grad_norm": 0.010336724109947681, "learning_rate": 1e-05, "loss": 0.036, "num_tokens": 10017960.0, "reward": 0.65625, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.559314489364624, "sampling/importance_sampling_ratio/mean": 0.999816358089447, "sampling/importance_sampling_ratio/min": 0.6663103699684143, "sampling/sampling_logp_difference/max": 0.4442462921142578, "sampling/sampling_logp_difference/mean": 0.013187477365136147, "step": 470 }, { "clip_ratio/high_max": 5.02008042531088e-05, "clip_ratio/high_mean": 5.02008042531088e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 5.02008042531088e-05, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 742.0, "completions/mean_length": 537.3125, "completions/mean_terminated_length": 504.357177734375, "completions/min_length": 300.0, "completions/min_terminated_length": 300.0, "entropy": 0.3696693442761898, "epoch": 0.25214132762312635, "frac_reward_zero_std": 0.5, "grad_norm": 0.0031507830135524273, "learning_rate": 1e-05, "loss": 0.0262, "num_tokens": 10038626.0, "reward": 0.78125, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.314389705657959, "sampling/importance_sampling_ratio/mean": 0.9997232556343079, "sampling/importance_sampling_ratio/min": 0.6920733451843262, "sampling/sampling_logp_difference/max": 0.3680633306503296, "sampling/sampling_logp_difference/mean": 0.011672900058329105, "step": 471 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0002072493007290177, "clip_ratio/low_min": 0.0002072493007290177, "clip_ratio/region_mean": 0.0002072493007290177, "completions/clipped_ratio": 0.40625, "completions/max_length": 768.0, "completions/max_terminated_length": 707.0, "completions/mean_length": 639.59375, "completions/mean_terminated_length": 551.73681640625, "completions/min_length": 250.0, "completions/min_terminated_length": 250.0, "entropy": 0.4280319772660732, "epoch": 0.25267665952890794, "frac_reward_zero_std": 0.25, "grad_norm": 0.004263668786734343, "learning_rate": 1e-05, "loss": 0.0685, "num_tokens": 10062741.0, "reward": 0.53125, "reward_std": 0.3377465009689331, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.4454926252365112, "sampling/importance_sampling_ratio/mean": 1.0005178451538086, "sampling/importance_sampling_ratio/min": 0.7545243501663208, "sampling/sampling_logp_difference/max": 0.3684501647949219, "sampling/sampling_logp_difference/mean": 0.01289775688201189, "step": 472 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.917543214512989e-05, "clip_ratio/low_min": 6.917543214512989e-05, "clip_ratio/region_mean": 6.917543214512989e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 636.0, "completions/max_terminated_length": 636.0, "completions/mean_length": 395.1875, "completions/mean_terminated_length": 395.1875, "completions/min_length": 248.0, "completions/min_terminated_length": 248.0, "entropy": 0.3105512037873268, "epoch": 0.25321199143468953, "frac_reward_zero_std": 0.25, "grad_norm": 0.011443409137427807, "learning_rate": 1e-05, "loss": -0.0437, "num_tokens": 10078795.0, "reward": 0.59375, "reward_std": 0.3061639964580536, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.2866857051849365, "sampling/importance_sampling_ratio/mean": 0.999954879283905, "sampling/importance_sampling_ratio/min": 0.7495657205581665, "sampling/sampling_logp_difference/max": 0.2882612943649292, "sampling/sampling_logp_difference/mean": 0.010995929129421711, "step": 473 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.58534411538858e-05, "clip_ratio/low_min": 5.58534411538858e-05, "clip_ratio/region_mean": 5.58534411538858e-05, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 694.0, "completions/mean_length": 481.34375, "completions/mean_terminated_length": 462.2333679199219, "completions/min_length": 304.0, "completions/min_terminated_length": 304.0, "entropy": 0.33481962233781815, "epoch": 0.25374732334047106, "frac_reward_zero_std": 0.5, "grad_norm": 0.009619650430977345, "learning_rate": 1e-05, "loss": 0.0067, "num_tokens": 10097694.0, "reward": 0.84375, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.4322324991226196, "sampling/importance_sampling_ratio/mean": 0.9997403025627136, "sampling/importance_sampling_ratio/min": 0.7026197910308838, "sampling/sampling_logp_difference/max": 0.3592343330383301, "sampling/sampling_logp_difference/mean": 0.011315671727061272, "step": 474 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00023391545983031392, "clip_ratio/low_min": 0.00023391545983031392, "clip_ratio/region_mean": 0.00023391545983031392, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 610.0, "completions/mean_length": 537.9375, "completions/mean_terminated_length": 461.25, "completions/min_length": 249.0, "completions/min_terminated_length": 249.0, "entropy": 0.4866362549364567, "epoch": 0.25428265524625265, "frac_reward_zero_std": 0.5, "grad_norm": 0.007133313920348883, "learning_rate": 1e-05, "loss": -0.0068, "num_tokens": 10118892.0, "reward": 0.71875, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.4732807874679565, "sampling/importance_sampling_ratio/mean": 1.0003376007080078, "sampling/importance_sampling_ratio/min": 0.5954201817512512, "sampling/sampling_logp_difference/max": 0.5184879302978516, "sampling/sampling_logp_difference/mean": 0.015425093472003937, "step": 475 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.645889905281365e-05, "clip_ratio/low_min": 5.645889905281365e-05, "clip_ratio/region_mean": 5.645889905281365e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 763.0, "completions/max_terminated_length": 763.0, "completions/mean_length": 486.96875, "completions/mean_terminated_length": 486.96875, "completions/min_length": 278.0, "completions/min_terminated_length": 278.0, "entropy": 0.3146357908844948, "epoch": 0.25481798715203424, "frac_reward_zero_std": 0.0, "grad_norm": 0.012660901062190533, "learning_rate": 1e-05, "loss": 0.0086, "num_tokens": 10138235.0, "reward": 0.65625, "reward_std": 0.4397946000099182, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.3446470499038696, "sampling/importance_sampling_ratio/mean": 0.9998399019241333, "sampling/importance_sampling_ratio/min": 0.6975634694099426, "sampling/sampling_logp_difference/max": 0.36016178131103516, "sampling/sampling_logp_difference/mean": 0.011733213439583778, "step": 476 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 763.0, "completions/max_terminated_length": 763.0, "completions/mean_length": 520.625, "completions/mean_terminated_length": 520.625, "completions/min_length": 242.0, "completions/min_terminated_length": 242.0, "entropy": 0.33602092042565346, "epoch": 0.25535331905781583, "frac_reward_zero_std": 0.5, "grad_norm": 0.0043482608161866665, "learning_rate": 1e-05, "loss": -0.0085, "num_tokens": 10158959.0, "reward": 0.78125, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.4700850248336792, "sampling/importance_sampling_ratio/mean": 1.0000849962234497, "sampling/importance_sampling_ratio/min": 0.6636093854904175, "sampling/sampling_logp_difference/max": 0.4100615978240967, "sampling/sampling_logp_difference/mean": 0.01105241198092699, "step": 477 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 687.0, "completions/mean_length": 485.6875, "completions/mean_terminated_length": 420.5384826660156, "completions/min_length": 303.0, "completions/min_terminated_length": 303.0, "entropy": 0.5346232987940311, "epoch": 0.2558886509635974, "frac_reward_zero_std": 0.25, "grad_norm": 0.01230322103947401, "learning_rate": 1e-05, "loss": 0.1196, "num_tokens": 10178821.0, "reward": 0.71875, "reward_std": 0.3608423173427582, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.7887417078018188, "sampling/importance_sampling_ratio/mean": 0.999797523021698, "sampling/importance_sampling_ratio/min": 0.7177157402038574, "sampling/sampling_logp_difference/max": 0.581512451171875, "sampling/sampling_logp_difference/mean": 0.016022939234972, "step": 478 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0001788809895515442, "clip_ratio/low_min": 0.0001788809895515442, "clip_ratio/region_mean": 0.0001788809895515442, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 745.0, "completions/mean_length": 501.0625, "completions/mean_terminated_length": 473.4482727050781, "completions/min_length": 323.0, "completions/min_terminated_length": 323.0, "entropy": 0.402830358594656, "epoch": 0.256423982869379, "frac_reward_zero_std": 0.5, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0022, "num_tokens": 10199175.0, "reward": 0.6875, "reward_std": 0.249358132481575, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.4302822351455688, "sampling/importance_sampling_ratio/mean": 0.9999926090240479, "sampling/importance_sampling_ratio/min": 0.5216460824012756, "sampling/sampling_logp_difference/max": 0.6507658958435059, "sampling/sampling_logp_difference/mean": 0.013747112825512886, "step": 479 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.610844553099014e-05, "clip_ratio/low_min": 4.610844553099014e-05, "clip_ratio/region_mean": 4.610844553099014e-05, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 743.0, "completions/mean_length": 561.1875, "completions/mean_terminated_length": 522.888916015625, "completions/min_length": 280.0, "completions/min_terminated_length": 280.0, "entropy": 0.33710160478949547, "epoch": 0.2569593147751606, "frac_reward_zero_std": 0.5, "grad_norm": 0.016739992424845695, "learning_rate": 1e-05, "loss": 0.0588, "num_tokens": 10220837.0, "reward": 0.78125, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0000311136245728, "sampling/importance_sampling_ratio/min": 0.72202467918396, "sampling/sampling_logp_difference/max": 0.849064826965332, "sampling/sampling_logp_difference/mean": 0.011132810264825821, "step": 480 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.106497312430292e-05, "clip_ratio/low_min": 6.106497312430292e-05, "clip_ratio/region_mean": 6.106497312430292e-05, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 761.0, "completions/mean_length": 488.09375, "completions/mean_terminated_length": 469.433349609375, "completions/min_length": 272.0, "completions/min_terminated_length": 272.0, "entropy": 0.4116736091673374, "epoch": 0.2574946466809422, "frac_reward_zero_std": 0.5, "grad_norm": 0.002297243569046259, "learning_rate": 1e-05, "loss": 0.008, "num_tokens": 10239800.0, "reward": 0.9375, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.9375, "rewards/accuracy_reward/std": 0.24593468010425568, "sampling/importance_sampling_ratio/max": 1.332295536994934, "sampling/importance_sampling_ratio/mean": 0.9997638463973999, "sampling/importance_sampling_ratio/min": 0.6012037396430969, "sampling/sampling_logp_difference/max": 0.5088214874267578, "sampling/sampling_logp_difference/mean": 0.01294345036149025, "step": 481 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 716.0, "completions/mean_length": 497.78125, "completions/mean_terminated_length": 489.06451416015625, "completions/min_length": 320.0, "completions/min_terminated_length": 320.0, "entropy": 0.29102923534810543, "epoch": 0.2580299785867238, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0257, "num_tokens": 10259297.0, "reward": 0.9375, "reward_std": 0.1157275140285492, "rewards/accuracy_reward/mean": 0.9375, "rewards/accuracy_reward/std": 0.24593468010425568, "sampling/importance_sampling_ratio/max": 1.4557756185531616, "sampling/importance_sampling_ratio/mean": 1.0003036260604858, "sampling/importance_sampling_ratio/min": 0.7349271774291992, "sampling/sampling_logp_difference/max": 0.37553882598876953, "sampling/sampling_logp_difference/mean": 0.010717620141804218, "step": 482 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.463286652229726e-05, "clip_ratio/low_min": 5.463286652229726e-05, "clip_ratio/region_mean": 5.463286652229726e-05, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 719.0, "completions/mean_length": 524.9375, "completions/mean_terminated_length": 456.8800048828125, "completions/min_length": 241.0, "completions/min_terminated_length": 241.0, "entropy": 0.3851854242384434, "epoch": 0.25856531049250536, "frac_reward_zero_std": 0.0, "grad_norm": 0.015280285850167274, "learning_rate": 1e-05, "loss": 0.0701, "num_tokens": 10280359.0, "reward": 0.71875, "reward_std": 0.4534739851951599, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.4300639629364014, "sampling/importance_sampling_ratio/mean": 1.0003355741500854, "sampling/importance_sampling_ratio/min": 0.687623918056488, "sampling/sampling_logp_difference/max": 0.3745131492614746, "sampling/sampling_logp_difference/mean": 0.01244751363992691, "step": 483 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 742.0, "completions/mean_length": 550.53125, "completions/mean_terminated_length": 519.4642944335938, "completions/min_length": 328.0, "completions/min_terminated_length": 328.0, "entropy": 0.32115957140922546, "epoch": 0.25910064239828695, "frac_reward_zero_std": 0.0, "grad_norm": 0.02360318973660469, "learning_rate": 1e-05, "loss": 0.0242, "num_tokens": 10301624.0, "reward": 0.75, "reward_std": 0.4261348247528076, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.4405555725097656, "sampling/importance_sampling_ratio/mean": 1.0000886917114258, "sampling/importance_sampling_ratio/min": 0.7017530798912048, "sampling/sampling_logp_difference/max": 0.36502885818481445, "sampling/sampling_logp_difference/mean": 0.011256097815930843, "step": 484 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.610413063550368e-05, "clip_ratio/low_min": 5.610413063550368e-05, "clip_ratio/region_mean": 5.610413063550368e-05, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 707.0, "completions/mean_length": 566.96875, "completions/mean_terminated_length": 499.9583435058594, "completions/min_length": 218.0, "completions/min_terminated_length": 218.0, "entropy": 0.4112725034356117, "epoch": 0.25963597430406854, "frac_reward_zero_std": 0.25, "grad_norm": 0.01185301411896944, "learning_rate": 1e-05, "loss": 0.0349, "num_tokens": 10323711.0, "reward": 0.75, "reward_std": 0.3745020925998688, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.6268253326416016, "sampling/importance_sampling_ratio/mean": 1.000152349472046, "sampling/importance_sampling_ratio/min": 0.7719011902809143, "sampling/sampling_logp_difference/max": 0.4866304397583008, "sampling/sampling_logp_difference/mean": 0.012860638089478016, "step": 485 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.52400849503465e-05, "clip_ratio/low_min": 6.52400849503465e-05, "clip_ratio/region_mean": 6.52400849503465e-05, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 671.0, "completions/mean_length": 445.53125, "completions/mean_terminated_length": 412.17242431640625, "completions/min_length": 138.0, "completions/min_terminated_length": 138.0, "entropy": 0.4744582735002041, "epoch": 0.26017130620985013, "frac_reward_zero_std": 0.25, "grad_norm": 0.021493274718523026, "learning_rate": 1e-05, "loss": 0.0594, "num_tokens": 10341472.0, "reward": 0.71875, "reward_std": 0.3471629321575165, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.3100839853286743, "sampling/importance_sampling_ratio/mean": 0.9998884797096252, "sampling/importance_sampling_ratio/min": 0.6793792843818665, "sampling/sampling_logp_difference/max": 0.38657569885253906, "sampling/sampling_logp_difference/mean": 0.013434859924018383, "step": 486 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 761.0, "completions/mean_length": 614.65625, "completions/mean_terminated_length": 586.25927734375, "completions/min_length": 364.0, "completions/min_terminated_length": 364.0, "entropy": 0.35626184195280075, "epoch": 0.2607066381156317, "frac_reward_zero_std": 0.5, "grad_norm": 0.004379448015242815, "learning_rate": 1e-05, "loss": 0.0517, "num_tokens": 10365389.0, "reward": 0.65625, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.442468523979187, "sampling/importance_sampling_ratio/mean": 1.000192403793335, "sampling/importance_sampling_ratio/min": 0.741629958152771, "sampling/sampling_logp_difference/max": 0.36635589599609375, "sampling/sampling_logp_difference/mean": 0.012494973838329315, "step": 487 }, { "clip_ratio/high_max": 6.145526276668534e-05, "clip_ratio/high_mean": 6.145526276668534e-05, "clip_ratio/low_mean": 0.00017596149700693786, "clip_ratio/low_min": 0.00017596149700693786, "clip_ratio/region_mean": 0.0002374167597736232, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 760.0, "completions/mean_length": 523.5625, "completions/mean_terminated_length": 498.2758483886719, "completions/min_length": 268.0, "completions/min_terminated_length": 268.0, "entropy": 0.42526591569185257, "epoch": 0.26124197002141325, "frac_reward_zero_std": 0.25, "grad_norm": 0.018124375492334366, "learning_rate": 1e-05, "loss": 0.0338, "num_tokens": 10386135.0, "reward": 0.6875, "reward_std": 0.2925041913986206, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.6005200147628784, "sampling/importance_sampling_ratio/mean": 1.0000367164611816, "sampling/importance_sampling_ratio/min": 0.5470401048660278, "sampling/sampling_logp_difference/max": 0.6032330989837646, "sampling/sampling_logp_difference/mean": 0.013319337740540504, "step": 488 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 719.0, "completions/mean_length": 593.21875, "completions/mean_terminated_length": 534.9583740234375, "completions/min_length": 365.0, "completions/min_terminated_length": 365.0, "entropy": 0.444037351757288, "epoch": 0.26177730192719484, "frac_reward_zero_std": 0.25, "grad_norm": 0.014569836668670177, "learning_rate": 1e-05, "loss": 0.0437, "num_tokens": 10409278.0, "reward": 0.625, "reward_std": 0.3745020925998688, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.433860421180725, "sampling/importance_sampling_ratio/mean": 1.000381350517273, "sampling/importance_sampling_ratio/min": 0.6514508128166199, "sampling/sampling_logp_difference/max": 0.42855334281921387, "sampling/sampling_logp_difference/mean": 0.013760481961071491, "step": 489 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 739.0, "completions/max_terminated_length": 739.0, "completions/mean_length": 475.1875, "completions/mean_terminated_length": 475.1875, "completions/min_length": 227.0, "completions/min_terminated_length": 227.0, "entropy": 0.4105031192302704, "epoch": 0.2623126338329764, "frac_reward_zero_std": 0.5, "grad_norm": 0.011106204241514206, "learning_rate": 1e-05, "loss": -0.0591, "num_tokens": 10428356.0, "reward": 0.71875, "reward_std": 0.2630178928375244, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.3552688360214233, "sampling/importance_sampling_ratio/mean": 1.000112771987915, "sampling/importance_sampling_ratio/min": 0.7021424174308777, "sampling/sampling_logp_difference/max": 0.3536190986633301, "sampling/sampling_logp_difference/mean": 0.013007367961108685, "step": 490 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 750.0, "completions/mean_length": 576.53125, "completions/mean_terminated_length": 512.7083740234375, "completions/min_length": 290.0, "completions/min_terminated_length": 290.0, "entropy": 0.40421775728464127, "epoch": 0.262847965738758, "frac_reward_zero_std": 0.5, "grad_norm": 0.007814164273440838, "learning_rate": 1e-05, "loss": 0.0464, "num_tokens": 10451053.0, "reward": 0.625, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.4316725730895996, "sampling/importance_sampling_ratio/mean": 1.0002444982528687, "sampling/importance_sampling_ratio/min": 0.6939866542816162, "sampling/sampling_logp_difference/max": 0.36530256271362305, "sampling/sampling_logp_difference/mean": 0.012022517621517181, "step": 491 }, { "clip_ratio/high_max": 4.22582816099748e-05, "clip_ratio/high_mean": 4.22582816099748e-05, "clip_ratio/low_mean": 0.00010322048183297738, "clip_ratio/low_min": 0.00010322048183297738, "clip_ratio/region_mean": 0.00014547876344295219, "completions/clipped_ratio": 0.34375, "completions/max_length": 768.0, "completions/max_terminated_length": 733.0, "completions/mean_length": 625.8125, "completions/mean_terminated_length": 551.3333740234375, "completions/min_length": 259.0, "completions/min_terminated_length": 259.0, "entropy": 0.3712773844599724, "epoch": 0.2633832976445396, "frac_reward_zero_std": 0.0, "grad_norm": 0.005484628025442362, "learning_rate": 1e-05, "loss": 0.0043, "num_tokens": 10474983.0, "reward": 0.5, "reward_std": 0.4082317352294922, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5080004930496216, "sampling/importance_sampling_ratio/max": 1.6351101398468018, "sampling/importance_sampling_ratio/mean": 0.9999654293060303, "sampling/importance_sampling_ratio/min": 0.6186308264732361, "sampling/sampling_logp_difference/max": 0.49171018600463867, "sampling/sampling_logp_difference/mean": 0.011516343802213669, "step": 492 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00014466403081314638, "clip_ratio/low_min": 0.00014466403081314638, "clip_ratio/region_mean": 0.00014466403081314638, "completions/clipped_ratio": 0.3125, "completions/max_length": 768.0, "completions/max_terminated_length": 735.0, "completions/mean_length": 602.125, "completions/mean_terminated_length": 526.727294921875, "completions/min_length": 395.0, "completions/min_terminated_length": 395.0, "entropy": 0.46679985895752907, "epoch": 0.2639186295503212, "frac_reward_zero_std": 0.25, "grad_norm": 0.009510909207165241, "learning_rate": 1e-05, "loss": 0.0668, "num_tokens": 10497787.0, "reward": 0.5625, "reward_std": 0.3471825420856476, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.4565629959106445, "sampling/importance_sampling_ratio/mean": 1.0000561475753784, "sampling/importance_sampling_ratio/min": 0.6113215684890747, "sampling/sampling_logp_difference/max": 0.49213218688964844, "sampling/sampling_logp_difference/mean": 0.014486799016594887, "step": 493 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.402958802529611e-05, "clip_ratio/low_min": 4.402958802529611e-05, "clip_ratio/region_mean": 4.402958802529611e-05, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 710.0, "completions/mean_length": 576.40625, "completions/mean_terminated_length": 532.1923217773438, "completions/min_length": 283.0, "completions/min_terminated_length": 283.0, "entropy": 0.4298512861132622, "epoch": 0.2644539614561028, "frac_reward_zero_std": 0.25, "grad_norm": 0.014777930453419685, "learning_rate": 1e-05, "loss": 0.0311, "num_tokens": 10519952.0, "reward": 0.46875, "reward_std": 0.378745436668396, "rewards/accuracy_reward/mean": 0.46875, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.420779824256897, "sampling/importance_sampling_ratio/mean": 1.0001146793365479, "sampling/importance_sampling_ratio/min": 0.5928531289100647, "sampling/sampling_logp_difference/max": 0.5228085517883301, "sampling/sampling_logp_difference/mean": 0.013614621013402939, "step": 494 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.510417006211355e-05, "clip_ratio/low_min": 6.510417006211355e-05, "clip_ratio/region_mean": 6.510417006211355e-05, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 750.0, "completions/mean_length": 543.71875, "completions/mean_terminated_length": 468.9583435058594, "completions/min_length": 288.0, "completions/min_terminated_length": 288.0, "entropy": 0.5003023743629456, "epoch": 0.26498929336188437, "frac_reward_zero_std": 0.25, "grad_norm": 0.013744396157562733, "learning_rate": 1e-05, "loss": 0.0594, "num_tokens": 10541527.0, "reward": 0.46875, "reward_std": 0.35564959049224854, "rewards/accuracy_reward/mean": 0.46875, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0005255937576294, "sampling/importance_sampling_ratio/min": 0.6238965392112732, "sampling/sampling_logp_difference/max": 0.695166826248169, "sampling/sampling_logp_difference/mean": 0.015651416033506393, "step": 495 }, { "clip_ratio/high_max": 6.148548709461465e-05, "clip_ratio/high_mean": 6.148548709461465e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 6.148548709461465e-05, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 729.0, "completions/mean_length": 455.9375, "completions/mean_terminated_length": 445.8709411621094, "completions/min_length": 161.0, "completions/min_terminated_length": 161.0, "entropy": 0.3635113351047039, "epoch": 0.26552462526766596, "frac_reward_zero_std": 0.25, "grad_norm": 0.015172747895121574, "learning_rate": 1e-05, "loss": -0.0157, "num_tokens": 10559381.0, "reward": 0.75, "reward_std": 0.2925041913986206, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0002249479293823, "sampling/importance_sampling_ratio/min": 0.19352464377880096, "sampling/sampling_logp_difference/max": 1.642350435256958, "sampling/sampling_logp_difference/mean": 0.012256532907485962, "step": 496 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.6159526391420513e-05, "clip_ratio/low_min": 4.6159526391420513e-05, "clip_ratio/region_mean": 4.6159526391420513e-05, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 766.0, "completions/mean_length": 635.875, "completions/mean_terminated_length": 611.4074096679688, "completions/min_length": 366.0, "completions/min_terminated_length": 366.0, "entropy": 0.32336436584591866, "epoch": 0.26605995717344755, "frac_reward_zero_std": 0.25, "grad_norm": 0.018259260803461075, "learning_rate": 1e-05, "loss": 0.0305, "num_tokens": 10583457.0, "reward": 0.78125, "reward_std": 0.3061639964580536, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.597190260887146, "sampling/importance_sampling_ratio/mean": 0.9999054074287415, "sampling/importance_sampling_ratio/min": 0.5457773804664612, "sampling/sampling_logp_difference/max": 0.6055440902709961, "sampling/sampling_logp_difference/mean": 0.010763855651021004, "step": 497 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 727.0, "completions/mean_length": 559.3125, "completions/mean_terminated_length": 552.5806274414062, "completions/min_length": 287.0, "completions/min_terminated_length": 287.0, "entropy": 0.3208707682788372, "epoch": 0.26659528907922914, "frac_reward_zero_std": 0.75, "grad_norm": 0.003419401589781046, "learning_rate": 1e-05, "loss": -0.0228, "num_tokens": 10604875.0, "reward": 0.96875, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.96875, "rewards/accuracy_reward/std": 0.1767766922712326, "sampling/importance_sampling_ratio/max": 1.5586161613464355, "sampling/importance_sampling_ratio/mean": 1.0004326105117798, "sampling/importance_sampling_ratio/min": 0.7551673650741577, "sampling/sampling_logp_difference/max": 0.443798303604126, "sampling/sampling_logp_difference/mean": 0.011351877823472023, "step": 498 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 616.0, "completions/mean_length": 436.875, "completions/mean_terminated_length": 426.19354248046875, "completions/min_length": 245.0, "completions/min_terminated_length": 245.0, "entropy": 0.3958047814667225, "epoch": 0.2671306209850107, "frac_reward_zero_std": 0.5, "grad_norm": 0.014426103793084621, "learning_rate": 1e-05, "loss": 0.0226, "num_tokens": 10622375.0, "reward": 0.75, "reward_std": 0.26726123690605164, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.4606636762619019, "sampling/importance_sampling_ratio/mean": 0.9991129040718079, "sampling/importance_sampling_ratio/min": 0.6251304149627686, "sampling/sampling_logp_difference/max": 0.46979498863220215, "sampling/sampling_logp_difference/mean": 0.01428675651550293, "step": 499 }, { "clip_ratio/high_max": 4.4452346628531814e-05, "clip_ratio/high_mean": 4.4452346628531814e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 4.4452346628531814e-05, "completions/clipped_ratio": 0.3125, "completions/max_length": 768.0, "completions/max_terminated_length": 744.0, "completions/mean_length": 583.375, "completions/mean_terminated_length": 499.4545593261719, "completions/min_length": 205.0, "completions/min_terminated_length": 205.0, "entropy": 0.6192383728921413, "epoch": 0.2676659528907923, "frac_reward_zero_std": 0.25, "grad_norm": 0.014534000307321548, "learning_rate": 1e-05, "loss": 0.1002, "num_tokens": 10645627.0, "reward": 0.59375, "reward_std": 0.3061639666557312, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.4215008020401, "sampling/importance_sampling_ratio/mean": 1.0001673698425293, "sampling/importance_sampling_ratio/min": 0.7219524383544922, "sampling/sampling_logp_difference/max": 0.3517131805419922, "sampling/sampling_logp_difference/mean": 0.016877198591828346, "step": 500 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.530973430722952e-05, "clip_ratio/low_min": 5.530973430722952e-05, "clip_ratio/region_mean": 5.530973430722952e-05, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 640.0, "completions/mean_length": 499.4375, "completions/mean_terminated_length": 449.7037048339844, "completions/min_length": 245.0, "completions/min_terminated_length": 245.0, "entropy": 0.5286917313933372, "epoch": 0.2682012847965739, "frac_reward_zero_std": 0.25, "grad_norm": 0.025257959961891174, "learning_rate": 1e-05, "loss": 0.0305, "num_tokens": 10665641.0, "reward": 0.6875, "reward_std": 0.292504221200943, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.4534956216812134, "sampling/importance_sampling_ratio/mean": 1.0004061460494995, "sampling/importance_sampling_ratio/min": 0.6537541151046753, "sampling/sampling_logp_difference/max": 0.42502403259277344, "sampling/sampling_logp_difference/mean": 0.015922224149107933, "step": 501 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 650.0, "completions/max_terminated_length": 650.0, "completions/mean_length": 443.59375, "completions/mean_terminated_length": 443.59375, "completions/min_length": 222.0, "completions/min_terminated_length": 222.0, "entropy": 0.34683436155319214, "epoch": 0.26873661670235544, "frac_reward_zero_std": 0.5, "grad_norm": 0.0054996381513774395, "learning_rate": 1e-05, "loss": -0.0156, "num_tokens": 10683708.0, "reward": 0.9375, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.9375, "rewards/accuracy_reward/std": 0.24593468010425568, "sampling/importance_sampling_ratio/max": 1.4331287145614624, "sampling/importance_sampling_ratio/mean": 0.999981164932251, "sampling/importance_sampling_ratio/min": 0.6877760887145996, "sampling/sampling_logp_difference/max": 0.37429189682006836, "sampling/sampling_logp_difference/mean": 0.01251487247645855, "step": 502 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 766.0, "completions/mean_length": 509.375, "completions/mean_terminated_length": 482.6206970214844, "completions/min_length": 222.0, "completions/min_terminated_length": 222.0, "entropy": 0.41795216500759125, "epoch": 0.269271948608137, "frac_reward_zero_std": 0.25, "grad_norm": 0.03215835988521576, "learning_rate": 1e-05, "loss": 0.0994, "num_tokens": 10704144.0, "reward": 0.875, "reward_std": 0.292504221200943, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 1.3911006450653076, "sampling/importance_sampling_ratio/mean": 0.9998789429664612, "sampling/importance_sampling_ratio/min": 0.7018213272094727, "sampling/sampling_logp_difference/max": 0.3540763854980469, "sampling/sampling_logp_difference/mean": 0.013545467518270016, "step": 503 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0001050420178216882, "clip_ratio/low_min": 0.0001050420178216882, "clip_ratio/region_mean": 0.0001050420178216882, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 768.0, "completions/mean_length": 517.96875, "completions/mean_terminated_length": 482.2500305175781, "completions/min_length": 282.0, "completions/min_terminated_length": 282.0, "entropy": 0.4228440932929516, "epoch": 0.2698072805139186, "frac_reward_zero_std": 0.5, "grad_norm": 0.013474730774760246, "learning_rate": 1e-05, "loss": -0.034, "num_tokens": 10724279.0, "reward": 0.75, "reward_std": 0.2587745785713196, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.6665763854980469, "sampling/importance_sampling_ratio/mean": 1.0000231266021729, "sampling/importance_sampling_ratio/min": 0.702028214931488, "sampling/sampling_logp_difference/max": 0.5107715129852295, "sampling/sampling_logp_difference/mean": 0.013398643583059311, "step": 504 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00020762170606758446, "clip_ratio/low_min": 0.00020762170606758446, "clip_ratio/region_mean": 0.00020762170606758446, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 763.0, "completions/mean_length": 609.875, "completions/mean_terminated_length": 565.5999755859375, "completions/min_length": 255.0, "completions/min_terminated_length": 255.0, "entropy": 0.4561927057802677, "epoch": 0.2703426124197002, "frac_reward_zero_std": 0.0, "grad_norm": 0.012891552411019802, "learning_rate": 1e-05, "loss": 0.0592, "num_tokens": 10747835.0, "reward": 0.53125, "reward_std": 0.521792471408844, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.4030518531799316, "sampling/importance_sampling_ratio/mean": 0.9998916983604431, "sampling/importance_sampling_ratio/min": 0.6945270895957947, "sampling/sampling_logp_difference/max": 0.36452412605285645, "sampling/sampling_logp_difference/mean": 0.013256196863949299, "step": 505 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.773672091891058e-05, "clip_ratio/low_min": 5.773672091891058e-05, "clip_ratio/region_mean": 5.773672091891058e-05, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 757.0, "completions/mean_length": 523.375, "completions/mean_terminated_length": 498.0689697265625, "completions/min_length": 310.0, "completions/min_terminated_length": 310.0, "entropy": 0.4539306275546551, "epoch": 0.2708779443254818, "frac_reward_zero_std": 0.25, "grad_norm": 0.012112719006836414, "learning_rate": 1e-05, "loss": 0.0053, "num_tokens": 10768615.0, "reward": 0.625, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.3568693399429321, "sampling/importance_sampling_ratio/mean": 1.0000436305999756, "sampling/importance_sampling_ratio/min": 0.6628824472427368, "sampling/sampling_logp_difference/max": 0.41115760803222656, "sampling/sampling_logp_difference/mean": 0.013466725125908852, "step": 506 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 734.0, "completions/mean_length": 534.6875, "completions/mean_terminated_length": 501.357177734375, "completions/min_length": 286.0, "completions/min_terminated_length": 286.0, "entropy": 0.35264989361166954, "epoch": 0.2714132762312634, "frac_reward_zero_std": 0.5, "grad_norm": 0.015589317306876183, "learning_rate": 1e-05, "loss": 0.052, "num_tokens": 10789813.0, "reward": 0.8125, "reward_std": 0.2587745785713196, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.425531029701233, "sampling/importance_sampling_ratio/mean": 1.0001498460769653, "sampling/importance_sampling_ratio/min": 0.6998605132102966, "sampling/sampling_logp_difference/max": 0.3568742275238037, "sampling/sampling_logp_difference/mean": 0.011807294562458992, "step": 507 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0001117068823077716, "clip_ratio/low_min": 0.0001117068823077716, "clip_ratio/region_mean": 0.0001117068823077716, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 759.0, "completions/mean_length": 581.4375, "completions/mean_terminated_length": 519.25, "completions/min_length": 277.0, "completions/min_terminated_length": 277.0, "entropy": 0.501917477697134, "epoch": 0.27194860813704497, "frac_reward_zero_std": 0.25, "grad_norm": 0.016425846144557, "learning_rate": 1e-05, "loss": 0.0573, "num_tokens": 10811963.0, "reward": 0.53125, "reward_std": 0.35564959049224854, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.439595341682434, "sampling/importance_sampling_ratio/mean": 1.0000243186950684, "sampling/importance_sampling_ratio/min": 0.7093459367752075, "sampling/sampling_logp_difference/max": 0.3643620014190674, "sampling/sampling_logp_difference/mean": 0.015107567422091961, "step": 508 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 8.520791016053408e-05, "clip_ratio/low_min": 8.520791016053408e-05, "clip_ratio/region_mean": 8.520791016053408e-05, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 764.0, "completions/mean_length": 475.34375, "completions/mean_terminated_length": 465.9031982421875, "completions/min_length": 238.0, "completions/min_terminated_length": 238.0, "entropy": 0.29503167793154716, "epoch": 0.27248394004282656, "frac_reward_zero_std": 0.75, "grad_norm": 0.005277770571410656, "learning_rate": 1e-05, "loss": -0.0383, "num_tokens": 10830950.0, "reward": 0.90625, "reward_std": 0.1293872892856598, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.4213597774505615, "sampling/importance_sampling_ratio/mean": 1.000784158706665, "sampling/importance_sampling_ratio/min": 0.7571797370910645, "sampling/sampling_logp_difference/max": 0.35161399841308594, "sampling/sampling_logp_difference/mean": 0.010759200900793076, "step": 509 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.390254591475241e-05, "clip_ratio/low_min": 5.390254591475241e-05, "clip_ratio/region_mean": 5.390254591475241e-05, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 705.0, "completions/mean_length": 499.03125, "completions/mean_terminated_length": 471.2069091796875, "completions/min_length": 177.0, "completions/min_terminated_length": 177.0, "entropy": 0.3587932847440243, "epoch": 0.27301927194860814, "frac_reward_zero_std": 0.25, "grad_norm": 0.005858589429408312, "learning_rate": 1e-05, "loss": 0.0437, "num_tokens": 10850343.0, "reward": 0.8125, "reward_std": 0.3104073107242584, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.356927514076233, "sampling/importance_sampling_ratio/mean": 1.0000473260879517, "sampling/importance_sampling_ratio/min": 0.5159573554992676, "sampling/sampling_logp_difference/max": 0.6617312431335449, "sampling/sampling_logp_difference/mean": 0.011431219056248665, "step": 510 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.076811041566543e-05, "clip_ratio/low_min": 6.076811041566543e-05, "clip_ratio/region_mean": 6.076811041566543e-05, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 690.0, "completions/mean_length": 530.6875, "completions/mean_terminated_length": 496.7857360839844, "completions/min_length": 268.0, "completions/min_terminated_length": 268.0, "entropy": 0.3550248220562935, "epoch": 0.27355460385438973, "frac_reward_zero_std": 0.5, "grad_norm": 0.01730632595717907, "learning_rate": 1e-05, "loss": 0.0231, "num_tokens": 10871165.0, "reward": 0.78125, "reward_std": 0.2630178928375244, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.4192852973937988, "sampling/importance_sampling_ratio/mean": 0.9999898672103882, "sampling/importance_sampling_ratio/min": 0.27570825815200806, "sampling/sampling_logp_difference/max": 1.2884119749069214, "sampling/sampling_logp_difference/mean": 0.011719655245542526, "step": 511 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 641.0, "completions/max_terminated_length": 641.0, "completions/mean_length": 407.25, "completions/mean_terminated_length": 407.25, "completions/min_length": 248.0, "completions/min_terminated_length": 248.0, "entropy": 0.45111938565969467, "epoch": 0.2740899357601713, "frac_reward_zero_std": 0.25, "grad_norm": 0.015476170927286148, "learning_rate": 1e-05, "loss": -0.0452, "num_tokens": 10887989.0, "reward": 0.53125, "reward_std": 0.3198433816432953, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.4185147285461426, "sampling/importance_sampling_ratio/mean": 1.0004252195358276, "sampling/importance_sampling_ratio/min": 0.5482162833213806, "sampling/sampling_logp_difference/max": 0.6010854244232178, "sampling/sampling_logp_difference/mean": 0.012268926948308945, "step": 512 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 678.0, "completions/max_terminated_length": 678.0, "completions/mean_length": 485.71875, "completions/mean_terminated_length": 485.71875, "completions/min_length": 258.0, "completions/min_terminated_length": 258.0, "entropy": 0.34177630580961704, "epoch": 0.2746252676659529, "frac_reward_zero_std": 0.25, "grad_norm": 0.008045018650591373, "learning_rate": 1e-05, "loss": 0.0137, "num_tokens": 10907468.0, "reward": 0.65625, "reward_std": 0.3787454068660736, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.4314159154891968, "sampling/importance_sampling_ratio/mean": 0.9998471736907959, "sampling/importance_sampling_ratio/min": 0.4723207354545593, "sampling/sampling_logp_difference/max": 0.7500970363616943, "sampling/sampling_logp_difference/mean": 0.012011357583105564, "step": 513 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 707.0, "completions/mean_length": 567.15625, "completions/mean_terminated_length": 520.8077392578125, "completions/min_length": 306.0, "completions/min_terminated_length": 306.0, "entropy": 0.3598812371492386, "epoch": 0.2751605995717345, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 10929745.0, "reward": 0.5, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5080004930496216, "sampling/importance_sampling_ratio/max": 1.4048995971679688, "sampling/importance_sampling_ratio/mean": 0.9999309182167053, "sampling/importance_sampling_ratio/min": 0.7220192551612854, "sampling/sampling_logp_difference/max": 0.3399658203125, "sampling/sampling_logp_difference/mean": 0.010898095555603504, "step": 514 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 9.611687710275874e-05, "clip_ratio/low_min": 9.611687710275874e-05, "clip_ratio/region_mean": 9.611687710275874e-05, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 676.0, "completions/mean_length": 573.25, "completions/mean_terminated_length": 518.719970703125, "completions/min_length": 253.0, "completions/min_terminated_length": 253.0, "entropy": 0.2508980128914118, "epoch": 0.2756959314775161, "frac_reward_zero_std": 0.5, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0214, "num_tokens": 10952049.0, "reward": 0.71875, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.2707433700561523, "sampling/importance_sampling_ratio/mean": 0.9999803900718689, "sampling/importance_sampling_ratio/min": 0.4754341244697571, "sampling/sampling_logp_difference/max": 0.7435269355773926, "sampling/sampling_logp_difference/mean": 0.008386613801121712, "step": 515 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00019170560699421912, "clip_ratio/low_min": 0.00019170560699421912, "clip_ratio/region_mean": 0.00019170560699421912, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 753.0, "completions/mean_length": 594.9375, "completions/mean_terminated_length": 562.888916015625, "completions/min_length": 257.0, "completions/min_terminated_length": 257.0, "entropy": 0.3323214463889599, "epoch": 0.2762312633832976, "frac_reward_zero_std": 0.25, "grad_norm": 0.019251735880970955, "learning_rate": 1e-05, "loss": 0.0146, "num_tokens": 10975007.0, "reward": 0.65625, "reward_std": 0.3608423173427582, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.4540327787399292, "sampling/importance_sampling_ratio/mean": 1.0000197887420654, "sampling/importance_sampling_ratio/min": 0.7002866864204407, "sampling/sampling_logp_difference/max": 0.37434089183807373, "sampling/sampling_logp_difference/mean": 0.011816645041108131, "step": 516 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 693.0, "completions/mean_length": 507.28125, "completions/mean_terminated_length": 459.0, "completions/min_length": 182.0, "completions/min_terminated_length": 182.0, "entropy": 0.44671597331762314, "epoch": 0.2767665952890792, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0322, "num_tokens": 10995560.0, "reward": 0.71875, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.6457420587539673, "sampling/importance_sampling_ratio/mean": 1.000201940536499, "sampling/importance_sampling_ratio/min": 0.6064935922622681, "sampling/sampling_logp_difference/max": 0.50006103515625, "sampling/sampling_logp_difference/mean": 0.014169696718454361, "step": 517 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.3125, "completions/max_length": 768.0, "completions/max_terminated_length": 747.0, "completions/mean_length": 580.875, "completions/mean_terminated_length": 495.8182067871094, "completions/min_length": 314.0, "completions/min_terminated_length": 314.0, "entropy": 0.38640883564949036, "epoch": 0.2773019271948608, "frac_reward_zero_std": 0.0, "grad_norm": 0.025809966027736664, "learning_rate": 1e-05, "loss": 0.0468, "num_tokens": 11018028.0, "reward": 0.5, "reward_std": 0.4671337604522705, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5080004930496216, "sampling/importance_sampling_ratio/max": 1.3842450380325317, "sampling/importance_sampling_ratio/mean": 1.0000641345977783, "sampling/importance_sampling_ratio/min": 0.7114458680152893, "sampling/sampling_logp_difference/max": 0.3404560089111328, "sampling/sampling_logp_difference/mean": 0.012557556852698326, "step": 518 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00011339053526171483, "clip_ratio/low_min": 0.00011339053526171483, "clip_ratio/region_mean": 0.00011339053526171483, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 758.0, "completions/mean_length": 542.625, "completions/mean_terminated_length": 519.3103637695312, "completions/min_length": 293.0, "completions/min_terminated_length": 293.0, "entropy": 0.426570650190115, "epoch": 0.2778372591006424, "frac_reward_zero_std": 0.25, "grad_norm": 0.010524033568799496, "learning_rate": 1e-05, "loss": -0.0037, "num_tokens": 11039184.0, "reward": 0.65625, "reward_std": 0.3471629321575165, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.4324995279312134, "sampling/importance_sampling_ratio/mean": 0.9998242855072021, "sampling/importance_sampling_ratio/min": 0.6445524096488953, "sampling/sampling_logp_difference/max": 0.43919920921325684, "sampling/sampling_logp_difference/mean": 0.013481113128364086, "step": 519 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.4229934903560206e-05, "clip_ratio/low_min": 5.4229934903560206e-05, "clip_ratio/region_mean": 5.4229934903560206e-05, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 698.0, "completions/mean_length": 452.375, "completions/mean_terminated_length": 442.19354248046875, "completions/min_length": 234.0, "completions/min_terminated_length": 234.0, "entropy": 0.3255032114684582, "epoch": 0.278372591006424, "frac_reward_zero_std": 0.5, "grad_norm": 0.012102417647838593, "learning_rate": 1e-05, "loss": 0.0121, "num_tokens": 11057236.0, "reward": 0.6875, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.5126256942749023, "sampling/importance_sampling_ratio/mean": 0.9996980428695679, "sampling/importance_sampling_ratio/min": 0.6459639668464661, "sampling/sampling_logp_difference/max": 0.43701159954071045, "sampling/sampling_logp_difference/mean": 0.010542369447648525, "step": 520 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.091649654787034e-05, "clip_ratio/low_min": 5.091649654787034e-05, "clip_ratio/region_mean": 5.091649654787034e-05, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 700.0, "completions/mean_length": 560.28125, "completions/mean_terminated_length": 491.04168701171875, "completions/min_length": 235.0, "completions/min_terminated_length": 235.0, "entropy": 0.3828450106084347, "epoch": 0.27890792291220556, "frac_reward_zero_std": 0.25, "grad_norm": 0.006895342841744423, "learning_rate": 1e-05, "loss": 0.01, "num_tokens": 11079325.0, "reward": 0.6875, "reward_std": 0.292504221200943, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.5072721242904663, "sampling/importance_sampling_ratio/mean": 1.000172734260559, "sampling/importance_sampling_ratio/min": 0.6726121306419373, "sampling/sampling_logp_difference/max": 0.41030144691467285, "sampling/sampling_logp_difference/mean": 0.011244766414165497, "step": 521 }, { "clip_ratio/high_max": 4.6227811253629625e-05, "clip_ratio/high_mean": 4.6227811253629625e-05, "clip_ratio/low_mean": 0.0002374210707785096, "clip_ratio/low_min": 0.0002374210707785096, "clip_ratio/region_mean": 0.0002836488820321392, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 766.0, "completions/mean_length": 612.96875, "completions/mean_terminated_length": 569.5599975585938, "completions/min_length": 383.0, "completions/min_terminated_length": 383.0, "entropy": 0.4224571939557791, "epoch": 0.27944325481798715, "frac_reward_zero_std": 0.25, "grad_norm": 0.02829921804368496, "learning_rate": 1e-05, "loss": 0.0335, "num_tokens": 11102820.0, "reward": 0.6875, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.4662220478057861, "sampling/importance_sampling_ratio/mean": 0.9996917843818665, "sampling/importance_sampling_ratio/min": 0.6035193204879761, "sampling/sampling_logp_difference/max": 0.5049772262573242, "sampling/sampling_logp_difference/mean": 0.01334192045032978, "step": 522 }, { "clip_ratio/high_max": 7.212925265775993e-05, "clip_ratio/high_mean": 7.212925265775993e-05, "clip_ratio/low_mean": 7.851758709875867e-05, "clip_ratio/low_min": 7.851758709875867e-05, "clip_ratio/region_mean": 0.0001506468397565186, "completions/clipped_ratio": 0.0, "completions/max_length": 676.0, "completions/max_terminated_length": 676.0, "completions/mean_length": 452.46875, "completions/mean_terminated_length": 452.46875, "completions/min_length": 287.0, "completions/min_terminated_length": 287.0, "entropy": 0.34968700259923935, "epoch": 0.27997858672376874, "frac_reward_zero_std": 0.25, "grad_norm": 0.014668166637420654, "learning_rate": 1e-05, "loss": 0.0144, "num_tokens": 11120483.0, "reward": 0.65625, "reward_std": 0.3061639964580536, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.4449950456619263, "sampling/importance_sampling_ratio/mean": 0.9998252987861633, "sampling/importance_sampling_ratio/min": 0.7312772274017334, "sampling/sampling_logp_difference/max": 0.3681058883666992, "sampling/sampling_logp_difference/mean": 0.011569187045097351, "step": 523 }, { "clip_ratio/high_max": 0.00010992674651788548, "clip_ratio/high_mean": 0.00010992674651788548, "clip_ratio/low_mean": 4.39831092080567e-05, "clip_ratio/low_min": 4.39831092080567e-05, "clip_ratio/region_mean": 0.00015390985572594218, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 757.0, "completions/mean_length": 553.1875, "completions/mean_terminated_length": 522.5, "completions/min_length": 240.0, "completions/min_terminated_length": 240.0, "entropy": 0.33637315034866333, "epoch": 0.28051391862955033, "frac_reward_zero_std": 0.0, "grad_norm": 0.010086177848279476, "learning_rate": 1e-05, "loss": 0.0157, "num_tokens": 11142033.0, "reward": 0.78125, "reward_std": 0.4218915104866028, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.4344085454940796, "sampling/importance_sampling_ratio/mean": 1.0000633001327515, "sampling/importance_sampling_ratio/min": 0.7605924606323242, "sampling/sampling_logp_difference/max": 0.36075258255004883, "sampling/sampling_logp_difference/mean": 0.011049482971429825, "step": 524 }, { "clip_ratio/high_max": 7.122506940504536e-05, "clip_ratio/high_mean": 7.122506940504536e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 7.122506940504536e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 703.0, "completions/max_terminated_length": 703.0, "completions/mean_length": 424.875, "completions/mean_terminated_length": 424.875, "completions/min_length": 97.0, "completions/min_terminated_length": 97.0, "entropy": 0.3426404930651188, "epoch": 0.2810492505353319, "frac_reward_zero_std": 0.5, "grad_norm": 0.0028623396065086126, "learning_rate": 1e-05, "loss": -0.0547, "num_tokens": 11158925.0, "reward": 0.90625, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.6245464086532593, "sampling/importance_sampling_ratio/mean": 0.9997588396072388, "sampling/importance_sampling_ratio/min": 0.5904847979545593, "sampling/sampling_logp_difference/max": 0.5268113613128662, "sampling/sampling_logp_difference/mean": 0.011864609085023403, "step": 525 }, { "clip_ratio/high_max": 5.600358417723328e-05, "clip_ratio/high_mean": 5.600358417723328e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 5.600358417723328e-05, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 755.0, "completions/mean_length": 551.84375, "completions/mean_terminated_length": 529.4827270507812, "completions/min_length": 318.0, "completions/min_terminated_length": 318.0, "entropy": 0.48494572564959526, "epoch": 0.2815845824411135, "frac_reward_zero_std": 0.0, "grad_norm": 0.018498221412301064, "learning_rate": 1e-05, "loss": 0.0008, "num_tokens": 11180128.0, "reward": 0.59375, "reward_std": 0.4628904461860657, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.4667741060256958, "sampling/importance_sampling_ratio/mean": 1.0001482963562012, "sampling/importance_sampling_ratio/min": 0.6934671998023987, "sampling/sampling_logp_difference/max": 0.38306546211242676, "sampling/sampling_logp_difference/mean": 0.014432166703045368, "step": 526 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 714.0, "completions/mean_length": 474.1875, "completions/mean_terminated_length": 419.77777099609375, "completions/min_length": 214.0, "completions/min_terminated_length": 214.0, "entropy": 0.31750617176294327, "epoch": 0.2821199143468951, "frac_reward_zero_std": 0.5, "grad_norm": 0.007369263097643852, "learning_rate": 1e-05, "loss": -0.0133, "num_tokens": 11198910.0, "reward": 0.8125, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.366921067237854, "sampling/importance_sampling_ratio/mean": 0.9999237656593323, "sampling/importance_sampling_ratio/min": 0.747006893157959, "sampling/sampling_logp_difference/max": 0.3125607967376709, "sampling/sampling_logp_difference/mean": 0.010578572750091553, "step": 527 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 722.0, "completions/mean_length": 485.90625, "completions/mean_terminated_length": 476.8064270019531, "completions/min_length": 239.0, "completions/min_terminated_length": 239.0, "entropy": 0.29819491878151894, "epoch": 0.2826552462526767, "frac_reward_zero_std": 0.75, "grad_norm": 0.006596745923161507, "learning_rate": 1e-05, "loss": 0.0212, "num_tokens": 11217907.0, "reward": 0.96875, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.96875, "rewards/accuracy_reward/std": 0.1767766922712326, "sampling/importance_sampling_ratio/max": 1.779435396194458, "sampling/importance_sampling_ratio/mean": 0.9997535347938538, "sampling/importance_sampling_ratio/min": 0.7113785147666931, "sampling/sampling_logp_difference/max": 0.5762960910797119, "sampling/sampling_logp_difference/mean": 0.01006353460252285, "step": 528 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00015349980094470084, "clip_ratio/low_min": 0.00015349980094470084, "clip_ratio/region_mean": 0.00015349980094470084, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 734.0, "completions/mean_length": 504.9375, "completions/mean_terminated_length": 477.72412109375, "completions/min_length": 251.0, "completions/min_terminated_length": 251.0, "entropy": 0.33065625093877316, "epoch": 0.2831905781584582, "frac_reward_zero_std": 0.5, "grad_norm": 0.01009507104754448, "learning_rate": 1e-05, "loss": 0.0059, "num_tokens": 11237785.0, "reward": 0.75, "reward_std": 0.2587745785713196, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.397469162940979, "sampling/importance_sampling_ratio/mean": 1.0005205869674683, "sampling/importance_sampling_ratio/min": 0.7454022169113159, "sampling/sampling_logp_difference/max": 0.33466291427612305, "sampling/sampling_logp_difference/mean": 0.010764190927147865, "step": 529 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00011261971667408943, "clip_ratio/low_min": 0.00011261971667408943, "clip_ratio/region_mean": 0.00011261971667408943, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 735.0, "completions/mean_length": 540.59375, "completions/mean_terminated_length": 498.4814758300781, "completions/min_length": 219.0, "completions/min_terminated_length": 219.0, "entropy": 0.3277864959090948, "epoch": 0.2837259100642398, "frac_reward_zero_std": 0.75, "grad_norm": 0.01308476086705923, "learning_rate": 1e-05, "loss": 0.0069, "num_tokens": 11259068.0, "reward": 0.84375, "reward_std": 0.1293872892856598, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.7836689949035645, "sampling/importance_sampling_ratio/mean": 1.0001953840255737, "sampling/importance_sampling_ratio/min": 0.731069803237915, "sampling/sampling_logp_difference/max": 0.5786724090576172, "sampling/sampling_logp_difference/mean": 0.010434354655444622, "step": 530 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00013345943807507865, "clip_ratio/low_min": 0.00013345943807507865, "clip_ratio/region_mean": 0.00013345943807507865, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 640.0, "completions/mean_length": 448.125, "completions/mean_terminated_length": 437.8064270019531, "completions/min_length": 263.0, "completions/min_terminated_length": 263.0, "entropy": 0.340240228921175, "epoch": 0.2842612419700214, "frac_reward_zero_std": 0.0, "grad_norm": 0.008644849993288517, "learning_rate": 1e-05, "loss": 0.0528, "num_tokens": 11277008.0, "reward": 0.78125, "reward_std": 0.4218915104866028, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.3213413953781128, "sampling/importance_sampling_ratio/mean": 1.000316858291626, "sampling/importance_sampling_ratio/min": 0.7618153691291809, "sampling/sampling_logp_difference/max": 0.27864742279052734, "sampling/sampling_logp_difference/mean": 0.010548721067607403, "step": 531 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 761.0, "completions/mean_length": 534.40625, "completions/mean_terminated_length": 501.0357360839844, "completions/min_length": 271.0, "completions/min_terminated_length": 271.0, "entropy": 0.37937711365520954, "epoch": 0.284796573875803, "frac_reward_zero_std": 0.0, "grad_norm": 0.007510210853070021, "learning_rate": 1e-05, "loss": 0.0653, "num_tokens": 11297765.0, "reward": 0.5625, "reward_std": 0.4261348247528076, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.3444803953170776, "sampling/importance_sampling_ratio/mean": 0.9998617768287659, "sampling/importance_sampling_ratio/min": 0.7237727642059326, "sampling/sampling_logp_difference/max": 0.3232778310775757, "sampling/sampling_logp_difference/mean": 0.011948785744607449, "step": 532 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00010255300730932504, "clip_ratio/low_min": 0.00010255300730932504, "clip_ratio/region_mean": 0.00010255300730932504, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 761.0, "completions/mean_length": 542.84375, "completions/mean_terminated_length": 510.6785888671875, "completions/min_length": 92.0, "completions/min_terminated_length": 92.0, "entropy": 0.33837086148560047, "epoch": 0.28533190578158457, "frac_reward_zero_std": 0.25, "grad_norm": 0.019465498626232147, "learning_rate": 1e-05, "loss": 0.0268, "num_tokens": 11318960.0, "reward": 0.75, "reward_std": 0.3650856614112854, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.383101463317871, "sampling/importance_sampling_ratio/mean": 1.000052571296692, "sampling/importance_sampling_ratio/min": 0.726313591003418, "sampling/sampling_logp_difference/max": 0.3243284225463867, "sampling/sampling_logp_difference/mean": 0.011019178666174412, "step": 533 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 748.0, "completions/mean_length": 462.71875, "completions/mean_terminated_length": 431.137939453125, "completions/min_length": 253.0, "completions/min_terminated_length": 253.0, "entropy": 0.38455117493867874, "epoch": 0.28586723768736616, "frac_reward_zero_std": 0.5, "grad_norm": 0.011966320686042309, "learning_rate": 1e-05, "loss": 0.0413, "num_tokens": 11337511.0, "reward": 0.71875, "reward_std": 0.2630178928375244, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.394614338874817, "sampling/importance_sampling_ratio/mean": 1.0001449584960938, "sampling/importance_sampling_ratio/min": 0.6992799043655396, "sampling/sampling_logp_difference/max": 0.35770416259765625, "sampling/sampling_logp_difference/mean": 0.01251339539885521, "step": 534 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 749.0, "completions/mean_length": 455.375, "completions/mean_terminated_length": 423.03448486328125, "completions/min_length": 183.0, "completions/min_terminated_length": 183.0, "entropy": 0.3394015394151211, "epoch": 0.28640256959314775, "frac_reward_zero_std": 0.75, "grad_norm": 0.003924597520381212, "learning_rate": 1e-05, "loss": -0.0078, "num_tokens": 11355939.0, "reward": 0.71875, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.4565744400024414, "sampling/importance_sampling_ratio/mean": 1.0002892017364502, "sampling/importance_sampling_ratio/min": 0.7144251465797424, "sampling/sampling_logp_difference/max": 0.3760874271392822, "sampling/sampling_logp_difference/mean": 0.011010273359715939, "step": 535 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 768.0, "completions/max_terminated_length": 768.0, "completions/mean_length": 473.28125, "completions/mean_terminated_length": 473.28125, "completions/min_length": 209.0, "completions/min_terminated_length": 209.0, "entropy": 0.35326141864061356, "epoch": 0.28693790149892934, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 11374788.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.2930437326431274, "sampling/importance_sampling_ratio/mean": 1.0008374452590942, "sampling/importance_sampling_ratio/min": 0.6965340971946716, "sampling/sampling_logp_difference/max": 0.36163854598999023, "sampling/sampling_logp_difference/mean": 0.011547897942364216, "step": 536 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.451373726828024e-05, "clip_ratio/low_min": 5.451373726828024e-05, "clip_ratio/region_mean": 5.451373726828024e-05, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 706.0, "completions/mean_length": 603.65625, "completions/mean_terminated_length": 557.6400146484375, "completions/min_length": 393.0, "completions/min_terminated_length": 393.0, "entropy": 0.2894803434610367, "epoch": 0.2874732334047109, "frac_reward_zero_std": 0.0, "grad_norm": 0.012875039130449295, "learning_rate": 1e-05, "loss": 0.0792, "num_tokens": 11398169.0, "reward": 0.4375, "reward_std": 0.49022960662841797, "rewards/accuracy_reward/mean": 0.4375, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.3899998664855957, "sampling/importance_sampling_ratio/mean": 0.9999532103538513, "sampling/importance_sampling_ratio/min": 0.697039783000946, "sampling/sampling_logp_difference/max": 0.3609127998352051, "sampling/sampling_logp_difference/mean": 0.010038269683718681, "step": 537 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00015962265024427325, "clip_ratio/low_min": 0.00015962265024427325, "clip_ratio/region_mean": 0.00015962265024427325, "completions/clipped_ratio": 0.40625, "completions/max_length": 768.0, "completions/max_terminated_length": 734.0, "completions/mean_length": 614.4375, "completions/mean_terminated_length": 509.3684387207031, "completions/min_length": 359.0, "completions/min_terminated_length": 359.0, "entropy": 0.576258696615696, "epoch": 0.2880085653104925, "frac_reward_zero_std": 0.25, "grad_norm": 0.013417059555649757, "learning_rate": 1e-05, "loss": -0.0048, "num_tokens": 11422311.0, "reward": 0.40625, "reward_std": 0.3198433816432953, "rewards/accuracy_reward/mean": 0.40625, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.3797985315322876, "sampling/importance_sampling_ratio/mean": 0.9999678134918213, "sampling/importance_sampling_ratio/min": 0.6583792567253113, "sampling/sampling_logp_difference/max": 0.41797423362731934, "sampling/sampling_logp_difference/mean": 0.015053062699735165, "step": 538 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 9.074409899767488e-05, "clip_ratio/low_min": 9.074409899767488e-05, "clip_ratio/region_mean": 9.074409899767488e-05, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 750.0, "completions/mean_length": 506.21875, "completions/mean_terminated_length": 445.8077087402344, "completions/min_length": 214.0, "completions/min_terminated_length": 214.0, "entropy": 0.5533407405018806, "epoch": 0.2885438972162741, "frac_reward_zero_std": 0.25, "grad_norm": 0.014826291240751743, "learning_rate": 1e-05, "loss": 0.0122, "num_tokens": 11442222.0, "reward": 0.59375, "reward_std": 0.3377464711666107, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.3939472436904907, "sampling/importance_sampling_ratio/mean": 1.0001685619354248, "sampling/importance_sampling_ratio/min": 0.7790518999099731, "sampling/sampling_logp_difference/max": 0.3321394920349121, "sampling/sampling_logp_difference/mean": 0.014903251081705093, "step": 539 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.4375, "completions/max_length": 768.0, "completions/max_terminated_length": 688.0, "completions/mean_length": 641.8125, "completions/mean_terminated_length": 543.6666870117188, "completions/min_length": 312.0, "completions/min_terminated_length": 312.0, "entropy": 0.41299524903297424, "epoch": 0.2890792291220557, "frac_reward_zero_std": 0.25, "grad_norm": 0.005552849732339382, "learning_rate": 1e-05, "loss": 0.0707, "num_tokens": 11466568.0, "reward": 0.53125, "reward_std": 0.3471629321575165, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.4220200777053833, "sampling/importance_sampling_ratio/mean": 0.9999431371688843, "sampling/importance_sampling_ratio/min": 0.6992934346199036, "sampling/sampling_logp_difference/max": 0.357684850692749, "sampling/sampling_logp_difference/mean": 0.012839055620133877, "step": 540 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 731.0, "completions/mean_length": 528.96875, "completions/mean_terminated_length": 484.7037048339844, "completions/min_length": 287.0, "completions/min_terminated_length": 287.0, "entropy": 0.36412297561764717, "epoch": 0.2896145610278373, "frac_reward_zero_std": 0.75, "grad_norm": 0.003376058069989085, "learning_rate": 1e-05, "loss": -0.021, "num_tokens": 11487335.0, "reward": 0.6875, "reward_std": 0.1157275140285492, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.3501732349395752, "sampling/importance_sampling_ratio/mean": 0.999661922454834, "sampling/importance_sampling_ratio/min": 0.7226549983024597, "sampling/sampling_logp_difference/max": 0.32482337951660156, "sampling/sampling_logp_difference/mean": 0.011531899683177471, "step": 541 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.430060991784558e-05, "clip_ratio/low_min": 5.430060991784558e-05, "clip_ratio/region_mean": 5.430060991784558e-05, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 758.0, "completions/mean_length": 505.3125, "completions/mean_terminated_length": 456.6666564941406, "completions/min_length": 282.0, "completions/min_terminated_length": 282.0, "entropy": 0.4022515006363392, "epoch": 0.29014989293361887, "frac_reward_zero_std": 0.25, "grad_norm": 0.019753659144043922, "learning_rate": 1e-05, "loss": -0.0232, "num_tokens": 11507697.0, "reward": 0.625, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0000683069229126, "sampling/importance_sampling_ratio/min": 0.7496616840362549, "sampling/sampling_logp_difference/max": 0.6967096328735352, "sampling/sampling_logp_difference/mean": 0.012148507870733738, "step": 542 }, { "clip_ratio/high_max": 4.701015495811589e-05, "clip_ratio/high_mean": 4.701015495811589e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 4.701015495811589e-05, "completions/clipped_ratio": 0.3125, "completions/max_length": 768.0, "completions/max_terminated_length": 760.0, "completions/mean_length": 566.90625, "completions/mean_terminated_length": 475.5, "completions/min_length": 187.0, "completions/min_terminated_length": 187.0, "entropy": 0.3782684691250324, "epoch": 0.2906852248394004, "frac_reward_zero_std": 0.5, "grad_norm": 0.013792258687317371, "learning_rate": 1e-05, "loss": 0.044, "num_tokens": 11529982.0, "reward": 0.53125, "reward_std": 0.2630178928375244, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.336808204650879, "sampling/importance_sampling_ratio/mean": 1.0000829696655273, "sampling/importance_sampling_ratio/min": 0.6733843088150024, "sampling/sampling_logp_difference/max": 0.39543914794921875, "sampling/sampling_logp_difference/mean": 0.012462123297154903, "step": 543 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.42673549009487e-05, "clip_ratio/low_min": 6.42673549009487e-05, "clip_ratio/region_mean": 6.42673549009487e-05, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 728.0, "completions/mean_length": 423.59375, "completions/mean_terminated_length": 412.4838562011719, "completions/min_length": 240.0, "completions/min_terminated_length": 240.0, "entropy": 0.3379148058593273, "epoch": 0.291220556745182, "frac_reward_zero_std": 0.5, "grad_norm": 0.040512192994356155, "learning_rate": 1e-05, "loss": 0.0553, "num_tokens": 11546929.0, "reward": 0.90625, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.434513807296753, "sampling/importance_sampling_ratio/mean": 1.000313401222229, "sampling/importance_sampling_ratio/min": 0.6895756125450134, "sampling/sampling_logp_difference/max": 0.37167888879776, "sampling/sampling_logp_difference/mean": 0.012057242915034294, "step": 544 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 723.0, "completions/mean_length": 558.34375, "completions/mean_terminated_length": 509.9615478515625, "completions/min_length": 335.0, "completions/min_terminated_length": 335.0, "entropy": 0.3759257663041353, "epoch": 0.2917558886509636, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 11568764.0, "reward": 0.75, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.4700766801834106, "sampling/importance_sampling_ratio/mean": 0.9996271133422852, "sampling/importance_sampling_ratio/min": 0.536020040512085, "sampling/sampling_logp_difference/max": 0.6235837936401367, "sampling/sampling_logp_difference/mean": 0.011745463125407696, "step": 545 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00014392330922419205, "clip_ratio/low_min": 0.00014392330922419205, "clip_ratio/region_mean": 0.00014392330922419205, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 756.0, "completions/mean_length": 561.84375, "completions/mean_terminated_length": 540.5172119140625, "completions/min_length": 258.0, "completions/min_terminated_length": 258.0, "entropy": 0.37757671996951103, "epoch": 0.29229122055674517, "frac_reward_zero_std": 0.25, "grad_norm": 0.033179812133312225, "learning_rate": 1e-05, "loss": 0.0605, "num_tokens": 11590695.0, "reward": 0.78125, "reward_std": 0.3377464711666107, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.7618918418884277, "sampling/importance_sampling_ratio/mean": 1.0002686977386475, "sampling/importance_sampling_ratio/min": 0.7163848280906677, "sampling/sampling_logp_difference/max": 0.5663881301879883, "sampling/sampling_logp_difference/mean": 0.011982995085418224, "step": 546 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 750.0, "completions/mean_length": 550.46875, "completions/mean_terminated_length": 527.9655151367188, "completions/min_length": 377.0, "completions/min_terminated_length": 377.0, "entropy": 0.30198042280972004, "epoch": 0.29282655246252676, "frac_reward_zero_std": 0.5, "grad_norm": 0.010587194934487343, "learning_rate": 1e-05, "loss": 0.0073, "num_tokens": 11611958.0, "reward": 0.875, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 1.424277901649475, "sampling/importance_sampling_ratio/mean": 0.9997079372406006, "sampling/importance_sampling_ratio/min": 0.68252032995224, "sampling/sampling_logp_difference/max": 0.38196301460266113, "sampling/sampling_logp_difference/mean": 0.01032177358865738, "step": 547 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.29089045105502e-05, "clip_ratio/low_min": 6.29089045105502e-05, "clip_ratio/region_mean": 6.29089045105502e-05, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 728.0, "completions/mean_length": 533.25, "completions/mean_terminated_length": 525.6774291992188, "completions/min_length": 319.0, "completions/min_terminated_length": 319.0, "entropy": 0.34730110689997673, "epoch": 0.29336188436830835, "frac_reward_zero_std": 0.5, "grad_norm": 0.0331089049577713, "learning_rate": 1e-05, "loss": 0.0456, "num_tokens": 11633478.0, "reward": 0.9375, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.9375, "rewards/accuracy_reward/std": 0.24593468010425568, "sampling/importance_sampling_ratio/max": 1.6550542116165161, "sampling/importance_sampling_ratio/mean": 1.0001535415649414, "sampling/importance_sampling_ratio/min": 0.6979007720947266, "sampling/sampling_logp_difference/max": 0.5038337707519531, "sampling/sampling_logp_difference/mean": 0.011588075198233128, "step": 548 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.760368730989285e-05, "clip_ratio/low_min": 5.760368730989285e-05, "clip_ratio/region_mean": 5.760368730989285e-05, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 579.0, "completions/mean_length": 417.65625, "completions/mean_terminated_length": 394.3000183105469, "completions/min_length": 211.0, "completions/min_terminated_length": 211.0, "entropy": 0.3705006167292595, "epoch": 0.29389721627408993, "frac_reward_zero_std": 0.25, "grad_norm": 0.023140978068113327, "learning_rate": 1e-05, "loss": 0.0602, "num_tokens": 11650819.0, "reward": 0.75, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.3773125410079956, "sampling/importance_sampling_ratio/mean": 1.0000128746032715, "sampling/importance_sampling_ratio/min": 0.7603497505187988, "sampling/sampling_logp_difference/max": 0.32013416290283203, "sampling/sampling_logp_difference/mean": 0.012140178121626377, "step": 549 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.545696694753133e-05, "clip_ratio/low_min": 5.545696694753133e-05, "clip_ratio/region_mean": 5.545696694753133e-05, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 715.0, "completions/mean_length": 449.4375, "completions/mean_terminated_length": 439.1612854003906, "completions/min_length": 231.0, "completions/min_terminated_length": 231.0, "entropy": 0.29957092367112637, "epoch": 0.2944325481798715, "frac_reward_zero_std": 0.5, "grad_norm": 0.004742546007037163, "learning_rate": 1e-05, "loss": 0.0437, "num_tokens": 11668473.0, "reward": 0.875, "reward_std": 0.2314550280570984, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 1.457090973854065, "sampling/importance_sampling_ratio/mean": 0.9997395873069763, "sampling/importance_sampling_ratio/min": 0.7257310748100281, "sampling/sampling_logp_difference/max": 0.37644195556640625, "sampling/sampling_logp_difference/mean": 0.011099728755652905, "step": 550 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.894649413879961e-05, "clip_ratio/low_min": 6.894649413879961e-05, "clip_ratio/region_mean": 6.894649413879961e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 766.0, "completions/max_terminated_length": 766.0, "completions/mean_length": 447.0625, "completions/mean_terminated_length": 447.0625, "completions/min_length": 222.0, "completions/min_terminated_length": 222.0, "entropy": 0.33931807056069374, "epoch": 0.2949678800856531, "frac_reward_zero_std": 0.0, "grad_norm": 0.02971675433218479, "learning_rate": 1e-05, "loss": 0.0353, "num_tokens": 11686419.0, "reward": 0.6875, "reward_std": 0.4355512857437134, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.3912973403930664, "sampling/importance_sampling_ratio/mean": 1.0001190900802612, "sampling/importance_sampling_ratio/min": 0.7627151608467102, "sampling/sampling_logp_difference/max": 0.33023667335510254, "sampling/sampling_logp_difference/mean": 0.0111058559268713, "step": 551 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 768.0, "completions/mean_length": 483.09375, "completions/mean_terminated_length": 473.9031982421875, "completions/min_length": 251.0, "completions/min_terminated_length": 251.0, "entropy": 0.31367006339132786, "epoch": 0.2955032119914347, "frac_reward_zero_std": 0.5, "grad_norm": 0.0038670250214636326, "learning_rate": 1e-05, "loss": 0.0014, "num_tokens": 11705734.0, "reward": 0.6875, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.4247621297836304, "sampling/importance_sampling_ratio/mean": 0.9998100996017456, "sampling/importance_sampling_ratio/min": 0.6891435980796814, "sampling/sampling_logp_difference/max": 0.37230563163757324, "sampling/sampling_logp_difference/mean": 0.010242590680718422, "step": 552 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 701.0, "completions/mean_length": 486.46875, "completions/mean_terminated_length": 477.3870849609375, "completions/min_length": 258.0, "completions/min_terminated_length": 258.0, "entropy": 0.33838013373315334, "epoch": 0.2960385438972163, "frac_reward_zero_std": 0.5, "grad_norm": 0.005681267008185387, "learning_rate": 1e-05, "loss": 0.0715, "num_tokens": 11724821.0, "reward": 0.90625, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.4111241102218628, "sampling/importance_sampling_ratio/mean": 1.0002086162567139, "sampling/importance_sampling_ratio/min": 0.7292938232421875, "sampling/sampling_logp_difference/max": 0.34438657760620117, "sampling/sampling_logp_difference/mean": 0.0109095498919487, "step": 553 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 700.0, "completions/mean_length": 453.09375, "completions/mean_terminated_length": 432.10003662109375, "completions/min_length": 239.0, "completions/min_terminated_length": 239.0, "entropy": 0.3436109106987715, "epoch": 0.2965738758029979, "frac_reward_zero_std": 0.5, "grad_norm": 0.006001697387546301, "learning_rate": 1e-05, "loss": 0.0345, "num_tokens": 11742920.0, "reward": 0.8125, "reward_std": 0.249358132481575, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.3577940464019775, "sampling/importance_sampling_ratio/mean": 1.0000940561294556, "sampling/importance_sampling_ratio/min": 0.6864410042762756, "sampling/sampling_logp_difference/max": 0.3762350082397461, "sampling/sampling_logp_difference/mean": 0.010374966077506542, "step": 554 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.885122300242074e-05, "clip_ratio/low_min": 5.885122300242074e-05, "clip_ratio/region_mean": 5.885122300242074e-05, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 750.0, "completions/mean_length": 474.84375, "completions/mean_terminated_length": 465.3870849609375, "completions/min_length": 330.0, "completions/min_terminated_length": 330.0, "entropy": 0.38593094795942307, "epoch": 0.29710920770877947, "frac_reward_zero_std": 0.25, "grad_norm": 0.017718249931931496, "learning_rate": 1e-05, "loss": 0.0331, "num_tokens": 11762171.0, "reward": 0.625, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.4624276161193848, "sampling/importance_sampling_ratio/mean": 0.9998676180839539, "sampling/importance_sampling_ratio/min": 0.711609423160553, "sampling/sampling_logp_difference/max": 0.3800978660583496, "sampling/sampling_logp_difference/mean": 0.011882642284035683, "step": 555 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 762.0, "completions/mean_length": 518.40625, "completions/mean_terminated_length": 460.8077087402344, "completions/min_length": 270.0, "completions/min_terminated_length": 270.0, "entropy": 0.3598509766161442, "epoch": 0.29764453961456105, "frac_reward_zero_std": 0.0, "grad_norm": 0.01213877834379673, "learning_rate": 1e-05, "loss": 0.031, "num_tokens": 11782272.0, "reward": 0.71875, "reward_std": 0.4628904461860657, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.3928390741348267, "sampling/importance_sampling_ratio/mean": 1.000434398651123, "sampling/importance_sampling_ratio/min": 0.7674993276596069, "sampling/sampling_logp_difference/max": 0.3313441276550293, "sampling/sampling_logp_difference/mean": 0.011637666262686253, "step": 556 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00012170137415523641, "clip_ratio/low_min": 0.00012170137415523641, "clip_ratio/region_mean": 0.00012170137415523641, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 704.0, "completions/mean_length": 542.25, "completions/mean_terminated_length": 467.0, "completions/min_length": 267.0, "completions/min_terminated_length": 267.0, "entropy": 0.3468864634633064, "epoch": 0.2981798715203426, "frac_reward_zero_std": 0.25, "grad_norm": 0.026100486516952515, "learning_rate": 1e-05, "loss": 0.0751, "num_tokens": 11803352.0, "reward": 0.59375, "reward_std": 0.3061639964580536, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.4118132591247559, "sampling/importance_sampling_ratio/mean": 1.0002919435501099, "sampling/importance_sampling_ratio/min": 0.7072765231132507, "sampling/sampling_logp_difference/max": 0.34633350372314453, "sampling/sampling_logp_difference/mean": 0.011029123328626156, "step": 557 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.3125, "completions/max_length": 768.0, "completions/max_terminated_length": 742.0, "completions/mean_length": 568.6875, "completions/mean_terminated_length": 478.0909118652344, "completions/min_length": 263.0, "completions/min_terminated_length": 263.0, "entropy": 0.33586900494992733, "epoch": 0.2987152034261242, "frac_reward_zero_std": 0.25, "grad_norm": 0.007690643426030874, "learning_rate": 1e-05, "loss": 0.0607, "num_tokens": 11825174.0, "reward": 0.375, "reward_std": 0.3104073107242584, "rewards/accuracy_reward/mean": 0.375, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.4109917879104614, "sampling/importance_sampling_ratio/mean": 1.0002493858337402, "sampling/importance_sampling_ratio/min": 0.6766687035560608, "sampling/sampling_logp_difference/max": 0.39057350158691406, "sampling/sampling_logp_difference/mean": 0.010549957863986492, "step": 558 }, { "clip_ratio/high_max": 5.171700468054041e-05, "clip_ratio/high_mean": 5.171700468054041e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 5.171700468054041e-05, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 696.0, "completions/mean_length": 549.125, "completions/mean_terminated_length": 526.4827270507812, "completions/min_length": 302.0, "completions/min_terminated_length": 302.0, "entropy": 0.29706745594739914, "epoch": 0.29925053533190576, "frac_reward_zero_std": 0.5, "grad_norm": 0.012323442846536636, "learning_rate": 1e-05, "loss": 0.042, "num_tokens": 11846578.0, "reward": 0.875, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 1.4247862100601196, "sampling/importance_sampling_ratio/mean": 1.0000526905059814, "sampling/importance_sampling_ratio/min": 0.6969889402389526, "sampling/sampling_logp_difference/max": 0.36098575592041016, "sampling/sampling_logp_difference/mean": 0.01032998226583004, "step": 559 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 7.327080675167963e-05, "clip_ratio/low_min": 7.327080675167963e-05, "clip_ratio/region_mean": 7.327080675167963e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 600.0, "completions/max_terminated_length": 600.0, "completions/mean_length": 401.125, "completions/mean_terminated_length": 401.125, "completions/min_length": 164.0, "completions/min_terminated_length": 164.0, "entropy": 0.28526004403829575, "epoch": 0.29978586723768735, "frac_reward_zero_std": 0.5, "grad_norm": 0.010400772094726562, "learning_rate": 1e-05, "loss": -0.0236, "num_tokens": 11862974.0, "reward": 0.84375, "reward_std": 0.22201895713806152, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.4232308864593506, "sampling/importance_sampling_ratio/mean": 0.9998368620872498, "sampling/importance_sampling_ratio/min": 0.763550341129303, "sampling/sampling_logp_difference/max": 0.35292959213256836, "sampling/sampling_logp_difference/mean": 0.009960007853806019, "step": 560 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.493170490604825e-05, "clip_ratio/low_min": 4.493170490604825e-05, "clip_ratio/region_mean": 4.493170490604825e-05, "completions/clipped_ratio": 0.28125, "completions/max_length": 768.0, "completions/max_terminated_length": 732.0, "completions/mean_length": 575.34375, "completions/mean_terminated_length": 499.95654296875, "completions/min_length": 345.0, "completions/min_terminated_length": 345.0, "entropy": 0.3350704815238714, "epoch": 0.30032119914346894, "frac_reward_zero_std": 0.25, "grad_norm": 0.01137653086334467, "learning_rate": 1e-05, "loss": 0.0114, "num_tokens": 11884577.0, "reward": 0.65625, "reward_std": 0.3377464711666107, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.2981727123260498, "sampling/importance_sampling_ratio/mean": 0.999843418598175, "sampling/importance_sampling_ratio/min": 0.7069149613380432, "sampling/sampling_logp_difference/max": 0.3468449115753174, "sampling/sampling_logp_difference/mean": 0.010699411854147911, "step": 561 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00011851915041916072, "clip_ratio/low_min": 0.00011851915041916072, "clip_ratio/region_mean": 0.00011851915041916072, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 706.0, "completions/mean_length": 506.1875, "completions/mean_terminated_length": 488.7333679199219, "completions/min_length": 312.0, "completions/min_terminated_length": 312.0, "entropy": 0.2681577932089567, "epoch": 0.30085653104925053, "frac_reward_zero_std": 0.5, "grad_norm": 0.008579129353165627, "learning_rate": 1e-05, "loss": 0.0102, "num_tokens": 11904759.0, "reward": 0.65625, "reward_std": 0.22201895713806152, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.4129170179367065, "sampling/importance_sampling_ratio/mean": 1.000343680381775, "sampling/importance_sampling_ratio/min": 0.7034191489219666, "sampling/sampling_logp_difference/max": 0.35180234909057617, "sampling/sampling_logp_difference/mean": 0.009591000154614449, "step": 562 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00010635163926053792, "clip_ratio/low_min": 0.00010635163926053792, "clip_ratio/region_mean": 0.00010635163926053792, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 717.0, "completions/mean_length": 560.59375, "completions/mean_terminated_length": 502.5199890136719, "completions/min_length": 282.0, "completions/min_terminated_length": 282.0, "entropy": 0.3293144442141056, "epoch": 0.3013918629550321, "frac_reward_zero_std": 0.5, "grad_norm": 0.003007086692377925, "learning_rate": 1e-05, "loss": -0.0021, "num_tokens": 11926234.0, "reward": 0.625, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.4711403846740723, "sampling/importance_sampling_ratio/mean": 1.000432014465332, "sampling/importance_sampling_ratio/min": 0.6421256065368652, "sampling/sampling_logp_difference/max": 0.4429713487625122, "sampling/sampling_logp_difference/mean": 0.010263259522616863, "step": 563 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00010347682109568268, "clip_ratio/low_min": 0.00010347682109568268, "clip_ratio/region_mean": 0.00010347682109568268, "completions/clipped_ratio": 0.28125, "completions/max_length": 768.0, "completions/max_terminated_length": 609.0, "completions/mean_length": 565.90625, "completions/mean_terminated_length": 486.82611083984375, "completions/min_length": 344.0, "completions/min_terminated_length": 344.0, "entropy": 0.5083322878926992, "epoch": 0.3019271948608137, "frac_reward_zero_std": 0.5, "grad_norm": 0.009941553696990013, "learning_rate": 1e-05, "loss": 0.0005, "num_tokens": 11948487.0, "reward": 0.59375, "reward_std": 0.22201895713806152, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.7080460786819458, "sampling/importance_sampling_ratio/mean": 1.0003758668899536, "sampling/importance_sampling_ratio/min": 0.6820323467254639, "sampling/sampling_logp_difference/max": 0.5353500843048096, "sampling/sampling_logp_difference/mean": 0.015410225838422775, "step": 564 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 681.0, "completions/mean_length": 476.21875, "completions/mean_terminated_length": 456.7666931152344, "completions/min_length": 227.0, "completions/min_terminated_length": 227.0, "entropy": 0.35852690041065216, "epoch": 0.3024625267665953, "frac_reward_zero_std": 0.5, "grad_norm": 0.012777484022080898, "learning_rate": 1e-05, "loss": 0.0032, "num_tokens": 11967254.0, "reward": 0.59375, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.363776445388794, "sampling/importance_sampling_ratio/mean": 1.0000510215759277, "sampling/importance_sampling_ratio/min": 0.6584330201148987, "sampling/sampling_logp_difference/max": 0.4178924560546875, "sampling/sampling_logp_difference/mean": 0.011805204674601555, "step": 565 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 678.0, "completions/max_terminated_length": 678.0, "completions/mean_length": 474.25, "completions/mean_terminated_length": 474.25, "completions/min_length": 217.0, "completions/min_terminated_length": 217.0, "entropy": 0.27754843421280384, "epoch": 0.3029978586723769, "frac_reward_zero_std": 0.5, "grad_norm": 0.004616685211658478, "learning_rate": 1e-05, "loss": -0.0059, "num_tokens": 11986078.0, "reward": 0.875, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 1.4319286346435547, "sampling/importance_sampling_ratio/mean": 1.0002179145812988, "sampling/importance_sampling_ratio/min": 0.7719475626945496, "sampling/sampling_logp_difference/max": 0.3590221405029297, "sampling/sampling_logp_difference/mean": 0.009731479920446873, "step": 566 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 9.335325012216344e-05, "clip_ratio/low_min": 9.335325012216344e-05, "clip_ratio/region_mean": 9.335325012216344e-05, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 626.0, "completions/mean_length": 438.65625, "completions/mean_terminated_length": 416.70001220703125, "completions/min_length": 214.0, "completions/min_terminated_length": 214.0, "entropy": 0.2740330286324024, "epoch": 0.3035331905781585, "frac_reward_zero_std": 0.5, "grad_norm": 0.004365598317235708, "learning_rate": 1e-05, "loss": 0.0706, "num_tokens": 12003803.0, "reward": 0.6875, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.346114993095398, "sampling/importance_sampling_ratio/mean": 1.0001251697540283, "sampling/importance_sampling_ratio/min": 0.7477982640266418, "sampling/sampling_logp_difference/max": 0.2972226142883301, "sampling/sampling_logp_difference/mean": 0.009908981621265411, "step": 567 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 660.0, "completions/max_terminated_length": 660.0, "completions/mean_length": 435.59375, "completions/mean_terminated_length": 435.59375, "completions/min_length": 101.0, "completions/min_terminated_length": 101.0, "entropy": 0.37362296879291534, "epoch": 0.30406852248394006, "frac_reward_zero_std": 0.5, "grad_norm": 0.022444570437073708, "learning_rate": 1e-05, "loss": -0.0767, "num_tokens": 12020990.0, "reward": 0.875, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 1.4383749961853027, "sampling/importance_sampling_ratio/mean": 1.0005102157592773, "sampling/importance_sampling_ratio/min": 0.619283139705658, "sampling/sampling_logp_difference/max": 0.47919273376464844, "sampling/sampling_logp_difference/mean": 0.010805340483784676, "step": 568 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.577866977546364e-05, "clip_ratio/low_min": 5.577866977546364e-05, "clip_ratio/region_mean": 5.577866977546364e-05, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 723.0, "completions/mean_length": 474.78125, "completions/mean_terminated_length": 465.32257080078125, "completions/min_length": 259.0, "completions/min_terminated_length": 259.0, "entropy": 0.3136964626610279, "epoch": 0.30460385438972165, "frac_reward_zero_std": 0.5, "grad_norm": 0.00746798375621438, "learning_rate": 1e-05, "loss": 0.0394, "num_tokens": 12039719.0, "reward": 0.6875, "reward_std": 0.2587745785713196, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.4146543741226196, "sampling/importance_sampling_ratio/mean": 1.0001848936080933, "sampling/importance_sampling_ratio/min": 0.7112087607383728, "sampling/sampling_logp_difference/max": 0.34688520431518555, "sampling/sampling_logp_difference/mean": 0.010922006331384182, "step": 569 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 7.171543256845325e-05, "clip_ratio/low_min": 7.171543256845325e-05, "clip_ratio/region_mean": 7.171543256845325e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 711.0, "completions/max_terminated_length": 711.0, "completions/mean_length": 420.5625, "completions/mean_terminated_length": 420.5625, "completions/min_length": 231.0, "completions/min_terminated_length": 231.0, "entropy": 0.3946829177439213, "epoch": 0.30513918629550324, "frac_reward_zero_std": 0.0, "grad_norm": 0.027273882180452347, "learning_rate": 1e-05, "loss": 0.0292, "num_tokens": 12057401.0, "reward": 0.65625, "reward_std": 0.3808925747871399, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.4265540838241577, "sampling/importance_sampling_ratio/mean": 1.0003025531768799, "sampling/importance_sampling_ratio/min": 0.699528694152832, "sampling/sampling_logp_difference/max": 0.3573484420776367, "sampling/sampling_logp_difference/mean": 0.013109739869832993, "step": 570 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 743.0, "completions/mean_length": 509.6875, "completions/mean_terminated_length": 501.3548278808594, "completions/min_length": 238.0, "completions/min_terminated_length": 238.0, "entropy": 0.3093193285167217, "epoch": 0.3056745182012848, "frac_reward_zero_std": 0.5, "grad_norm": 0.0035189527552574873, "learning_rate": 1e-05, "loss": -0.0725, "num_tokens": 12077711.0, "reward": 0.75, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.3714594841003418, "sampling/importance_sampling_ratio/mean": 1.000036358833313, "sampling/importance_sampling_ratio/min": 0.6436813473701477, "sampling/sampling_logp_difference/max": 0.4405515193939209, "sampling/sampling_logp_difference/mean": 0.010766547173261642, "step": 571 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00021212662977632135, "clip_ratio/low_min": 0.00021212662977632135, "clip_ratio/region_mean": 0.00021212662977632135, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 660.0, "completions/mean_length": 479.53125, "completions/mean_terminated_length": 460.3000183105469, "completions/min_length": 171.0, "completions/min_terminated_length": 171.0, "entropy": 0.3884589597582817, "epoch": 0.30620985010706636, "frac_reward_zero_std": 0.25, "grad_norm": 0.006360316649079323, "learning_rate": 1e-05, "loss": -0.0034, "num_tokens": 12096544.0, "reward": 0.71875, "reward_std": 0.3198433816432953, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.4243212938308716, "sampling/importance_sampling_ratio/mean": 0.9998013973236084, "sampling/importance_sampling_ratio/min": 0.7178239226341248, "sampling/sampling_logp_difference/max": 0.3536953926086426, "sampling/sampling_logp_difference/mean": 0.01242065615952015, "step": 572 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 7.066139369271696e-05, "clip_ratio/low_min": 7.066139369271696e-05, "clip_ratio/region_mean": 7.066139369271696e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 700.0, "completions/max_terminated_length": 700.0, "completions/mean_length": 416.84375, "completions/mean_terminated_length": 416.84375, "completions/min_length": 245.0, "completions/min_terminated_length": 245.0, "entropy": 0.32096610218286514, "epoch": 0.30674518201284795, "frac_reward_zero_std": 0.5, "grad_norm": 0.026198146864771843, "learning_rate": 1e-05, "loss": -0.0354, "num_tokens": 12113083.0, "reward": 0.5, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5080004930496216, "sampling/importance_sampling_ratio/max": 1.3706988096237183, "sampling/importance_sampling_ratio/mean": 1.0001565217971802, "sampling/importance_sampling_ratio/min": 0.7378323078155518, "sampling/sampling_logp_difference/max": 0.3153207302093506, "sampling/sampling_logp_difference/mean": 0.011015393771231174, "step": 573 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 762.0, "completions/mean_length": 495.1875, "completions/mean_terminated_length": 456.21429443359375, "completions/min_length": 248.0, "completions/min_terminated_length": 248.0, "entropy": 0.34507349506020546, "epoch": 0.30728051391862954, "frac_reward_zero_std": 0.5, "grad_norm": 0.014050091616809368, "learning_rate": 1e-05, "loss": 0.0363, "num_tokens": 12132921.0, "reward": 0.75, "reward_std": 0.2587745785713196, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.451440453529358, "sampling/importance_sampling_ratio/mean": 0.9998599290847778, "sampling/importance_sampling_ratio/min": 0.7007318735122681, "sampling/sampling_logp_difference/max": 0.3725564479827881, "sampling/sampling_logp_difference/mean": 0.011650852859020233, "step": 574 }, { "clip_ratio/high_max": 8.350033749593422e-05, "clip_ratio/high_mean": 8.350033749593422e-05, "clip_ratio/low_mean": 6.551362457685173e-05, "clip_ratio/low_min": 6.551362457685173e-05, "clip_ratio/region_mean": 0.00014901396207278594, "completions/clipped_ratio": 0.0, "completions/max_length": 659.0, "completions/max_terminated_length": 659.0, "completions/mean_length": 412.4375, "completions/mean_terminated_length": 412.4375, "completions/min_length": 151.0, "completions/min_terminated_length": 151.0, "entropy": 0.40968939661979675, "epoch": 0.3078158458244111, "frac_reward_zero_std": 0.0, "grad_norm": 0.014298057183623314, "learning_rate": 1e-05, "loss": -0.0156, "num_tokens": 12149927.0, "reward": 0.65625, "reward_std": 0.4807935357093811, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.3289611339569092, "sampling/importance_sampling_ratio/mean": 0.9999315142631531, "sampling/importance_sampling_ratio/min": 0.6963590383529663, "sampling/sampling_logp_difference/max": 0.3618898391723633, "sampling/sampling_logp_difference/mean": 0.013523918576538563, "step": 575 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00021885034948354587, "clip_ratio/low_min": 0.00021885034948354587, "clip_ratio/region_mean": 0.00021885034948354587, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 610.0, "completions/mean_length": 454.34375, "completions/mean_terminated_length": 433.433349609375, "completions/min_length": 293.0, "completions/min_terminated_length": 293.0, "entropy": 0.5596118047833443, "epoch": 0.3083511777301927, "frac_reward_zero_std": 0.25, "grad_norm": 0.017974786460399628, "learning_rate": 1e-05, "loss": 0.0508, "num_tokens": 12168386.0, "reward": 0.71875, "reward_std": 0.3198433816432953, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.6629281044006348, "sampling/importance_sampling_ratio/mean": 1.000468134880066, "sampling/importance_sampling_ratio/min": 0.6651910543441772, "sampling/sampling_logp_difference/max": 0.5085799694061279, "sampling/sampling_logp_difference/mean": 0.015394064597785473, "step": 576 }, { "clip_ratio/high_max": 4.936808909405954e-05, "clip_ratio/high_mean": 4.936808909405954e-05, "clip_ratio/low_mean": 8.355615136679262e-05, "clip_ratio/low_min": 8.355615136679262e-05, "clip_ratio/region_mean": 0.00013292424046085216, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 681.0, "completions/mean_length": 494.8125, "completions/mean_terminated_length": 455.7857360839844, "completions/min_length": 234.0, "completions/min_terminated_length": 234.0, "entropy": 0.41458289697766304, "epoch": 0.3088865096359743, "frac_reward_zero_std": 0.75, "grad_norm": 0.010435909032821655, "learning_rate": 1e-05, "loss": -0.0022, "num_tokens": 12188092.0, "reward": 0.625, "reward_std": 0.13363061845302582, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9999361634254456, "sampling/importance_sampling_ratio/min": 0.651833176612854, "sampling/sampling_logp_difference/max": 0.710848331451416, "sampling/sampling_logp_difference/mean": 0.013841825537383556, "step": 577 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 752.0, "completions/mean_length": 445.75, "completions/mean_terminated_length": 435.3548278808594, "completions/min_length": 235.0, "completions/min_terminated_length": 235.0, "entropy": 0.2802911829203367, "epoch": 0.3094218415417559, "frac_reward_zero_std": 0.25, "grad_norm": 0.009748955257236958, "learning_rate": 1e-05, "loss": 0.0593, "num_tokens": 12205708.0, "reward": 0.78125, "reward_std": 0.3061639964580536, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.4513030052185059, "sampling/importance_sampling_ratio/mean": 1.0002527236938477, "sampling/importance_sampling_ratio/min": 0.7443873882293701, "sampling/sampling_logp_difference/max": 0.37246179580688477, "sampling/sampling_logp_difference/mean": 0.01012650690972805, "step": 578 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 618.0, "completions/max_terminated_length": 618.0, "completions/mean_length": 433.96875, "completions/mean_terminated_length": 433.96875, "completions/min_length": 264.0, "completions/min_terminated_length": 264.0, "entropy": 0.2798552177846432, "epoch": 0.3099571734475375, "frac_reward_zero_std": 0.75, "grad_norm": 0.007197749800980091, "learning_rate": 1e-05, "loss": 0.0012, "num_tokens": 12223843.0, "reward": 0.9375, "reward_std": 0.1157275140285492, "rewards/accuracy_reward/mean": 0.9375, "rewards/accuracy_reward/std": 0.24593468010425568, "sampling/importance_sampling_ratio/max": 1.3898836374282837, "sampling/importance_sampling_ratio/mean": 0.9999532103538513, "sampling/importance_sampling_ratio/min": 0.6809445023536682, "sampling/sampling_logp_difference/max": 0.3842744827270508, "sampling/sampling_logp_difference/mean": 0.010100327432155609, "step": 579 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 694.0, "completions/max_terminated_length": 694.0, "completions/mean_length": 477.6875, "completions/mean_terminated_length": 477.6875, "completions/min_length": 329.0, "completions/min_terminated_length": 329.0, "entropy": 0.3741901181638241, "epoch": 0.31049250535331907, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0116, "num_tokens": 12243473.0, "reward": 0.96875, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.96875, "rewards/accuracy_reward/std": 0.1767766922712326, "sampling/importance_sampling_ratio/max": 1.6115895509719849, "sampling/importance_sampling_ratio/mean": 0.9998815655708313, "sampling/importance_sampling_ratio/min": 0.7135924100875854, "sampling/sampling_logp_difference/max": 0.4772210121154785, "sampling/sampling_logp_difference/mean": 0.012397648766636848, "step": 580 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00014245013881009072, "clip_ratio/low_min": 0.00014245013881009072, "clip_ratio/region_mean": 0.00014245013881009072, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 700.0, "completions/mean_length": 458.46875, "completions/mean_terminated_length": 437.8333435058594, "completions/min_length": 224.0, "completions/min_terminated_length": 224.0, "entropy": 0.36436559818685055, "epoch": 0.31102783725910066, "frac_reward_zero_std": 0.75, "grad_norm": 0.008065787144005299, "learning_rate": 1e-05, "loss": 0.0445, "num_tokens": 12261968.0, "reward": 0.90625, "reward_std": 0.1293872892856598, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.4186723232269287, "sampling/importance_sampling_ratio/mean": 0.9999932646751404, "sampling/importance_sampling_ratio/min": 0.5788215398788452, "sampling/sampling_logp_difference/max": 0.5467610359191895, "sampling/sampling_logp_difference/mean": 0.011901299469172955, "step": 581 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.628095459542237e-05, "clip_ratio/low_min": 5.628095459542237e-05, "clip_ratio/region_mean": 5.628095459542237e-05, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 703.0, "completions/mean_length": 489.34375, "completions/mean_terminated_length": 470.7666931152344, "completions/min_length": 313.0, "completions/min_terminated_length": 313.0, "entropy": 0.31481700018048286, "epoch": 0.31156316916488225, "frac_reward_zero_std": 0.5, "grad_norm": 0.009348850697278976, "learning_rate": 1e-05, "loss": 0.0264, "num_tokens": 12280947.0, "reward": 0.84375, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.4497952461242676, "sampling/importance_sampling_ratio/mean": 0.9998577237129211, "sampling/importance_sampling_ratio/min": 0.6553128957748413, "sampling/sampling_logp_difference/max": 0.42264246940612793, "sampling/sampling_logp_difference/mean": 0.010949268937110901, "step": 582 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 660.0, "completions/mean_length": 483.28125, "completions/mean_terminated_length": 417.5769348144531, "completions/min_length": 163.0, "completions/min_terminated_length": 163.0, "entropy": 0.4111689142882824, "epoch": 0.31209850107066384, "frac_reward_zero_std": 0.5, "grad_norm": 0.006937266327440739, "learning_rate": 1e-05, "loss": 0.0841, "num_tokens": 12299828.0, "reward": 0.625, "reward_std": 0.2314550280570984, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.3981070518493652, "sampling/importance_sampling_ratio/mean": 1.0002576112747192, "sampling/importance_sampling_ratio/min": 0.6941964626312256, "sampling/sampling_logp_difference/max": 0.36500024795532227, "sampling/sampling_logp_difference/mean": 0.012879115529358387, "step": 583 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 580.0, "completions/max_terminated_length": 580.0, "completions/mean_length": 362.75, "completions/mean_terminated_length": 362.75, "completions/min_length": 194.0, "completions/min_terminated_length": 194.0, "entropy": 0.3672071844339371, "epoch": 0.31263383297644537, "frac_reward_zero_std": 0.5, "grad_norm": 0.0040225074626505375, "learning_rate": 1e-05, "loss": 0.0381, "num_tokens": 12314796.0, "reward": 0.9375, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.9375, "rewards/accuracy_reward/std": 0.24593468010425568, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0000457763671875, "sampling/importance_sampling_ratio/min": 0.685818076133728, "sampling/sampling_logp_difference/max": 0.959202766418457, "sampling/sampling_logp_difference/mean": 0.01274156291037798, "step": 584 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 700.0, "completions/max_terminated_length": 700.0, "completions/mean_length": 425.46875, "completions/mean_terminated_length": 425.46875, "completions/min_length": 229.0, "completions/min_terminated_length": 229.0, "entropy": 0.3684750981628895, "epoch": 0.31316916488222696, "frac_reward_zero_std": 0.5, "grad_norm": 0.006314178463071585, "learning_rate": 1e-05, "loss": -0.0178, "num_tokens": 12331947.0, "reward": 0.9375, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.9375, "rewards/accuracy_reward/std": 0.24593468010425568, "sampling/importance_sampling_ratio/max": 1.5478490591049194, "sampling/importance_sampling_ratio/mean": 1.0000567436218262, "sampling/importance_sampling_ratio/min": 0.608217716217041, "sampling/sampling_logp_difference/max": 0.4972224235534668, "sampling/sampling_logp_difference/mean": 0.012741255573928356, "step": 585 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.7313158322358504e-05, "clip_ratio/low_min": 5.7313158322358504e-05, "clip_ratio/region_mean": 5.7313158322358504e-05, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 763.0, "completions/mean_length": 495.5, "completions/mean_terminated_length": 486.70965576171875, "completions/min_length": 305.0, "completions/min_terminated_length": 305.0, "entropy": 0.2861919142305851, "epoch": 0.31370449678800855, "frac_reward_zero_std": 0.25, "grad_norm": 0.01361022423952818, "learning_rate": 1e-05, "loss": -0.0097, "num_tokens": 12351307.0, "reward": 0.84375, "reward_std": 0.3061639964580536, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.4768742322921753, "sampling/importance_sampling_ratio/mean": 0.9997875690460205, "sampling/importance_sampling_ratio/min": 0.6761652231216431, "sampling/sampling_logp_difference/max": 0.39131784439086914, "sampling/sampling_logp_difference/mean": 0.010119804181158543, "step": 586 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.276428444427438e-05, "clip_ratio/low_min": 4.276428444427438e-05, "clip_ratio/region_mean": 4.276428444427438e-05, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 730.0, "completions/mean_length": 541.78125, "completions/mean_terminated_length": 478.44000244140625, "completions/min_length": 301.0, "completions/min_terminated_length": 301.0, "entropy": 0.3428086116909981, "epoch": 0.31423982869379014, "frac_reward_zero_std": 0.25, "grad_norm": 0.010882562957704067, "learning_rate": 1e-05, "loss": 0.081, "num_tokens": 12372084.0, "reward": 0.53125, "reward_std": 0.378745436668396, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.6300252676010132, "sampling/importance_sampling_ratio/mean": 1.0001062154769897, "sampling/importance_sampling_ratio/min": 0.6990644931793213, "sampling/sampling_logp_difference/max": 0.48859548568725586, "sampling/sampling_logp_difference/mean": 0.010662877932190895, "step": 587 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 595.0, "completions/max_terminated_length": 595.0, "completions/mean_length": 408.125, "completions/mean_terminated_length": 408.125, "completions/min_length": 240.0, "completions/min_terminated_length": 240.0, "entropy": 0.2970775030553341, "epoch": 0.3147751605995717, "frac_reward_zero_std": 0.75, "grad_norm": 0.0053651840426027775, "learning_rate": 1e-05, "loss": 0.0052, "num_tokens": 12388208.0, "reward": 0.90625, "reward_std": 0.1293872892856598, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.4733755588531494, "sampling/importance_sampling_ratio/mean": 0.9997366070747375, "sampling/importance_sampling_ratio/min": 0.7308172583580017, "sampling/sampling_logp_difference/max": 0.3875560760498047, "sampling/sampling_logp_difference/mean": 0.009974067099392414, "step": 588 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00017644630133872852, "clip_ratio/low_min": 0.00017644630133872852, "clip_ratio/region_mean": 0.00017644630133872852, "completions/clipped_ratio": 0.0, "completions/max_length": 574.0, "completions/max_terminated_length": 574.0, "completions/mean_length": 352.3125, "completions/mean_terminated_length": 352.3125, "completions/min_length": 204.0, "completions/min_terminated_length": 204.0, "entropy": 0.3529447540640831, "epoch": 0.3153104925053533, "frac_reward_zero_std": 0.25, "grad_norm": 0.006927943788468838, "learning_rate": 1e-05, "loss": -0.0377, "num_tokens": 12403074.0, "reward": 0.875, "reward_std": 0.292504221200943, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 1.2885662317276, "sampling/importance_sampling_ratio/mean": 0.9999823570251465, "sampling/importance_sampling_ratio/min": 0.727958619594574, "sampling/sampling_logp_difference/max": 0.31751108169555664, "sampling/sampling_logp_difference/mean": 0.011859443970024586, "step": 589 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00014966265007387847, "clip_ratio/low_min": 0.00014966265007387847, "clip_ratio/region_mean": 0.00014966265007387847, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 726.0, "completions/mean_length": 456.875, "completions/mean_terminated_length": 436.13336181640625, "completions/min_length": 260.0, "completions/min_terminated_length": 260.0, "entropy": 0.302200585603714, "epoch": 0.3158458244111349, "frac_reward_zero_std": 0.5, "grad_norm": 0.008140325546264648, "learning_rate": 1e-05, "loss": -0.0021, "num_tokens": 12421270.0, "reward": 0.71875, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.3672969341278076, "sampling/importance_sampling_ratio/mean": 0.999929666519165, "sampling/importance_sampling_ratio/min": 0.7372152805328369, "sampling/sampling_logp_difference/max": 0.312835693359375, "sampling/sampling_logp_difference/mean": 0.010572618804872036, "step": 590 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 697.0, "completions/mean_length": 456.34375, "completions/mean_terminated_length": 435.5666809082031, "completions/min_length": 223.0, "completions/min_terminated_length": 223.0, "entropy": 0.3139870762825012, "epoch": 0.3163811563169165, "frac_reward_zero_std": 0.5, "grad_norm": 0.00583472428843379, "learning_rate": 1e-05, "loss": 0.0876, "num_tokens": 12439761.0, "reward": 0.6875, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.4433567523956299, "sampling/importance_sampling_ratio/mean": 0.9998515844345093, "sampling/importance_sampling_ratio/min": 0.5484488606452942, "sampling/sampling_logp_difference/max": 0.6006612777709961, "sampling/sampling_logp_difference/mean": 0.011254210956394672, "step": 591 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 628.0, "completions/max_terminated_length": 628.0, "completions/mean_length": 406.25, "completions/mean_terminated_length": 406.25, "completions/min_length": 267.0, "completions/min_terminated_length": 267.0, "entropy": 0.3209523782134056, "epoch": 0.3169164882226981, "frac_reward_zero_std": 0.5, "grad_norm": 0.0074307601898908615, "learning_rate": 1e-05, "loss": 0.0107, "num_tokens": 12456321.0, "reward": 0.90625, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.3655701875686646, "sampling/importance_sampling_ratio/mean": 0.9996328949928284, "sampling/importance_sampling_ratio/min": 0.6671118140220642, "sampling/sampling_logp_difference/max": 0.4047975540161133, "sampling/sampling_logp_difference/mean": 0.011223303154110909, "step": 592 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 678.0, "completions/mean_length": 434.53125, "completions/mean_terminated_length": 386.89288330078125, "completions/min_length": 229.0, "completions/min_terminated_length": 229.0, "entropy": 0.5003904700279236, "epoch": 0.31745182012847967, "frac_reward_zero_std": 0.5, "grad_norm": 0.03328745439648628, "learning_rate": 1e-05, "loss": -0.0125, "num_tokens": 12474082.0, "reward": 0.53125, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.4050369262695312, "sampling/importance_sampling_ratio/mean": 1.0001511573791504, "sampling/importance_sampling_ratio/min": 0.6716787219047546, "sampling/sampling_logp_difference/max": 0.39797520637512207, "sampling/sampling_logp_difference/mean": 0.015175948850810528, "step": 593 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00012303800394874997, "clip_ratio/low_min": 0.00012303800394874997, "clip_ratio/region_mean": 0.00012303800394874997, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 751.0, "completions/mean_length": 544.3125, "completions/mean_terminated_length": 469.75, "completions/min_length": 297.0, "completions/min_terminated_length": 297.0, "entropy": 0.36914557218551636, "epoch": 0.31798715203426126, "frac_reward_zero_std": 0.0, "grad_norm": 0.017193464562296867, "learning_rate": 1e-05, "loss": 0.0734, "num_tokens": 12495924.0, "reward": 0.53125, "reward_std": 0.4397946000099182, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.4910573959350586, "sampling/importance_sampling_ratio/mean": 0.9997447729110718, "sampling/importance_sampling_ratio/min": 0.6404514908790588, "sampling/sampling_logp_difference/max": 0.44558191299438477, "sampling/sampling_logp_difference/mean": 0.012314929626882076, "step": 594 }, { "clip_ratio/high_max": 6.256256165215746e-05, "clip_ratio/high_mean": 6.256256165215746e-05, "clip_ratio/low_mean": 5.827505810884759e-05, "clip_ratio/low_min": 5.827505810884759e-05, "clip_ratio/region_mean": 0.00012083761976100504, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 727.0, "completions/mean_length": 478.6875, "completions/mean_terminated_length": 437.3571472167969, "completions/min_length": 171.0, "completions/min_terminated_length": 171.0, "entropy": 0.36259258911013603, "epoch": 0.31852248394004284, "frac_reward_zero_std": 0.0, "grad_norm": 0.019357332959771156, "learning_rate": 1e-05, "loss": 0.0489, "num_tokens": 12515138.0, "reward": 0.8125, "reward_std": 0.3945523500442505, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.5570584535598755, "sampling/importance_sampling_ratio/mean": 0.9998414516448975, "sampling/importance_sampling_ratio/min": 0.317865252494812, "sampling/sampling_logp_difference/max": 1.146127700805664, "sampling/sampling_logp_difference/mean": 0.012102233245968819, "step": 595 }, { "clip_ratio/high_max": 8.05931631475687e-05, "clip_ratio/high_mean": 8.05931631475687e-05, "clip_ratio/low_mean": 0.00011983118019998074, "clip_ratio/low_min": 0.00011983118019998074, "clip_ratio/region_mean": 0.00020042434334754944, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 754.0, "completions/mean_length": 471.0625, "completions/mean_terminated_length": 440.3448181152344, "completions/min_length": 202.0, "completions/min_terminated_length": 202.0, "entropy": 0.43530962988734245, "epoch": 0.31905781584582443, "frac_reward_zero_std": 0.25, "grad_norm": 0.010029680095613003, "learning_rate": 1e-05, "loss": 0.0822, "num_tokens": 12534300.0, "reward": 0.625, "reward_std": 0.3745020925998688, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.3254101276397705, "sampling/importance_sampling_ratio/mean": 1.0000773668289185, "sampling/importance_sampling_ratio/min": 0.6802136301994324, "sampling/sampling_logp_difference/max": 0.3853483200073242, "sampling/sampling_logp_difference/mean": 0.014276028610765934, "step": 596 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 670.0, "completions/mean_length": 455.8125, "completions/mean_terminated_length": 423.5172424316406, "completions/min_length": 176.0, "completions/min_terminated_length": 176.0, "entropy": 0.3696868233382702, "epoch": 0.319593147751606, "frac_reward_zero_std": 0.75, "grad_norm": 0.0108862966299057, "learning_rate": 1e-05, "loss": 0.0345, "num_tokens": 12552694.0, "reward": 0.59375, "reward_std": 0.1293872892856598, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.4264345169067383, "sampling/importance_sampling_ratio/mean": 0.999952495098114, "sampling/importance_sampling_ratio/min": 0.6310193538665771, "sampling/sampling_logp_difference/max": 0.460418701171875, "sampling/sampling_logp_difference/mean": 0.012025072239339352, "step": 597 }, { "clip_ratio/high_max": 8.591065125074238e-05, "clip_ratio/high_mean": 8.591065125074238e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 8.591065125074238e-05, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 683.0, "completions/mean_length": 485.125, "completions/mean_terminated_length": 466.2666931152344, "completions/min_length": 180.0, "completions/min_terminated_length": 180.0, "entropy": 0.3785792216658592, "epoch": 0.32012847965738755, "frac_reward_zero_std": 0.0, "grad_norm": 0.02048811875283718, "learning_rate": 1e-05, "loss": -0.0037, "num_tokens": 12572498.0, "reward": 0.53125, "reward_std": 0.4628904461860657, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.4059467315673828, "sampling/importance_sampling_ratio/mean": 0.9998643398284912, "sampling/importance_sampling_ratio/min": 0.6858024001121521, "sampling/sampling_logp_difference/max": 0.3771657943725586, "sampling/sampling_logp_difference/mean": 0.012960119172930717, "step": 598 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.234413740457967e-05, "clip_ratio/low_min": 6.234413740457967e-05, "clip_ratio/region_mean": 6.234413740457967e-05, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 698.0, "completions/mean_length": 445.625, "completions/mean_terminated_length": 435.2257995605469, "completions/min_length": 212.0, "completions/min_terminated_length": 212.0, "entropy": 0.36770792305469513, "epoch": 0.32066381156316914, "frac_reward_zero_std": 0.25, "grad_norm": 0.00507653784006834, "learning_rate": 1e-05, "loss": -0.0499, "num_tokens": 12591094.0, "reward": 0.6875, "reward_std": 0.2925041913986206, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.405523419380188, "sampling/importance_sampling_ratio/mean": 0.9998577833175659, "sampling/importance_sampling_ratio/min": 0.5965756773948669, "sampling/sampling_logp_difference/max": 0.5165491104125977, "sampling/sampling_logp_difference/mean": 0.012210875749588013, "step": 599 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 571.0, "completions/max_terminated_length": 571.0, "completions/mean_length": 367.21875, "completions/mean_terminated_length": 367.21875, "completions/min_length": 236.0, "completions/min_terminated_length": 236.0, "entropy": 0.28999911807477474, "epoch": 0.32119914346895073, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 12606389.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.4321156740188599, "sampling/importance_sampling_ratio/mean": 0.9997718334197998, "sampling/importance_sampling_ratio/min": 0.6788029074668884, "sampling/sampling_logp_difference/max": 0.3874244689941406, "sampling/sampling_logp_difference/mean": 0.01094890758395195, "step": 600 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 720.0, "completions/mean_length": 502.46875, "completions/mean_terminated_length": 464.5357360839844, "completions/min_length": 270.0, "completions/min_terminated_length": 270.0, "entropy": 0.37995409965515137, "epoch": 0.3217344753747323, "frac_reward_zero_std": 0.0, "grad_norm": 0.010627315379679203, "learning_rate": 1e-05, "loss": 0.0361, "num_tokens": 12626204.0, "reward": 0.6875, "reward_std": 0.4671337604522705, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.3667848110198975, "sampling/importance_sampling_ratio/mean": 1.0004326105117798, "sampling/importance_sampling_ratio/min": 0.548538327217102, "sampling/sampling_logp_difference/max": 0.6004981398582458, "sampling/sampling_logp_difference/mean": 0.011942821554839611, "step": 601 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 7.749535143375397e-05, "clip_ratio/low_min": 7.749535143375397e-05, "clip_ratio/region_mean": 7.749535143375397e-05, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 691.0, "completions/mean_length": 455.09375, "completions/mean_terminated_length": 445.0, "completions/min_length": 220.0, "completions/min_terminated_length": 220.0, "entropy": 0.3100219089537859, "epoch": 0.3222698072805139, "frac_reward_zero_std": 0.25, "grad_norm": 0.01845543086528778, "learning_rate": 1e-05, "loss": 0.0445, "num_tokens": 12644647.0, "reward": 0.71875, "reward_std": 0.3198433816432953, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.4430947303771973, "sampling/importance_sampling_ratio/mean": 1.0000067949295044, "sampling/importance_sampling_ratio/min": 0.699577271938324, "sampling/sampling_logp_difference/max": 0.36678993701934814, "sampling/sampling_logp_difference/mean": 0.010655754245817661, "step": 602 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 615.0, "completions/mean_length": 445.53125, "completions/mean_terminated_length": 435.1290283203125, "completions/min_length": 274.0, "completions/min_terminated_length": 274.0, "entropy": 0.3358000721782446, "epoch": 0.3228051391862955, "frac_reward_zero_std": 0.5, "grad_norm": 0.0026316565927118063, "learning_rate": 1e-05, "loss": -0.0196, "num_tokens": 12662824.0, "reward": 0.8125, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.4125378131866455, "sampling/importance_sampling_ratio/mean": 0.999963104724884, "sampling/importance_sampling_ratio/min": 0.704192042350769, "sampling/sampling_logp_difference/max": 0.3507041931152344, "sampling/sampling_logp_difference/mean": 0.011649207212030888, "step": 603 }, { "clip_ratio/high_max": 5.3717230912297964e-05, "clip_ratio/high_mean": 5.3717230912297964e-05, "clip_ratio/low_mean": 0.00017549217227497138, "clip_ratio/low_min": 0.00017549217227497138, "clip_ratio/region_mean": 0.00022920940318726934, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 674.0, "completions/mean_length": 523.59375, "completions/mean_terminated_length": 467.19232177734375, "completions/min_length": 278.0, "completions/min_terminated_length": 278.0, "entropy": 0.5017894580960274, "epoch": 0.3233404710920771, "frac_reward_zero_std": 0.5, "grad_norm": 0.007616211660206318, "learning_rate": 1e-05, "loss": 0.009, "num_tokens": 12683963.0, "reward": 0.5, "reward_std": 0.2314550280570984, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5080004930496216, "sampling/importance_sampling_ratio/max": 1.6693872213363647, "sampling/importance_sampling_ratio/mean": 0.9996455311775208, "sampling/importance_sampling_ratio/min": 0.6767191886901855, "sampling/sampling_logp_difference/max": 0.5124566555023193, "sampling/sampling_logp_difference/mean": 0.014794141054153442, "step": 604 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 756.0, "completions/mean_length": 505.15625, "completions/mean_terminated_length": 487.63336181640625, "completions/min_length": 275.0, "completions/min_terminated_length": 275.0, "entropy": 0.37051649391651154, "epoch": 0.3238758029978587, "frac_reward_zero_std": 0.5, "grad_norm": 0.017546482384204865, "learning_rate": 1e-05, "loss": 0.0457, "num_tokens": 12703776.0, "reward": 0.75, "reward_std": 0.2587745785713196, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.2899664640426636, "sampling/importance_sampling_ratio/mean": 0.9997596144676208, "sampling/importance_sampling_ratio/min": 0.6594014167785645, "sampling/sampling_logp_difference/max": 0.41642284393310547, "sampling/sampling_logp_difference/mean": 0.012441151775419712, "step": 605 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 748.0, "completions/mean_length": 490.15625, "completions/mean_terminated_length": 450.46429443359375, "completions/min_length": 249.0, "completions/min_terminated_length": 249.0, "entropy": 0.39272466488182545, "epoch": 0.32441113490364026, "frac_reward_zero_std": 0.25, "grad_norm": 0.011300858110189438, "learning_rate": 1e-05, "loss": 0.0577, "num_tokens": 12722837.0, "reward": 0.8125, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.4034194946289062, "sampling/importance_sampling_ratio/mean": 1.0000683069229126, "sampling/importance_sampling_ratio/min": 0.6582999229431152, "sampling/sampling_logp_difference/max": 0.4180946350097656, "sampling/sampling_logp_difference/mean": 0.01232908759266138, "step": 606 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 678.0, "completions/mean_length": 505.625, "completions/mean_terminated_length": 457.03704833984375, "completions/min_length": 242.0, "completions/min_terminated_length": 242.0, "entropy": 0.4399935882538557, "epoch": 0.32494646680942185, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 12742385.0, "reward": 0.75, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.3815072774887085, "sampling/importance_sampling_ratio/mean": 0.9998868107795715, "sampling/importance_sampling_ratio/min": 0.6501041054725647, "sampling/sampling_logp_difference/max": 0.4306226968765259, "sampling/sampling_logp_difference/mean": 0.013430907391011715, "step": 607 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.693954178947024e-05, "clip_ratio/low_min": 4.693954178947024e-05, "clip_ratio/region_mean": 4.693954178947024e-05, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 752.0, "completions/mean_length": 494.875, "completions/mean_terminated_length": 444.2962951660156, "completions/min_length": 268.0, "completions/min_terminated_length": 268.0, "entropy": 0.3446064256131649, "epoch": 0.32548179871520344, "frac_reward_zero_std": 0.5, "grad_norm": 0.02724466659128666, "learning_rate": 1e-05, "loss": -0.0147, "num_tokens": 12761701.0, "reward": 0.71875, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.4556853771209717, "sampling/importance_sampling_ratio/mean": 0.9998579025268555, "sampling/importance_sampling_ratio/min": 0.7118197679519653, "sampling/sampling_logp_difference/max": 0.3754768371582031, "sampling/sampling_logp_difference/mean": 0.01101600844413042, "step": 608 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 733.0, "completions/mean_length": 490.375, "completions/mean_terminated_length": 461.6551818847656, "completions/min_length": 280.0, "completions/min_terminated_length": 280.0, "entropy": 0.4092586301267147, "epoch": 0.32601713062098503, "frac_reward_zero_std": 0.75, "grad_norm": 0.006393893621861935, "learning_rate": 1e-05, "loss": 0.0006, "num_tokens": 12781009.0, "reward": 0.59375, "reward_std": 0.1293872892856598, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.5612653493881226, "sampling/importance_sampling_ratio/mean": 0.999735951423645, "sampling/importance_sampling_ratio/min": 0.7179129123687744, "sampling/sampling_logp_difference/max": 0.4454965591430664, "sampling/sampling_logp_difference/mean": 0.012787237763404846, "step": 609 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 722.0, "completions/mean_length": 398.625, "completions/mean_terminated_length": 386.70965576171875, "completions/min_length": 191.0, "completions/min_terminated_length": 191.0, "entropy": 0.314617732539773, "epoch": 0.3265524625267666, "frac_reward_zero_std": 0.75, "grad_norm": 0.0067096371203660965, "learning_rate": 1e-05, "loss": 0.0434, "num_tokens": 12797181.0, "reward": 0.96875, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.96875, "rewards/accuracy_reward/std": 0.1767766922712326, "sampling/importance_sampling_ratio/max": 1.3729749917984009, "sampling/importance_sampling_ratio/mean": 0.9996647834777832, "sampling/importance_sampling_ratio/min": 0.7131001949310303, "sampling/sampling_logp_difference/max": 0.3381333351135254, "sampling/sampling_logp_difference/mean": 0.010695677250623703, "step": 610 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00012370113108772784, "clip_ratio/low_min": 0.00012370113108772784, "clip_ratio/region_mean": 0.00012370113108772784, "completions/clipped_ratio": 0.0, "completions/max_length": 736.0, "completions/max_terminated_length": 736.0, "completions/mean_length": 428.125, "completions/mean_terminated_length": 428.125, "completions/min_length": 258.0, "completions/min_terminated_length": 258.0, "entropy": 0.2924865297973156, "epoch": 0.3270877944325482, "frac_reward_zero_std": 0.5, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0315, "num_tokens": 12814377.0, "reward": 0.6875, "reward_std": 0.2587745785713196, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.4198427200317383, "sampling/importance_sampling_ratio/mean": 0.9998385906219482, "sampling/importance_sampling_ratio/min": 0.6952728033065796, "sampling/sampling_logp_difference/max": 0.3634510040283203, "sampling/sampling_logp_difference/mean": 0.010244255885481834, "step": 611 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 634.0, "completions/mean_length": 473.34375, "completions/mean_terminated_length": 453.70001220703125, "completions/min_length": 237.0, "completions/min_terminated_length": 237.0, "entropy": 0.3630869872868061, "epoch": 0.32762312633832974, "frac_reward_zero_std": 0.5, "grad_norm": 0.012424632906913757, "learning_rate": 1e-05, "loss": 0.0484, "num_tokens": 12833252.0, "reward": 0.46875, "reward_std": 0.2630178928375244, "rewards/accuracy_reward/mean": 0.46875, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.430388331413269, "sampling/importance_sampling_ratio/mean": 0.999660074710846, "sampling/importance_sampling_ratio/min": 0.6966915130615234, "sampling/sampling_logp_difference/max": 0.36141252517700195, "sampling/sampling_logp_difference/mean": 0.010973338969051838, "step": 612 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 9.038322605192661e-05, "clip_ratio/low_min": 9.038322605192661e-05, "clip_ratio/region_mean": 9.038322605192661e-05, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 598.0, "completions/mean_length": 396.25, "completions/mean_terminated_length": 343.14288330078125, "completions/min_length": 226.0, "completions/min_terminated_length": 226.0, "entropy": 0.39421503990888596, "epoch": 0.32815845824411133, "frac_reward_zero_std": 0.5, "grad_norm": 0.015410852618515491, "learning_rate": 1e-05, "loss": -0.0137, "num_tokens": 12849428.0, "reward": 0.5625, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.4275332689285278, "sampling/importance_sampling_ratio/mean": 1.0005714893341064, "sampling/importance_sampling_ratio/min": 0.7691166400909424, "sampling/sampling_logp_difference/max": 0.35594797134399414, "sampling/sampling_logp_difference/mean": 0.012892412021756172, "step": 613 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00013157616558601148, "clip_ratio/low_min": 0.00013157616558601148, "clip_ratio/region_mean": 0.00013157616558601148, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 716.0, "completions/mean_length": 493.5625, "completions/mean_terminated_length": 454.357177734375, "completions/min_length": 184.0, "completions/min_terminated_length": 184.0, "entropy": 0.38468278013169765, "epoch": 0.3286937901498929, "frac_reward_zero_std": 0.0, "grad_norm": 0.03001777082681656, "learning_rate": 1e-05, "loss": 0.0024, "num_tokens": 12868870.0, "reward": 0.46875, "reward_std": 0.4397946000099182, "rewards/accuracy_reward/mean": 0.46875, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.455602765083313, "sampling/importance_sampling_ratio/mean": 1.0003761053085327, "sampling/importance_sampling_ratio/min": 0.6040632128715515, "sampling/sampling_logp_difference/max": 0.5040764808654785, "sampling/sampling_logp_difference/mean": 0.012294515036046505, "step": 614 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 8.017960499273613e-05, "clip_ratio/low_min": 8.017960499273613e-05, "clip_ratio/region_mean": 8.017960499273613e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 661.0, "completions/max_terminated_length": 661.0, "completions/mean_length": 385.34375, "completions/mean_terminated_length": 385.34375, "completions/min_length": 258.0, "completions/min_terminated_length": 258.0, "entropy": 0.3371674194931984, "epoch": 0.3292291220556745, "frac_reward_zero_std": 0.75, "grad_norm": 0.009930756874382496, "learning_rate": 1e-05, "loss": 0.0399, "num_tokens": 12884737.0, "reward": 0.90625, "reward_std": 0.1293872892856598, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.4359911680221558, "sampling/importance_sampling_ratio/mean": 1.0005505084991455, "sampling/importance_sampling_ratio/min": 0.7238682508468628, "sampling/sampling_logp_difference/max": 0.36185526847839355, "sampling/sampling_logp_difference/mean": 0.01123226061463356, "step": 615 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00019472106214379892, "clip_ratio/low_min": 0.00019472106214379892, "clip_ratio/region_mean": 0.00019472106214379892, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 647.0, "completions/mean_length": 483.875, "completions/mean_terminated_length": 474.70965576171875, "completions/min_length": 284.0, "completions/min_terminated_length": 284.0, "entropy": 0.3057300020009279, "epoch": 0.3297644539614561, "frac_reward_zero_std": 0.5, "grad_norm": 0.026799941435456276, "learning_rate": 1e-05, "loss": 0.0185, "num_tokens": 12904085.0, "reward": 0.75, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.359426736831665, "sampling/importance_sampling_ratio/mean": 0.9996913075447083, "sampling/importance_sampling_ratio/min": 0.6805739402770996, "sampling/sampling_logp_difference/max": 0.38481879234313965, "sampling/sampling_logp_difference/mean": 0.009956633672118187, "step": 616 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 741.0, "completions/mean_length": 495.125, "completions/mean_terminated_length": 486.32257080078125, "completions/min_length": 276.0, "completions/min_terminated_length": 276.0, "entropy": 0.33366139978170395, "epoch": 0.3302997858672377, "frac_reward_zero_std": 0.25, "grad_norm": 0.008611881174147129, "learning_rate": 1e-05, "loss": 0.0004, "num_tokens": 12924049.0, "reward": 0.875, "reward_std": 0.292504221200943, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 1.3542838096618652, "sampling/importance_sampling_ratio/mean": 1.0000187158584595, "sampling/importance_sampling_ratio/min": 0.5206681489944458, "sampling/sampling_logp_difference/max": 0.6526424884796143, "sampling/sampling_logp_difference/mean": 0.011231090873479843, "step": 617 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 557.0, "completions/max_terminated_length": 557.0, "completions/mean_length": 367.75, "completions/mean_terminated_length": 367.75, "completions/min_length": 220.0, "completions/min_terminated_length": 220.0, "entropy": 0.25993393547832966, "epoch": 0.33083511777301927, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": -0.0004, "num_tokens": 12939537.0, "reward": 0.96875, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.96875, "rewards/accuracy_reward/std": 0.1767766922712326, "sampling/importance_sampling_ratio/max": 1.6012578010559082, "sampling/importance_sampling_ratio/mean": 1.000769019126892, "sampling/importance_sampling_ratio/min": 0.6878544688224792, "sampling/sampling_logp_difference/max": 0.47078943252563477, "sampling/sampling_logp_difference/mean": 0.010146456770598888, "step": 618 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 738.0, "completions/max_terminated_length": 738.0, "completions/mean_length": 414.75, "completions/mean_terminated_length": 414.75, "completions/min_length": 151.0, "completions/min_terminated_length": 151.0, "entropy": 0.29927484318614006, "epoch": 0.33137044967880086, "frac_reward_zero_std": 0.5, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": -0.0156, "num_tokens": 12956089.0, "reward": 0.9375, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.9375, "rewards/accuracy_reward/std": 0.24593468010425568, "sampling/importance_sampling_ratio/max": 1.3900020122528076, "sampling/importance_sampling_ratio/mean": 0.9998085498809814, "sampling/importance_sampling_ratio/min": 0.6901939511299133, "sampling/sampling_logp_difference/max": 0.37078261375427246, "sampling/sampling_logp_difference/mean": 0.009291633032262325, "step": 619 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00012921850793645717, "clip_ratio/low_min": 0.00012921850793645717, "clip_ratio/region_mean": 0.00012921850793645717, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 734.0, "completions/mean_length": 503.5, "completions/mean_terminated_length": 465.71429443359375, "completions/min_length": 278.0, "completions/min_terminated_length": 278.0, "entropy": 0.29361248947679996, "epoch": 0.33190578158458245, "frac_reward_zero_std": 0.25, "grad_norm": 0.007103252224624157, "learning_rate": 1e-05, "loss": 0.0687, "num_tokens": 12976161.0, "reward": 0.75, "reward_std": 0.292504221200943, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.4360511302947998, "sampling/importance_sampling_ratio/mean": 1.0004138946533203, "sampling/importance_sampling_ratio/min": 0.7299440503120422, "sampling/sampling_logp_difference/max": 0.36189699172973633, "sampling/sampling_logp_difference/mean": 0.009897296316921711, "step": 620 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 762.0, "completions/max_terminated_length": 762.0, "completions/mean_length": 415.625, "completions/mean_terminated_length": 415.625, "completions/min_length": 254.0, "completions/min_terminated_length": 254.0, "entropy": 0.3415104281157255, "epoch": 0.33244111349036404, "frac_reward_zero_std": 0.75, "grad_norm": 0.00380335608497262, "learning_rate": 1e-05, "loss": -0.0231, "num_tokens": 12993245.0, "reward": 0.71875, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.4889551401138306, "sampling/importance_sampling_ratio/mean": 0.9999030828475952, "sampling/importance_sampling_ratio/min": 0.7551504969596863, "sampling/sampling_logp_difference/max": 0.3980746269226074, "sampling/sampling_logp_difference/mean": 0.011102134361863136, "step": 621 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 8.778089977568015e-05, "clip_ratio/low_min": 8.778089977568015e-05, "clip_ratio/region_mean": 8.778089977568015e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 536.0, "completions/max_terminated_length": 536.0, "completions/mean_length": 375.71875, "completions/mean_terminated_length": 375.71875, "completions/min_length": 234.0, "completions/min_terminated_length": 234.0, "entropy": 0.33459382504224777, "epoch": 0.3329764453961456, "frac_reward_zero_std": 0.25, "grad_norm": 0.020825320854783058, "learning_rate": 1e-05, "loss": -0.0145, "num_tokens": 13008980.0, "reward": 0.6875, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.4972962141036987, "sampling/importance_sampling_ratio/mean": 1.000253438949585, "sampling/importance_sampling_ratio/min": 0.6408749222755432, "sampling/sampling_logp_difference/max": 0.44492101669311523, "sampling/sampling_logp_difference/mean": 0.011420239694416523, "step": 622 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 510.0, "completions/max_terminated_length": 510.0, "completions/mean_length": 312.71875, "completions/mean_terminated_length": 312.71875, "completions/min_length": 142.0, "completions/min_terminated_length": 142.0, "entropy": 0.29183613508939743, "epoch": 0.3335117773019272, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 13022371.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.3589096069335938, "sampling/importance_sampling_ratio/mean": 1.000159740447998, "sampling/importance_sampling_ratio/min": 0.7225852012634277, "sampling/sampling_logp_difference/max": 0.3249199390411377, "sampling/sampling_logp_difference/mean": 0.010989728383719921, "step": 623 }, { "clip_ratio/high_max": 0.00012599398905877024, "clip_ratio/high_mean": 0.00012599398905877024, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00012599398905877024, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 712.0, "completions/mean_length": 474.375, "completions/mean_terminated_length": 444.0, "completions/min_length": 229.0, "completions/min_terminated_length": 229.0, "entropy": 0.3476267158985138, "epoch": 0.3340471092077088, "frac_reward_zero_std": 0.25, "grad_norm": 0.03480792045593262, "learning_rate": 1e-05, "loss": 0.0719, "num_tokens": 13041231.0, "reward": 0.75, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.4345088005065918, "sampling/importance_sampling_ratio/mean": 0.9998278021812439, "sampling/importance_sampling_ratio/min": 0.5451315641403198, "sampling/sampling_logp_difference/max": 0.6067280769348145, "sampling/sampling_logp_difference/mean": 0.011186945252120495, "step": 624 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 756.0, "completions/mean_length": 505.90625, "completions/mean_terminated_length": 497.45159912109375, "completions/min_length": 239.0, "completions/min_terminated_length": 239.0, "entropy": 0.292398190125823, "epoch": 0.33458244111349034, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 13061828.0, "reward": 0.75, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.5679001808166504, "sampling/importance_sampling_ratio/mean": 0.9997112154960632, "sampling/importance_sampling_ratio/min": 0.7010811567306519, "sampling/sampling_logp_difference/max": 0.4497373104095459, "sampling/sampling_logp_difference/mean": 0.010309663601219654, "step": 625 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 624.0, "completions/mean_length": 376.25, "completions/mean_terminated_length": 363.6128845214844, "completions/min_length": 186.0, "completions/min_terminated_length": 186.0, "entropy": 0.36883340403437614, "epoch": 0.3351177730192719, "frac_reward_zero_std": 0.25, "grad_norm": 0.00871721189469099, "learning_rate": 1e-05, "loss": -0.0147, "num_tokens": 13077196.0, "reward": 0.5625, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.686894178390503, "sampling/importance_sampling_ratio/mean": 0.9995442628860474, "sampling/importance_sampling_ratio/min": 0.6272304654121399, "sampling/sampling_logp_difference/max": 0.5228891372680664, "sampling/sampling_logp_difference/mean": 0.013311648741364479, "step": 626 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0001392210542690009, "clip_ratio/low_min": 0.0001392210542690009, "clip_ratio/region_mean": 0.0001392210542690009, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 756.0, "completions/mean_length": 485.75, "completions/mean_terminated_length": 420.6153869628906, "completions/min_length": 108.0, "completions/min_terminated_length": 108.0, "entropy": 0.446913231164217, "epoch": 0.3356531049250535, "frac_reward_zero_std": 0.0, "grad_norm": 0.008604291826486588, "learning_rate": 1e-05, "loss": 0.034, "num_tokens": 13096524.0, "reward": 0.65625, "reward_std": 0.3808925747871399, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.5247750282287598, "sampling/importance_sampling_ratio/mean": 1.0002942085266113, "sampling/importance_sampling_ratio/min": 0.7407239675521851, "sampling/sampling_logp_difference/max": 0.421846866607666, "sampling/sampling_logp_difference/mean": 0.013524254783987999, "step": 627 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 8.573388186050579e-05, "clip_ratio/low_min": 8.573388186050579e-05, "clip_ratio/region_mean": 8.573388186050579e-05, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 713.0, "completions/mean_length": 456.59375, "completions/mean_terminated_length": 446.5483703613281, "completions/min_length": 236.0, "completions/min_terminated_length": 236.0, "entropy": 0.3660719469189644, "epoch": 0.3361884368308351, "frac_reward_zero_std": 0.25, "grad_norm": 0.0075892615132033825, "learning_rate": 1e-05, "loss": 0.0429, "num_tokens": 13114503.0, "reward": 0.875, "reward_std": 0.2925041913986206, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 1.3400059938430786, "sampling/importance_sampling_ratio/mean": 0.9996907711029053, "sampling/importance_sampling_ratio/min": 0.4545471668243408, "sampling/sampling_logp_difference/max": 0.7884535789489746, "sampling/sampling_logp_difference/mean": 0.011270992457866669, "step": 628 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.000156465066538658, "clip_ratio/low_min": 0.000156465066538658, "clip_ratio/region_mean": 0.000156465066538658, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 736.0, "completions/mean_length": 529.25, "completions/mean_terminated_length": 495.14288330078125, "completions/min_length": 266.0, "completions/min_terminated_length": 266.0, "entropy": 0.2408799361437559, "epoch": 0.3367237687366167, "frac_reward_zero_std": 0.25, "grad_norm": 0.003707155818119645, "learning_rate": 1e-05, "loss": 0.0326, "num_tokens": 13135175.0, "reward": 0.75, "reward_std": 0.292504221200943, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.9237333536148071, "sampling/importance_sampling_ratio/mean": 1.0000091791152954, "sampling/importance_sampling_ratio/min": 0.7001087069511414, "sampling/sampling_logp_difference/max": 0.6542677879333496, "sampling/sampling_logp_difference/mean": 0.008334740065038204, "step": 629 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 7.301401637960225e-05, "clip_ratio/low_min": 7.301401637960225e-05, "clip_ratio/region_mean": 7.301401637960225e-05, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 560.0, "completions/mean_length": 410.5625, "completions/mean_terminated_length": 399.0322570800781, "completions/min_length": 210.0, "completions/min_terminated_length": 210.0, "entropy": 0.23404156975448132, "epoch": 0.3372591006423983, "frac_reward_zero_std": 0.5, "grad_norm": 0.013698897324502468, "learning_rate": 1e-05, "loss": -0.007, "num_tokens": 13151961.0, "reward": 0.8125, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.6546179056167603, "sampling/importance_sampling_ratio/mean": 0.9999274611473083, "sampling/importance_sampling_ratio/min": 0.7193414568901062, "sampling/sampling_logp_difference/max": 0.5035700798034668, "sampling/sampling_logp_difference/mean": 0.008526804856956005, "step": 630 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 503.0, "completions/max_terminated_length": 503.0, "completions/mean_length": 345.625, "completions/mean_terminated_length": 345.625, "completions/min_length": 220.0, "completions/min_terminated_length": 220.0, "entropy": 0.3241981565952301, "epoch": 0.33779443254817987, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 13166741.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.339516520500183, "sampling/importance_sampling_ratio/mean": 0.9997711181640625, "sampling/importance_sampling_ratio/min": 0.5709845423698425, "sampling/sampling_logp_difference/max": 0.5603930950164795, "sampling/sampling_logp_difference/mean": 0.011668842285871506, "step": 631 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 504.0, "completions/max_terminated_length": 504.0, "completions/mean_length": 408.96875, "completions/mean_terminated_length": 408.96875, "completions/min_length": 249.0, "completions/min_terminated_length": 249.0, "entropy": 0.2496287040412426, "epoch": 0.33832976445396146, "frac_reward_zero_std": 0.75, "grad_norm": 0.0036416437942534685, "learning_rate": 1e-05, "loss": -0.0081, "num_tokens": 13183428.0, "reward": 0.96875, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.96875, "rewards/accuracy_reward/std": 0.1767766922712326, "sampling/importance_sampling_ratio/max": 1.3984626531600952, "sampling/importance_sampling_ratio/mean": 0.9999477863311768, "sampling/importance_sampling_ratio/min": 0.7157694101333618, "sampling/sampling_logp_difference/max": 0.3353736400604248, "sampling/sampling_logp_difference/mean": 0.008716152049601078, "step": 632 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 754.0, "completions/mean_length": 431.5, "completions/mean_terminated_length": 409.0666809082031, "completions/min_length": 184.0, "completions/min_terminated_length": 184.0, "entropy": 0.5070165768265724, "epoch": 0.33886509635974305, "frac_reward_zero_std": 0.5, "grad_norm": 0.022825445979833603, "learning_rate": 1e-05, "loss": -0.0167, "num_tokens": 13201492.0, "reward": 0.625, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.369990587234497, "sampling/importance_sampling_ratio/mean": 0.9995102882385254, "sampling/importance_sampling_ratio/min": 0.6855331063270569, "sampling/sampling_logp_difference/max": 0.37755846977233887, "sampling/sampling_logp_difference/mean": 0.01544769573956728, "step": 633 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 658.0, "completions/max_terminated_length": 658.0, "completions/mean_length": 337.65625, "completions/mean_terminated_length": 337.65625, "completions/min_length": 153.0, "completions/min_terminated_length": 153.0, "entropy": 0.39696030132472515, "epoch": 0.33940042826552463, "frac_reward_zero_std": 0.5, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0322, "num_tokens": 13216009.0, "reward": 0.8125, "reward_std": 0.249358132481575, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.6422919034957886, "sampling/importance_sampling_ratio/mean": 1.0000416040420532, "sampling/importance_sampling_ratio/min": 0.6801885962486267, "sampling/sampling_logp_difference/max": 0.4960927963256836, "sampling/sampling_logp_difference/mean": 0.012387676164507866, "step": 634 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00012491943198256195, "clip_ratio/low_min": 0.00012491943198256195, "clip_ratio/region_mean": 0.00012491943198256195, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 763.0, "completions/mean_length": 494.375, "completions/mean_terminated_length": 431.23077392578125, "completions/min_length": 243.0, "completions/min_terminated_length": 243.0, "entropy": 0.38411999866366386, "epoch": 0.3399357601713062, "frac_reward_zero_std": 0.25, "grad_norm": 0.017900027334690094, "learning_rate": 1e-05, "loss": 0.0241, "num_tokens": 13235501.0, "reward": 0.6875, "reward_std": 0.3924051821231842, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.5086416006088257, "sampling/importance_sampling_ratio/mean": 1.0000834465026855, "sampling/importance_sampling_ratio/min": 0.5438719391822815, "sampling/sampling_logp_difference/max": 0.6090414524078369, "sampling/sampling_logp_difference/mean": 0.012571083381772041, "step": 635 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 596.0, "completions/max_terminated_length": 596.0, "completions/mean_length": 348.5625, "completions/mean_terminated_length": 348.5625, "completions/min_length": 96.0, "completions/min_terminated_length": 96.0, "entropy": 0.2587822787463665, "epoch": 0.3404710920770878, "frac_reward_zero_std": 0.75, "grad_norm": 0.0038941828534007072, "learning_rate": 1e-05, "loss": -0.0019, "num_tokens": 13250751.0, "reward": 0.96875, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.96875, "rewards/accuracy_reward/std": 0.1767766922712326, "sampling/importance_sampling_ratio/max": 1.6068605184555054, "sampling/importance_sampling_ratio/mean": 1.0001121759414673, "sampling/importance_sampling_ratio/min": 0.6504940390586853, "sampling/sampling_logp_difference/max": 0.47428226470947266, "sampling/sampling_logp_difference/mean": 0.008922860026359558, "step": 636 }, { "clip_ratio/high_max": 5.816658813273534e-05, "clip_ratio/high_mean": 5.816658813273534e-05, "clip_ratio/low_mean": 0.00019885157598764636, "clip_ratio/low_min": 0.00019885157598764636, "clip_ratio/region_mean": 0.0002570181641203817, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 662.0, "completions/mean_length": 478.53125, "completions/mean_terminated_length": 437.1785888671875, "completions/min_length": 223.0, "completions/min_terminated_length": 223.0, "entropy": 0.4376033619046211, "epoch": 0.3410064239828694, "frac_reward_zero_std": 0.0, "grad_norm": 0.022219238802790642, "learning_rate": 1e-05, "loss": 0.0423, "num_tokens": 13270656.0, "reward": 0.5, "reward_std": 0.5081326961517334, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5080004930496216, "sampling/importance_sampling_ratio/max": 1.4648514986038208, "sampling/importance_sampling_ratio/mean": 0.9998618960380554, "sampling/importance_sampling_ratio/min": 0.6825798749923706, "sampling/sampling_logp_difference/max": 0.38187575340270996, "sampling/sampling_logp_difference/mean": 0.013110549189150333, "step": 637 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 761.0, "completions/mean_length": 486.96875, "completions/mean_terminated_length": 446.8214416503906, "completions/min_length": 328.0, "completions/min_terminated_length": 328.0, "entropy": 0.46911646053195, "epoch": 0.341541755888651, "frac_reward_zero_std": 0.25, "grad_norm": 0.030147511512041092, "learning_rate": 1e-05, "loss": 0.028, "num_tokens": 13290447.0, "reward": 0.5625, "reward_std": 0.3104073107242584, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.4902180433273315, "sampling/importance_sampling_ratio/mean": 0.9998611211776733, "sampling/importance_sampling_ratio/min": 0.658909797668457, "sampling/sampling_logp_difference/max": 0.41716861724853516, "sampling/sampling_logp_difference/mean": 0.01516705472022295, "step": 638 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 686.0, "completions/max_terminated_length": 686.0, "completions/mean_length": 418.5625, "completions/mean_terminated_length": 418.5625, "completions/min_length": 245.0, "completions/min_terminated_length": 245.0, "entropy": 0.2731906659901142, "epoch": 0.3420770877944325, "frac_reward_zero_std": 0.25, "grad_norm": 0.02774636074900627, "learning_rate": 1e-05, "loss": -0.0124, "num_tokens": 13307305.0, "reward": 0.5625, "reward_std": 0.2925041913986206, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.4035072326660156, "sampling/importance_sampling_ratio/mean": 1.0002001523971558, "sampling/importance_sampling_ratio/min": 0.6635493040084839, "sampling/sampling_logp_difference/max": 0.4101520776748657, "sampling/sampling_logp_difference/mean": 0.010155044496059418, "step": 639 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 8.491847984259948e-05, "clip_ratio/low_min": 8.491847984259948e-05, "clip_ratio/region_mean": 8.491847984259948e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 582.0, "completions/max_terminated_length": 582.0, "completions/mean_length": 336.75, "completions/mean_terminated_length": 336.75, "completions/min_length": 115.0, "completions/min_terminated_length": 115.0, "entropy": 0.3048718683421612, "epoch": 0.3426124197002141, "frac_reward_zero_std": 0.75, "grad_norm": 0.018667301163077354, "learning_rate": 1e-05, "loss": -0.0075, "num_tokens": 13321665.0, "reward": 0.90625, "reward_std": 0.1293872892856598, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.9191588163375854, "sampling/importance_sampling_ratio/mean": 0.9999879598617554, "sampling/importance_sampling_ratio/min": 0.7230131030082703, "sampling/sampling_logp_difference/max": 0.6518869400024414, "sampling/sampling_logp_difference/mean": 0.010768414475023746, "step": 640 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 574.0, "completions/max_terminated_length": 574.0, "completions/mean_length": 409.96875, "completions/mean_terminated_length": 409.96875, "completions/min_length": 239.0, "completions/min_terminated_length": 239.0, "entropy": 0.27836749143898487, "epoch": 0.3431477516059957, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0146, "num_tokens": 13338288.0, "reward": 0.9375, "reward_std": 0.1157275140285492, "rewards/accuracy_reward/mean": 0.9375, "rewards/accuracy_reward/std": 0.24593468010425568, "sampling/importance_sampling_ratio/max": 1.4447996616363525, "sampling/importance_sampling_ratio/mean": 0.9999314546585083, "sampling/importance_sampling_ratio/min": 0.7261752486228943, "sampling/sampling_logp_difference/max": 0.36797070503234863, "sampling/sampling_logp_difference/mean": 0.010056580416858196, "step": 641 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 9.667439735494554e-05, "clip_ratio/low_min": 9.667439735494554e-05, "clip_ratio/region_mean": 9.667439735494554e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 592.0, "completions/max_terminated_length": 592.0, "completions/mean_length": 362.3125, "completions/mean_terminated_length": 362.3125, "completions/min_length": 205.0, "completions/min_terminated_length": 205.0, "entropy": 0.33984828367829323, "epoch": 0.3436830835117773, "frac_reward_zero_std": 0.75, "grad_norm": 0.009369701147079468, "learning_rate": 1e-05, "loss": -0.0089, "num_tokens": 13353946.0, "reward": 0.9375, "reward_std": 0.1157275140285492, "rewards/accuracy_reward/mean": 0.9375, "rewards/accuracy_reward/std": 0.24593468010425568, "sampling/importance_sampling_ratio/max": 1.4577983617782593, "sampling/importance_sampling_ratio/mean": 0.9998650550842285, "sampling/importance_sampling_ratio/min": 0.7225463390350342, "sampling/sampling_logp_difference/max": 0.37692737579345703, "sampling/sampling_logp_difference/mean": 0.011347470805048943, "step": 642 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00012793374844477512, "clip_ratio/low_min": 0.00012793374844477512, "clip_ratio/region_mean": 0.00012793374844477512, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 648.0, "completions/mean_length": 473.6875, "completions/mean_terminated_length": 405.7692565917969, "completions/min_length": 228.0, "completions/min_terminated_length": 228.0, "entropy": 0.48161710426211357, "epoch": 0.3442184154175589, "frac_reward_zero_std": 0.25, "grad_norm": 0.051371026784181595, "learning_rate": 1e-05, "loss": 0.0165, "num_tokens": 13373248.0, "reward": 0.71875, "reward_std": 0.2651650309562683, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.6313735246658325, "sampling/importance_sampling_ratio/mean": 1.0000813007354736, "sampling/importance_sampling_ratio/min": 0.48223310708999634, "sampling/sampling_logp_difference/max": 0.7293276786804199, "sampling/sampling_logp_difference/mean": 0.014763213694095612, "step": 643 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00018642803479451686, "clip_ratio/low_min": 0.00018642803479451686, "clip_ratio/region_mean": 0.00018642803479451686, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 705.0, "completions/mean_length": 392.8125, "completions/mean_terminated_length": 354.0, "completions/min_length": 115.0, "completions/min_terminated_length": 115.0, "entropy": 0.40540508180856705, "epoch": 0.34475374732334046, "frac_reward_zero_std": 0.25, "grad_norm": 0.015619847923517227, "learning_rate": 1e-05, "loss": 0.0966, "num_tokens": 13389274.0, "reward": 0.84375, "reward_std": 0.3198433816432953, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.7861274480819702, "sampling/importance_sampling_ratio/mean": 1.0002065896987915, "sampling/importance_sampling_ratio/min": 0.7051683068275452, "sampling/sampling_logp_difference/max": 0.5800497531890869, "sampling/sampling_logp_difference/mean": 0.012819117866456509, "step": 644 }, { "clip_ratio/high_max": 5.6407941883662716e-05, "clip_ratio/high_mean": 5.6407941883662716e-05, "clip_ratio/low_mean": 0.00013486891839420423, "clip_ratio/low_min": 0.00013486891839420423, "clip_ratio/region_mean": 0.00019127686027786694, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 738.0, "completions/mean_length": 427.0, "completions/mean_terminated_length": 391.7241516113281, "completions/min_length": 207.0, "completions/min_terminated_length": 207.0, "entropy": 0.3380007464438677, "epoch": 0.34528907922912205, "frac_reward_zero_std": 0.5, "grad_norm": 0.018019359558820724, "learning_rate": 1e-05, "loss": 0.0676, "num_tokens": 13406490.0, "reward": 0.75, "reward_std": 0.2587745785713196, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.3706324100494385, "sampling/importance_sampling_ratio/mean": 0.9995419979095459, "sampling/importance_sampling_ratio/min": 0.7309109568595886, "sampling/sampling_logp_difference/max": 0.31527233123779297, "sampling/sampling_logp_difference/mean": 0.011513370089232922, "step": 645 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.902263703523204e-05, "clip_ratio/low_min": 6.902263703523204e-05, "clip_ratio/region_mean": 6.902263703523204e-05, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 651.0, "completions/mean_length": 441.53125, "completions/mean_terminated_length": 407.75860595703125, "completions/min_length": 239.0, "completions/min_terminated_length": 239.0, "entropy": 0.3893450815230608, "epoch": 0.34582441113490364, "frac_reward_zero_std": 0.25, "grad_norm": 0.01541195809841156, "learning_rate": 1e-05, "loss": 0.0126, "num_tokens": 13424499.0, "reward": 0.625, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.5599128007888794, "sampling/importance_sampling_ratio/mean": 1.000036358833313, "sampling/importance_sampling_ratio/min": 0.7086088061332703, "sampling/sampling_logp_difference/max": 0.4446299076080322, "sampling/sampling_logp_difference/mean": 0.01274517085403204, "step": 646 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00019918624457204714, "clip_ratio/low_min": 0.00019918624457204714, "clip_ratio/region_mean": 0.00019918624457204714, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 679.0, "completions/mean_length": 492.40625, "completions/mean_terminated_length": 441.370361328125, "completions/min_length": 183.0, "completions/min_terminated_length": 183.0, "entropy": 0.5329282619059086, "epoch": 0.34635974304068523, "frac_reward_zero_std": 0.0, "grad_norm": 0.01550175342708826, "learning_rate": 1e-05, "loss": 0.074, "num_tokens": 13444560.0, "reward": 0.34375, "reward_std": 0.4944729208946228, "rewards/accuracy_reward/mean": 0.34375, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.900555968284607, "sampling/importance_sampling_ratio/mean": 1.0000826120376587, "sampling/importance_sampling_ratio/min": 0.3233708143234253, "sampling/sampling_logp_difference/max": 1.128955602645874, "sampling/sampling_logp_difference/mean": 0.016209932044148445, "step": 647 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 712.0, "completions/mean_length": 478.375, "completions/mean_terminated_length": 469.0322570800781, "completions/min_length": 296.0, "completions/min_terminated_length": 296.0, "entropy": 0.28995392099022865, "epoch": 0.3468950749464668, "frac_reward_zero_std": 0.25, "grad_norm": 0.007992844097316265, "learning_rate": 1e-05, "loss": 0.0169, "num_tokens": 13463636.0, "reward": 0.75, "reward_std": 0.292504221200943, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.9551899433135986, "sampling/importance_sampling_ratio/mean": 0.99978107213974, "sampling/importance_sampling_ratio/min": 0.6795781850814819, "sampling/sampling_logp_difference/max": 0.6704874038696289, "sampling/sampling_logp_difference/mean": 0.010118094272911549, "step": 648 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.487269663717598e-05, "clip_ratio/low_min": 5.487269663717598e-05, "clip_ratio/region_mean": 5.487269663717598e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 688.0, "completions/max_terminated_length": 688.0, "completions/mean_length": 428.0625, "completions/mean_terminated_length": 428.0625, "completions/min_length": 244.0, "completions/min_terminated_length": 244.0, "entropy": 0.26039893738925457, "epoch": 0.3474304068522484, "frac_reward_zero_std": 0.75, "grad_norm": 0.01623428985476494, "learning_rate": 1e-05, "loss": 0.0077, "num_tokens": 13480558.0, "reward": 0.8125, "reward_std": 0.1157275140285492, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.528709053993225, "sampling/importance_sampling_ratio/mean": 1.0003135204315186, "sampling/importance_sampling_ratio/min": 0.7110331654548645, "sampling/sampling_logp_difference/max": 0.4244236946105957, "sampling/sampling_logp_difference/mean": 0.009109492413699627, "step": 649 }, { "clip_ratio/high_max": 9.984025382436812e-05, "clip_ratio/high_mean": 9.984025382436812e-05, "clip_ratio/low_mean": 0.00010296540131093934, "clip_ratio/low_min": 0.00010296540131093934, "clip_ratio/region_mean": 0.00020280565513530746, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 588.0, "completions/mean_length": 381.125, "completions/mean_terminated_length": 325.8571472167969, "completions/min_length": 147.0, "completions/min_terminated_length": 147.0, "entropy": 0.41817745566368103, "epoch": 0.34796573875803, "frac_reward_zero_std": 0.75, "grad_norm": 0.020147062838077545, "learning_rate": 1e-05, "loss": 0.0932, "num_tokens": 13496930.0, "reward": 0.8125, "reward_std": 0.1157275140285492, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.4589701890945435, "sampling/importance_sampling_ratio/mean": 1.000463843345642, "sampling/importance_sampling_ratio/min": 0.6899737119674683, "sampling/sampling_logp_difference/max": 0.3777308464050293, "sampling/sampling_logp_difference/mean": 0.014919880777597427, "step": 650 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 743.0, "completions/mean_length": 482.9375, "completions/mean_terminated_length": 442.21429443359375, "completions/min_length": 177.0, "completions/min_terminated_length": 177.0, "entropy": 0.4246320314705372, "epoch": 0.3485010706638116, "frac_reward_zero_std": 0.75, "grad_norm": 0.00700872577726841, "learning_rate": 1e-05, "loss": -0.0015, "num_tokens": 13516568.0, "reward": 0.40625, "reward_std": 0.1293872892856598, "rewards/accuracy_reward/mean": 0.40625, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.6444509029388428, "sampling/importance_sampling_ratio/mean": 0.9996163845062256, "sampling/importance_sampling_ratio/min": 0.6946247816085815, "sampling/sampling_logp_difference/max": 0.4974064826965332, "sampling/sampling_logp_difference/mean": 0.014024922624230385, "step": 651 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00015512068785028532, "clip_ratio/low_min": 0.00015512068785028532, "clip_ratio/region_mean": 0.00015512068785028532, "completions/clipped_ratio": 0.0, "completions/max_length": 551.0, "completions/max_terminated_length": 551.0, "completions/mean_length": 348.03125, "completions/mean_terminated_length": 348.03125, "completions/min_length": 190.0, "completions/min_terminated_length": 190.0, "entropy": 0.35516560450196266, "epoch": 0.3490364025695932, "frac_reward_zero_std": 0.5, "grad_norm": 0.0200100876390934, "learning_rate": 1e-05, "loss": -0.004, "num_tokens": 13531425.0, "reward": 0.6875, "reward_std": 0.249358132481575, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.5035443305969238, "sampling/importance_sampling_ratio/mean": 0.9996904730796814, "sampling/importance_sampling_ratio/min": 0.413336843252182, "sampling/sampling_logp_difference/max": 0.8834924697875977, "sampling/sampling_logp_difference/mean": 0.012606313452124596, "step": 652 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00016290302301058546, "clip_ratio/low_min": 0.00016290302301058546, "clip_ratio/region_mean": 0.00016290302301058546, "completions/clipped_ratio": 0.0, "completions/max_length": 564.0, "completions/max_terminated_length": 564.0, "completions/mean_length": 436.4375, "completions/mean_terminated_length": 436.4375, "completions/min_length": 307.0, "completions/min_terminated_length": 307.0, "entropy": 0.259181784465909, "epoch": 0.3495717344753747, "frac_reward_zero_std": 0.0, "grad_norm": 0.014700076542794704, "learning_rate": 1e-05, "loss": 0.0024, "num_tokens": 13549503.0, "reward": 0.59375, "reward_std": 0.4534739851951599, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.2649004459381104, "sampling/importance_sampling_ratio/mean": 0.9999638795852661, "sampling/importance_sampling_ratio/min": 0.6338332891464233, "sampling/sampling_logp_difference/max": 0.45596933364868164, "sampling/sampling_logp_difference/mean": 0.008763880468904972, "step": 653 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 760.0, "completions/mean_length": 393.5625, "completions/mean_terminated_length": 381.4838562011719, "completions/min_length": 118.0, "completions/min_terminated_length": 118.0, "entropy": 0.460235420614481, "epoch": 0.3501070663811563, "frac_reward_zero_std": 0.5, "grad_norm": 0.004804051946848631, "learning_rate": 1e-05, "loss": -0.0044, "num_tokens": 13565617.0, "reward": 0.75, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.5459589958190918, "sampling/importance_sampling_ratio/mean": 1.0002037286758423, "sampling/importance_sampling_ratio/min": 0.7705621719360352, "sampling/sampling_logp_difference/max": 0.43564438819885254, "sampling/sampling_logp_difference/mean": 0.013562281616032124, "step": 654 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0001592706612427719, "clip_ratio/low_min": 0.0001592706612427719, "clip_ratio/region_mean": 0.0001592706612427719, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 693.0, "completions/mean_length": 445.21875, "completions/mean_terminated_length": 399.1071472167969, "completions/min_length": 186.0, "completions/min_terminated_length": 186.0, "entropy": 0.39120227843523026, "epoch": 0.3506423982869379, "frac_reward_zero_std": 0.25, "grad_norm": 0.01896318420767784, "learning_rate": 1e-05, "loss": 0.0684, "num_tokens": 13583144.0, "reward": 0.6875, "reward_std": 0.3471825420856476, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.3875007629394531, "sampling/importance_sampling_ratio/mean": 0.9998114109039307, "sampling/importance_sampling_ratio/min": 0.6009103655815125, "sampling/sampling_logp_difference/max": 0.5093095302581787, "sampling/sampling_logp_difference/mean": 0.012231100350618362, "step": 655 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0001298701245104894, "clip_ratio/low_min": 0.0001298701245104894, "clip_ratio/region_mean": 0.0001298701245104894, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 695.0, "completions/mean_length": 502.5625, "completions/mean_terminated_length": 484.86669921875, "completions/min_length": 317.0, "completions/min_terminated_length": 317.0, "entropy": 0.2781757991760969, "epoch": 0.3511777301927195, "frac_reward_zero_std": 0.25, "grad_norm": 0.007967548444867134, "learning_rate": 1e-05, "loss": 0.0895, "num_tokens": 13602442.0, "reward": 0.78125, "reward_std": 0.3377464711666107, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.4814249277114868, "sampling/importance_sampling_ratio/mean": 0.9999395608901978, "sampling/importance_sampling_ratio/min": 0.6002770066261292, "sampling/sampling_logp_difference/max": 0.5103640556335449, "sampling/sampling_logp_difference/mean": 0.009514091536402702, "step": 656 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00014287418889580294, "clip_ratio/low_min": 0.00014287418889580294, "clip_ratio/region_mean": 0.00014287418889580294, "completions/clipped_ratio": 0.0, "completions/max_length": 744.0, "completions/max_terminated_length": 744.0, "completions/mean_length": 344.28125, "completions/mean_terminated_length": 344.28125, "completions/min_length": 157.0, "completions/min_terminated_length": 157.0, "entropy": 0.4778929203748703, "epoch": 0.35171306209850106, "frac_reward_zero_std": 0.25, "grad_norm": 0.03561371564865112, "learning_rate": 1e-05, "loss": -0.0209, "num_tokens": 13618115.0, "reward": 0.75, "reward_std": 0.3745020925998688, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9999615550041199, "sampling/importance_sampling_ratio/min": 0.6989265084266663, "sampling/sampling_logp_difference/max": 0.8352646827697754, "sampling/sampling_logp_difference/mean": 0.014193162322044373, "step": 657 }, { "clip_ratio/high_max": 8.043758134590462e-05, "clip_ratio/high_mean": 8.043758134590462e-05, "clip_ratio/low_mean": 5.885122300242074e-05, "clip_ratio/low_min": 5.885122300242074e-05, "clip_ratio/region_mean": 0.00013928880434832536, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 669.0, "completions/mean_length": 422.8125, "completions/mean_terminated_length": 411.6773986816406, "completions/min_length": 236.0, "completions/min_terminated_length": 236.0, "entropy": 0.27686809934675694, "epoch": 0.35224839400428265, "frac_reward_zero_std": 0.5, "grad_norm": 0.010238280519843102, "learning_rate": 1e-05, "loss": 0.0128, "num_tokens": 13635869.0, "reward": 0.875, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 1.4051257371902466, "sampling/importance_sampling_ratio/mean": 1.0004974603652954, "sampling/importance_sampling_ratio/min": 0.5414208173751831, "sampling/sampling_logp_difference/max": 0.6135585308074951, "sampling/sampling_logp_difference/mean": 0.010490876622498035, "step": 658 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 743.0, "completions/mean_length": 496.59375, "completions/mean_terminated_length": 420.5999755859375, "completions/min_length": 228.0, "completions/min_terminated_length": 228.0, "entropy": 0.4050780236721039, "epoch": 0.35278372591006424, "frac_reward_zero_std": 0.5, "grad_norm": 0.01573796197772026, "learning_rate": 1e-05, "loss": 0.0283, "num_tokens": 13655752.0, "reward": 0.53125, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.6682548522949219, "sampling/importance_sampling_ratio/mean": 0.9996892809867859, "sampling/importance_sampling_ratio/min": 0.6459711194038391, "sampling/sampling_logp_difference/max": 0.5117781162261963, "sampling/sampling_logp_difference/mean": 0.01291722059249878, "step": 659 }, { "clip_ratio/high_max": 6.909894727868959e-05, "clip_ratio/high_mean": 6.909894727868959e-05, "clip_ratio/low_mean": 0.00014629126235377043, "clip_ratio/low_min": 0.00014629126235377043, "clip_ratio/region_mean": 0.00021539020963246003, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 611.0, "completions/mean_length": 446.09375, "completions/mean_terminated_length": 424.63336181640625, "completions/min_length": 191.0, "completions/min_terminated_length": 191.0, "entropy": 0.3119290601462126, "epoch": 0.3533190578158458, "frac_reward_zero_std": 0.0, "grad_norm": 0.04319221153855324, "learning_rate": 1e-05, "loss": 0.0081, "num_tokens": 13674315.0, "reward": 0.59375, "reward_std": 0.3987956643104553, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.6307077407836914, "sampling/importance_sampling_ratio/mean": 1.0006777048110962, "sampling/importance_sampling_ratio/min": 0.6531321406364441, "sampling/sampling_logp_difference/max": 0.4890141487121582, "sampling/sampling_logp_difference/mean": 0.010623999871313572, "step": 660 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 8.729050023248419e-05, "clip_ratio/low_min": 8.729050023248419e-05, "clip_ratio/region_mean": 8.729050023248419e-05, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 576.0, "completions/mean_length": 379.75, "completions/mean_terminated_length": 367.2257995605469, "completions/min_length": 235.0, "completions/min_terminated_length": 235.0, "entropy": 0.3697916939854622, "epoch": 0.3538543897216274, "frac_reward_zero_std": 0.0, "grad_norm": 0.04080353304743767, "learning_rate": 1e-05, "loss": 0.0464, "num_tokens": 13690123.0, "reward": 0.53125, "reward_std": 0.4355708956718445, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.438925862312317, "sampling/importance_sampling_ratio/mean": 0.9999911189079285, "sampling/importance_sampling_ratio/min": 0.6053933501243591, "sampling/sampling_logp_difference/max": 0.5018768310546875, "sampling/sampling_logp_difference/mean": 0.01327377837151289, "step": 661 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 681.0, "completions/mean_length": 397.125, "completions/mean_terminated_length": 385.1612854003906, "completions/min_length": 150.0, "completions/min_terminated_length": 150.0, "entropy": 0.26181492395699024, "epoch": 0.354389721627409, "frac_reward_zero_std": 0.75, "grad_norm": 0.0094644445925951, "learning_rate": 1e-05, "loss": -0.0471, "num_tokens": 13705999.0, "reward": 0.9375, "reward_std": 0.1157275140285492, "rewards/accuracy_reward/mean": 0.9375, "rewards/accuracy_reward/std": 0.24593468010425568, "sampling/importance_sampling_ratio/max": 1.5801801681518555, "sampling/importance_sampling_ratio/mean": 0.9998408555984497, "sampling/importance_sampling_ratio/min": 0.3525858521461487, "sampling/sampling_logp_difference/max": 1.0424611568450928, "sampling/sampling_logp_difference/mean": 0.009918587282299995, "step": 662 }, { "clip_ratio/high_max": 9.110787505051121e-05, "clip_ratio/high_mean": 9.110787505051121e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 9.110787505051121e-05, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 672.0, "completions/mean_length": 495.75, "completions/mean_terminated_length": 419.5199890136719, "completions/min_length": 275.0, "completions/min_terminated_length": 275.0, "entropy": 0.5153794698417187, "epoch": 0.3549250535331906, "frac_reward_zero_std": 0.25, "grad_norm": 0.027701010927557945, "learning_rate": 1e-05, "loss": 0.0769, "num_tokens": 13725967.0, "reward": 0.46875, "reward_std": 0.3198433816432953, "rewards/accuracy_reward/mean": 0.46875, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.5489026308059692, "sampling/importance_sampling_ratio/mean": 0.9996196627616882, "sampling/importance_sampling_ratio/min": 0.6261435747146606, "sampling/sampling_logp_difference/max": 0.4681755304336548, "sampling/sampling_logp_difference/mean": 0.015272910706698895, "step": 663 }, { "clip_ratio/high_max": 6.789788312744349e-05, "clip_ratio/high_mean": 6.789788312744349e-05, "clip_ratio/low_mean": 6.789788312744349e-05, "clip_ratio/low_min": 6.789788312744349e-05, "clip_ratio/region_mean": 0.00013579576625488698, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 614.0, "completions/mean_length": 449.65625, "completions/mean_terminated_length": 404.1785888671875, "completions/min_length": 246.0, "completions/min_terminated_length": 246.0, "entropy": 0.5399555154144764, "epoch": 0.3554603854389722, "frac_reward_zero_std": 0.0, "grad_norm": 0.012038396671414375, "learning_rate": 1e-05, "loss": 0.1164, "num_tokens": 13744804.0, "reward": 0.59375, "reward_std": 0.4807935357093811, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.999747097492218, "sampling/importance_sampling_ratio/min": 0.4197999835014343, "sampling/sampling_logp_difference/max": 0.8679769039154053, "sampling/sampling_logp_difference/mean": 0.016079533845186234, "step": 664 }, { "clip_ratio/high_max": 7.494004967156798e-05, "clip_ratio/high_mean": 7.494004967156798e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 7.494004967156798e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 637.0, "completions/max_terminated_length": 637.0, "completions/mean_length": 377.65625, "completions/mean_terminated_length": 377.65625, "completions/min_length": 108.0, "completions/min_terminated_length": 108.0, "entropy": 0.2771514691412449, "epoch": 0.35599571734475377, "frac_reward_zero_std": 0.5, "grad_norm": 0.005888278130441904, "learning_rate": 1e-05, "loss": 0.0078, "num_tokens": 13760321.0, "reward": 0.75, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.4778823852539062, "sampling/importance_sampling_ratio/mean": 0.9998041391372681, "sampling/importance_sampling_ratio/min": 0.6845996379852295, "sampling/sampling_logp_difference/max": 0.3906102180480957, "sampling/sampling_logp_difference/mean": 0.010204792954027653, "step": 665 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.000129915468278341, "clip_ratio/low_min": 0.000129915468278341, "clip_ratio/region_mean": 0.000129915468278341, "completions/clipped_ratio": 0.0, "completions/max_length": 714.0, "completions/max_terminated_length": 714.0, "completions/mean_length": 444.40625, "completions/mean_terminated_length": 444.40625, "completions/min_length": 207.0, "completions/min_terminated_length": 207.0, "entropy": 0.3105376996099949, "epoch": 0.35653104925053536, "frac_reward_zero_std": 0.25, "grad_norm": 0.016817567870020866, "learning_rate": 1e-05, "loss": -0.007, "num_tokens": 13778270.0, "reward": 0.78125, "reward_std": 0.3471629321575165, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.6711481809616089, "sampling/importance_sampling_ratio/mean": 0.999639630317688, "sampling/importance_sampling_ratio/min": 0.6574479341506958, "sampling/sampling_logp_difference/max": 0.5135109424591064, "sampling/sampling_logp_difference/mean": 0.011440515518188477, "step": 666 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 568.0, "completions/max_terminated_length": 568.0, "completions/mean_length": 364.625, "completions/mean_terminated_length": 364.625, "completions/min_length": 176.0, "completions/min_terminated_length": 176.0, "entropy": 0.3822487108409405, "epoch": 0.3570663811563169, "frac_reward_zero_std": 0.25, "grad_norm": 0.022095561027526855, "learning_rate": 1e-05, "loss": -0.0609, "num_tokens": 13793434.0, "reward": 0.65625, "reward_std": 0.3377464711666107, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.9472275972366333, "sampling/importance_sampling_ratio/mean": 1.0000711679458618, "sampling/importance_sampling_ratio/min": 0.6074678897857666, "sampling/sampling_logp_difference/max": 0.6664066314697266, "sampling/sampling_logp_difference/mean": 0.01337414886802435, "step": 667 }, { "clip_ratio/high_max": 6.537656736327335e-05, "clip_ratio/high_mean": 6.537656736327335e-05, "clip_ratio/low_mean": 0.00015136247384361923, "clip_ratio/low_min": 0.00015136247384361923, "clip_ratio/region_mean": 0.00021673904120689258, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 740.0, "completions/mean_length": 434.9375, "completions/mean_terminated_length": 424.19354248046875, "completions/min_length": 163.0, "completions/min_terminated_length": 163.0, "entropy": 0.2582597993314266, "epoch": 0.3576017130620985, "frac_reward_zero_std": 0.0, "grad_norm": 0.048923395574092865, "learning_rate": 1e-05, "loss": 0.0356, "num_tokens": 13810568.0, "reward": 0.65625, "reward_std": 0.3808925747871399, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.4425057172775269, "sampling/importance_sampling_ratio/mean": 1.0000734329223633, "sampling/importance_sampling_ratio/min": 0.5596014857292175, "sampling/sampling_logp_difference/max": 0.5805304050445557, "sampling/sampling_logp_difference/mean": 0.009786554612219334, "step": 668 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 546.0, "completions/max_terminated_length": 546.0, "completions/mean_length": 367.53125, "completions/mean_terminated_length": 367.53125, "completions/min_length": 201.0, "completions/min_terminated_length": 201.0, "entropy": 0.315692201256752, "epoch": 0.35813704496788007, "frac_reward_zero_std": 0.5, "grad_norm": 0.00875961035490036, "learning_rate": 1e-05, "loss": 0.0145, "num_tokens": 13826417.0, "reward": 0.8125, "reward_std": 0.2587745785713196, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.6791738271713257, "sampling/importance_sampling_ratio/mean": 0.9998831748962402, "sampling/importance_sampling_ratio/min": 0.6887439489364624, "sampling/sampling_logp_difference/max": 0.5183019638061523, "sampling/sampling_logp_difference/mean": 0.012204647995531559, "step": 669 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 7.678132533328608e-05, "clip_ratio/low_min": 7.678132533328608e-05, "clip_ratio/region_mean": 7.678132533328608e-05, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 754.0, "completions/mean_length": 431.78125, "completions/mean_terminated_length": 409.36669921875, "completions/min_length": 198.0, "completions/min_terminated_length": 198.0, "entropy": 0.3540220931172371, "epoch": 0.35867237687366166, "frac_reward_zero_std": 0.5, "grad_norm": 0.012857015244662762, "learning_rate": 1e-05, "loss": 0.0082, "num_tokens": 13844394.0, "reward": 0.6875, "reward_std": 0.249358132481575, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.58338463306427, "sampling/importance_sampling_ratio/mean": 1.00005042552948, "sampling/importance_sampling_ratio/min": 0.64851313829422, "sampling/sampling_logp_difference/max": 0.4595646858215332, "sampling/sampling_logp_difference/mean": 0.011811037547886372, "step": 670 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 613.0, "completions/mean_length": 414.375, "completions/mean_terminated_length": 402.9677429199219, "completions/min_length": 202.0, "completions/min_terminated_length": 202.0, "entropy": 0.2561605256050825, "epoch": 0.35920770877944325, "frac_reward_zero_std": 0.5, "grad_norm": 0.005706072319298983, "learning_rate": 1e-05, "loss": 0.078, "num_tokens": 13861166.0, "reward": 0.8125, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.402472972869873, "sampling/importance_sampling_ratio/mean": 0.999974250793457, "sampling/importance_sampling_ratio/min": 0.5753530263900757, "sampling/sampling_logp_difference/max": 0.5527715682983398, "sampling/sampling_logp_difference/mean": 0.00912991538643837, "step": 671 }, { "clip_ratio/high_max": 0.00017741198098519817, "clip_ratio/high_mean": 0.00017741198098519817, "clip_ratio/low_mean": 0.00018907566845882684, "clip_ratio/low_min": 0.00018907566845882684, "clip_ratio/region_mean": 0.000366487649444025, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 666.0, "completions/mean_length": 401.0, "completions/mean_terminated_length": 389.1612854003906, "completions/min_length": 118.0, "completions/min_terminated_length": 118.0, "entropy": 0.38735750690102577, "epoch": 0.35974304068522484, "frac_reward_zero_std": 0.0, "grad_norm": 0.008876451291143894, "learning_rate": 1e-05, "loss": -0.0949, "num_tokens": 13878022.0, "reward": 0.84375, "reward_std": 0.3808925747871399, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.535284161567688, "sampling/importance_sampling_ratio/mean": 1.0001300573349, "sampling/importance_sampling_ratio/min": 0.45783501863479614, "sampling/sampling_logp_difference/max": 0.7812464237213135, "sampling/sampling_logp_difference/mean": 0.01291007548570633, "step": 672 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 526.0, "completions/max_terminated_length": 526.0, "completions/mean_length": 359.6875, "completions/mean_terminated_length": 359.6875, "completions/min_length": 128.0, "completions/min_terminated_length": 128.0, "entropy": 0.29211121797561646, "epoch": 0.3602783725910064, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 13893076.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.832126498222351, "sampling/importance_sampling_ratio/mean": 0.9998286366462708, "sampling/importance_sampling_ratio/min": 0.6737604737281799, "sampling/sampling_logp_difference/max": 0.6054773330688477, "sampling/sampling_logp_difference/mean": 0.010949667543172836, "step": 673 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00013384085468715057, "clip_ratio/low_min": 0.00013384085468715057, "clip_ratio/region_mean": 0.00013384085468715057, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 764.0, "completions/mean_length": 478.375, "completions/mean_terminated_length": 459.0666809082031, "completions/min_length": 296.0, "completions/min_terminated_length": 296.0, "entropy": 0.3839053250849247, "epoch": 0.360813704496788, "frac_reward_zero_std": 0.25, "grad_norm": 0.015960123389959335, "learning_rate": 1e-05, "loss": 0.1086, "num_tokens": 13912512.0, "reward": 0.6875, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.4055489301681519, "sampling/importance_sampling_ratio/mean": 0.9999480843544006, "sampling/importance_sampling_ratio/min": 0.5603016018867493, "sampling/sampling_logp_difference/max": 0.5792800188064575, "sampling/sampling_logp_difference/mean": 0.013557146303355694, "step": 674 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00015738436923129484, "clip_ratio/low_min": 0.00015738436923129484, "clip_ratio/region_mean": 0.00015738436923129484, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 742.0, "completions/mean_length": 375.71875, "completions/mean_terminated_length": 363.06451416015625, "completions/min_length": 232.0, "completions/min_terminated_length": 232.0, "entropy": 0.3089521825313568, "epoch": 0.3613490364025696, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": -0.0096, "num_tokens": 13928359.0, "reward": 0.90625, "reward_std": 0.1293872892856598, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.8923866748809814, "sampling/importance_sampling_ratio/mean": 1.0001062154769897, "sampling/importance_sampling_ratio/min": 0.6327048540115356, "sampling/sampling_logp_difference/max": 0.6378388404846191, "sampling/sampling_logp_difference/mean": 0.012162160128355026, "step": 675 }, { "clip_ratio/high_max": 0.00012065636838087812, "clip_ratio/high_mean": 0.00012065636838087812, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00012065636838087812, "completions/clipped_ratio": 0.0, "completions/max_length": 727.0, "completions/max_terminated_length": 727.0, "completions/mean_length": 314.125, "completions/mean_terminated_length": 314.125, "completions/min_length": 120.0, "completions/min_terminated_length": 120.0, "entropy": 0.34653982892632484, "epoch": 0.3618843683083512, "frac_reward_zero_std": 0.5, "grad_norm": 0.004590500611811876, "learning_rate": 1e-05, "loss": 0.0698, "num_tokens": 13942379.0, "reward": 0.9375, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.9375, "rewards/accuracy_reward/std": 0.24593468010425568, "sampling/importance_sampling_ratio/max": 1.4619866609573364, "sampling/importance_sampling_ratio/mean": 0.9997506141662598, "sampling/importance_sampling_ratio/min": 0.546380877494812, "sampling/sampling_logp_difference/max": 0.6044389009475708, "sampling/sampling_logp_difference/mean": 0.011684027500450611, "step": 676 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 605.0, "completions/mean_length": 433.34375, "completions/mean_terminated_length": 385.5357360839844, "completions/min_length": 239.0, "completions/min_terminated_length": 239.0, "entropy": 0.40856448002159595, "epoch": 0.3624197002141328, "frac_reward_zero_std": 0.75, "grad_norm": 0.003538036486133933, "learning_rate": 1e-05, "loss": -0.0164, "num_tokens": 13960094.0, "reward": 0.71875, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.4316827058792114, "sampling/importance_sampling_ratio/mean": 0.9999025464057922, "sampling/importance_sampling_ratio/min": 0.5827119946479797, "sampling/sampling_logp_difference/max": 0.5400621891021729, "sampling/sampling_logp_difference/mean": 0.013678331859409809, "step": 677 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 7.78331232140772e-05, "clip_ratio/low_min": 7.78331232140772e-05, "clip_ratio/region_mean": 7.78331232140772e-05, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 755.0, "completions/mean_length": 416.875, "completions/mean_terminated_length": 393.4666748046875, "completions/min_length": 79.0, "completions/min_terminated_length": 79.0, "entropy": 0.279329190030694, "epoch": 0.36295503211991437, "frac_reward_zero_std": 0.75, "grad_norm": 0.009009398519992828, "learning_rate": 1e-05, "loss": 0.0378, "num_tokens": 13976618.0, "reward": 0.84375, "reward_std": 0.1293872892856598, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.7500592470169067, "sampling/importance_sampling_ratio/mean": 1.0001730918884277, "sampling/importance_sampling_ratio/min": 0.5929864645004272, "sampling/sampling_logp_difference/max": 0.5596495866775513, "sampling/sampling_logp_difference/mean": 0.009725116193294525, "step": 678 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00016687424067640677, "clip_ratio/low_min": 0.00016687424067640677, "clip_ratio/region_mean": 0.00016687424067640677, "completions/clipped_ratio": 0.0, "completions/max_length": 628.0, "completions/max_terminated_length": 628.0, "completions/mean_length": 373.75, "completions/mean_terminated_length": 373.75, "completions/min_length": 110.0, "completions/min_terminated_length": 110.0, "entropy": 0.2835135981440544, "epoch": 0.36349036402569596, "frac_reward_zero_std": 0.5, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": -0.1194, "num_tokens": 13992162.0, "reward": 0.75, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.4620192050933838, "sampling/importance_sampling_ratio/mean": 0.9996680021286011, "sampling/importance_sampling_ratio/min": 0.5569161772727966, "sampling/sampling_logp_difference/max": 0.5853404998779297, "sampling/sampling_logp_difference/mean": 0.011053714901208878, "step": 679 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00014245013881009072, "clip_ratio/low_min": 0.00014245013881009072, "clip_ratio/region_mean": 0.00014245013881009072, "completions/clipped_ratio": 0.0, "completions/max_length": 633.0, "completions/max_terminated_length": 633.0, "completions/mean_length": 419.59375, "completions/mean_terminated_length": 419.59375, "completions/min_length": 272.0, "completions/min_terminated_length": 272.0, "entropy": 0.26943989656865597, "epoch": 0.3640256959314775, "frac_reward_zero_std": 0.25, "grad_norm": 0.022552641108632088, "learning_rate": 1e-05, "loss": 0.0257, "num_tokens": 14009085.0, "reward": 0.625, "reward_std": 0.3745020925998688, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9997143745422363, "sampling/importance_sampling_ratio/min": 0.4835348427295685, "sampling/sampling_logp_difference/max": 0.7421181201934814, "sampling/sampling_logp_difference/mean": 0.010526628233492374, "step": 680 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00015492347301915288, "clip_ratio/low_min": 0.00015492347301915288, "clip_ratio/region_mean": 0.00015492347301915288, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 723.0, "completions/mean_length": 451.34375, "completions/mean_terminated_length": 418.5862121582031, "completions/min_length": 244.0, "completions/min_terminated_length": 244.0, "entropy": 0.47729120403528214, "epoch": 0.3645610278372591, "frac_reward_zero_std": 0.0, "grad_norm": 0.016579551622271538, "learning_rate": 1e-05, "loss": 0.0787, "num_tokens": 14027240.0, "reward": 0.53125, "reward_std": 0.4397946000099182, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.8460532426834106, "sampling/importance_sampling_ratio/mean": 1.0000616312026978, "sampling/importance_sampling_ratio/min": 0.6330141425132751, "sampling/sampling_logp_difference/max": 0.6130499839782715, "sampling/sampling_logp_difference/mean": 0.014952778816223145, "step": 681 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 586.0, "completions/max_terminated_length": 586.0, "completions/mean_length": 394.84375, "completions/mean_terminated_length": 394.84375, "completions/min_length": 183.0, "completions/min_terminated_length": 183.0, "entropy": 0.27810865454375744, "epoch": 0.36509635974304067, "frac_reward_zero_std": 0.25, "grad_norm": 0.012250752188265324, "learning_rate": 1e-05, "loss": -0.0525, "num_tokens": 14043987.0, "reward": 0.78125, "reward_std": 0.3061639666557312, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.499852180480957, "sampling/importance_sampling_ratio/mean": 1.0003774166107178, "sampling/importance_sampling_ratio/min": 0.46095797419548035, "sampling/sampling_logp_difference/max": 0.7744483947753906, "sampling/sampling_logp_difference/mean": 0.010378003120422363, "step": 682 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 479.0, "completions/max_terminated_length": 479.0, "completions/mean_length": 305.1875, "completions/mean_terminated_length": 305.1875, "completions/min_length": 118.0, "completions/min_terminated_length": 118.0, "entropy": 0.3248937167227268, "epoch": 0.36563169164882225, "frac_reward_zero_std": 0.5, "grad_norm": 0.02616657130420208, "learning_rate": 1e-05, "loss": 0.0288, "num_tokens": 14057617.0, "reward": 0.625, "reward_std": 0.2314550280570984, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.5338889360427856, "sampling/importance_sampling_ratio/mean": 1.0000650882720947, "sampling/importance_sampling_ratio/min": 0.6569048762321472, "sampling/sampling_logp_difference/max": 0.4278062582015991, "sampling/sampling_logp_difference/mean": 0.012229588814079762, "step": 683 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 637.0, "completions/mean_length": 374.53125, "completions/mean_terminated_length": 361.83868408203125, "completions/min_length": 175.0, "completions/min_terminated_length": 175.0, "entropy": 0.3952745087444782, "epoch": 0.36616702355460384, "frac_reward_zero_std": 0.25, "grad_norm": 0.0198307354003191, "learning_rate": 1e-05, "loss": 0.028, "num_tokens": 14073274.0, "reward": 0.71875, "reward_std": 0.378745436668396, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.6379166841506958, "sampling/importance_sampling_ratio/mean": 0.9999538064002991, "sampling/importance_sampling_ratio/min": 0.6694729924201965, "sampling/sampling_logp_difference/max": 0.4934251308441162, "sampling/sampling_logp_difference/mean": 0.013427930884063244, "step": 684 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00014134941739030182, "clip_ratio/low_min": 0.00014134941739030182, "clip_ratio/region_mean": 0.00014134941739030182, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 754.0, "completions/mean_length": 445.09375, "completions/mean_terminated_length": 423.5666809082031, "completions/min_length": 146.0, "completions/min_terminated_length": 146.0, "entropy": 0.37205213122069836, "epoch": 0.36670235546038543, "frac_reward_zero_std": 0.25, "grad_norm": 0.02514619193971157, "learning_rate": 1e-05, "loss": -0.0186, "num_tokens": 14091501.0, "reward": 0.625, "reward_std": 0.3104073107242584, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.000280737876892, "sampling/importance_sampling_ratio/min": 0.3590173125267029, "sampling/sampling_logp_difference/max": 1.0243847370147705, "sampling/sampling_logp_difference/mean": 0.013068465515971184, "step": 685 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 586.0, "completions/mean_length": 360.78125, "completions/mean_terminated_length": 347.6451416015625, "completions/min_length": 225.0, "completions/min_terminated_length": 225.0, "entropy": 0.34020672738552094, "epoch": 0.367237687366167, "frac_reward_zero_std": 0.25, "grad_norm": 0.0432632714509964, "learning_rate": 1e-05, "loss": 0.0745, "num_tokens": 14106646.0, "reward": 0.6875, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.7226487398147583, "sampling/importance_sampling_ratio/mean": 1.0001485347747803, "sampling/importance_sampling_ratio/min": 0.6301517486572266, "sampling/sampling_logp_difference/max": 0.54386305809021, "sampling/sampling_logp_difference/mean": 0.01216810755431652, "step": 686 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 8.366800466319546e-05, "clip_ratio/low_min": 8.366800466319546e-05, "clip_ratio/region_mean": 8.366800466319546e-05, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 630.0, "completions/mean_length": 417.0, "completions/mean_terminated_length": 380.6896667480469, "completions/min_length": 220.0, "completions/min_terminated_length": 220.0, "entropy": 0.3689681924879551, "epoch": 0.3677730192719486, "frac_reward_zero_std": 0.5, "grad_norm": 0.017116356641054153, "learning_rate": 1e-05, "loss": 0.0792, "num_tokens": 14123014.0, "reward": 0.84375, "reward_std": 0.22201895713806152, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.6711039543151855, "sampling/importance_sampling_ratio/mean": 0.9996310472488403, "sampling/importance_sampling_ratio/min": 0.5510715842247009, "sampling/sampling_logp_difference/max": 0.5958905220031738, "sampling/sampling_logp_difference/mean": 0.012507694773375988, "step": 687 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 730.0, "completions/mean_length": 429.4375, "completions/mean_terminated_length": 406.86669921875, "completions/min_length": 234.0, "completions/min_terminated_length": 234.0, "entropy": 0.36085673049092293, "epoch": 0.3683083511777302, "frac_reward_zero_std": 0.75, "grad_norm": 0.026025017723441124, "learning_rate": 1e-05, "loss": 0.0327, "num_tokens": 14140332.0, "reward": 0.65625, "reward_std": 0.1293872892856598, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.9497008323669434, "sampling/importance_sampling_ratio/mean": 0.9995465874671936, "sampling/importance_sampling_ratio/min": 0.5762607455253601, "sampling/sampling_logp_difference/max": 0.6676759719848633, "sampling/sampling_logp_difference/mean": 0.01313733495771885, "step": 688 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 7.263218867592514e-05, "clip_ratio/low_min": 7.263218867592514e-05, "clip_ratio/region_mean": 7.263218867592514e-05, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 722.0, "completions/mean_length": 487.40625, "completions/mean_terminated_length": 435.4444580078125, "completions/min_length": 219.0, "completions/min_terminated_length": 219.0, "entropy": 0.382045628502965, "epoch": 0.3688436830835118, "frac_reward_zero_std": 0.25, "grad_norm": 0.019538233056664467, "learning_rate": 1e-05, "loss": 0.0505, "num_tokens": 14160465.0, "reward": 0.5625, "reward_std": 0.292504221200943, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.953310489654541, "sampling/importance_sampling_ratio/mean": 1.0001777410507202, "sampling/importance_sampling_ratio/min": 0.5836070775985718, "sampling/sampling_logp_difference/max": 0.6695256233215332, "sampling/sampling_logp_difference/mean": 0.014216558076441288, "step": 689 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 746.0, "completions/mean_length": 377.59375, "completions/mean_terminated_length": 351.5666809082031, "completions/min_length": 154.0, "completions/min_terminated_length": 154.0, "entropy": 0.35210950300097466, "epoch": 0.3693790149892934, "frac_reward_zero_std": 0.5, "grad_norm": 0.00846430379897356, "learning_rate": 1e-05, "loss": 0.0782, "num_tokens": 14176300.0, "reward": 0.65625, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9999990463256836, "sampling/importance_sampling_ratio/min": 0.5328687429428101, "sampling/sampling_logp_difference/max": 0.7396030426025391, "sampling/sampling_logp_difference/mean": 0.01267185527831316, "step": 690 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 7.53012063796632e-05, "clip_ratio/low_min": 7.53012063796632e-05, "clip_ratio/region_mean": 7.53012063796632e-05, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 722.0, "completions/mean_length": 502.0, "completions/mean_terminated_length": 464.0000305175781, "completions/min_length": 222.0, "completions/min_terminated_length": 222.0, "entropy": 0.482187956571579, "epoch": 0.36991434689507496, "frac_reward_zero_std": 0.25, "grad_norm": 0.0195754524320364, "learning_rate": 1e-05, "loss": 0.0041, "num_tokens": 14196060.0, "reward": 0.5, "reward_std": 0.3514062762260437, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5080004930496216, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9999594688415527, "sampling/importance_sampling_ratio/min": 0.5215132832527161, "sampling/sampling_logp_difference/max": 0.8177294731140137, "sampling/sampling_logp_difference/mean": 0.0157401692122221, "step": 691 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 597.0, "completions/max_terminated_length": 597.0, "completions/mean_length": 447.1875, "completions/mean_terminated_length": 447.1875, "completions/min_length": 276.0, "completions/min_terminated_length": 276.0, "entropy": 0.23954479582607746, "epoch": 0.37044967880085655, "frac_reward_zero_std": 0.5, "grad_norm": 0.009266071952879429, "learning_rate": 1e-05, "loss": 0.0032, "num_tokens": 14214330.0, "reward": 0.6875, "reward_std": 0.249358132481575, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.872075080871582, "sampling/importance_sampling_ratio/mean": 0.9997952580451965, "sampling/importance_sampling_ratio/min": 0.6269657611846924, "sampling/sampling_logp_difference/max": 0.6270475387573242, "sampling/sampling_logp_difference/mean": 0.009122824296355247, "step": 692 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00010674636723706499, "clip_ratio/low_min": 0.00010674636723706499, "clip_ratio/region_mean": 0.00010674636723706499, "completions/clipped_ratio": 0.0, "completions/max_length": 562.0, "completions/max_terminated_length": 562.0, "completions/mean_length": 325.84375, "completions/mean_terminated_length": 325.84375, "completions/min_length": 72.0, "completions/min_terminated_length": 72.0, "entropy": 0.30397594533860683, "epoch": 0.37098501070663814, "frac_reward_zero_std": 0.25, "grad_norm": 0.027942130342125893, "learning_rate": 1e-05, "loss": -0.0269, "num_tokens": 14228373.0, "reward": 0.6875, "reward_std": 0.38298875093460083, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.5171500444412231, "sampling/importance_sampling_ratio/mean": 1.000351071357727, "sampling/importance_sampling_ratio/min": 0.5740833282470703, "sampling/sampling_logp_difference/max": 0.5549807548522949, "sampling/sampling_logp_difference/mean": 0.011213074438273907, "step": 693 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 762.0, "completions/mean_length": 511.625, "completions/mean_terminated_length": 475.0000305175781, "completions/min_length": 282.0, "completions/min_terminated_length": 282.0, "entropy": 0.376805879175663, "epoch": 0.3715203426124197, "frac_reward_zero_std": 0.5, "grad_norm": 0.00635635107755661, "learning_rate": 1e-05, "loss": 0.0007, "num_tokens": 14248377.0, "reward": 0.75, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.5212767124176025, "sampling/importance_sampling_ratio/mean": 0.9996983408927917, "sampling/importance_sampling_ratio/min": 0.7199479937553406, "sampling/sampling_logp_difference/max": 0.41954994201660156, "sampling/sampling_logp_difference/mean": 0.01242767833173275, "step": 694 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 658.0, "completions/mean_length": 414.53125, "completions/mean_terminated_length": 403.1290283203125, "completions/min_length": 168.0, "completions/min_terminated_length": 168.0, "entropy": 0.32878440991044044, "epoch": 0.37205567451820126, "frac_reward_zero_std": 0.25, "grad_norm": 0.014503064565360546, "learning_rate": 1e-05, "loss": -0.0431, "num_tokens": 14264962.0, "reward": 0.71875, "reward_std": 0.3787454068660736, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.730616807937622, "sampling/importance_sampling_ratio/mean": 0.9998878240585327, "sampling/importance_sampling_ratio/min": 0.5685059428215027, "sampling/sampling_logp_difference/max": 0.5647435188293457, "sampling/sampling_logp_difference/mean": 0.011785061098635197, "step": 695 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0001320291485171765, "clip_ratio/low_min": 0.0001320291485171765, "clip_ratio/region_mean": 0.0001320291485171765, "completions/clipped_ratio": 0.0, "completions/max_length": 700.0, "completions/max_terminated_length": 700.0, "completions/mean_length": 455.59375, "completions/mean_terminated_length": 455.59375, "completions/min_length": 271.0, "completions/min_terminated_length": 271.0, "entropy": 0.2597401551902294, "epoch": 0.37259100642398285, "frac_reward_zero_std": 0.25, "grad_norm": 0.01776432991027832, "learning_rate": 1e-05, "loss": -0.0185, "num_tokens": 14283445.0, "reward": 0.78125, "reward_std": 0.3608423173427582, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.6622546911239624, "sampling/importance_sampling_ratio/mean": 1.000230312347412, "sampling/importance_sampling_ratio/min": 0.5818818807601929, "sampling/sampling_logp_difference/max": 0.5414879322052002, "sampling/sampling_logp_difference/mean": 0.010266934521496296, "step": 696 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 655.0, "completions/max_terminated_length": 655.0, "completions/mean_length": 371.25, "completions/mean_terminated_length": 371.25, "completions/min_length": 153.0, "completions/min_terminated_length": 153.0, "entropy": 0.3413393497467041, "epoch": 0.37312633832976444, "frac_reward_zero_std": 0.0, "grad_norm": 0.021115630865097046, "learning_rate": 1e-05, "loss": -0.0336, "num_tokens": 14298893.0, "reward": 0.59375, "reward_std": 0.4534739851951599, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.779329776763916, "sampling/importance_sampling_ratio/mean": 1.000458836555481, "sampling/importance_sampling_ratio/min": 0.5514275431632996, "sampling/sampling_logp_difference/max": 0.5952447652816772, "sampling/sampling_logp_difference/mean": 0.012435673736035824, "step": 697 }, { "clip_ratio/high_max": 6.800870323786512e-05, "clip_ratio/high_mean": 6.800870323786512e-05, "clip_ratio/low_mean": 0.00015918692224659026, "clip_ratio/low_min": 0.00015918692224659026, "clip_ratio/region_mean": 0.00022719562548445538, "completions/clipped_ratio": 0.0, "completions/max_length": 562.0, "completions/max_terminated_length": 562.0, "completions/mean_length": 422.375, "completions/mean_terminated_length": 422.375, "completions/min_length": 293.0, "completions/min_terminated_length": 293.0, "entropy": 0.3052702955901623, "epoch": 0.37366167023554603, "frac_reward_zero_std": 0.25, "grad_norm": 0.021168548613786697, "learning_rate": 1e-05, "loss": -0.019, "num_tokens": 14316409.0, "reward": 0.71875, "reward_std": 0.378745436668396, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.7156918048858643, "sampling/importance_sampling_ratio/mean": 1.0006873607635498, "sampling/importance_sampling_ratio/min": 0.6708534955978394, "sampling/sampling_logp_difference/max": 0.5398163795471191, "sampling/sampling_logp_difference/mean": 0.010774184949696064, "step": 698 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 656.0, "completions/max_terminated_length": 656.0, "completions/mean_length": 353.90625, "completions/mean_terminated_length": 353.90625, "completions/min_length": 200.0, "completions/min_terminated_length": 200.0, "entropy": 0.2485798615962267, "epoch": 0.3741970021413276, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 14331086.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.2952086925506592, "sampling/importance_sampling_ratio/mean": 0.999834418296814, "sampling/importance_sampling_ratio/min": 0.46038660407066345, "sampling/sampling_logp_difference/max": 0.775688648223877, "sampling/sampling_logp_difference/mean": 0.009568484500050545, "step": 699 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 480.0, "completions/max_terminated_length": 480.0, "completions/mean_length": 271.09375, "completions/mean_terminated_length": 271.09375, "completions/min_length": 58.0, "completions/min_terminated_length": 58.0, "entropy": 0.3374805971980095, "epoch": 0.3747323340471092, "frac_reward_zero_std": 0.25, "grad_norm": 0.03927936777472496, "learning_rate": 1e-05, "loss": -0.034, "num_tokens": 14342969.0, "reward": 0.75, "reward_std": 0.3650856614112854, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.6612154245376587, "sampling/importance_sampling_ratio/mean": 0.9998925924301147, "sampling/importance_sampling_ratio/min": 0.4506168067455292, "sampling/sampling_logp_difference/max": 0.797137975692749, "sampling/sampling_logp_difference/mean": 0.012434404343366623, "step": 700 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 642.0, "completions/max_terminated_length": 642.0, "completions/mean_length": 398.375, "completions/mean_terminated_length": 398.375, "completions/min_length": 181.0, "completions/min_terminated_length": 181.0, "entropy": 0.2815327737480402, "epoch": 0.3752676659528908, "frac_reward_zero_std": 0.5, "grad_norm": 0.005384574178606272, "learning_rate": 1e-05, "loss": -0.0201, "num_tokens": 14359309.0, "reward": 0.6875, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.9044970273971558, "sampling/importance_sampling_ratio/mean": 1.0000734329223633, "sampling/importance_sampling_ratio/min": 0.4690447151660919, "sampling/sampling_logp_difference/max": 0.7570571899414062, "sampling/sampling_logp_difference/mean": 0.010723521932959557, "step": 701 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 659.0, "completions/mean_length": 435.71875, "completions/mean_terminated_length": 359.0384826660156, "completions/min_length": 93.0, "completions/min_terminated_length": 93.0, "entropy": 0.49586478620767593, "epoch": 0.3758029978586724, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0055, "num_tokens": 14377364.0, "reward": 0.59375, "reward_std": 0.1293872892856598, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.000350832939148, "sampling/importance_sampling_ratio/min": 0.6016424894332886, "sampling/sampling_logp_difference/max": 0.7710986137390137, "sampling/sampling_logp_difference/mean": 0.014931955374777317, "step": 702 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 469.0, "completions/max_terminated_length": 469.0, "completions/mean_length": 303.875, "completions/mean_terminated_length": 303.875, "completions/min_length": 126.0, "completions/min_terminated_length": 126.0, "entropy": 0.37582309544086456, "epoch": 0.37633832976445397, "frac_reward_zero_std": 0.25, "grad_norm": 0.016746819019317627, "learning_rate": 1e-05, "loss": 0.0071, "num_tokens": 14390600.0, "reward": 0.78125, "reward_std": 0.3471629321575165, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.5888361930847168, "sampling/importance_sampling_ratio/mean": 1.0002955198287964, "sampling/importance_sampling_ratio/min": 0.4946345090866089, "sampling/sampling_logp_difference/max": 0.7039362192153931, "sampling/sampling_logp_difference/mean": 0.01304656732827425, "step": 703 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 726.0, "completions/mean_length": 506.15625, "completions/mean_terminated_length": 445.73077392578125, "completions/min_length": 200.0, "completions/min_terminated_length": 200.0, "entropy": 0.5743923224508762, "epoch": 0.37687366167023556, "frac_reward_zero_std": 0.5, "grad_norm": 0.01048553641885519, "learning_rate": 1e-05, "loss": -0.0655, "num_tokens": 14410789.0, "reward": 0.375, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.375, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.9519473314285278, "sampling/importance_sampling_ratio/mean": 1.0001193284988403, "sampling/importance_sampling_ratio/min": 0.6970666646957397, "sampling/sampling_logp_difference/max": 0.6688275337219238, "sampling/sampling_logp_difference/mean": 0.016540801152586937, "step": 704 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 8.305647497763857e-05, "clip_ratio/low_min": 8.305647497763857e-05, "clip_ratio/region_mean": 8.305647497763857e-05, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 766.0, "completions/mean_length": 408.5, "completions/mean_terminated_length": 384.5333557128906, "completions/min_length": 225.0, "completions/min_terminated_length": 225.0, "entropy": 0.31146340630948544, "epoch": 0.37740899357601715, "frac_reward_zero_std": 0.25, "grad_norm": 0.020676637068390846, "learning_rate": 1e-05, "loss": 0.0654, "num_tokens": 14427789.0, "reward": 0.6875, "reward_std": 0.3514062464237213, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.3920773267745972, "sampling/importance_sampling_ratio/mean": 1.0001630783081055, "sampling/importance_sampling_ratio/min": 0.4027249217033386, "sampling/sampling_logp_difference/max": 0.9095015525817871, "sampling/sampling_logp_difference/mean": 0.011563870124518871, "step": 705 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 8.96057317731902e-05, "clip_ratio/low_min": 8.96057317731902e-05, "clip_ratio/region_mean": 8.96057317731902e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 615.0, "completions/max_terminated_length": 615.0, "completions/mean_length": 338.0625, "completions/mean_terminated_length": 338.0625, "completions/min_length": 144.0, "completions/min_terminated_length": 144.0, "entropy": 0.3190981522202492, "epoch": 0.37794432548179874, "frac_reward_zero_std": 0.25, "grad_norm": 0.011464577168226242, "learning_rate": 1e-05, "loss": -0.024, "num_tokens": 14442351.0, "reward": 0.6875, "reward_std": 0.2925041913986206, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.728134036064148, "sampling/importance_sampling_ratio/mean": 1.0003695487976074, "sampling/importance_sampling_ratio/min": 0.6184782385826111, "sampling/sampling_logp_difference/max": 0.5470422506332397, "sampling/sampling_logp_difference/mean": 0.010759817436337471, "step": 706 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00011438337605795823, "clip_ratio/low_min": 0.00011438337605795823, "clip_ratio/region_mean": 0.00011438337605795823, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 725.0, "completions/mean_length": 482.375, "completions/mean_terminated_length": 452.82757568359375, "completions/min_length": 276.0, "completions/min_terminated_length": 276.0, "entropy": 0.2971290107816458, "epoch": 0.3784796573875803, "frac_reward_zero_std": 0.25, "grad_norm": 0.0201345793902874, "learning_rate": 1e-05, "loss": 0.0308, "num_tokens": 14461387.0, "reward": 0.71875, "reward_std": 0.3787454068660736, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.3662681579589844, "sampling/importance_sampling_ratio/mean": 0.9997867345809937, "sampling/importance_sampling_ratio/min": 0.5406734943389893, "sampling/sampling_logp_difference/max": 0.6149396896362305, "sampling/sampling_logp_difference/mean": 0.010508444160223007, "step": 707 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 7.977025961736217e-05, "clip_ratio/low_min": 7.977025961736217e-05, "clip_ratio/region_mean": 7.977025961736217e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 584.0, "completions/max_terminated_length": 584.0, "completions/mean_length": 363.0625, "completions/mean_terminated_length": 363.0625, "completions/min_length": 177.0, "completions/min_terminated_length": 177.0, "entropy": 0.26614492386579514, "epoch": 0.37901498929336186, "frac_reward_zero_std": 0.25, "grad_norm": 0.028473904356360435, "learning_rate": 1e-05, "loss": -0.0015, "num_tokens": 14476405.0, "reward": 0.90625, "reward_std": 0.2651650309562683, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.7788512706756592, "sampling/importance_sampling_ratio/mean": 1.0002089738845825, "sampling/importance_sampling_ratio/min": 0.30601656436920166, "sampling/sampling_logp_difference/max": 1.1841161251068115, "sampling/sampling_logp_difference/mean": 0.010216321796178818, "step": 708 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 673.0, "completions/max_terminated_length": 673.0, "completions/mean_length": 346.21875, "completions/mean_terminated_length": 346.21875, "completions/min_length": 213.0, "completions/min_terminated_length": 213.0, "entropy": 0.37835797667503357, "epoch": 0.37955032119914345, "frac_reward_zero_std": 0.25, "grad_norm": 0.007868225686252117, "learning_rate": 1e-05, "loss": -0.0044, "num_tokens": 14491556.0, "reward": 0.84375, "reward_std": 0.3061639964580536, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.4122024774551392, "sampling/importance_sampling_ratio/mean": 0.9999848008155823, "sampling/importance_sampling_ratio/min": 0.3929634988307953, "sampling/sampling_logp_difference/max": 0.9340386390686035, "sampling/sampling_logp_difference/mean": 0.01364780031144619, "step": 709 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00013705813034903258, "clip_ratio/low_min": 0.00013705813034903258, "clip_ratio/region_mean": 0.00013705813034903258, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 649.0, "completions/mean_length": 417.03125, "completions/mean_terminated_length": 405.70965576171875, "completions/min_length": 208.0, "completions/min_terminated_length": 208.0, "entropy": 0.3463163934648037, "epoch": 0.38008565310492504, "frac_reward_zero_std": 0.25, "grad_norm": 0.01991078443825245, "learning_rate": 1e-05, "loss": 0.0074, "num_tokens": 14509581.0, "reward": 0.65625, "reward_std": 0.3377465009689331, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.5266883373260498, "sampling/importance_sampling_ratio/mean": 0.9995376467704773, "sampling/importance_sampling_ratio/min": 0.4936709403991699, "sampling/sampling_logp_difference/max": 0.7058861255645752, "sampling/sampling_logp_difference/mean": 0.013335020281374454, "step": 710 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00023627230984857306, "clip_ratio/low_min": 0.00023627230984857306, "clip_ratio/region_mean": 0.00023627230984857306, "completions/clipped_ratio": 0.0, "completions/max_length": 605.0, "completions/max_terminated_length": 605.0, "completions/mean_length": 379.28125, "completions/mean_terminated_length": 379.28125, "completions/min_length": 230.0, "completions/min_terminated_length": 230.0, "entropy": 0.29287885315716267, "epoch": 0.3806209850107066, "frac_reward_zero_std": 0.5, "grad_norm": 0.00403599301353097, "learning_rate": 1e-05, "loss": 0.0136, "num_tokens": 14525126.0, "reward": 0.84375, "reward_std": 0.22201895713806152, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.4173802137374878, "sampling/importance_sampling_ratio/mean": 1.0004808902740479, "sampling/importance_sampling_ratio/min": 0.7535046935081482, "sampling/sampling_logp_difference/max": 0.34881019592285156, "sampling/sampling_logp_difference/mean": 0.010657152161002159, "step": 711 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.7313158322358504e-05, "clip_ratio/low_min": 5.7313158322358504e-05, "clip_ratio/region_mean": 5.7313158322358504e-05, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 664.0, "completions/mean_length": 449.125, "completions/mean_terminated_length": 403.5714416503906, "completions/min_length": 268.0, "completions/min_terminated_length": 268.0, "entropy": 0.42011337354779243, "epoch": 0.3811563169164882, "frac_reward_zero_std": 0.5, "grad_norm": 0.02013416960835457, "learning_rate": 1e-05, "loss": 0.0405, "num_tokens": 14543698.0, "reward": 0.625, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.5694878101348877, "sampling/importance_sampling_ratio/mean": 1.000623345375061, "sampling/importance_sampling_ratio/min": 0.685395359992981, "sampling/sampling_logp_difference/max": 0.45074939727783203, "sampling/sampling_logp_difference/mean": 0.013381236232817173, "step": 712 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0001274974856642075, "clip_ratio/low_min": 0.0001274974856642075, "clip_ratio/region_mean": 0.0001274974856642075, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 707.0, "completions/mean_length": 415.0, "completions/mean_terminated_length": 364.5714416503906, "completions/min_length": 196.0, "completions/min_terminated_length": 196.0, "entropy": 0.33021325804293156, "epoch": 0.3816916488222698, "frac_reward_zero_std": 0.25, "grad_norm": 0.03616506606340408, "learning_rate": 1e-05, "loss": 0.1167, "num_tokens": 14560242.0, "reward": 0.59375, "reward_std": 0.378745436668396, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.5868172645568848, "sampling/importance_sampling_ratio/mean": 1.0000605583190918, "sampling/importance_sampling_ratio/min": 0.6655277609825134, "sampling/sampling_logp_difference/max": 0.4617302417755127, "sampling/sampling_logp_difference/mean": 0.011609706096351147, "step": 713 }, { "clip_ratio/high_max": 0.000141934011480771, "clip_ratio/high_mean": 0.000141934011480771, "clip_ratio/low_mean": 6.433350790757686e-05, "clip_ratio/low_min": 6.433350790757686e-05, "clip_ratio/region_mean": 0.00020626751938834786, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 762.0, "completions/mean_length": 497.1875, "completions/mean_terminated_length": 479.13336181640625, "completions/min_length": 213.0, "completions/min_terminated_length": 213.0, "entropy": 0.2680256310850382, "epoch": 0.3822269807280514, "frac_reward_zero_std": 0.0, "grad_norm": 0.018306700512766838, "learning_rate": 1e-05, "loss": 0.0252, "num_tokens": 14579952.0, "reward": 0.75, "reward_std": 0.4261348247528076, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.8021780252456665, "sampling/importance_sampling_ratio/mean": 1.0002920627593994, "sampling/importance_sampling_ratio/min": 0.6692163944244385, "sampling/sampling_logp_difference/max": 0.5889959335327148, "sampling/sampling_logp_difference/mean": 0.010151496157050133, "step": 714 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00035480190490488894, "clip_ratio/low_min": 0.00035480190490488894, "clip_ratio/region_mean": 0.00035480190490488894, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 768.0, "completions/mean_length": 491.53125, "completions/mean_terminated_length": 427.73077392578125, "completions/min_length": 179.0, "completions/min_terminated_length": 179.0, "entropy": 0.4604201167821884, "epoch": 0.382762312633833, "frac_reward_zero_std": 0.5, "grad_norm": 0.020163601264357567, "learning_rate": 1e-05, "loss": 0.0457, "num_tokens": 14599729.0, "reward": 0.59375, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0004055500030518, "sampling/importance_sampling_ratio/min": 0.6129925847053528, "sampling/sampling_logp_difference/max": 0.7964284420013428, "sampling/sampling_logp_difference/mean": 0.013950590044260025, "step": 715 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 766.0, "completions/max_terminated_length": 766.0, "completions/mean_length": 446.25, "completions/mean_terminated_length": 446.25, "completions/min_length": 258.0, "completions/min_terminated_length": 258.0, "entropy": 0.2816047314554453, "epoch": 0.38329764453961457, "frac_reward_zero_std": 0.75, "grad_norm": 0.016802972182631493, "learning_rate": 1e-05, "loss": 0.0159, "num_tokens": 14617649.0, "reward": 0.875, "reward_std": 0.13363061845302582, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 1.6031620502471924, "sampling/importance_sampling_ratio/mean": 0.9994713068008423, "sampling/importance_sampling_ratio/min": 0.5475712418556213, "sampling/sampling_logp_difference/max": 0.6022627353668213, "sampling/sampling_logp_difference/mean": 0.01150565966963768, "step": 716 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 755.0, "completions/mean_length": 473.0625, "completions/mean_terminated_length": 442.5517272949219, "completions/min_length": 197.0, "completions/min_terminated_length": 197.0, "entropy": 0.32550000585615635, "epoch": 0.38383297644539616, "frac_reward_zero_std": 0.25, "grad_norm": 0.018743062391877174, "learning_rate": 1e-05, "loss": 0.0594, "num_tokens": 14636843.0, "reward": 0.84375, "reward_std": 0.3061639666557312, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.4701733589172363, "sampling/importance_sampling_ratio/mean": 0.9999250173568726, "sampling/importance_sampling_ratio/min": 0.6543622612953186, "sampling/sampling_logp_difference/max": 0.42409420013427734, "sampling/sampling_logp_difference/mean": 0.011421573348343372, "step": 717 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00010977495912811719, "clip_ratio/low_min": 0.00010977495912811719, "clip_ratio/region_mean": 0.00010977495912811719, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 740.0, "completions/mean_length": 517.25, "completions/mean_terminated_length": 481.4285888671875, "completions/min_length": 348.0, "completions/min_terminated_length": 348.0, "entropy": 0.2791203670203686, "epoch": 0.38436830835117775, "frac_reward_zero_std": 0.0, "grad_norm": 0.016800623387098312, "learning_rate": 1e-05, "loss": -0.0087, "num_tokens": 14657019.0, "reward": 0.71875, "reward_std": 0.3808925747871399, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.4731851816177368, "sampling/importance_sampling_ratio/mean": 1.000221848487854, "sampling/importance_sampling_ratio/min": 0.6232567429542542, "sampling/sampling_logp_difference/max": 0.4727966785430908, "sampling/sampling_logp_difference/mean": 0.009981920011341572, "step": 718 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 724.0, "completions/max_terminated_length": 724.0, "completions/mean_length": 447.59375, "completions/mean_terminated_length": 447.59375, "completions/min_length": 194.0, "completions/min_terminated_length": 194.0, "entropy": 0.3053096607327461, "epoch": 0.38490364025695933, "frac_reward_zero_std": 0.5, "grad_norm": 0.016528841108083725, "learning_rate": 1e-05, "loss": 0.0071, "num_tokens": 14675134.0, "reward": 0.75, "reward_std": 0.2587745785713196, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.2990198135375977, "sampling/importance_sampling_ratio/mean": 0.9999493956565857, "sampling/importance_sampling_ratio/min": 0.6751003861427307, "sampling/sampling_logp_difference/max": 0.39289385080337524, "sampling/sampling_logp_difference/mean": 0.011058874428272247, "step": 719 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 582.0, "completions/max_terminated_length": 582.0, "completions/mean_length": 375.625, "completions/mean_terminated_length": 375.625, "completions/min_length": 44.0, "completions/min_terminated_length": 44.0, "entropy": 0.28763924166560173, "epoch": 0.3854389721627409, "frac_reward_zero_std": 0.25, "grad_norm": 0.007499333005398512, "learning_rate": 1e-05, "loss": 0.0178, "num_tokens": 14690354.0, "reward": 0.75, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.4215821027755737, "sampling/importance_sampling_ratio/mean": 1.000325083732605, "sampling/importance_sampling_ratio/min": 0.6132378578186035, "sampling/sampling_logp_difference/max": 0.4890024662017822, "sampling/sampling_logp_difference/mean": 0.010743832215666771, "step": 720 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 542.0, "completions/mean_length": 366.25, "completions/mean_terminated_length": 353.2903137207031, "completions/min_length": 180.0, "completions/min_terminated_length": 180.0, "entropy": 0.3741515651345253, "epoch": 0.3859743040685225, "frac_reward_zero_std": 0.5, "grad_norm": 0.054097432643175125, "learning_rate": 1e-05, "loss": 0.0758, "num_tokens": 14705674.0, "reward": 0.875, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9993348121643066, "sampling/importance_sampling_ratio/min": 0.6482955813407898, "sampling/sampling_logp_difference/max": 0.7959752082824707, "sampling/sampling_logp_difference/mean": 0.012856943532824516, "step": 721 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 7.876496238168329e-05, "clip_ratio/low_min": 7.876496238168329e-05, "clip_ratio/region_mean": 7.876496238168329e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 640.0, "completions/max_terminated_length": 640.0, "completions/mean_length": 384.59375, "completions/mean_terminated_length": 384.59375, "completions/min_length": 240.0, "completions/min_terminated_length": 240.0, "entropy": 0.3284129276871681, "epoch": 0.38650963597430404, "frac_reward_zero_std": 0.5, "grad_norm": 0.04252352938055992, "learning_rate": 1e-05, "loss": 0.0663, "num_tokens": 14721941.0, "reward": 0.65625, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.455653429031372, "sampling/importance_sampling_ratio/mean": 1.000124454498291, "sampling/importance_sampling_ratio/min": 0.7095494270324707, "sampling/sampling_logp_difference/max": 0.3754549026489258, "sampling/sampling_logp_difference/mean": 0.01177291665226221, "step": 722 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 718.0, "completions/max_terminated_length": 718.0, "completions/mean_length": 446.90625, "completions/mean_terminated_length": 446.90625, "completions/min_length": 182.0, "completions/min_terminated_length": 182.0, "entropy": 0.33111138828098774, "epoch": 0.38704496788008563, "frac_reward_zero_std": 0.25, "grad_norm": 0.03499797731637955, "learning_rate": 1e-05, "loss": 0.0411, "num_tokens": 14740330.0, "reward": 0.8125, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.595922589302063, "sampling/importance_sampling_ratio/mean": 0.9998600482940674, "sampling/importance_sampling_ratio/min": 0.6625149250030518, "sampling/sampling_logp_difference/max": 0.4674520492553711, "sampling/sampling_logp_difference/mean": 0.011430527083575726, "step": 723 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 8.37801635498181e-05, "clip_ratio/low_min": 8.37801635498181e-05, "clip_ratio/region_mean": 8.37801635498181e-05, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 693.0, "completions/mean_length": 456.625, "completions/mean_terminated_length": 435.86669921875, "completions/min_length": 226.0, "completions/min_terminated_length": 226.0, "entropy": 0.3971633203327656, "epoch": 0.3875802997858672, "frac_reward_zero_std": 0.0, "grad_norm": 0.024260831996798515, "learning_rate": 1e-05, "loss": 0.0843, "num_tokens": 14758574.0, "reward": 0.59375, "reward_std": 0.3987956643104553, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0007160902023315, "sampling/importance_sampling_ratio/min": 0.6169441938400269, "sampling/sampling_logp_difference/max": 0.7076516151428223, "sampling/sampling_logp_difference/mean": 0.013493540696799755, "step": 724 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 745.0, "completions/max_terminated_length": 745.0, "completions/mean_length": 378.5, "completions/mean_terminated_length": 378.5, "completions/min_length": 201.0, "completions/min_terminated_length": 201.0, "entropy": 0.31588367745280266, "epoch": 0.3881156316916488, "frac_reward_zero_std": 0.75, "grad_norm": 0.0039367713034152985, "learning_rate": 1e-05, "loss": 0.016, "num_tokens": 14774078.0, "reward": 0.96875, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.96875, "rewards/accuracy_reward/std": 0.1767766922712326, "sampling/importance_sampling_ratio/max": 1.409155249595642, "sampling/importance_sampling_ratio/mean": 0.999642550945282, "sampling/importance_sampling_ratio/min": 0.5600169897079468, "sampling/sampling_logp_difference/max": 0.5797882080078125, "sampling/sampling_logp_difference/mean": 0.01088521908968687, "step": 725 }, { "clip_ratio/high_max": 7.413997809635475e-05, "clip_ratio/high_mean": 7.413997809635475e-05, "clip_ratio/low_mean": 5.592841262114234e-05, "clip_ratio/low_min": 5.592841262114234e-05, "clip_ratio/region_mean": 0.0001300683907174971, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 642.0, "completions/mean_length": 407.03125, "completions/mean_terminated_length": 355.46429443359375, "completions/min_length": 177.0, "completions/min_terminated_length": 177.0, "entropy": 0.3510789033025503, "epoch": 0.3886509635974304, "frac_reward_zero_std": 0.25, "grad_norm": 0.026060735806822777, "learning_rate": 1e-05, "loss": 0.1434, "num_tokens": 14790591.0, "reward": 0.78125, "reward_std": 0.3608423173427582, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.9262347221374512, "sampling/importance_sampling_ratio/mean": 0.9992910027503967, "sampling/importance_sampling_ratio/min": 0.5453320145606995, "sampling/sampling_logp_difference/max": 0.6555671691894531, "sampling/sampling_logp_difference/mean": 0.01253750640898943, "step": 726 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 9.335325012216344e-05, "clip_ratio/low_min": 9.335325012216344e-05, "clip_ratio/region_mean": 9.335325012216344e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 537.0, "completions/max_terminated_length": 537.0, "completions/mean_length": 336.75, "completions/mean_terminated_length": 336.75, "completions/min_length": 123.0, "completions/min_terminated_length": 123.0, "entropy": 0.3309592194855213, "epoch": 0.389186295503212, "frac_reward_zero_std": 0.5, "grad_norm": 0.03679654002189636, "learning_rate": 1e-05, "loss": -0.0065, "num_tokens": 14805127.0, "reward": 0.625, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9996213912963867, "sampling/importance_sampling_ratio/min": 0.7002161741256714, "sampling/sampling_logp_difference/max": 0.8192439079284668, "sampling/sampling_logp_difference/mean": 0.01202657725661993, "step": 727 }, { "clip_ratio/high_max": 7.598783849971369e-05, "clip_ratio/high_mean": 7.598783849971369e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 7.598783849971369e-05, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 646.0, "completions/mean_length": 479.09375, "completions/mean_terminated_length": 449.2069091796875, "completions/min_length": 245.0, "completions/min_terminated_length": 245.0, "entropy": 0.29027774184942245, "epoch": 0.3897216274089936, "frac_reward_zero_std": 0.5, "grad_norm": 0.01727694272994995, "learning_rate": 1e-05, "loss": 0.0732, "num_tokens": 14824298.0, "reward": 0.8125, "reward_std": 0.2587745785713196, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.7989857196807861, "sampling/importance_sampling_ratio/mean": 1.0000174045562744, "sampling/importance_sampling_ratio/min": 0.56607586145401, "sampling/sampling_logp_difference/max": 0.5872230529785156, "sampling/sampling_logp_difference/mean": 0.010853583924472332, "step": 728 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 735.0, "completions/mean_length": 506.6875, "completions/mean_terminated_length": 479.6551818847656, "completions/min_length": 311.0, "completions/min_terminated_length": 311.0, "entropy": 0.418092992156744, "epoch": 0.39025695931477516, "frac_reward_zero_std": 0.0, "grad_norm": 0.03450678288936615, "learning_rate": 1e-05, "loss": 0.0174, "num_tokens": 14844528.0, "reward": 0.40625, "reward_std": 0.4628904461860657, "rewards/accuracy_reward/mean": 0.40625, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.4291409254074097, "sampling/importance_sampling_ratio/mean": 1.0002530813217163, "sampling/importance_sampling_ratio/min": 0.4035452604293823, "sampling/sampling_logp_difference/max": 0.9074666500091553, "sampling/sampling_logp_difference/mean": 0.01486382819712162, "step": 729 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 540.0, "completions/mean_length": 368.375, "completions/mean_terminated_length": 355.4838562011719, "completions/min_length": 174.0, "completions/min_terminated_length": 174.0, "entropy": 0.268840491771698, "epoch": 0.39079229122055675, "frac_reward_zero_std": 0.75, "grad_norm": 0.015092047862708569, "learning_rate": 1e-05, "loss": 0.0295, "num_tokens": 14860092.0, "reward": 0.90625, "reward_std": 0.1293872892856598, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.7625010013580322, "sampling/importance_sampling_ratio/mean": 1.0003862380981445, "sampling/importance_sampling_ratio/min": 0.5371314883232117, "sampling/sampling_logp_difference/max": 0.6215124130249023, "sampling/sampling_logp_difference/mean": 0.01023824792355299, "step": 730 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0001708475756458938, "clip_ratio/low_min": 0.0001708475756458938, "clip_ratio/region_mean": 0.0001708475756458938, "completions/clipped_ratio": 0.0, "completions/max_length": 676.0, "completions/max_terminated_length": 676.0, "completions/mean_length": 406.59375, "completions/mean_terminated_length": 406.59375, "completions/min_length": 161.0, "completions/min_terminated_length": 161.0, "entropy": 0.305848591029644, "epoch": 0.39132762312633834, "frac_reward_zero_std": 0.5, "grad_norm": 0.010290650650858879, "learning_rate": 1e-05, "loss": -0.0165, "num_tokens": 14876815.0, "reward": 0.84375, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.9217058420181274, "sampling/importance_sampling_ratio/mean": 1.0004491806030273, "sampling/importance_sampling_ratio/min": 0.6322147846221924, "sampling/sampling_logp_difference/max": 0.6532132625579834, "sampling/sampling_logp_difference/mean": 0.012023229151964188, "step": 731 }, { "clip_ratio/high_max": 7.246376480907202e-05, "clip_ratio/high_mean": 7.246376480907202e-05, "clip_ratio/low_mean": 0.000138184055685997, "clip_ratio/low_min": 0.000138184055685997, "clip_ratio/region_mean": 0.00021064782049506903, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 567.0, "completions/mean_length": 426.53125, "completions/mean_terminated_length": 377.7500305175781, "completions/min_length": 225.0, "completions/min_terminated_length": 225.0, "entropy": 0.3874700292944908, "epoch": 0.39186295503211993, "frac_reward_zero_std": 0.25, "grad_norm": 0.030863050371408463, "learning_rate": 1e-05, "loss": 0.1072, "num_tokens": 14894096.0, "reward": 0.53125, "reward_std": 0.3471629321575165, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.5078504085540771, "sampling/importance_sampling_ratio/mean": 0.9998968243598938, "sampling/importance_sampling_ratio/min": 0.5072235465049744, "sampling/sampling_logp_difference/max": 0.6788034439086914, "sampling/sampling_logp_difference/mean": 0.012878618203103542, "step": 732 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 659.0, "completions/mean_length": 415.5, "completions/mean_terminated_length": 392.0000305175781, "completions/min_length": 234.0, "completions/min_terminated_length": 234.0, "entropy": 0.406641460955143, "epoch": 0.3923982869379015, "frac_reward_zero_std": 0.25, "grad_norm": 0.01745034195482731, "learning_rate": 1e-05, "loss": 0.0481, "num_tokens": 14911208.0, "reward": 0.5, "reward_std": 0.3514062464237213, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5080004930496216, "sampling/importance_sampling_ratio/max": 1.880895972251892, "sampling/importance_sampling_ratio/mean": 1.0001474618911743, "sampling/importance_sampling_ratio/min": 0.4862619638442993, "sampling/sampling_logp_difference/max": 0.7210078239440918, "sampling/sampling_logp_difference/mean": 0.01395708043128252, "step": 733 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 572.0, "completions/mean_length": 393.875, "completions/mean_terminated_length": 381.8064270019531, "completions/min_length": 213.0, "completions/min_terminated_length": 213.0, "entropy": 0.30655495449900627, "epoch": 0.3929336188436831, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": -0.0009, "num_tokens": 14927036.0, "reward": 0.78125, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0001674890518188, "sampling/importance_sampling_ratio/min": 0.5404631495475769, "sampling/sampling_logp_difference/max": 0.9599714279174805, "sampling/sampling_logp_difference/mean": 0.01090756431221962, "step": 734 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 8.256275032181293e-05, "clip_ratio/low_min": 8.256275032181293e-05, "clip_ratio/region_mean": 8.256275032181293e-05, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 691.0, "completions/mean_length": 480.03125, "completions/mean_terminated_length": 438.89288330078125, "completions/min_length": 196.0, "completions/min_terminated_length": 196.0, "entropy": 0.32466258853673935, "epoch": 0.39346895074946464, "frac_reward_zero_std": 0.5, "grad_norm": 0.004461785312741995, "learning_rate": 1e-05, "loss": 0.0427, "num_tokens": 14946549.0, "reward": 0.59375, "reward_std": 0.22201895713806152, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.7895437479019165, "sampling/importance_sampling_ratio/mean": 1.000295877456665, "sampling/importance_sampling_ratio/min": 0.5543748140335083, "sampling/sampling_logp_difference/max": 0.5899143218994141, "sampling/sampling_logp_difference/mean": 0.011947096325457096, "step": 735 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00014100589760346338, "clip_ratio/low_min": 0.00014100589760346338, "clip_ratio/region_mean": 0.00014100589760346338, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 646.0, "completions/mean_length": 438.15625, "completions/mean_terminated_length": 427.51611328125, "completions/min_length": 193.0, "completions/min_terminated_length": 193.0, "entropy": 0.2634274195879698, "epoch": 0.39400428265524623, "frac_reward_zero_std": 0.0, "grad_norm": 0.017593378201127052, "learning_rate": 1e-05, "loss": 0.0109, "num_tokens": 14964250.0, "reward": 0.53125, "reward_std": 0.47137707471847534, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.4653797149658203, "sampling/importance_sampling_ratio/mean": 0.9998362064361572, "sampling/importance_sampling_ratio/min": 0.6434476375579834, "sampling/sampling_logp_difference/max": 0.4409146308898926, "sampling/sampling_logp_difference/mean": 0.010042332112789154, "step": 736 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 618.0, "completions/max_terminated_length": 618.0, "completions/mean_length": 379.1875, "completions/mean_terminated_length": 379.1875, "completions/min_length": 195.0, "completions/min_terminated_length": 195.0, "entropy": 0.31994888186454773, "epoch": 0.3945396145610278, "frac_reward_zero_std": 0.5, "grad_norm": 0.008601024746894836, "learning_rate": 1e-05, "loss": 0.0103, "num_tokens": 14979848.0, "reward": 0.90625, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.6797938346862793, "sampling/importance_sampling_ratio/mean": 1.0005078315734863, "sampling/importance_sampling_ratio/min": 0.617961585521698, "sampling/sampling_logp_difference/max": 0.5186710357666016, "sampling/sampling_logp_difference/mean": 0.011722034774720669, "step": 737 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 524.0, "completions/max_terminated_length": 524.0, "completions/mean_length": 369.28125, "completions/mean_terminated_length": 369.28125, "completions/min_length": 183.0, "completions/min_terminated_length": 183.0, "entropy": 0.26390823535621166, "epoch": 0.3950749464668094, "frac_reward_zero_std": 0.5, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": -0.006, "num_tokens": 14995105.0, "reward": 0.78125, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.510011076927185, "sampling/importance_sampling_ratio/mean": 1.0000381469726562, "sampling/importance_sampling_ratio/min": 0.5908969044685364, "sampling/sampling_logp_difference/max": 0.5261136293411255, "sampling/sampling_logp_difference/mean": 0.010361729189753532, "step": 738 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 565.0, "completions/max_terminated_length": 565.0, "completions/mean_length": 337.65625, "completions/mean_terminated_length": 337.65625, "completions/min_length": 161.0, "completions/min_terminated_length": 161.0, "entropy": 0.3128814436495304, "epoch": 0.395610278372591, "frac_reward_zero_std": 0.5, "grad_norm": 0.009559151716530323, "learning_rate": 1e-05, "loss": 0.0669, "num_tokens": 15009862.0, "reward": 0.71875, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.8868142366409302, "sampling/importance_sampling_ratio/mean": 1.0002555847167969, "sampling/importance_sampling_ratio/min": 0.5747727751731873, "sampling/sampling_logp_difference/max": 0.6348898410797119, "sampling/sampling_logp_difference/mean": 0.01171938981860876, "step": 739 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 686.0, "completions/mean_length": 440.875, "completions/mean_terminated_length": 419.0666809082031, "completions/min_length": 225.0, "completions/min_terminated_length": 225.0, "entropy": 0.31444643810391426, "epoch": 0.3961456102783726, "frac_reward_zero_std": 0.25, "grad_norm": 0.021820830181241035, "learning_rate": 1e-05, "loss": 0.0243, "num_tokens": 15027850.0, "reward": 0.625, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.4483706951141357, "sampling/importance_sampling_ratio/mean": 0.999547004699707, "sampling/importance_sampling_ratio/min": 0.33352625370025635, "sampling/sampling_logp_difference/max": 1.0980336666107178, "sampling/sampling_logp_difference/mean": 0.011443336494266987, "step": 740 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00018385963267064653, "clip_ratio/low_min": 0.00018385963267064653, "clip_ratio/region_mean": 0.00018385963267064653, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 679.0, "completions/mean_length": 470.875, "completions/mean_terminated_length": 451.0666809082031, "completions/min_length": 229.0, "completions/min_terminated_length": 229.0, "entropy": 0.3229607306420803, "epoch": 0.3966809421841542, "frac_reward_zero_std": 0.5, "grad_norm": 0.034369517117738724, "learning_rate": 1e-05, "loss": 0.0112, "num_tokens": 15047286.0, "reward": 0.75, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.4677540063858032, "sampling/importance_sampling_ratio/mean": 1.0001025199890137, "sampling/importance_sampling_ratio/min": 0.43193694949150085, "sampling/sampling_logp_difference/max": 0.8394756317138672, "sampling/sampling_logp_difference/mean": 0.012847675010561943, "step": 741 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 717.0, "completions/max_terminated_length": 717.0, "completions/mean_length": 410.4375, "completions/mean_terminated_length": 410.4375, "completions/min_length": 106.0, "completions/min_terminated_length": 106.0, "entropy": 0.2223656103014946, "epoch": 0.39721627408993576, "frac_reward_zero_std": 0.5, "grad_norm": 0.00397397531196475, "learning_rate": 1e-05, "loss": -0.0189, "num_tokens": 15063660.0, "reward": 0.9375, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.9375, "rewards/accuracy_reward/std": 0.24593468010425568, "sampling/importance_sampling_ratio/max": 1.66285240650177, "sampling/importance_sampling_ratio/mean": 0.999877393245697, "sampling/importance_sampling_ratio/min": 0.6022452116012573, "sampling/sampling_logp_difference/max": 0.5085344314575195, "sampling/sampling_logp_difference/mean": 0.008522205986082554, "step": 742 }, { "clip_ratio/high_max": 6.742178811691701e-05, "clip_ratio/high_mean": 6.742178811691701e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 6.742178811691701e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 578.0, "completions/max_terminated_length": 578.0, "completions/mean_length": 378.65625, "completions/mean_terminated_length": 378.65625, "completions/min_length": 143.0, "completions/min_terminated_length": 143.0, "entropy": 0.27840827964246273, "epoch": 0.39775160599571735, "frac_reward_zero_std": 0.25, "grad_norm": 0.02370659075677395, "learning_rate": 1e-05, "loss": -0.0534, "num_tokens": 15079385.0, "reward": 0.75, "reward_std": 0.3514062464237213, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.4989700317382812, "sampling/importance_sampling_ratio/mean": 0.9998046159744263, "sampling/importance_sampling_ratio/min": 0.4175216853618622, "sampling/sampling_logp_difference/max": 0.8734188079833984, "sampling/sampling_logp_difference/mean": 0.010278995148837566, "step": 743 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 757.0, "completions/max_terminated_length": 757.0, "completions/mean_length": 387.59375, "completions/mean_terminated_length": 387.59375, "completions/min_length": 209.0, "completions/min_terminated_length": 209.0, "entropy": 0.2853036727756262, "epoch": 0.39828693790149894, "frac_reward_zero_std": 0.75, "grad_norm": 0.036839559674263, "learning_rate": 1e-05, "loss": -0.0005, "num_tokens": 15095156.0, "reward": 0.96875, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.96875, "rewards/accuracy_reward/std": 0.1767766922712326, "sampling/importance_sampling_ratio/max": 1.6023297309875488, "sampling/importance_sampling_ratio/mean": 0.999885082244873, "sampling/importance_sampling_ratio/min": 0.5930861234664917, "sampling/sampling_logp_difference/max": 0.5224156379699707, "sampling/sampling_logp_difference/mean": 0.010929422453045845, "step": 744 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00016292092186631635, "clip_ratio/low_min": 0.00016292092186631635, "clip_ratio/region_mean": 0.00016292092186631635, "completions/clipped_ratio": 0.0, "completions/max_length": 539.0, "completions/max_terminated_length": 539.0, "completions/mean_length": 363.09375, "completions/mean_terminated_length": 363.09375, "completions/min_length": 208.0, "completions/min_terminated_length": 208.0, "entropy": 0.26418436504900455, "epoch": 0.3988222698072805, "frac_reward_zero_std": 0.25, "grad_norm": 0.035045281052589417, "learning_rate": 1e-05, "loss": 0.0432, "num_tokens": 15110391.0, "reward": 0.8125, "reward_std": 0.3104073107242584, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.4381076097488403, "sampling/importance_sampling_ratio/mean": 0.9996638298034668, "sampling/importance_sampling_ratio/min": 0.6747323870658875, "sampling/sampling_logp_difference/max": 0.3934391736984253, "sampling/sampling_logp_difference/mean": 0.010694892145693302, "step": 745 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 468.0, "completions/max_terminated_length": 468.0, "completions/mean_length": 323.21875, "completions/mean_terminated_length": 323.21875, "completions/min_length": 193.0, "completions/min_terminated_length": 193.0, "entropy": 0.31976547837257385, "epoch": 0.3993576017130621, "frac_reward_zero_std": 0.25, "grad_norm": 0.028641268610954285, "learning_rate": 1e-05, "loss": 0.0052, "num_tokens": 15124222.0, "reward": 0.84375, "reward_std": 0.3198433816432953, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0001282691955566, "sampling/importance_sampling_ratio/min": 0.601650595664978, "sampling/sampling_logp_difference/max": 0.8825466632843018, "sampling/sampling_logp_difference/mean": 0.012408490292727947, "step": 746 }, { "clip_ratio/high_max": 0.00011574073869269341, "clip_ratio/high_mean": 0.00011574073869269341, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00011574073869269341, "completions/clipped_ratio": 0.0, "completions/max_length": 490.0, "completions/max_terminated_length": 490.0, "completions/mean_length": 346.59375, "completions/mean_terminated_length": 346.59375, "completions/min_length": 162.0, "completions/min_terminated_length": 162.0, "entropy": 0.38410582952201366, "epoch": 0.3998929336188437, "frac_reward_zero_std": 0.25, "grad_norm": 0.05293239653110504, "learning_rate": 1e-05, "loss": -0.0311, "num_tokens": 15139073.0, "reward": 0.875, "reward_std": 0.292504221200943, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 1.3349459171295166, "sampling/importance_sampling_ratio/mean": 0.9997342228889465, "sampling/importance_sampling_ratio/min": 0.6193897128105164, "sampling/sampling_logp_difference/max": 0.4790205955505371, "sampling/sampling_logp_difference/mean": 0.011812268756330013, "step": 747 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00016867216618265957, "clip_ratio/low_min": 0.00016867216618265957, "clip_ratio/region_mean": 0.00016867216618265957, "completions/clipped_ratio": 0.0, "completions/max_length": 619.0, "completions/max_terminated_length": 619.0, "completions/mean_length": 384.25, "completions/mean_terminated_length": 384.25, "completions/min_length": 256.0, "completions/min_terminated_length": 256.0, "entropy": 0.2777410428971052, "epoch": 0.4004282655246253, "frac_reward_zero_std": 0.5, "grad_norm": 0.03175799176096916, "learning_rate": 1e-05, "loss": 0.017, "num_tokens": 15154881.0, "reward": 0.6875, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.5922999382019043, "sampling/importance_sampling_ratio/mean": 0.9996979236602783, "sampling/importance_sampling_ratio/min": 0.6333731412887573, "sampling/sampling_logp_difference/max": 0.465179443359375, "sampling/sampling_logp_difference/mean": 0.010911596938967705, "step": 748 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 9.938473522197455e-05, "clip_ratio/low_min": 9.938473522197455e-05, "clip_ratio/region_mean": 9.938473522197455e-05, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 742.0, "completions/mean_length": 500.90625, "completions/mean_terminated_length": 462.7500305175781, "completions/min_length": 195.0, "completions/min_terminated_length": 195.0, "entropy": 0.39744829572737217, "epoch": 0.4009635974304068, "frac_reward_zero_std": 0.25, "grad_norm": 0.016104180365800858, "learning_rate": 1e-05, "loss": 0.0094, "num_tokens": 15175198.0, "reward": 0.59375, "reward_std": 0.3608423173427582, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.7563401460647583, "sampling/importance_sampling_ratio/mean": 1.0001014471054077, "sampling/importance_sampling_ratio/min": 0.32650068402290344, "sampling/sampling_logp_difference/max": 1.1193232536315918, "sampling/sampling_logp_difference/mean": 0.013287602923810482, "step": 749 }, { "clip_ratio/high_max": 0.00015699722280260175, "clip_ratio/high_mean": 0.00015699722280260175, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00015699722280260175, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 630.0, "completions/mean_length": 432.28125, "completions/mean_terminated_length": 409.9000244140625, "completions/min_length": 216.0, "completions/min_terminated_length": 216.0, "entropy": 0.25118602253496647, "epoch": 0.4014989293361884, "frac_reward_zero_std": 0.0, "grad_norm": 0.00824292004108429, "learning_rate": 1e-05, "loss": 0.0091, "num_tokens": 15192655.0, "reward": 0.6875, "reward_std": 0.4082317352294922, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0002760887145996, "sampling/importance_sampling_ratio/min": 0.7011640667915344, "sampling/sampling_logp_difference/max": 0.707486629486084, "sampling/sampling_logp_difference/mean": 0.0095582390204072, "step": 750 }, { "clip_ratio/high_max": 7.911392458481714e-05, "clip_ratio/high_mean": 7.911392458481714e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 7.911392458481714e-05, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 654.0, "completions/mean_length": 436.78125, "completions/mean_terminated_length": 389.46429443359375, "completions/min_length": 236.0, "completions/min_terminated_length": 236.0, "entropy": 0.3411795999854803, "epoch": 0.40203426124197, "frac_reward_zero_std": 0.5, "grad_norm": 0.037053268402814865, "learning_rate": 1e-05, "loss": 0.1044, "num_tokens": 15210064.0, "reward": 0.71875, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.9501639604568481, "sampling/importance_sampling_ratio/mean": 0.9999677538871765, "sampling/importance_sampling_ratio/min": 0.6075533032417297, "sampling/sampling_logp_difference/max": 0.6679134368896484, "sampling/sampling_logp_difference/mean": 0.013313813135027885, "step": 751 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 740.0, "completions/max_terminated_length": 740.0, "completions/mean_length": 421.78125, "completions/mean_terminated_length": 421.78125, "completions/min_length": 274.0, "completions/min_terminated_length": 274.0, "entropy": 0.3252240139991045, "epoch": 0.4025695931477516, "frac_reward_zero_std": 0.5, "grad_norm": 0.017057444900274277, "learning_rate": 1e-05, "loss": -0.0399, "num_tokens": 15227369.0, "reward": 0.53125, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9996662735939026, "sampling/importance_sampling_ratio/min": 0.22936411201953888, "sampling/sampling_logp_difference/max": 1.4724445343017578, "sampling/sampling_logp_difference/mean": 0.01190988253802061, "step": 752 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 705.0, "completions/max_terminated_length": 705.0, "completions/mean_length": 394.625, "completions/mean_terminated_length": 394.625, "completions/min_length": 246.0, "completions/min_terminated_length": 246.0, "entropy": 0.28018102422356606, "epoch": 0.4031049250535332, "frac_reward_zero_std": 0.5, "grad_norm": 0.008426283486187458, "learning_rate": 1e-05, "loss": -0.0432, "num_tokens": 15243709.0, "reward": 0.84375, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.3750056028366089, "sampling/importance_sampling_ratio/mean": 0.9998921751976013, "sampling/importance_sampling_ratio/min": 0.5485638976097107, "sampling/sampling_logp_difference/max": 0.6004514694213867, "sampling/sampling_logp_difference/mean": 0.010837670415639877, "step": 753 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 766.0, "completions/mean_length": 509.65625, "completions/mean_terminated_length": 461.8148193359375, "completions/min_length": 239.0, "completions/min_terminated_length": 239.0, "entropy": 0.39862199127674103, "epoch": 0.40364025695931477, "frac_reward_zero_std": 0.5, "grad_norm": 0.018544510006904602, "learning_rate": 1e-05, "loss": 0.0106, "num_tokens": 15264162.0, "reward": 0.5625, "reward_std": 0.2587745785713196, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.8872061967849731, "sampling/importance_sampling_ratio/mean": 1.000205159187317, "sampling/importance_sampling_ratio/min": 0.4344595670700073, "sampling/sampling_logp_difference/max": 0.8336523771286011, "sampling/sampling_logp_difference/mean": 0.01450402569025755, "step": 754 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00021248032862786204, "clip_ratio/low_min": 0.00021248032862786204, "clip_ratio/region_mean": 0.00021248032862786204, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 749.0, "completions/mean_length": 392.21875, "completions/mean_terminated_length": 338.5357360839844, "completions/min_length": 132.0, "completions/min_terminated_length": 132.0, "entropy": 0.3706481456756592, "epoch": 0.40417558886509636, "frac_reward_zero_std": 0.25, "grad_norm": 0.01635662093758583, "learning_rate": 1e-05, "loss": 0.0605, "num_tokens": 15279929.0, "reward": 0.4375, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.4375, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.7147916555404663, "sampling/importance_sampling_ratio/mean": 0.9998767971992493, "sampling/importance_sampling_ratio/min": 0.502638041973114, "sampling/sampling_logp_difference/max": 0.687885046005249, "sampling/sampling_logp_difference/mean": 0.01205374300479889, "step": 755 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0001334980333922431, "clip_ratio/low_min": 0.0001334980333922431, "clip_ratio/region_mean": 0.0001334980333922431, "completions/clipped_ratio": 0.0, "completions/max_length": 702.0, "completions/max_terminated_length": 702.0, "completions/mean_length": 411.90625, "completions/mean_terminated_length": 411.90625, "completions/min_length": 163.0, "completions/min_terminated_length": 163.0, "entropy": 0.37051665410399437, "epoch": 0.40471092077087795, "frac_reward_zero_std": 0.5, "grad_norm": 0.01385048869997263, "learning_rate": 1e-05, "loss": 0.0404, "num_tokens": 15297390.0, "reward": 0.71875, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0005110502243042, "sampling/importance_sampling_ratio/min": 0.43829840421676636, "sampling/sampling_logp_difference/max": 0.8248553276062012, "sampling/sampling_logp_difference/mean": 0.011706370860338211, "step": 756 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 605.0, "completions/max_terminated_length": 605.0, "completions/mean_length": 383.75, "completions/mean_terminated_length": 383.75, "completions/min_length": 176.0, "completions/min_terminated_length": 176.0, "entropy": 0.2585982959717512, "epoch": 0.40524625267665954, "frac_reward_zero_std": 0.25, "grad_norm": 0.01750893145799637, "learning_rate": 1e-05, "loss": -0.0102, "num_tokens": 15313430.0, "reward": 0.78125, "reward_std": 0.3377465009689331, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.4770658016204834, "sampling/importance_sampling_ratio/mean": 1.0001264810562134, "sampling/importance_sampling_ratio/min": 0.6391497254371643, "sampling/sampling_logp_difference/max": 0.4476165771484375, "sampling/sampling_logp_difference/mean": 0.010347745381295681, "step": 757 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 638.0, "completions/mean_length": 366.125, "completions/mean_terminated_length": 353.1612854003906, "completions/min_length": 230.0, "completions/min_terminated_length": 230.0, "entropy": 0.26789403706789017, "epoch": 0.4057815845824411, "frac_reward_zero_std": 0.0, "grad_norm": 0.02607472613453865, "learning_rate": 1e-05, "loss": 0.0282, "num_tokens": 15328610.0, "reward": 0.71875, "reward_std": 0.4218915104866028, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0002455711364746, "sampling/importance_sampling_ratio/min": 0.3001675307750702, "sampling/sampling_logp_difference/max": 1.2034145593643188, "sampling/sampling_logp_difference/mean": 0.009937835857272148, "step": 758 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 698.0, "completions/mean_length": 476.0, "completions/mean_terminated_length": 456.5333557128906, "completions/min_length": 232.0, "completions/min_terminated_length": 232.0, "entropy": 0.2855139411985874, "epoch": 0.4063169164882227, "frac_reward_zero_std": 0.25, "grad_norm": 0.02170407772064209, "learning_rate": 1e-05, "loss": 0.0117, "num_tokens": 15348266.0, "reward": 0.65625, "reward_std": 0.38816186785697937, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.4933019876480103, "sampling/importance_sampling_ratio/mean": 1.0000791549682617, "sampling/importance_sampling_ratio/min": 0.516124427318573, "sampling/sampling_logp_difference/max": 0.661407470703125, "sampling/sampling_logp_difference/mean": 0.01078470703214407, "step": 759 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.765682726632804e-05, "clip_ratio/low_min": 5.765682726632804e-05, "clip_ratio/region_mean": 5.765682726632804e-05, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 753.0, "completions/mean_length": 503.5, "completions/mean_terminated_length": 485.86669921875, "completions/min_length": 257.0, "completions/min_terminated_length": 257.0, "entropy": 0.25144183821976185, "epoch": 0.4068522483940043, "frac_reward_zero_std": 0.25, "grad_norm": 0.00593197625130415, "learning_rate": 1e-05, "loss": 0.008, "num_tokens": 15367922.0, "reward": 0.5, "reward_std": 0.3514062762260437, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5080004930496216, "sampling/importance_sampling_ratio/max": 1.6604912281036377, "sampling/importance_sampling_ratio/mean": 1.000133991241455, "sampling/importance_sampling_ratio/min": 0.4977162480354309, "sampling/sampling_logp_difference/max": 0.6977251768112183, "sampling/sampling_logp_difference/mean": 0.00899930763989687, "step": 760 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 561.0, "completions/mean_length": 429.46875, "completions/mean_terminated_length": 418.5483703613281, "completions/min_length": 246.0, "completions/min_terminated_length": 246.0, "entropy": 0.24014792777597904, "epoch": 0.4073875802997859, "frac_reward_zero_std": 0.25, "grad_norm": 0.01803966425359249, "learning_rate": 1e-05, "loss": 0.0309, "num_tokens": 15385553.0, "reward": 0.6875, "reward_std": 0.38298875093460083, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0003598928451538, "sampling/importance_sampling_ratio/min": 0.5966231822967529, "sampling/sampling_logp_difference/max": 0.7804055213928223, "sampling/sampling_logp_difference/mean": 0.009317105636000633, "step": 761 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00011831928350147791, "clip_ratio/low_min": 0.00011831928350147791, "clip_ratio/region_mean": 0.00011831928350147791, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 745.0, "completions/mean_length": 540.46875, "completions/mean_terminated_length": 507.96429443359375, "completions/min_length": 128.0, "completions/min_terminated_length": 128.0, "entropy": 0.3541956767439842, "epoch": 0.4079229122055675, "frac_reward_zero_std": 0.5, "grad_norm": 0.007222883403301239, "learning_rate": 1e-05, "loss": 0.0316, "num_tokens": 15406136.0, "reward": 0.78125, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.6312808990478516, "sampling/importance_sampling_ratio/mean": 1.000046730041504, "sampling/importance_sampling_ratio/min": 0.5512207746505737, "sampling/sampling_logp_difference/max": 0.5956199169158936, "sampling/sampling_logp_difference/mean": 0.01018043328076601, "step": 762 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 584.0, "completions/mean_length": 420.5625, "completions/mean_terminated_length": 409.3548278808594, "completions/min_length": 249.0, "completions/min_terminated_length": 249.0, "entropy": 0.244696456938982, "epoch": 0.408458244111349, "frac_reward_zero_std": 0.5, "grad_norm": 0.0164913572371006, "learning_rate": 1e-05, "loss": -0.0001, "num_tokens": 15423402.0, "reward": 0.71875, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.335079312324524, "sampling/importance_sampling_ratio/mean": 0.9999603629112244, "sampling/importance_sampling_ratio/min": 0.5473047494888306, "sampling/sampling_logp_difference/max": 0.6027494668960571, "sampling/sampling_logp_difference/mean": 0.009655633009970188, "step": 763 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 699.0, "completions/mean_length": 454.375, "completions/mean_terminated_length": 433.4666748046875, "completions/min_length": 205.0, "completions/min_terminated_length": 205.0, "entropy": 0.32273755595088005, "epoch": 0.4089935760171306, "frac_reward_zero_std": 0.5, "grad_norm": 0.029607856646180153, "learning_rate": 1e-05, "loss": 0.0472, "num_tokens": 15441606.0, "reward": 0.8125, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.4689022302627563, "sampling/importance_sampling_ratio/mean": 1.0001521110534668, "sampling/importance_sampling_ratio/min": 0.33022794127464294, "sampling/sampling_logp_difference/max": 1.1079721450805664, "sampling/sampling_logp_difference/mean": 0.010980737395584583, "step": 764 }, { "clip_ratio/high_max": 9.384384611621499e-05, "clip_ratio/high_mean": 9.384384611621499e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 9.384384611621499e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 674.0, "completions/max_terminated_length": 674.0, "completions/mean_length": 377.90625, "completions/mean_terminated_length": 377.90625, "completions/min_length": 202.0, "completions/min_terminated_length": 202.0, "entropy": 0.24666313268244267, "epoch": 0.4095289079229122, "frac_reward_zero_std": 0.5, "grad_norm": 0.0074622477404773235, "learning_rate": 1e-05, "loss": 0.0023, "num_tokens": 15457755.0, "reward": 0.9375, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.9375, "rewards/accuracy_reward/std": 0.24593468010425568, "sampling/importance_sampling_ratio/max": 1.3826043605804443, "sampling/importance_sampling_ratio/mean": 0.9999831914901733, "sampling/importance_sampling_ratio/min": 0.4515019953250885, "sampling/sampling_logp_difference/max": 0.7951755523681641, "sampling/sampling_logp_difference/mean": 0.009976961649954319, "step": 765 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00010403341366327368, "clip_ratio/low_min": 0.00010403341366327368, "clip_ratio/region_mean": 0.00010403341366327368, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 682.0, "completions/mean_length": 537.21875, "completions/mean_terminated_length": 494.4814758300781, "completions/min_length": 250.0, "completions/min_terminated_length": 250.0, "entropy": 0.23389310017228127, "epoch": 0.4100642398286938, "frac_reward_zero_std": 0.5, "grad_norm": 0.01662544719874859, "learning_rate": 1e-05, "loss": 0.0479, "num_tokens": 15479738.0, "reward": 0.6875, "reward_std": 0.249358132481575, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.617555856704712, "sampling/importance_sampling_ratio/mean": 0.9997934103012085, "sampling/importance_sampling_ratio/min": 0.6282559037208557, "sampling/sampling_logp_difference/max": 0.48091626167297363, "sampling/sampling_logp_difference/mean": 0.009437416680157185, "step": 766 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 8.305647497763857e-05, "clip_ratio/low_min": 8.305647497763857e-05, "clip_ratio/region_mean": 8.305647497763857e-05, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 675.0, "completions/mean_length": 437.34375, "completions/mean_terminated_length": 426.6773986816406, "completions/min_length": 220.0, "completions/min_terminated_length": 220.0, "entropy": 0.305824164301157, "epoch": 0.41059957173447537, "frac_reward_zero_std": 0.25, "grad_norm": 0.01872457191348076, "learning_rate": 1e-05, "loss": 0.0059, "num_tokens": 15497637.0, "reward": 0.6875, "reward_std": 0.292504221200943, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.000535249710083, "sampling/importance_sampling_ratio/min": 0.4695589542388916, "sampling/sampling_logp_difference/max": 0.7767643928527832, "sampling/sampling_logp_difference/mean": 0.012080066837370396, "step": 767 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 613.0, "completions/max_terminated_length": 613.0, "completions/mean_length": 355.78125, "completions/mean_terminated_length": 355.78125, "completions/min_length": 191.0, "completions/min_terminated_length": 191.0, "entropy": 0.25859714299440384, "epoch": 0.41113490364025695, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 15512774.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.000051736831665, "sampling/importance_sampling_ratio/min": 0.39346182346343994, "sampling/sampling_logp_difference/max": 0.9327712059020996, "sampling/sampling_logp_difference/mean": 0.010906939394772053, "step": 768 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 693.0, "completions/max_terminated_length": 693.0, "completions/mean_length": 387.84375, "completions/mean_terminated_length": 387.84375, "completions/min_length": 146.0, "completions/min_terminated_length": 146.0, "entropy": 0.27067269943654537, "epoch": 0.41167023554603854, "frac_reward_zero_std": 0.25, "grad_norm": 0.024706391617655754, "learning_rate": 1e-05, "loss": 0.0289, "num_tokens": 15528361.0, "reward": 0.71875, "reward_std": 0.3198433816432953, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.6198508739471436, "sampling/importance_sampling_ratio/mean": 1.0004866123199463, "sampling/importance_sampling_ratio/min": 0.36960989236831665, "sampling/sampling_logp_difference/max": 0.995307207107544, "sampling/sampling_logp_difference/mean": 0.010347813367843628, "step": 769 }, { "clip_ratio/high_max": 6.433350790757686e-05, "clip_ratio/high_mean": 6.433350790757686e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 6.433350790757686e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 543.0, "completions/max_terminated_length": 543.0, "completions/mean_length": 384.5625, "completions/mean_terminated_length": 384.5625, "completions/min_length": 104.0, "completions/min_terminated_length": 104.0, "entropy": 0.3696562983095646, "epoch": 0.41220556745182013, "frac_reward_zero_std": 0.25, "grad_norm": 0.007952449843287468, "learning_rate": 1e-05, "loss": -0.0314, "num_tokens": 15544723.0, "reward": 0.53125, "reward_std": 0.3198433816432953, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.5049047470092773, "sampling/importance_sampling_ratio/mean": 0.9992790818214417, "sampling/importance_sampling_ratio/min": 0.45941370725631714, "sampling/sampling_logp_difference/max": 0.7778041362762451, "sampling/sampling_logp_difference/mean": 0.012590371072292328, "step": 770 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 697.0, "completions/mean_length": 444.5, "completions/mean_terminated_length": 422.933349609375, "completions/min_length": 205.0, "completions/min_terminated_length": 205.0, "entropy": 0.304232407361269, "epoch": 0.4127408993576017, "frac_reward_zero_std": 0.5, "grad_norm": 0.007635242305696011, "learning_rate": 1e-05, "loss": 0.0027, "num_tokens": 15563411.0, "reward": 0.53125, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.6617324352264404, "sampling/importance_sampling_ratio/mean": 0.9998767375946045, "sampling/importance_sampling_ratio/min": 0.5821569561958313, "sampling/sampling_logp_difference/max": 0.5410151481628418, "sampling/sampling_logp_difference/mean": 0.010919096879661083, "step": 771 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 704.0, "completions/max_terminated_length": 704.0, "completions/mean_length": 442.40625, "completions/mean_terminated_length": 442.40625, "completions/min_length": 191.0, "completions/min_terminated_length": 191.0, "entropy": 0.23302766494452953, "epoch": 0.4132762312633833, "frac_reward_zero_std": 0.5, "grad_norm": 0.004053059499710798, "learning_rate": 1e-05, "loss": -0.0086, "num_tokens": 15581904.0, "reward": 0.625, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9999101161956787, "sampling/importance_sampling_ratio/min": 0.5091020464897156, "sampling/sampling_logp_difference/max": 0.7373270988464355, "sampling/sampling_logp_difference/mean": 0.009034702554345131, "step": 772 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00020220431179041043, "clip_ratio/low_min": 0.00020220431179041043, "clip_ratio/region_mean": 0.00020220431179041043, "completions/clipped_ratio": 0.0, "completions/max_length": 519.0, "completions/max_terminated_length": 519.0, "completions/mean_length": 345.46875, "completions/mean_terminated_length": 345.46875, "completions/min_length": 182.0, "completions/min_terminated_length": 182.0, "entropy": 0.2553048860281706, "epoch": 0.4138115631691649, "frac_reward_zero_std": 0.5, "grad_norm": 0.03436039760708809, "learning_rate": 1e-05, "loss": -0.0183, "num_tokens": 15596871.0, "reward": 0.9375, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.9375, "rewards/accuracy_reward/std": 0.24593468010425568, "sampling/importance_sampling_ratio/max": 1.488649845123291, "sampling/importance_sampling_ratio/mean": 0.9997662901878357, "sampling/importance_sampling_ratio/min": 0.4369298219680786, "sampling/sampling_logp_difference/max": 0.8279826641082764, "sampling/sampling_logp_difference/mean": 0.010252413339912891, "step": 773 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00018368846212979406, "clip_ratio/low_min": 0.00018368846212979406, "clip_ratio/region_mean": 0.00018368846212979406, "completions/clipped_ratio": 0.0, "completions/max_length": 500.0, "completions/max_terminated_length": 500.0, "completions/mean_length": 328.0625, "completions/mean_terminated_length": 328.0625, "completions/min_length": 127.0, "completions/min_terminated_length": 127.0, "entropy": 0.27277201041579247, "epoch": 0.4143468950749465, "frac_reward_zero_std": 0.5, "grad_norm": 0.016949793323874474, "learning_rate": 1e-05, "loss": -0.0561, "num_tokens": 15610977.0, "reward": 0.71875, "reward_std": 0.2630178928375244, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.9013302326202393, "sampling/importance_sampling_ratio/mean": 0.9998557567596436, "sampling/importance_sampling_ratio/min": 0.6245912313461304, "sampling/sampling_logp_difference/max": 0.6425538063049316, "sampling/sampling_logp_difference/mean": 0.010486333630979061, "step": 774 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00037325148878153414, "clip_ratio/low_min": 0.00037325148878153414, "clip_ratio/region_mean": 0.00037325148878153414, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 636.0, "completions/mean_length": 413.875, "completions/mean_terminated_length": 402.45159912109375, "completions/min_length": 129.0, "completions/min_terminated_length": 129.0, "entropy": 0.40199044346809387, "epoch": 0.4148822269807281, "frac_reward_zero_std": 0.25, "grad_norm": 0.024073319509625435, "learning_rate": 1e-05, "loss": 0.0021, "num_tokens": 15628629.0, "reward": 0.65625, "reward_std": 0.3608423173427582, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9997939467430115, "sampling/importance_sampling_ratio/min": 0.38455668091773987, "sampling/sampling_logp_difference/max": 0.9556640982627869, "sampling/sampling_logp_difference/mean": 0.01387203112244606, "step": 775 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00013112008309690282, "clip_ratio/low_min": 0.00013112008309690282, "clip_ratio/region_mean": 0.00013112008309690282, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 763.0, "completions/mean_length": 529.0, "completions/mean_terminated_length": 494.857177734375, "completions/min_length": 322.0, "completions/min_terminated_length": 322.0, "entropy": 0.2845477182418108, "epoch": 0.41541755888650966, "frac_reward_zero_std": 0.25, "grad_norm": 0.0260250773280859, "learning_rate": 1e-05, "loss": 0.0167, "num_tokens": 15649469.0, "reward": 0.59375, "reward_std": 0.378745436668396, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.4169169664382935, "sampling/importance_sampling_ratio/mean": 0.9998788833618164, "sampling/importance_sampling_ratio/min": 0.36439773440361023, "sampling/sampling_logp_difference/max": 1.0095093250274658, "sampling/sampling_logp_difference/mean": 0.010882158763706684, "step": 776 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0002542619840824045, "clip_ratio/low_min": 0.0002542619840824045, "clip_ratio/region_mean": 0.0002542619840824045, "completions/clipped_ratio": 0.0, "completions/max_length": 569.0, "completions/max_terminated_length": 569.0, "completions/mean_length": 376.15625, "completions/mean_terminated_length": 376.15625, "completions/min_length": 220.0, "completions/min_terminated_length": 220.0, "entropy": 0.2752981185913086, "epoch": 0.4159528907922912, "frac_reward_zero_std": 0.25, "grad_norm": 0.021978862583637238, "learning_rate": 1e-05, "loss": 0.0158, "num_tokens": 15664810.0, "reward": 0.65625, "reward_std": 0.38816186785697937, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.4845292568206787, "sampling/importance_sampling_ratio/mean": 1.0002071857452393, "sampling/importance_sampling_ratio/min": 0.6175705194473267, "sampling/sampling_logp_difference/max": 0.4819619655609131, "sampling/sampling_logp_difference/mean": 0.010415537282824516, "step": 777 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.383290408644825e-05, "clip_ratio/low_min": 5.383290408644825e-05, "clip_ratio/region_mean": 5.383290408644825e-05, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 722.0, "completions/mean_length": 488.78125, "completions/mean_terminated_length": 437.0740661621094, "completions/min_length": 142.0, "completions/min_terminated_length": 142.0, "entropy": 0.3771573305130005, "epoch": 0.4164882226980728, "frac_reward_zero_std": 0.25, "grad_norm": 0.0047351461835205555, "learning_rate": 1e-05, "loss": 0.0318, "num_tokens": 15684283.0, "reward": 0.46875, "reward_std": 0.2651650309562683, "rewards/accuracy_reward/mean": 0.46875, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.717313289642334, "sampling/importance_sampling_ratio/mean": 0.999986469745636, "sampling/importance_sampling_ratio/min": 0.593487560749054, "sampling/sampling_logp_difference/max": 0.5407609939575195, "sampling/sampling_logp_difference/mean": 0.012063194066286087, "step": 778 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 579.0, "completions/mean_length": 358.0, "completions/mean_terminated_length": 330.66668701171875, "completions/min_length": 115.0, "completions/min_terminated_length": 115.0, "entropy": 0.3367639631032944, "epoch": 0.4170235546038544, "frac_reward_zero_std": 0.25, "grad_norm": 0.025062229484319687, "learning_rate": 1e-05, "loss": -0.0248, "num_tokens": 15699363.0, "reward": 0.65625, "reward_std": 0.3966485261917114, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.511836051940918, "sampling/importance_sampling_ratio/mean": 1.0002409219741821, "sampling/importance_sampling_ratio/min": 0.5363574624061584, "sampling/sampling_logp_difference/max": 0.6229543685913086, "sampling/sampling_logp_difference/mean": 0.0124586820602417, "step": 779 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00021610707335639745, "clip_ratio/low_min": 0.00021610707335639745, "clip_ratio/region_mean": 0.00021610707335639745, "completions/clipped_ratio": 0.0, "completions/max_length": 681.0, "completions/max_terminated_length": 681.0, "completions/mean_length": 373.28125, "completions/mean_terminated_length": 373.28125, "completions/min_length": 244.0, "completions/min_terminated_length": 244.0, "entropy": 0.25080471858382225, "epoch": 0.41755888650963596, "frac_reward_zero_std": 0.5, "grad_norm": 0.005736921913921833, "learning_rate": 1e-05, "loss": 0.008, "num_tokens": 15714508.0, "reward": 0.75, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9998818635940552, "sampling/importance_sampling_ratio/min": 0.5307590365409851, "sampling/sampling_logp_difference/max": 0.7510523796081543, "sampling/sampling_logp_difference/mean": 0.010422402992844582, "step": 780 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 583.0, "completions/mean_length": 482.5, "completions/mean_terminated_length": 429.629638671875, "completions/min_length": 255.0, "completions/min_terminated_length": 255.0, "entropy": 0.4434059262275696, "epoch": 0.41809421841541755, "frac_reward_zero_std": 0.5, "grad_norm": 0.0073394631035625935, "learning_rate": 1e-05, "loss": 0.0235, "num_tokens": 15733892.0, "reward": 0.59375, "reward_std": 0.22201895713806152, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.9157443046569824, "sampling/importance_sampling_ratio/mean": 0.9997337460517883, "sampling/importance_sampling_ratio/min": 0.38598206639289856, "sampling/sampling_logp_difference/max": 0.9519643783569336, "sampling/sampling_logp_difference/mean": 0.013192393817007542, "step": 781 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 7.571168680442497e-05, "clip_ratio/low_min": 7.571168680442497e-05, "clip_ratio/region_mean": 7.571168680442497e-05, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 734.0, "completions/mean_length": 471.0625, "completions/mean_terminated_length": 461.4838562011719, "completions/min_length": 330.0, "completions/min_terminated_length": 330.0, "entropy": 0.33475570380687714, "epoch": 0.41862955032119914, "frac_reward_zero_std": 0.5, "grad_norm": 0.007119338493794203, "learning_rate": 1e-05, "loss": -0.0001, "num_tokens": 15752534.0, "reward": 0.5625, "reward_std": 0.249358132481575, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.4272743463516235, "sampling/importance_sampling_ratio/mean": 0.9999619722366333, "sampling/importance_sampling_ratio/min": 0.5658502578735352, "sampling/sampling_logp_difference/max": 0.5694258213043213, "sampling/sampling_logp_difference/mean": 0.011955409310758114, "step": 782 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 659.0, "completions/max_terminated_length": 659.0, "completions/mean_length": 384.8125, "completions/mean_terminated_length": 384.8125, "completions/min_length": 132.0, "completions/min_terminated_length": 132.0, "entropy": 0.2584788240492344, "epoch": 0.41916488222698073, "frac_reward_zero_std": 0.5, "grad_norm": 0.0038310440722852945, "learning_rate": 1e-05, "loss": -0.0908, "num_tokens": 15768888.0, "reward": 0.75, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.5195432901382446, "sampling/importance_sampling_ratio/mean": 0.9996142983436584, "sampling/importance_sampling_ratio/min": 0.5287886261940002, "sampling/sampling_logp_difference/max": 0.6371665000915527, "sampling/sampling_logp_difference/mean": 0.010382593609392643, "step": 783 }, { "clip_ratio/high_max": 0.0001403668211423792, "clip_ratio/high_mean": 0.0001403668211423792, "clip_ratio/low_mean": 5.643340773531236e-05, "clip_ratio/low_min": 5.643340773531236e-05, "clip_ratio/region_mean": 0.00019680022887769155, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 609.0, "completions/mean_length": 424.15625, "completions/mean_terminated_length": 375.0357360839844, "completions/min_length": 149.0, "completions/min_terminated_length": 149.0, "entropy": 0.39270056039094925, "epoch": 0.4197002141327623, "frac_reward_zero_std": 0.25, "grad_norm": 0.02223135344684124, "learning_rate": 1e-05, "loss": -0.0231, "num_tokens": 15786613.0, "reward": 0.65625, "reward_std": 0.3198433816432953, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0002459287643433, "sampling/importance_sampling_ratio/min": 0.5257071256637573, "sampling/sampling_logp_difference/max": 0.9976482391357422, "sampling/sampling_logp_difference/mean": 0.014230569824576378, "step": 784 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 7.896398892626166e-05, "clip_ratio/low_min": 7.896398892626166e-05, "clip_ratio/region_mean": 7.896398892626166e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 646.0, "completions/max_terminated_length": 646.0, "completions/mean_length": 415.34375, "completions/mean_terminated_length": 415.34375, "completions/min_length": 227.0, "completions/min_terminated_length": 227.0, "entropy": 0.35414634086191654, "epoch": 0.4202355460385439, "frac_reward_zero_std": 0.25, "grad_norm": 0.0093699861317873, "learning_rate": 1e-05, "loss": 0.0066, "num_tokens": 15804024.0, "reward": 0.84375, "reward_std": 0.3061639964580536, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9995676279067993, "sampling/importance_sampling_ratio/min": 0.5625743269920349, "sampling/sampling_logp_difference/max": 1.251774787902832, "sampling/sampling_logp_difference/mean": 0.0111850555986166, "step": 785 }, { "clip_ratio/high_max": 9.689922444522381e-05, "clip_ratio/high_mean": 9.689922444522381e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 9.689922444522381e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 589.0, "completions/max_terminated_length": 589.0, "completions/mean_length": 378.53125, "completions/mean_terminated_length": 378.53125, "completions/min_length": 156.0, "completions/min_terminated_length": 156.0, "entropy": 0.26351975463330746, "epoch": 0.4207708779443255, "frac_reward_zero_std": 0.75, "grad_norm": 0.008030323311686516, "learning_rate": 1e-05, "loss": -0.0131, "num_tokens": 15819897.0, "reward": 0.90625, "reward_std": 0.1293872892856598, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.9409581422805786, "sampling/importance_sampling_ratio/mean": 0.9995185732841492, "sampling/importance_sampling_ratio/min": 0.5583049654960632, "sampling/sampling_logp_difference/max": 0.6631817817687988, "sampling/sampling_logp_difference/mean": 0.011047407053411007, "step": 786 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 714.0, "completions/max_terminated_length": 714.0, "completions/mean_length": 380.34375, "completions/mean_terminated_length": 380.34375, "completions/min_length": 103.0, "completions/min_terminated_length": 103.0, "entropy": 0.26448206044733524, "epoch": 0.4213062098501071, "frac_reward_zero_std": 0.25, "grad_norm": 0.002527264878153801, "learning_rate": 1e-05, "loss": 0.0025, "num_tokens": 15835484.0, "reward": 0.8125, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.9102119207382202, "sampling/importance_sampling_ratio/mean": 1.0003339052200317, "sampling/importance_sampling_ratio/min": 0.5570401549339294, "sampling/sampling_logp_difference/max": 0.6472141742706299, "sampling/sampling_logp_difference/mean": 0.011137889698147774, "step": 787 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00015303726104320958, "clip_ratio/low_min": 0.00015303726104320958, "clip_ratio/region_mean": 0.00015303726104320958, "completions/clipped_ratio": 0.0, "completions/max_length": 633.0, "completions/max_terminated_length": 633.0, "completions/mean_length": 377.53125, "completions/mean_terminated_length": 377.53125, "completions/min_length": 221.0, "completions/min_terminated_length": 221.0, "entropy": 0.2805671710520983, "epoch": 0.42184154175588867, "frac_reward_zero_std": 0.5, "grad_norm": 0.025459816679358482, "learning_rate": 1e-05, "loss": -0.0408, "num_tokens": 15851317.0, "reward": 0.71875, "reward_std": 0.2630178928375244, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.5567599534988403, "sampling/importance_sampling_ratio/mean": 0.9995025396347046, "sampling/importance_sampling_ratio/min": 0.6074600219726562, "sampling/sampling_logp_difference/max": 0.49846887588500977, "sampling/sampling_logp_difference/mean": 0.011317256838083267, "step": 788 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 8.153946691891178e-05, "clip_ratio/low_min": 8.153946691891178e-05, "clip_ratio/region_mean": 8.153946691891178e-05, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 698.0, "completions/mean_length": 454.96875, "completions/mean_terminated_length": 422.5862121582031, "completions/min_length": 221.0, "completions/min_terminated_length": 221.0, "entropy": 0.3912490103393793, "epoch": 0.42237687366167026, "frac_reward_zero_std": 0.5, "grad_norm": 0.004201194737106562, "learning_rate": 1e-05, "loss": 0.0112, "num_tokens": 15870156.0, "reward": 0.3125, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.3125, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.5440092086791992, "sampling/importance_sampling_ratio/mean": 0.9996799230575562, "sampling/importance_sampling_ratio/min": 0.5852885246276855, "sampling/sampling_logp_difference/max": 0.5356502532958984, "sampling/sampling_logp_difference/mean": 0.013936889357864857, "step": 789 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 8.445946150459349e-05, "clip_ratio/low_min": 8.445946150459349e-05, "clip_ratio/region_mean": 8.445946150459349e-05, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 753.0, "completions/mean_length": 429.375, "completions/mean_terminated_length": 406.8000183105469, "completions/min_length": 233.0, "completions/min_terminated_length": 233.0, "entropy": 0.28259735368192196, "epoch": 0.4229122055674518, "frac_reward_zero_std": 0.5, "grad_norm": 0.013709502294659615, "learning_rate": 1e-05, "loss": 0.0427, "num_tokens": 15887672.0, "reward": 0.84375, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.6652288436889648, "sampling/importance_sampling_ratio/mean": 1.0003029108047485, "sampling/importance_sampling_ratio/min": 0.5105920433998108, "sampling/sampling_logp_difference/max": 0.6721844673156738, "sampling/sampling_logp_difference/mean": 0.011160780675709248, "step": 790 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00018738366634352133, "clip_ratio/low_min": 0.00018738366634352133, "clip_ratio/region_mean": 0.00018738366634352133, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 628.0, "completions/mean_length": 462.78125, "completions/mean_terminated_length": 442.433349609375, "completions/min_length": 265.0, "completions/min_terminated_length": 265.0, "entropy": 0.3385087884962559, "epoch": 0.4234475374732334, "frac_reward_zero_std": 0.5, "grad_norm": 0.023096540942788124, "learning_rate": 1e-05, "loss": 0.0255, "num_tokens": 15906681.0, "reward": 0.71875, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.5910266637802124, "sampling/importance_sampling_ratio/mean": 1.0004183053970337, "sampling/importance_sampling_ratio/min": 0.5628833174705505, "sampling/sampling_logp_difference/max": 0.5746829509735107, "sampling/sampling_logp_difference/mean": 0.011910232715308666, "step": 791 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 7.485030073439702e-05, "clip_ratio/low_min": 7.485030073439702e-05, "clip_ratio/region_mean": 7.485030073439702e-05, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 675.0, "completions/mean_length": 431.625, "completions/mean_terminated_length": 409.20001220703125, "completions/min_length": 230.0, "completions/min_terminated_length": 230.0, "entropy": 0.3437661435455084, "epoch": 0.42398286937901497, "frac_reward_zero_std": 0.0, "grad_norm": 0.03732145577669144, "learning_rate": 1e-05, "loss": 0.0649, "num_tokens": 15924069.0, "reward": 0.78125, "reward_std": 0.4218915104866028, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9997829794883728, "sampling/importance_sampling_ratio/min": 0.5115315318107605, "sampling/sampling_logp_difference/max": 1.006821632385254, "sampling/sampling_logp_difference/mean": 0.012319444678723812, "step": 792 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 575.0, "completions/max_terminated_length": 575.0, "completions/mean_length": 340.375, "completions/mean_terminated_length": 340.375, "completions/min_length": 33.0, "completions/min_terminated_length": 33.0, "entropy": 0.2199815083295107, "epoch": 0.42451820128479656, "frac_reward_zero_std": 0.75, "grad_norm": 0.022471919655799866, "learning_rate": 1e-05, "loss": -0.005, "num_tokens": 15938449.0, "reward": 0.9375, "reward_std": 0.1157275140285492, "rewards/accuracy_reward/mean": 0.9375, "rewards/accuracy_reward/std": 0.24593468010425568, "sampling/importance_sampling_ratio/max": 1.6085669994354248, "sampling/importance_sampling_ratio/mean": 1.0001497268676758, "sampling/importance_sampling_ratio/min": 0.500328540802002, "sampling/sampling_logp_difference/max": 0.6924903392791748, "sampling/sampling_logp_difference/mean": 0.008921113796532154, "step": 793 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00010339123400626704, "clip_ratio/low_min": 0.00010339123400626704, "clip_ratio/region_mean": 0.00010339123400626704, "completions/clipped_ratio": 0.0, "completions/max_length": 482.0, "completions/max_terminated_length": 482.0, "completions/mean_length": 292.40625, "completions/mean_terminated_length": 292.40625, "completions/min_length": 137.0, "completions/min_terminated_length": 137.0, "entropy": 0.24353780783712864, "epoch": 0.42505353319057815, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": -0.0156, "num_tokens": 15951358.0, "reward": 0.84375, "reward_std": 0.1293872892856598, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.457561731338501, "sampling/importance_sampling_ratio/mean": 1.0000132322311401, "sampling/importance_sampling_ratio/min": 0.5762859582901001, "sampling/sampling_logp_difference/max": 0.5511512756347656, "sampling/sampling_logp_difference/mean": 0.009628204628825188, "step": 794 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 644.0, "completions/max_terminated_length": 644.0, "completions/mean_length": 434.03125, "completions/mean_terminated_length": 434.03125, "completions/min_length": 223.0, "completions/min_terminated_length": 223.0, "entropy": 0.3120315596461296, "epoch": 0.42558886509635974, "frac_reward_zero_std": 0.0, "grad_norm": 0.01217488944530487, "learning_rate": 1e-05, "loss": 0.0066, "num_tokens": 15969039.0, "reward": 0.4375, "reward_std": 0.4261348247528076, "rewards/accuracy_reward/mean": 0.4375, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.461539387702942, "sampling/importance_sampling_ratio/mean": 1.0000052452087402, "sampling/importance_sampling_ratio/min": 0.5028045177459717, "sampling/sampling_logp_difference/max": 0.687553882598877, "sampling/sampling_logp_difference/mean": 0.012451590970158577, "step": 795 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.656016921624541e-05, "clip_ratio/low_min": 6.656016921624541e-05, "clip_ratio/region_mean": 6.656016921624541e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 615.0, "completions/max_terminated_length": 615.0, "completions/mean_length": 401.78125, "completions/mean_terminated_length": 401.78125, "completions/min_length": 247.0, "completions/min_terminated_length": 247.0, "entropy": 0.28384484723210335, "epoch": 0.4261241970021413, "frac_reward_zero_std": 0.5, "grad_norm": 0.003345826640725136, "learning_rate": 1e-05, "loss": -0.015, "num_tokens": 15985536.0, "reward": 0.875, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 1.8771435022354126, "sampling/importance_sampling_ratio/mean": 0.9998907446861267, "sampling/importance_sampling_ratio/min": 0.615212619304657, "sampling/sampling_logp_difference/max": 0.6297512054443359, "sampling/sampling_logp_difference/mean": 0.011010140180587769, "step": 796 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 588.0, "completions/mean_length": 318.53125, "completions/mean_terminated_length": 304.0322570800781, "completions/min_length": 132.0, "completions/min_terminated_length": 132.0, "entropy": 0.2827566247433424, "epoch": 0.4266595289079229, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": -0.0147, "num_tokens": 15999217.0, "reward": 0.90625, "reward_std": 0.1293872892856598, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.7602216005325317, "sampling/importance_sampling_ratio/mean": 1.0001087188720703, "sampling/importance_sampling_ratio/min": 0.44405969977378845, "sampling/sampling_logp_difference/max": 0.8117963075637817, "sampling/sampling_logp_difference/mean": 0.011242852546274662, "step": 797 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 596.0, "completions/max_terminated_length": 596.0, "completions/mean_length": 385.71875, "completions/mean_terminated_length": 385.71875, "completions/min_length": 134.0, "completions/min_terminated_length": 134.0, "entropy": 0.33430335484445095, "epoch": 0.4271948608137045, "frac_reward_zero_std": 0.0, "grad_norm": 0.019825590774416924, "learning_rate": 1e-05, "loss": -0.0319, "num_tokens": 16015280.0, "reward": 0.65625, "reward_std": 0.3808925747871399, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.4260501861572266, "sampling/importance_sampling_ratio/mean": 1.0001707077026367, "sampling/importance_sampling_ratio/min": 0.511563777923584, "sampling/sampling_logp_difference/max": 0.6702830791473389, "sampling/sampling_logp_difference/mean": 0.011605370789766312, "step": 798 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.755064557888545e-05, "clip_ratio/low_min": 5.755064557888545e-05, "clip_ratio/region_mean": 5.755064557888545e-05, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 759.0, "completions/mean_length": 500.46875, "completions/mean_terminated_length": 491.83868408203125, "completions/min_length": 251.0, "completions/min_terminated_length": 251.0, "entropy": 0.28260005451738834, "epoch": 0.4277301927194861, "frac_reward_zero_std": 0.25, "grad_norm": 0.009162155911326408, "learning_rate": 1e-05, "loss": 0.0544, "num_tokens": 16035271.0, "reward": 0.78125, "reward_std": 0.3377465009689331, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.4117162227630615, "sampling/importance_sampling_ratio/mean": 0.9999385476112366, "sampling/importance_sampling_ratio/min": 0.3373129665851593, "sampling/sampling_logp_difference/max": 1.0867441892623901, "sampling/sampling_logp_difference/mean": 0.011142052710056305, "step": 799 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 8.778089977568015e-05, "clip_ratio/low_min": 8.778089977568015e-05, "clip_ratio/region_mean": 8.778089977568015e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 604.0, "completions/max_terminated_length": 604.0, "completions/mean_length": 387.125, "completions/mean_terminated_length": 387.125, "completions/min_length": 140.0, "completions/min_terminated_length": 140.0, "entropy": 0.27057322300970554, "epoch": 0.4282655246252677, "frac_reward_zero_std": 0.25, "grad_norm": 0.028597332537174225, "learning_rate": 1e-05, "loss": 0.0163, "num_tokens": 16051723.0, "reward": 0.8125, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0005011558532715, "sampling/importance_sampling_ratio/min": 0.6890622973442078, "sampling/sampling_logp_difference/max": 0.8228888511657715, "sampling/sampling_logp_difference/mean": 0.011211972683668137, "step": 800 }, { "clip_ratio/high_max": 7.00672680977732e-05, "clip_ratio/high_mean": 7.00672680977732e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 7.00672680977732e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 640.0, "completions/max_terminated_length": 640.0, "completions/mean_length": 444.125, "completions/mean_terminated_length": 444.125, "completions/min_length": 279.0, "completions/min_terminated_length": 279.0, "entropy": 0.2429835107177496, "epoch": 0.42880085653104927, "frac_reward_zero_std": 0.25, "grad_norm": 0.015095447190105915, "learning_rate": 1e-05, "loss": -0.039, "num_tokens": 16069567.0, "reward": 0.6875, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.426932454109192, "sampling/importance_sampling_ratio/mean": 1.0000156164169312, "sampling/importance_sampling_ratio/min": 0.3773273229598999, "sampling/sampling_logp_difference/max": 0.974642276763916, "sampling/sampling_logp_difference/mean": 0.009842971339821815, "step": 801 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.827505810884759e-05, "clip_ratio/low_min": 5.827505810884759e-05, "clip_ratio/region_mean": 5.827505810884759e-05, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 702.0, "completions/mean_length": 450.90625, "completions/mean_terminated_length": 440.6773986816406, "completions/min_length": 218.0, "completions/min_terminated_length": 218.0, "entropy": 0.26054478995501995, "epoch": 0.42933618843683086, "frac_reward_zero_std": 0.5, "grad_norm": 0.0036774608306586742, "learning_rate": 1e-05, "loss": 0.0426, "num_tokens": 16088276.0, "reward": 0.6875, "reward_std": 0.249358132481575, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.48768949508667, "sampling/importance_sampling_ratio/mean": 0.9999830722808838, "sampling/importance_sampling_ratio/min": 0.6813663840293884, "sampling/sampling_logp_difference/max": 0.39722418785095215, "sampling/sampling_logp_difference/mean": 0.009339271113276482, "step": 802 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 7.17565999366343e-05, "clip_ratio/low_min": 7.17565999366343e-05, "clip_ratio/region_mean": 7.17565999366343e-05, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 585.0, "completions/mean_length": 406.53125, "completions/mean_terminated_length": 394.8709411621094, "completions/min_length": 215.0, "completions/min_terminated_length": 215.0, "entropy": 0.2608866672962904, "epoch": 0.42987152034261245, "frac_reward_zero_std": 0.5, "grad_norm": 0.014327667653560638, "learning_rate": 1e-05, "loss": -0.0132, "num_tokens": 16104981.0, "reward": 0.8125, "reward_std": 0.249358132481575, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.6868841648101807, "sampling/importance_sampling_ratio/mean": 0.9999310374259949, "sampling/importance_sampling_ratio/min": 0.4192970097064972, "sampling/sampling_logp_difference/max": 0.8691757917404175, "sampling/sampling_logp_difference/mean": 0.01003206241875887, "step": 803 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 589.0, "completions/mean_length": 396.625, "completions/mean_terminated_length": 327.85186767578125, "completions/min_length": 216.0, "completions/min_terminated_length": 216.0, "entropy": 0.31929980032145977, "epoch": 0.430406852248394, "frac_reward_zero_std": 0.75, "grad_norm": 0.004386443179100752, "learning_rate": 1e-05, "loss": 0.0304, "num_tokens": 16121689.0, "reward": 0.78125, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.8314975500106812, "sampling/importance_sampling_ratio/mean": 1.000288963317871, "sampling/importance_sampling_ratio/min": 0.6134749054908752, "sampling/sampling_logp_difference/max": 0.6051340103149414, "sampling/sampling_logp_difference/mean": 0.01133585162460804, "step": 804 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 7.720815483480692e-05, "clip_ratio/low_min": 7.720815483480692e-05, "clip_ratio/region_mean": 7.720815483480692e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 623.0, "completions/max_terminated_length": 623.0, "completions/mean_length": 379.34375, "completions/mean_terminated_length": 379.34375, "completions/min_length": 135.0, "completions/min_terminated_length": 135.0, "entropy": 0.25638870522379875, "epoch": 0.43094218415417557, "frac_reward_zero_std": 0.25, "grad_norm": 0.020009981468319893, "learning_rate": 1e-05, "loss": -0.0083, "num_tokens": 16137164.0, "reward": 0.75, "reward_std": 0.3514062762260437, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.7594571113586426, "sampling/importance_sampling_ratio/mean": 1.0000542402267456, "sampling/importance_sampling_ratio/min": 0.5219526290893555, "sampling/sampling_logp_difference/max": 0.6501784324645996, "sampling/sampling_logp_difference/mean": 0.010279352776706219, "step": 805 }, { "clip_ratio/high_max": 9.65996878221631e-05, "clip_ratio/high_mean": 9.65996878221631e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 9.65996878221631e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 575.0, "completions/max_terminated_length": 575.0, "completions/mean_length": 345.78125, "completions/mean_terminated_length": 345.78125, "completions/min_length": 195.0, "completions/min_terminated_length": 195.0, "entropy": 0.24040366522967815, "epoch": 0.43147751605995716, "frac_reward_zero_std": 0.0, "grad_norm": 0.038052044808864594, "learning_rate": 1e-05, "loss": 0.0042, "num_tokens": 16151581.0, "reward": 0.71875, "reward_std": 0.3808925747871399, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.432822346687317, "sampling/importance_sampling_ratio/mean": 0.9999960064888, "sampling/importance_sampling_ratio/min": 0.5406076908111572, "sampling/sampling_logp_difference/max": 0.6150614023208618, "sampling/sampling_logp_difference/mean": 0.00955849140882492, "step": 806 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 8.127438195515424e-05, "clip_ratio/low_min": 8.127438195515424e-05, "clip_ratio/region_mean": 8.127438195515424e-05, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 695.0, "completions/mean_length": 404.4375, "completions/mean_terminated_length": 392.70965576171875, "completions/min_length": 252.0, "completions/min_terminated_length": 252.0, "entropy": 0.2859781552106142, "epoch": 0.43201284796573874, "frac_reward_zero_std": 0.25, "grad_norm": 0.006455859635025263, "learning_rate": 1e-05, "loss": 0.0495, "num_tokens": 16168123.0, "reward": 0.6875, "reward_std": 0.2925041913986206, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.4351640939712524, "sampling/importance_sampling_ratio/mean": 1.0002063512802124, "sampling/importance_sampling_ratio/min": 0.5675336122512817, "sampling/sampling_logp_difference/max": 0.5664552450180054, "sampling/sampling_logp_difference/mean": 0.011134625412523746, "step": 807 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 684.0, "completions/mean_length": 405.0, "completions/mean_terminated_length": 380.8000183105469, "completions/min_length": 213.0, "completions/min_terminated_length": 213.0, "entropy": 0.35887262411415577, "epoch": 0.43254817987152033, "frac_reward_zero_std": 0.5, "grad_norm": 0.005303828977048397, "learning_rate": 1e-05, "loss": 0.0038, "num_tokens": 16184803.0, "reward": 0.78125, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.824461817741394, "sampling/importance_sampling_ratio/mean": 0.9998227953910828, "sampling/importance_sampling_ratio/min": 0.5102828741073608, "sampling/sampling_logp_difference/max": 0.6727900505065918, "sampling/sampling_logp_difference/mean": 0.012198110111057758, "step": 808 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 768.0, "completions/mean_length": 500.8125, "completions/mean_terminated_length": 462.64288330078125, "completions/min_length": 220.0, "completions/min_terminated_length": 220.0, "entropy": 0.2566366661339998, "epoch": 0.4330835117773019, "frac_reward_zero_std": 0.5, "grad_norm": 0.0034935621079057455, "learning_rate": 1e-05, "loss": 0.0774, "num_tokens": 16204661.0, "reward": 0.875, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 1.5814470052719116, "sampling/importance_sampling_ratio/mean": 0.999999463558197, "sampling/importance_sampling_ratio/min": 0.6377719640731812, "sampling/sampling_logp_difference/max": 0.4583401679992676, "sampling/sampling_logp_difference/mean": 0.0101484265178442, "step": 809 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 668.0, "completions/mean_length": 435.875, "completions/mean_terminated_length": 425.1612854003906, "completions/min_length": 163.0, "completions/min_terminated_length": 163.0, "entropy": 0.2633406352251768, "epoch": 0.4336188436830835, "frac_reward_zero_std": 0.25, "grad_norm": 0.021491779014468193, "learning_rate": 1e-05, "loss": 0.0484, "num_tokens": 16222193.0, "reward": 0.71875, "reward_std": 0.35564959049224854, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.5119787454605103, "sampling/importance_sampling_ratio/mean": 0.9998757243156433, "sampling/importance_sampling_ratio/min": 0.30166324973106384, "sampling/sampling_logp_difference/max": 1.19844388961792, "sampling/sampling_logp_difference/mean": 0.01047497894614935, "step": 810 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 8.207485370803624e-05, "clip_ratio/low_min": 8.207485370803624e-05, "clip_ratio/region_mean": 8.207485370803624e-05, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 738.0, "completions/mean_length": 428.21875, "completions/mean_terminated_length": 405.5666809082031, "completions/min_length": 139.0, "completions/min_terminated_length": 139.0, "entropy": 0.25248922407627106, "epoch": 0.4341541755888651, "frac_reward_zero_std": 0.5, "grad_norm": 0.01177243608981371, "learning_rate": 1e-05, "loss": 0.0085, "num_tokens": 16239528.0, "reward": 0.65625, "reward_std": 0.22201895713806152, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.000215768814087, "sampling/importance_sampling_ratio/min": 0.7016726136207581, "sampling/sampling_logp_difference/max": 0.7899539470672607, "sampling/sampling_logp_difference/mean": 0.009390268474817276, "step": 811 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.0444875998655334e-05, "clip_ratio/low_min": 6.0444875998655334e-05, "clip_ratio/region_mean": 6.0444875998655334e-05, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 703.0, "completions/mean_length": 489.96875, "completions/mean_terminated_length": 471.433349609375, "completions/min_length": 277.0, "completions/min_terminated_length": 277.0, "entropy": 0.28769651986658573, "epoch": 0.4346895074946467, "frac_reward_zero_std": 0.5, "grad_norm": 0.0207393579185009, "learning_rate": 1e-05, "loss": 0.0106, "num_tokens": 16258999.0, "reward": 0.75, "reward_std": 0.2587745785713196, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.388028860092163, "sampling/importance_sampling_ratio/mean": 0.9997342228889465, "sampling/importance_sampling_ratio/min": 0.5272652506828308, "sampling/sampling_logp_difference/max": 0.6400516033172607, "sampling/sampling_logp_difference/mean": 0.01168107334524393, "step": 812 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 734.0, "completions/mean_length": 503.59375, "completions/mean_terminated_length": 465.8214416503906, "completions/min_length": 252.0, "completions/min_terminated_length": 252.0, "entropy": 0.32064057886600494, "epoch": 0.4352248394004283, "frac_reward_zero_std": 0.25, "grad_norm": 0.025505701079964638, "learning_rate": 1e-05, "loss": -0.0651, "num_tokens": 16278738.0, "reward": 0.40625, "reward_std": 0.3061639666557312, "rewards/accuracy_reward/mean": 0.40625, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.4701932668685913, "sampling/importance_sampling_ratio/mean": 1.000252366065979, "sampling/importance_sampling_ratio/min": 0.4715697467327118, "sampling/sampling_logp_difference/max": 0.7516883611679077, "sampling/sampling_logp_difference/mean": 0.011873779818415642, "step": 813 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 9.342301927972585e-05, "clip_ratio/low_min": 9.342301927972585e-05, "clip_ratio/region_mean": 9.342301927972585e-05, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 525.0, "completions/mean_length": 366.25, "completions/mean_terminated_length": 353.2903137207031, "completions/min_length": 117.0, "completions/min_terminated_length": 117.0, "entropy": 0.2732634488493204, "epoch": 0.43576017130620986, "frac_reward_zero_std": 0.0, "grad_norm": 0.012402699328958988, "learning_rate": 1e-05, "loss": 0.0694, "num_tokens": 16294066.0, "reward": 0.59375, "reward_std": 0.4534739851951599, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.687206745147705, "sampling/importance_sampling_ratio/mean": 1.0001044273376465, "sampling/importance_sampling_ratio/min": 0.4808214604854584, "sampling/sampling_logp_difference/max": 0.7322592735290527, "sampling/sampling_logp_difference/mean": 0.01016717217862606, "step": 814 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00013087536353850737, "clip_ratio/low_min": 0.00013087536353850737, "clip_ratio/region_mean": 0.00013087536353850737, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 656.0, "completions/mean_length": 473.28125, "completions/mean_terminated_length": 442.7930908203125, "completions/min_length": 280.0, "completions/min_terminated_length": 280.0, "entropy": 0.3047125283628702, "epoch": 0.43629550321199145, "frac_reward_zero_std": 0.0, "grad_norm": 0.015324367210268974, "learning_rate": 1e-05, "loss": 0.0015, "num_tokens": 16312779.0, "reward": 0.65625, "reward_std": 0.4397946000099182, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.4569432735443115, "sampling/importance_sampling_ratio/mean": 1.0001065731048584, "sampling/importance_sampling_ratio/min": 0.5850861668586731, "sampling/sampling_logp_difference/max": 0.5359960794448853, "sampling/sampling_logp_difference/mean": 0.010756208561360836, "step": 815 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 9.117432637140155e-05, "clip_ratio/low_min": 9.117432637140155e-05, "clip_ratio/region_mean": 9.117432637140155e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 496.0, "completions/max_terminated_length": 496.0, "completions/mean_length": 350.40625, "completions/mean_terminated_length": 350.40625, "completions/min_length": 239.0, "completions/min_terminated_length": 239.0, "entropy": 0.3057396877557039, "epoch": 0.43683083511777304, "frac_reward_zero_std": 0.5, "grad_norm": 0.015559912659227848, "learning_rate": 1e-05, "loss": 0.03, "num_tokens": 16327880.0, "reward": 0.875, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 1.5554462671279907, "sampling/importance_sampling_ratio/mean": 1.000300407409668, "sampling/importance_sampling_ratio/min": 0.6858586072921753, "sampling/sampling_logp_difference/max": 0.44176244735717773, "sampling/sampling_logp_difference/mean": 0.011645578779280186, "step": 816 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 7.866582745919004e-05, "clip_ratio/low_min": 7.866582745919004e-05, "clip_ratio/region_mean": 7.866582745919004e-05, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 734.0, "completions/mean_length": 362.53125, "completions/mean_terminated_length": 335.5000305175781, "completions/min_length": 122.0, "completions/min_terminated_length": 122.0, "entropy": 0.32446580566465855, "epoch": 0.43736616702355463, "frac_reward_zero_std": 0.5, "grad_norm": 0.028071025386452675, "learning_rate": 1e-05, "loss": 0.058, "num_tokens": 16343377.0, "reward": 0.90625, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.6221219301223755, "sampling/importance_sampling_ratio/mean": 1.0005759000778198, "sampling/importance_sampling_ratio/min": 0.5040833353996277, "sampling/sampling_logp_difference/max": 0.6850136518478394, "sampling/sampling_logp_difference/mean": 0.011989017017185688, "step": 817 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00027901858265977353, "clip_ratio/low_min": 0.00027901858265977353, "clip_ratio/region_mean": 0.00027901858265977353, "completions/clipped_ratio": 0.0, "completions/max_length": 664.0, "completions/max_terminated_length": 664.0, "completions/mean_length": 462.9375, "completions/mean_terminated_length": 462.9375, "completions/min_length": 327.0, "completions/min_terminated_length": 327.0, "entropy": 0.3302745521068573, "epoch": 0.43790149892933616, "frac_reward_zero_std": 0.25, "grad_norm": 0.010575388558208942, "learning_rate": 1e-05, "loss": -0.0575, "num_tokens": 16362679.0, "reward": 0.65625, "reward_std": 0.3198433816432953, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.608583927154541, "sampling/importance_sampling_ratio/mean": 1.000242829322815, "sampling/importance_sampling_ratio/min": 0.6750907897949219, "sampling/sampling_logp_difference/max": 0.4753541946411133, "sampling/sampling_logp_difference/mean": 0.011667050421237946, "step": 818 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.71037016925402e-05, "clip_ratio/low_min": 5.71037016925402e-05, "clip_ratio/region_mean": 5.71037016925402e-05, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 764.0, "completions/mean_length": 489.4375, "completions/mean_terminated_length": 470.86669921875, "completions/min_length": 272.0, "completions/min_terminated_length": 272.0, "entropy": 0.3521044850349426, "epoch": 0.43843683083511775, "frac_reward_zero_std": 0.5, "grad_norm": 0.012330119498074055, "learning_rate": 1e-05, "loss": 0.0152, "num_tokens": 16382245.0, "reward": 0.75, "reward_std": 0.2587745785713196, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.6827696561813354, "sampling/importance_sampling_ratio/mean": 0.9994314908981323, "sampling/importance_sampling_ratio/min": 0.6177808046340942, "sampling/sampling_logp_difference/max": 0.5204410552978516, "sampling/sampling_logp_difference/mean": 0.012086994014680386, "step": 819 }, { "clip_ratio/high_max": 8.164597966242582e-05, "clip_ratio/high_mean": 8.164597966242582e-05, "clip_ratio/low_mean": 0.00023453583708032966, "clip_ratio/low_min": 0.00023453583708032966, "clip_ratio/region_mean": 0.00031618181674275547, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 708.0, "completions/mean_length": 459.09375, "completions/mean_terminated_length": 427.137939453125, "completions/min_length": 212.0, "completions/min_terminated_length": 212.0, "entropy": 0.3209436070173979, "epoch": 0.43897216274089934, "frac_reward_zero_std": 0.0, "grad_norm": 0.018147796392440796, "learning_rate": 1e-05, "loss": 0.114, "num_tokens": 16400768.0, "reward": 0.65625, "reward_std": 0.4628904461860657, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.606744647026062, "sampling/importance_sampling_ratio/mean": 1.0000828504562378, "sampling/importance_sampling_ratio/min": 0.5109875202178955, "sampling/sampling_logp_difference/max": 0.671410083770752, "sampling/sampling_logp_difference/mean": 0.011897324584424496, "step": 820 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0001959901928785257, "clip_ratio/low_min": 0.0001959901928785257, "clip_ratio/region_mean": 0.0001959901928785257, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 737.0, "completions/mean_length": 576.0, "completions/mean_terminated_length": 522.239990234375, "completions/min_length": 203.0, "completions/min_terminated_length": 203.0, "entropy": 0.3952139653265476, "epoch": 0.43950749464668093, "frac_reward_zero_std": 0.0, "grad_norm": 0.01596885733306408, "learning_rate": 1e-05, "loss": 0.038, "num_tokens": 16424760.0, "reward": 0.46875, "reward_std": 0.47137707471847534, "rewards/accuracy_reward/mean": 0.46875, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.8614606857299805, "sampling/importance_sampling_ratio/mean": 1.0003068447113037, "sampling/importance_sampling_ratio/min": 0.613844633102417, "sampling/sampling_logp_difference/max": 0.621361494064331, "sampling/sampling_logp_difference/mean": 0.014272034168243408, "step": 821 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 619.0, "completions/mean_length": 380.3125, "completions/mean_terminated_length": 367.8064270019531, "completions/min_length": 158.0, "completions/min_terminated_length": 158.0, "entropy": 0.27511613443493843, "epoch": 0.4400428265524625, "frac_reward_zero_std": 0.75, "grad_norm": 0.023855295032262802, "learning_rate": 1e-05, "loss": 0.0241, "num_tokens": 16440602.0, "reward": 0.90625, "reward_std": 0.1293872892856598, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.6222468614578247, "sampling/importance_sampling_ratio/mean": 1.000023365020752, "sampling/importance_sampling_ratio/min": 0.40764185786247253, "sampling/sampling_logp_difference/max": 0.8973662853240967, "sampling/sampling_logp_difference/mean": 0.010873417370021343, "step": 822 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00024445212693535723, "clip_ratio/low_min": 0.00024445212693535723, "clip_ratio/region_mean": 0.00024445212693535723, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 682.0, "completions/mean_length": 501.6875, "completions/mean_terminated_length": 452.370361328125, "completions/min_length": 233.0, "completions/min_terminated_length": 233.0, "entropy": 0.3429740257561207, "epoch": 0.4405781584582441, "frac_reward_zero_std": 0.25, "grad_norm": 0.0056623490527272224, "learning_rate": 1e-05, "loss": 0.0578, "num_tokens": 16460352.0, "reward": 0.5625, "reward_std": 0.292504221200943, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.5929806232452393, "sampling/importance_sampling_ratio/mean": 0.999823808670044, "sampling/importance_sampling_ratio/min": 0.554802656173706, "sampling/sampling_logp_difference/max": 0.5891427993774414, "sampling/sampling_logp_difference/mean": 0.012048034928739071, "step": 823 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.259389192564413e-05, "clip_ratio/low_min": 6.259389192564413e-05, "clip_ratio/region_mean": 6.259389192564413e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 605.0, "completions/max_terminated_length": 605.0, "completions/mean_length": 408.71875, "completions/mean_terminated_length": 408.71875, "completions/min_length": 120.0, "completions/min_terminated_length": 120.0, "entropy": 0.2218430433422327, "epoch": 0.4411134903640257, "frac_reward_zero_std": 0.75, "grad_norm": 0.008542133495211601, "learning_rate": 1e-05, "loss": -0.0061, "num_tokens": 16476727.0, "reward": 0.875, "reward_std": 0.13363061845302582, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 1.6581839323043823, "sampling/importance_sampling_ratio/mean": 0.9997780323028564, "sampling/importance_sampling_ratio/min": 0.5478554964065552, "sampling/sampling_logp_difference/max": 0.6017436981201172, "sampling/sampling_logp_difference/mean": 0.009161354042589664, "step": 824 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0001178507627628278, "clip_ratio/low_min": 0.0001178507627628278, "clip_ratio/region_mean": 0.0001178507627628278, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 738.0, "completions/mean_length": 515.1875, "completions/mean_terminated_length": 468.370361328125, "completions/min_length": 184.0, "completions/min_terminated_length": 184.0, "entropy": 0.3462813012301922, "epoch": 0.4416488222698073, "frac_reward_zero_std": 0.25, "grad_norm": 0.021326366811990738, "learning_rate": 1e-05, "loss": 0.0152, "num_tokens": 16497197.0, "reward": 0.6875, "reward_std": 0.3924051821231842, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.4823440313339233, "sampling/importance_sampling_ratio/mean": 1.000078797340393, "sampling/importance_sampling_ratio/min": 0.6797965168952942, "sampling/sampling_logp_difference/max": 0.3936246633529663, "sampling/sampling_logp_difference/mean": 0.012647273950278759, "step": 825 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00017289073730353266, "clip_ratio/low_min": 0.00017289073730353266, "clip_ratio/region_mean": 0.00017289073730353266, "completions/clipped_ratio": 0.0, "completions/max_length": 595.0, "completions/max_terminated_length": 595.0, "completions/mean_length": 362.25, "completions/mean_terminated_length": 362.25, "completions/min_length": 141.0, "completions/min_terminated_length": 141.0, "entropy": 0.2770177647471428, "epoch": 0.4421841541755889, "frac_reward_zero_std": 0.5, "grad_norm": 0.018211321905255318, "learning_rate": 1e-05, "loss": 0.0242, "num_tokens": 16512157.0, "reward": 0.75, "reward_std": 0.2314550280570984, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.622521162033081, "sampling/importance_sampling_ratio/mean": 1.0004990100860596, "sampling/importance_sampling_ratio/min": 0.46850597858428955, "sampling/sampling_logp_difference/max": 0.7582063674926758, "sampling/sampling_logp_difference/mean": 0.011050463654100895, "step": 826 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 7.649938925169408e-05, "clip_ratio/low_min": 7.649938925169408e-05, "clip_ratio/region_mean": 7.649938925169408e-05, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 638.0, "completions/mean_length": 422.59375, "completions/mean_terminated_length": 386.862060546875, "completions/min_length": 193.0, "completions/min_terminated_length": 193.0, "entropy": 0.48511166125535965, "epoch": 0.44271948608137046, "frac_reward_zero_std": 0.25, "grad_norm": 0.019551947712898254, "learning_rate": 1e-05, "loss": 0.0532, "num_tokens": 16529360.0, "reward": 0.625, "reward_std": 0.3104073107242584, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.5627206563949585, "sampling/importance_sampling_ratio/mean": 1.0000622272491455, "sampling/importance_sampling_ratio/min": 0.6936726570129395, "sampling/sampling_logp_difference/max": 0.4464282989501953, "sampling/sampling_logp_difference/mean": 0.014928963035345078, "step": 827 }, { "clip_ratio/high_max": 6.917543214512989e-05, "clip_ratio/high_mean": 6.917543214512989e-05, "clip_ratio/low_mean": 9.742790280142799e-05, "clip_ratio/low_min": 9.742790280142799e-05, "clip_ratio/region_mean": 0.00016660333494655788, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 639.0, "completions/mean_length": 380.5, "completions/mean_terminated_length": 354.66668701171875, "completions/min_length": 192.0, "completions/min_terminated_length": 192.0, "entropy": 0.348480436950922, "epoch": 0.44325481798715205, "frac_reward_zero_std": 0.25, "grad_norm": 0.021738076582551003, "learning_rate": 1e-05, "loss": -0.0607, "num_tokens": 16545160.0, "reward": 0.71875, "reward_std": 0.3471629321575165, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.485531210899353, "sampling/importance_sampling_ratio/mean": 1.0001122951507568, "sampling/importance_sampling_ratio/min": 0.6237407922744751, "sampling/sampling_logp_difference/max": 0.47202038764953613, "sampling/sampling_logp_difference/mean": 0.012043345719575882, "step": 828 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 732.0, "completions/mean_length": 491.90625, "completions/mean_terminated_length": 463.3448181152344, "completions/min_length": 247.0, "completions/min_terminated_length": 247.0, "entropy": 0.25553832203149796, "epoch": 0.44379014989293364, "frac_reward_zero_std": 0.25, "grad_norm": 0.004612386226654053, "learning_rate": 1e-05, "loss": 0.0455, "num_tokens": 16564685.0, "reward": 0.65625, "reward_std": 0.2651650309562683, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.000020146369934, "sampling/importance_sampling_ratio/min": 0.6000391840934753, "sampling/sampling_logp_difference/max": 0.7492682933807373, "sampling/sampling_logp_difference/mean": 0.010166849941015244, "step": 829 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.1295010305475444e-05, "clip_ratio/low_min": 4.1295010305475444e-05, "clip_ratio/region_mean": 4.1295010305475444e-05, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 724.0, "completions/mean_length": 505.21875, "completions/mean_terminated_length": 431.6399841308594, "completions/min_length": 185.0, "completions/min_terminated_length": 185.0, "entropy": 0.48429663851857185, "epoch": 0.4443254817987152, "frac_reward_zero_std": 0.0, "grad_norm": 0.029873324558138847, "learning_rate": 1e-05, "loss": 0.0091, "num_tokens": 16585540.0, "reward": 0.59375, "reward_std": 0.4628904461860657, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.414668083190918, "sampling/importance_sampling_ratio/mean": 0.9995958209037781, "sampling/importance_sampling_ratio/min": 0.5749284625053406, "sampling/sampling_logp_difference/max": 0.5535097122192383, "sampling/sampling_logp_difference/mean": 0.015926599502563477, "step": 830 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.466632476076484e-05, "clip_ratio/low_min": 6.466632476076484e-05, "clip_ratio/region_mean": 6.466632476076484e-05, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 719.0, "completions/mean_length": 503.65625, "completions/mean_terminated_length": 454.7037048339844, "completions/min_length": 201.0, "completions/min_terminated_length": 201.0, "entropy": 0.3221813887357712, "epoch": 0.44486081370449676, "frac_reward_zero_std": 0.25, "grad_norm": 0.010256975889205933, "learning_rate": 1e-05, "loss": 0.0358, "num_tokens": 16605745.0, "reward": 0.71875, "reward_std": 0.3471629321575165, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.736228585243225, "sampling/importance_sampling_ratio/mean": 1.0005266666412354, "sampling/importance_sampling_ratio/min": 0.6991448402404785, "sampling/sampling_logp_difference/max": 0.5517152547836304, "sampling/sampling_logp_difference/mean": 0.011762522161006927, "step": 831 }, { "clip_ratio/high_max": 0.00014737022866029292, "clip_ratio/high_mean": 0.00014737022866029292, "clip_ratio/low_mean": 0.0002193370382883586, "clip_ratio/low_min": 0.0002193370382883586, "clip_ratio/region_mean": 0.0003667072669486515, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 731.0, "completions/mean_length": 431.375, "completions/mean_terminated_length": 396.5517272949219, "completions/min_length": 155.0, "completions/min_terminated_length": 155.0, "entropy": 0.3429530579596758, "epoch": 0.44539614561027835, "frac_reward_zero_std": 0.25, "grad_norm": 0.011722094379365444, "learning_rate": 1e-05, "loss": 0.0277, "num_tokens": 16622845.0, "reward": 0.78125, "reward_std": 0.3608423173427582, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.451257348060608, "sampling/importance_sampling_ratio/mean": 0.9998858571052551, "sampling/importance_sampling_ratio/min": 0.3799667954444885, "sampling/sampling_logp_difference/max": 0.9676713943481445, "sampling/sampling_logp_difference/mean": 0.011598000302910805, "step": 832 }, { "clip_ratio/high_max": 6.811989442212507e-05, "clip_ratio/high_mean": 6.811989442212507e-05, "clip_ratio/low_mean": 6.380806735251099e-05, "clip_ratio/low_min": 6.380806735251099e-05, "clip_ratio/region_mean": 0.00013192796177463606, "completions/clipped_ratio": 0.0, "completions/max_length": 759.0, "completions/max_terminated_length": 759.0, "completions/mean_length": 451.6875, "completions/mean_terminated_length": 451.6875, "completions/min_length": 176.0, "completions/min_terminated_length": 176.0, "entropy": 0.24354325979948044, "epoch": 0.44593147751605994, "frac_reward_zero_std": 0.5, "grad_norm": 0.010895389132201672, "learning_rate": 1e-05, "loss": -0.0495, "num_tokens": 16641283.0, "reward": 0.875, "reward_std": 0.2314550280570984, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 1.2997705936431885, "sampling/importance_sampling_ratio/mean": 0.9999913573265076, "sampling/importance_sampling_ratio/min": 0.308633416891098, "sampling/sampling_logp_difference/max": 1.1756010055541992, "sampling/sampling_logp_difference/mean": 0.00954093225300312, "step": 833 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 7.327080675167963e-05, "clip_ratio/low_min": 7.327080675167963e-05, "clip_ratio/region_mean": 7.327080675167963e-05, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 743.0, "completions/mean_length": 440.09375, "completions/mean_terminated_length": 406.17242431640625, "completions/min_length": 237.0, "completions/min_terminated_length": 237.0, "entropy": 0.4135780967772007, "epoch": 0.4464668094218415, "frac_reward_zero_std": 0.25, "grad_norm": 0.02072235755622387, "learning_rate": 1e-05, "loss": 0.0604, "num_tokens": 16659102.0, "reward": 0.5625, "reward_std": 0.3514062464237213, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.000566840171814, "sampling/importance_sampling_ratio/min": 0.5816425085067749, "sampling/sampling_logp_difference/max": 0.7674446105957031, "sampling/sampling_logp_difference/mean": 0.013563827611505985, "step": 834 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 749.0, "completions/mean_length": 479.90625, "completions/mean_terminated_length": 413.423095703125, "completions/min_length": 242.0, "completions/min_terminated_length": 242.0, "entropy": 0.2676452100276947, "epoch": 0.4470021413276231, "frac_reward_zero_std": 0.25, "grad_norm": 0.019833054393529892, "learning_rate": 1e-05, "loss": 0.026, "num_tokens": 16678307.0, "reward": 0.40625, "reward_std": 0.378745436668396, "rewards/accuracy_reward/mean": 0.40625, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0006767511367798, "sampling/importance_sampling_ratio/min": 0.6516871452331543, "sampling/sampling_logp_difference/max": 0.9411327838897705, "sampling/sampling_logp_difference/mean": 0.009629964828491211, "step": 835 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 530.0, "completions/max_terminated_length": 530.0, "completions/mean_length": 354.15625, "completions/mean_terminated_length": 354.15625, "completions/min_length": 208.0, "completions/min_terminated_length": 208.0, "entropy": 0.22570085898041725, "epoch": 0.4475374732334047, "frac_reward_zero_std": 0.75, "grad_norm": 0.03132038190960884, "learning_rate": 1e-05, "loss": 0.0139, "num_tokens": 16692928.0, "reward": 0.96875, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.96875, "rewards/accuracy_reward/std": 0.1767766922712326, "sampling/importance_sampling_ratio/max": 1.7164597511291504, "sampling/importance_sampling_ratio/mean": 1.0002472400665283, "sampling/importance_sampling_ratio/min": 0.4757560193538666, "sampling/sampling_logp_difference/max": 0.7428500652313232, "sampling/sampling_logp_difference/mean": 0.00921986810863018, "step": 836 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 716.0, "completions/mean_length": 530.15625, "completions/mean_terminated_length": 463.55999755859375, "completions/min_length": 233.0, "completions/min_terminated_length": 233.0, "entropy": 0.3691186085343361, "epoch": 0.4480728051391863, "frac_reward_zero_std": 0.25, "grad_norm": 0.017191870138049126, "learning_rate": 1e-05, "loss": 0.0078, "num_tokens": 16713805.0, "reward": 0.5, "reward_std": 0.3650856614112854, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5080004930496216, "sampling/importance_sampling_ratio/max": 1.7796034812927246, "sampling/importance_sampling_ratio/mean": 1.000518560409546, "sampling/importance_sampling_ratio/min": 0.7095998525619507, "sampling/sampling_logp_difference/max": 0.5763905048370361, "sampling/sampling_logp_difference/mean": 0.012961843982338905, "step": 837 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.380806735251099e-05, "clip_ratio/low_min": 6.380806735251099e-05, "clip_ratio/region_mean": 6.380806735251099e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 724.0, "completions/max_terminated_length": 724.0, "completions/mean_length": 519.8125, "completions/mean_terminated_length": 519.8125, "completions/min_length": 295.0, "completions/min_terminated_length": 295.0, "entropy": 0.26343732699751854, "epoch": 0.4486081370449679, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0167, "num_tokens": 16734303.0, "reward": 0.90625, "reward_std": 0.1293872892856598, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.9722262620925903, "sampling/importance_sampling_ratio/mean": 0.9998405575752258, "sampling/importance_sampling_ratio/min": 0.5341039299964905, "sampling/sampling_logp_difference/max": 0.6791629791259766, "sampling/sampling_logp_difference/mean": 0.011166399344801903, "step": 838 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00021222411305643618, "clip_ratio/low_min": 0.00021222411305643618, "clip_ratio/region_mean": 0.00021222411305643618, "completions/clipped_ratio": 0.0, "completions/max_length": 622.0, "completions/max_terminated_length": 622.0, "completions/mean_length": 388.5625, "completions/mean_terminated_length": 388.5625, "completions/min_length": 92.0, "completions/min_terminated_length": 92.0, "entropy": 0.34673846885561943, "epoch": 0.44914346895074947, "frac_reward_zero_std": 0.25, "grad_norm": 0.02089758776128292, "learning_rate": 1e-05, "loss": 0.0058, "num_tokens": 16750809.0, "reward": 0.8125, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0003050565719604, "sampling/importance_sampling_ratio/min": 0.33226293325424194, "sampling/sampling_logp_difference/max": 1.1018285751342773, "sampling/sampling_logp_difference/mean": 0.012727145105600357, "step": 839 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00020833333837799728, "clip_ratio/low_min": 0.00020833333837799728, "clip_ratio/region_mean": 0.00020833333837799728, "completions/clipped_ratio": 0.0, "completions/max_length": 716.0, "completions/max_terminated_length": 716.0, "completions/mean_length": 465.59375, "completions/mean_terminated_length": 465.59375, "completions/min_length": 134.0, "completions/min_terminated_length": 134.0, "entropy": 0.21557242050766945, "epoch": 0.44967880085653106, "frac_reward_zero_std": 0.75, "grad_norm": 0.01053936593234539, "learning_rate": 1e-05, "loss": 0.0065, "num_tokens": 16769740.0, "reward": 0.875, "reward_std": 0.13363061845302582, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 1.4637336730957031, "sampling/importance_sampling_ratio/mean": 1.000529408454895, "sampling/importance_sampling_ratio/min": 0.6094999313354492, "sampling/sampling_logp_difference/max": 0.4951164722442627, "sampling/sampling_logp_difference/mean": 0.00881272740662098, "step": 840 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.3125, "completions/max_length": 768.0, "completions/max_terminated_length": 746.0, "completions/mean_length": 579.09375, "completions/mean_terminated_length": 493.227294921875, "completions/min_length": 254.0, "completions/min_terminated_length": 254.0, "entropy": 0.3651367239654064, "epoch": 0.45021413276231265, "frac_reward_zero_std": 0.5, "grad_norm": 0.002016344340518117, "learning_rate": 1e-05, "loss": 0.0184, "num_tokens": 16792871.0, "reward": 0.4375, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.4375, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.6037328243255615, "sampling/importance_sampling_ratio/mean": 1.0000085830688477, "sampling/importance_sampling_ratio/min": 0.6860070824623108, "sampling/sampling_logp_difference/max": 0.4723339080810547, "sampling/sampling_logp_difference/mean": 0.012109356932342052, "step": 841 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 715.0, "completions/mean_length": 440.125, "completions/mean_terminated_length": 429.5483703613281, "completions/min_length": 156.0, "completions/min_terminated_length": 156.0, "entropy": 0.31424909457564354, "epoch": 0.45074946466809424, "frac_reward_zero_std": 0.25, "grad_norm": 0.02117704227566719, "learning_rate": 1e-05, "loss": 0.013, "num_tokens": 16810555.0, "reward": 0.75, "reward_std": 0.3514062464237213, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.5482828617095947, "sampling/importance_sampling_ratio/mean": 0.9997996091842651, "sampling/importance_sampling_ratio/min": 0.5919291973114014, "sampling/sampling_logp_difference/max": 0.5243682861328125, "sampling/sampling_logp_difference/mean": 0.011482161469757557, "step": 842 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 630.0, "completions/max_terminated_length": 630.0, "completions/mean_length": 384.0, "completions/mean_terminated_length": 384.0, "completions/min_length": 224.0, "completions/min_terminated_length": 224.0, "entropy": 0.2951059080660343, "epoch": 0.4512847965738758, "frac_reward_zero_std": 0.75, "grad_norm": 0.014482242986559868, "learning_rate": 1e-05, "loss": 0.007, "num_tokens": 16826579.0, "reward": 0.84375, "reward_std": 0.1293872892856598, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.599572777748108, "sampling/importance_sampling_ratio/mean": 0.9997570514678955, "sampling/importance_sampling_ratio/min": 0.7156587839126587, "sampling/sampling_logp_difference/max": 0.46973657608032227, "sampling/sampling_logp_difference/mean": 0.010345741175115108, "step": 843 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 7.431628910126165e-05, "clip_ratio/low_min": 7.431628910126165e-05, "clip_ratio/region_mean": 7.431628910126165e-05, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 573.0, "completions/mean_length": 405.4375, "completions/mean_terminated_length": 367.9310302734375, "completions/min_length": 187.0, "completions/min_terminated_length": 187.0, "entropy": 0.45297108963131905, "epoch": 0.4518201284796574, "frac_reward_zero_std": 0.75, "grad_norm": 0.008480343036353588, "learning_rate": 1e-05, "loss": -0.013, "num_tokens": 16843049.0, "reward": 0.65625, "reward_std": 0.1293872892856598, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.5063261985778809, "sampling/importance_sampling_ratio/mean": 0.9999780058860779, "sampling/importance_sampling_ratio/min": 0.5270180106163025, "sampling/sampling_logp_difference/max": 0.6405205726623535, "sampling/sampling_logp_difference/mean": 0.015170668251812458, "step": 844 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 679.0, "completions/mean_length": 519.03125, "completions/mean_terminated_length": 483.46429443359375, "completions/min_length": 230.0, "completions/min_terminated_length": 230.0, "entropy": 0.3445496968924999, "epoch": 0.45235546038543895, "frac_reward_zero_std": 0.5, "grad_norm": 0.004610029514878988, "learning_rate": 1e-05, "loss": -0.008, "num_tokens": 16863986.0, "reward": 0.75, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.4119755029678345, "sampling/importance_sampling_ratio/mean": 0.9999881386756897, "sampling/importance_sampling_ratio/min": 0.6070384383201599, "sampling/sampling_logp_difference/max": 0.4991631507873535, "sampling/sampling_logp_difference/mean": 0.012054568156599998, "step": 845 }, { "clip_ratio/high_max": 7.720815483480692e-05, "clip_ratio/high_mean": 7.720815483480692e-05, "clip_ratio/low_mean": 0.00013008988025831059, "clip_ratio/low_min": 0.00013008988025831059, "clip_ratio/region_mean": 0.0002072980350931175, "completions/clipped_ratio": 0.0, "completions/max_length": 698.0, "completions/max_terminated_length": 698.0, "completions/mean_length": 464.65625, "completions/mean_terminated_length": 464.65625, "completions/min_length": 262.0, "completions/min_terminated_length": 262.0, "entropy": 0.2506127431988716, "epoch": 0.45289079229122053, "frac_reward_zero_std": 0.5, "grad_norm": 0.011979878880083561, "learning_rate": 1e-05, "loss": -0.0336, "num_tokens": 16882423.0, "reward": 0.6875, "reward_std": 0.249358132481575, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.5981649160385132, "sampling/importance_sampling_ratio/mean": 1.0001941919326782, "sampling/importance_sampling_ratio/min": 0.6572299003601074, "sampling/sampling_logp_difference/max": 0.4688560962677002, "sampling/sampling_logp_difference/mean": 0.01008039154112339, "step": 846 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.830601341789588e-05, "clip_ratio/low_min": 6.830601341789588e-05, "clip_ratio/region_mean": 6.830601341789588e-05, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 681.0, "completions/mean_length": 472.125, "completions/mean_terminated_length": 462.58062744140625, "completions/min_length": 275.0, "completions/min_terminated_length": 275.0, "entropy": 0.3238905295729637, "epoch": 0.4534261241970021, "frac_reward_zero_std": 0.5, "grad_norm": 0.0057082343846559525, "learning_rate": 1e-05, "loss": 0.0592, "num_tokens": 16901515.0, "reward": 0.75, "reward_std": 0.2314550280570984, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.5296075344085693, "sampling/importance_sampling_ratio/mean": 0.9998763203620911, "sampling/importance_sampling_ratio/min": 0.7057211995124817, "sampling/sampling_logp_difference/max": 0.42501115798950195, "sampling/sampling_logp_difference/mean": 0.011838079430162907, "step": 847 }, { "clip_ratio/high_max": 4.5620436139870435e-05, "clip_ratio/high_mean": 4.5620436139870435e-05, "clip_ratio/low_mean": 0.00015090232773218304, "clip_ratio/low_min": 0.00015090232773218304, "clip_ratio/region_mean": 0.00019652276387205347, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 728.0, "completions/mean_length": 601.40625, "completions/mean_terminated_length": 562.9615478515625, "completions/min_length": 369.0, "completions/min_terminated_length": 369.0, "entropy": 0.32016063295304775, "epoch": 0.4539614561027837, "frac_reward_zero_std": 0.5, "grad_norm": 0.0071393828839063644, "learning_rate": 1e-05, "loss": -0.0026, "num_tokens": 16925272.0, "reward": 0.71875, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.461082100868225, "sampling/importance_sampling_ratio/mean": 0.9995551109313965, "sampling/importance_sampling_ratio/min": 0.680505633354187, "sampling/sampling_logp_difference/max": 0.3849191665649414, "sampling/sampling_logp_difference/mean": 0.011756669729948044, "step": 848 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.641870277235284e-05, "clip_ratio/low_min": 6.641870277235284e-05, "clip_ratio/region_mean": 6.641870277235284e-05, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 669.0, "completions/mean_length": 453.71875, "completions/mean_terminated_length": 443.58062744140625, "completions/min_length": 241.0, "completions/min_terminated_length": 241.0, "entropy": 0.29842460341751575, "epoch": 0.4544967880085653, "frac_reward_zero_std": 0.0, "grad_norm": 0.01773042045533657, "learning_rate": 1e-05, "loss": 0.0629, "num_tokens": 16943479.0, "reward": 0.78125, "reward_std": 0.4355708956718445, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.3509780168533325, "sampling/importance_sampling_ratio/mean": 0.9998967051506042, "sampling/importance_sampling_ratio/min": 0.5830528736114502, "sampling/sampling_logp_difference/max": 0.5394773483276367, "sampling/sampling_logp_difference/mean": 0.010512765496969223, "step": 849 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 746.0, "completions/mean_length": 468.65625, "completions/mean_terminated_length": 459.0, "completions/min_length": 242.0, "completions/min_terminated_length": 242.0, "entropy": 0.19418418407440186, "epoch": 0.4550321199143469, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0081, "num_tokens": 16962076.0, "reward": 0.84375, "reward_std": 0.1293872892856598, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.3339976072311401, "sampling/importance_sampling_ratio/mean": 0.9999716877937317, "sampling/importance_sampling_ratio/min": 0.6263445615768433, "sampling/sampling_logp_difference/max": 0.46785473823547363, "sampling/sampling_logp_difference/mean": 0.00778750516474247, "step": 850 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 746.0, "completions/mean_length": 510.375, "completions/mean_terminated_length": 493.20001220703125, "completions/min_length": 251.0, "completions/min_terminated_length": 251.0, "entropy": 0.3192160576581955, "epoch": 0.4555674518201285, "frac_reward_zero_std": 0.0, "grad_norm": 0.041246507316827774, "learning_rate": 1e-05, "loss": 0.0158, "num_tokens": 16982248.0, "reward": 0.75, "reward_std": 0.4492306709289551, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.451161503791809, "sampling/importance_sampling_ratio/mean": 0.9997957348823547, "sampling/importance_sampling_ratio/min": 0.311160147190094, "sampling/sampling_logp_difference/max": 1.167447566986084, "sampling/sampling_logp_difference/mean": 0.011297615244984627, "step": 851 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00018371613987255841, "clip_ratio/low_min": 0.00018371613987255841, "clip_ratio/region_mean": 0.00018371613987255841, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 744.0, "completions/mean_length": 465.21875, "completions/mean_terminated_length": 421.96429443359375, "completions/min_length": 241.0, "completions/min_terminated_length": 241.0, "entropy": 0.29270661622285843, "epoch": 0.45610278372591007, "frac_reward_zero_std": 0.5, "grad_norm": 0.017661161720752716, "learning_rate": 1e-05, "loss": 0.0628, "num_tokens": 17000943.0, "reward": 0.5, "reward_std": 0.2587745785713196, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5080004930496216, "sampling/importance_sampling_ratio/max": 1.434083104133606, "sampling/importance_sampling_ratio/mean": 0.999801516532898, "sampling/importance_sampling_ratio/min": 0.6706390976905823, "sampling/sampling_logp_difference/max": 0.3995242118835449, "sampling/sampling_logp_difference/mean": 0.010368138551712036, "step": 852 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.404236799222417e-05, "clip_ratio/low_min": 5.404236799222417e-05, "clip_ratio/region_mean": 5.404236799222417e-05, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 736.0, "completions/mean_length": 436.625, "completions/mean_terminated_length": 389.2857360839844, "completions/min_length": 243.0, "completions/min_terminated_length": 243.0, "entropy": 0.34419820457696915, "epoch": 0.45663811563169165, "frac_reward_zero_std": 0.5, "grad_norm": 0.0037945820949971676, "learning_rate": 1e-05, "loss": 0.0854, "num_tokens": 17018275.0, "reward": 0.65625, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.6534979343414307, "sampling/importance_sampling_ratio/mean": 1.0004469156265259, "sampling/importance_sampling_ratio/min": 0.514949381351471, "sampling/sampling_logp_difference/max": 0.6636867523193359, "sampling/sampling_logp_difference/mean": 0.011727089993655682, "step": 853 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 687.0, "completions/mean_length": 428.0625, "completions/mean_terminated_length": 417.0967712402344, "completions/min_length": 203.0, "completions/min_terminated_length": 203.0, "entropy": 0.3591027185320854, "epoch": 0.45717344753747324, "frac_reward_zero_std": 0.25, "grad_norm": 0.0074991523288190365, "learning_rate": 1e-05, "loss": 0.0344, "num_tokens": 17035901.0, "reward": 0.6875, "reward_std": 0.292504221200943, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.604923129081726, "sampling/importance_sampling_ratio/mean": 0.9997456073760986, "sampling/importance_sampling_ratio/min": 0.635753333568573, "sampling/sampling_logp_difference/max": 0.47307586669921875, "sampling/sampling_logp_difference/mean": 0.012660740874707699, "step": 854 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 758.0, "completions/mean_length": 458.46875, "completions/mean_terminated_length": 401.1481628417969, "completions/min_length": 250.0, "completions/min_terminated_length": 250.0, "entropy": 0.3138784375041723, "epoch": 0.45770877944325483, "frac_reward_zero_std": 0.5, "grad_norm": 0.005024927668273449, "learning_rate": 1e-05, "loss": 0.0949, "num_tokens": 17054196.0, "reward": 0.8125, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.7859833240509033, "sampling/importance_sampling_ratio/mean": 0.9999326467514038, "sampling/importance_sampling_ratio/min": 0.6756333112716675, "sampling/sampling_logp_difference/max": 0.5799691677093506, "sampling/sampling_logp_difference/mean": 0.011334572918713093, "step": 855 }, { "clip_ratio/high_max": 6.998880417086184e-05, "clip_ratio/high_mean": 6.998880417086184e-05, "clip_ratio/low_mean": 7.817385630914941e-05, "clip_ratio/low_min": 7.817385630914941e-05, "clip_ratio/region_mean": 0.00014816266048001125, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 745.0, "completions/mean_length": 460.0, "completions/mean_terminated_length": 450.06451416015625, "completions/min_length": 291.0, "completions/min_terminated_length": 291.0, "entropy": 0.31339183636009693, "epoch": 0.4582441113490364, "frac_reward_zero_std": 0.5, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0246, "num_tokens": 17072284.0, "reward": 0.90625, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.5173048973083496, "sampling/importance_sampling_ratio/mean": 0.9996866583824158, "sampling/importance_sampling_ratio/min": 0.716724157333374, "sampling/sampling_logp_difference/max": 0.41693568229675293, "sampling/sampling_logp_difference/mean": 0.010737800039350986, "step": 856 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0002016609869315289, "clip_ratio/low_min": 0.0002016609869315289, "clip_ratio/region_mean": 0.0002016609869315289, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 725.0, "completions/mean_length": 431.15625, "completions/mean_terminated_length": 396.3103332519531, "completions/min_length": 226.0, "completions/min_terminated_length": 226.0, "entropy": 0.30677182972431183, "epoch": 0.458779443254818, "frac_reward_zero_std": 0.5, "grad_norm": 0.02329942211508751, "learning_rate": 1e-05, "loss": 0.0093, "num_tokens": 17089393.0, "reward": 0.75, "reward_std": 0.2587745785713196, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.6312874555587769, "sampling/importance_sampling_ratio/mean": 1.000083088874817, "sampling/importance_sampling_ratio/min": 0.655465841293335, "sampling/sampling_logp_difference/max": 0.48936963081359863, "sampling/sampling_logp_difference/mean": 0.010949400253593922, "step": 857 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.72868921153713e-05, "clip_ratio/low_min": 5.72868921153713e-05, "clip_ratio/region_mean": 5.72868921153713e-05, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 728.0, "completions/mean_length": 460.71875, "completions/mean_terminated_length": 440.2333679199219, "completions/min_length": 242.0, "completions/min_terminated_length": 242.0, "entropy": 0.28196910955011845, "epoch": 0.4593147751605996, "frac_reward_zero_std": 0.25, "grad_norm": 0.006725633516907692, "learning_rate": 1e-05, "loss": 0.03, "num_tokens": 17108288.0, "reward": 0.75, "reward_std": 0.292504221200943, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9996670484542847, "sampling/importance_sampling_ratio/min": 0.513340175151825, "sampling/sampling_logp_difference/max": 0.9619584083557129, "sampling/sampling_logp_difference/mean": 0.011167189106345177, "step": 858 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.521201455849223e-05, "clip_ratio/low_min": 5.521201455849223e-05, "clip_ratio/region_mean": 5.521201455849223e-05, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 713.0, "completions/mean_length": 456.15625, "completions/mean_terminated_length": 423.89654541015625, "completions/min_length": 116.0, "completions/min_terminated_length": 116.0, "entropy": 0.3315916433930397, "epoch": 0.45985010706638113, "frac_reward_zero_std": 0.25, "grad_norm": 0.026618383824825287, "learning_rate": 1e-05, "loss": 0.0142, "num_tokens": 17126133.0, "reward": 0.65625, "reward_std": 0.3061639964580536, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.5123687982559204, "sampling/importance_sampling_ratio/mean": 0.9998215436935425, "sampling/importance_sampling_ratio/min": 0.6895114183425903, "sampling/sampling_logp_difference/max": 0.4136772155761719, "sampling/sampling_logp_difference/mean": 0.011936474591493607, "step": 859 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 718.0, "completions/mean_length": 422.4375, "completions/mean_terminated_length": 386.6896667480469, "completions/min_length": 222.0, "completions/min_terminated_length": 222.0, "entropy": 0.35348470509052277, "epoch": 0.4603854389721627, "frac_reward_zero_std": 0.0, "grad_norm": 0.021620135754346848, "learning_rate": 1e-05, "loss": 0.0112, "num_tokens": 17143011.0, "reward": 0.6875, "reward_std": 0.44403791427612305, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.7950732707977295, "sampling/importance_sampling_ratio/mean": 0.9996082782745361, "sampling/importance_sampling_ratio/min": 0.586424708366394, "sampling/sampling_logp_difference/max": 0.5850458145141602, "sampling/sampling_logp_difference/mean": 0.012666827067732811, "step": 860 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.364562432281673e-05, "clip_ratio/low_min": 6.364562432281673e-05, "clip_ratio/region_mean": 6.364562432281673e-05, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 741.0, "completions/mean_length": 511.96875, "completions/mean_terminated_length": 485.4827575683594, "completions/min_length": 344.0, "completions/min_terminated_length": 344.0, "entropy": 0.22219637222588062, "epoch": 0.4609207708779443, "frac_reward_zero_std": 0.25, "grad_norm": 0.0042509338818490505, "learning_rate": 1e-05, "loss": 0.0104, "num_tokens": 17162690.0, "reward": 0.6875, "reward_std": 0.292504221200943, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.33616042137146, "sampling/importance_sampling_ratio/mean": 1.000166416168213, "sampling/importance_sampling_ratio/min": 0.6979255676269531, "sampling/sampling_logp_difference/max": 0.3596428632736206, "sampling/sampling_logp_difference/mean": 0.007480310276150703, "step": 861 }, { "clip_ratio/high_max": 6.406971078831702e-05, "clip_ratio/high_mean": 6.406971078831702e-05, "clip_ratio/low_mean": 9.430403588339686e-05, "clip_ratio/low_min": 9.430403588339686e-05, "clip_ratio/region_mean": 0.0001583737466717139, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 658.0, "completions/mean_length": 481.96875, "completions/mean_terminated_length": 462.9000244140625, "completions/min_length": 278.0, "completions/min_terminated_length": 278.0, "entropy": 0.24463251046836376, "epoch": 0.4614561027837259, "frac_reward_zero_std": 0.0, "grad_norm": 0.009309791028499603, "learning_rate": 1e-05, "loss": 0.0239, "num_tokens": 17181721.0, "reward": 0.6875, "reward_std": 0.44403791427612305, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.488285779953003, "sampling/importance_sampling_ratio/mean": 1.000335454940796, "sampling/importance_sampling_ratio/min": 0.639365553855896, "sampling/sampling_logp_difference/max": 0.4472789764404297, "sampling/sampling_logp_difference/mean": 0.009572452865540981, "step": 862 }, { "clip_ratio/high_max": 6.42673549009487e-05, "clip_ratio/high_mean": 6.42673549009487e-05, "clip_ratio/low_mean": 5.681818220182322e-05, "clip_ratio/low_min": 5.681818220182322e-05, "clip_ratio/region_mean": 0.00012108553710277192, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 734.0, "completions/mean_length": 522.75, "completions/mean_terminated_length": 466.15386962890625, "completions/min_length": 164.0, "completions/min_terminated_length": 164.0, "entropy": 0.3433074038475752, "epoch": 0.4619914346895075, "frac_reward_zero_std": 0.25, "grad_norm": 0.018548304215073586, "learning_rate": 1e-05, "loss": 0.0291, "num_tokens": 17202217.0, "reward": 0.6875, "reward_std": 0.3745020925998688, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.8536497354507446, "sampling/importance_sampling_ratio/mean": 0.9996280074119568, "sampling/importance_sampling_ratio/min": 0.5494065880775452, "sampling/sampling_logp_difference/max": 0.6171565055847168, "sampling/sampling_logp_difference/mean": 0.012258768081665039, "step": 863 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 9.018759010359645e-05, "clip_ratio/low_min": 9.018759010359645e-05, "clip_ratio/region_mean": 9.018759010359645e-05, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 550.0, "completions/mean_length": 348.75, "completions/mean_terminated_length": 335.2257995605469, "completions/min_length": 208.0, "completions/min_terminated_length": 208.0, "entropy": 0.3397308196872473, "epoch": 0.4625267665952891, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.026, "num_tokens": 17216633.0, "reward": 0.8125, "reward_std": 0.1157275140285492, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.7661750316619873, "sampling/importance_sampling_ratio/mean": 1.0003950595855713, "sampling/importance_sampling_ratio/min": 0.5858866572380066, "sampling/sampling_logp_difference/max": 0.5688161849975586, "sampling/sampling_logp_difference/mean": 0.01183544285595417, "step": 864 }, { "clip_ratio/high_max": 6.262525130296126e-05, "clip_ratio/high_mean": 6.262525130296126e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 6.262525130296126e-05, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 682.0, "completions/mean_length": 481.1875, "completions/mean_terminated_length": 428.0740661621094, "completions/min_length": 114.0, "completions/min_terminated_length": 114.0, "entropy": 0.3127823583781719, "epoch": 0.46306209850107066, "frac_reward_zero_std": 0.0, "grad_norm": 0.009447633288800716, "learning_rate": 1e-05, "loss": 0.0104, "num_tokens": 17236047.0, "reward": 0.59375, "reward_std": 0.3987956643104553, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.560917854309082, "sampling/importance_sampling_ratio/mean": 1.000223159790039, "sampling/importance_sampling_ratio/min": 0.6159913539886475, "sampling/sampling_logp_difference/max": 0.48452234268188477, "sampling/sampling_logp_difference/mean": 0.011035087518393993, "step": 865 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 687.0, "completions/max_terminated_length": 687.0, "completions/mean_length": 407.5, "completions/mean_terminated_length": 407.5, "completions/min_length": 172.0, "completions/min_terminated_length": 172.0, "entropy": 0.23410633765161037, "epoch": 0.46359743040685225, "frac_reward_zero_std": 0.5, "grad_norm": 0.006699251011013985, "learning_rate": 1e-05, "loss": 0.0114, "num_tokens": 17252615.0, "reward": 0.9375, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.9375, "rewards/accuracy_reward/std": 0.24593468010425568, "sampling/importance_sampling_ratio/max": 1.3216911554336548, "sampling/importance_sampling_ratio/mean": 1.0000323057174683, "sampling/importance_sampling_ratio/min": 0.7802764773368835, "sampling/sampling_logp_difference/max": 0.2789120674133301, "sampling/sampling_logp_difference/mean": 0.009173828177154064, "step": 866 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 8.765778329689056e-05, "clip_ratio/low_min": 8.765778329689056e-05, "clip_ratio/region_mean": 8.765778329689056e-05, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 756.0, "completions/mean_length": 521.8125, "completions/mean_terminated_length": 496.3448181152344, "completions/min_length": 205.0, "completions/min_terminated_length": 205.0, "entropy": 0.3517126627266407, "epoch": 0.46413276231263384, "frac_reward_zero_std": 0.75, "grad_norm": 0.01118876039981842, "learning_rate": 1e-05, "loss": 0.0676, "num_tokens": 17273049.0, "reward": 0.90625, "reward_std": 0.1293872892856598, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.4212286472320557, "sampling/importance_sampling_ratio/mean": 0.9997889399528503, "sampling/importance_sampling_ratio/min": 0.6396047472953796, "sampling/sampling_logp_difference/max": 0.44690489768981934, "sampling/sampling_logp_difference/mean": 0.011829306371510029, "step": 867 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 755.0, "completions/mean_length": 431.21875, "completions/mean_terminated_length": 420.3548278808594, "completions/min_length": 180.0, "completions/min_terminated_length": 180.0, "entropy": 0.2854548469185829, "epoch": 0.46466809421841543, "frac_reward_zero_std": 0.75, "grad_norm": 0.020841553807258606, "learning_rate": 1e-05, "loss": 0.0479, "num_tokens": 17290680.0, "reward": 0.6875, "reward_std": 0.1157275140285492, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.4288326501846313, "sampling/importance_sampling_ratio/mean": 0.9994004368782043, "sampling/importance_sampling_ratio/min": 0.7336400747299194, "sampling/sampling_logp_difference/max": 0.3568577766418457, "sampling/sampling_logp_difference/mean": 0.010053771547973156, "step": 868 }, { "clip_ratio/high_max": 5.540779966395348e-05, "clip_ratio/high_mean": 5.540779966395348e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 5.540779966395348e-05, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 745.0, "completions/mean_length": 459.375, "completions/mean_terminated_length": 438.8000183105469, "completions/min_length": 232.0, "completions/min_terminated_length": 232.0, "entropy": 0.28289372473955154, "epoch": 0.465203426124197, "frac_reward_zero_std": 0.5, "grad_norm": 0.029098093509674072, "learning_rate": 1e-05, "loss": -0.0066, "num_tokens": 17309308.0, "reward": 0.75, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.379487156867981, "sampling/importance_sampling_ratio/mean": 0.9998158812522888, "sampling/importance_sampling_ratio/min": 0.7158328294754028, "sampling/sampling_logp_difference/max": 0.3343086242675781, "sampling/sampling_logp_difference/mean": 0.010313146747648716, "step": 869 }, { "clip_ratio/high_max": 5.972288636257872e-05, "clip_ratio/high_mean": 5.972288636257872e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 5.972288636257872e-05, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 750.0, "completions/mean_length": 569.125, "completions/mean_terminated_length": 562.7096557617188, "completions/min_length": 404.0, "completions/min_terminated_length": 404.0, "entropy": 0.27961366064846516, "epoch": 0.4657387580299786, "frac_reward_zero_std": 0.5, "grad_norm": 0.015332610346376896, "learning_rate": 1e-05, "loss": -0.0162, "num_tokens": 17331784.0, "reward": 0.875, "reward_std": 0.2314550280570984, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 1.443011999130249, "sampling/importance_sampling_ratio/mean": 1.000084400177002, "sampling/importance_sampling_ratio/min": 0.6820155382156372, "sampling/sampling_logp_difference/max": 0.3827028274536133, "sampling/sampling_logp_difference/mean": 0.009687328711152077, "step": 870 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00014738050958840176, "clip_ratio/low_min": 0.00014738050958840176, "clip_ratio/region_mean": 0.00014738050958840176, "completions/clipped_ratio": 0.0, "completions/max_length": 716.0, "completions/max_terminated_length": 716.0, "completions/mean_length": 462.0625, "completions/mean_terminated_length": 462.0625, "completions/min_length": 313.0, "completions/min_terminated_length": 313.0, "entropy": 0.2723985593765974, "epoch": 0.4662740899357602, "frac_reward_zero_std": 0.25, "grad_norm": 0.013609047047793865, "learning_rate": 1e-05, "loss": -0.026, "num_tokens": 17350794.0, "reward": 0.78125, "reward_std": 0.3608423173427582, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.4104193449020386, "sampling/importance_sampling_ratio/mean": 1.0001128911972046, "sampling/importance_sampling_ratio/min": 0.7268655300140381, "sampling/sampling_logp_difference/max": 0.3438870906829834, "sampling/sampling_logp_difference/mean": 0.009979769587516785, "step": 871 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.9780009905807674e-05, "clip_ratio/low_min": 5.9780009905807674e-05, "clip_ratio/region_mean": 5.9780009905807674e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 716.0, "completions/max_terminated_length": 716.0, "completions/mean_length": 503.375, "completions/mean_terminated_length": 503.375, "completions/min_length": 335.0, "completions/min_terminated_length": 335.0, "entropy": 0.25760078616440296, "epoch": 0.4668094218415418, "frac_reward_zero_std": 0.5, "grad_norm": 0.015023523010313511, "learning_rate": 1e-05, "loss": 0.0011, "num_tokens": 17370846.0, "reward": 0.78125, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.3430633544921875, "sampling/importance_sampling_ratio/mean": 1.000095009803772, "sampling/importance_sampling_ratio/min": 0.6748009324073792, "sampling/sampling_logp_difference/max": 0.393337607383728, "sampling/sampling_logp_difference/mean": 0.009723112918436527, "step": 872 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00011422744137234986, "clip_ratio/low_min": 0.00011422744137234986, "clip_ratio/region_mean": 0.00011422744137234986, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 758.0, "completions/mean_length": 494.75, "completions/mean_terminated_length": 444.1481628417969, "completions/min_length": 215.0, "completions/min_terminated_length": 215.0, "entropy": 0.44542980566620827, "epoch": 0.4673447537473233, "frac_reward_zero_std": 0.25, "grad_norm": 0.00849328562617302, "learning_rate": 1e-05, "loss": 0.0638, "num_tokens": 17390166.0, "reward": 0.71875, "reward_std": 0.3198433816432953, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.4321115016937256, "sampling/importance_sampling_ratio/mean": 1.000278353691101, "sampling/importance_sampling_ratio/min": 0.7282505631446838, "sampling/sampling_logp_difference/max": 0.3591499328613281, "sampling/sampling_logp_difference/mean": 0.01396699994802475, "step": 873 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 569.0, "completions/max_terminated_length": 569.0, "completions/mean_length": 403.6875, "completions/mean_terminated_length": 403.6875, "completions/min_length": 262.0, "completions/min_terminated_length": 262.0, "entropy": 0.24195192381739616, "epoch": 0.4678800856531049, "frac_reward_zero_std": 0.75, "grad_norm": 0.005018971394747496, "learning_rate": 1e-05, "loss": 0.0197, "num_tokens": 17406484.0, "reward": 0.96875, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.96875, "rewards/accuracy_reward/std": 0.1767766922712326, "sampling/importance_sampling_ratio/max": 1.4348853826522827, "sampling/importance_sampling_ratio/mean": 0.9999661445617676, "sampling/importance_sampling_ratio/min": 0.603873074054718, "sampling/sampling_logp_difference/max": 0.5043913125991821, "sampling/sampling_logp_difference/mean": 0.009061934426426888, "step": 874 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00021602207198156975, "clip_ratio/low_min": 0.00021602207198156975, "clip_ratio/region_mean": 0.00021602207198156975, "completions/clipped_ratio": 0.34375, "completions/max_length": 768.0, "completions/max_terminated_length": 730.0, "completions/mean_length": 574.59375, "completions/mean_terminated_length": 473.2857360839844, "completions/min_length": 292.0, "completions/min_terminated_length": 292.0, "entropy": 0.37176984921097755, "epoch": 0.4684154175588865, "frac_reward_zero_std": 0.0, "grad_norm": 0.007806619629263878, "learning_rate": 1e-05, "loss": 0.0187, "num_tokens": 17428535.0, "reward": 0.34375, "reward_std": 0.3808925747871399, "rewards/accuracy_reward/mean": 0.34375, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.633134126663208, "sampling/importance_sampling_ratio/mean": 1.0000697374343872, "sampling/importance_sampling_ratio/min": 0.6994459629058838, "sampling/sampling_logp_difference/max": 0.49050092697143555, "sampling/sampling_logp_difference/mean": 0.012547642923891544, "step": 875 }, { "clip_ratio/high_max": 0.00012194439477752894, "clip_ratio/high_mean": 0.00012194439477752894, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00012194439477752894, "completions/clipped_ratio": 0.0, "completions/max_length": 701.0, "completions/max_terminated_length": 701.0, "completions/mean_length": 476.375, "completions/mean_terminated_length": 476.375, "completions/min_length": 226.0, "completions/min_terminated_length": 226.0, "entropy": 0.30392312444746494, "epoch": 0.4689507494646681, "frac_reward_zero_std": 0.5, "grad_norm": 0.005617944058030844, "learning_rate": 1e-05, "loss": -0.007, "num_tokens": 17447283.0, "reward": 0.6875, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.538852334022522, "sampling/importance_sampling_ratio/mean": 1.0001436471939087, "sampling/importance_sampling_ratio/min": 0.6784803867340088, "sampling/sampling_logp_difference/max": 0.43103694915771484, "sampling/sampling_logp_difference/mean": 0.011121723800897598, "step": 876 }, { "clip_ratio/high_max": 6.278252112679183e-05, "clip_ratio/high_mean": 6.278252112679183e-05, "clip_ratio/low_mean": 6.278252112679183e-05, "clip_ratio/low_min": 6.278252112679183e-05, "clip_ratio/region_mean": 0.00012556504225358367, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 687.0, "completions/mean_length": 563.75, "completions/mean_terminated_length": 506.55999755859375, "completions/min_length": 304.0, "completions/min_terminated_length": 304.0, "entropy": 0.28311794623732567, "epoch": 0.46948608137044967, "frac_reward_zero_std": 0.0, "grad_norm": 0.02786269783973694, "learning_rate": 1e-05, "loss": 0.0943, "num_tokens": 17469267.0, "reward": 0.6875, "reward_std": 0.4082317352294922, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.8582525253295898, "sampling/importance_sampling_ratio/mean": 0.9999716877937317, "sampling/importance_sampling_ratio/min": 0.6807003617286682, "sampling/sampling_logp_difference/max": 0.6196365356445312, "sampling/sampling_logp_difference/mean": 0.009687327779829502, "step": 877 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.374299118760973e-05, "clip_ratio/low_min": 6.374299118760973e-05, "clip_ratio/region_mean": 6.374299118760973e-05, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 686.0, "completions/mean_length": 469.0625, "completions/mean_terminated_length": 438.137939453125, "completions/min_length": 279.0, "completions/min_terminated_length": 279.0, "entropy": 0.3062589019536972, "epoch": 0.47002141327623126, "frac_reward_zero_std": 0.5, "grad_norm": 0.029604416340589523, "learning_rate": 1e-05, "loss": 0.0823, "num_tokens": 17487821.0, "reward": 0.84375, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.6236634254455566, "sampling/importance_sampling_ratio/mean": 1.0003081560134888, "sampling/importance_sampling_ratio/min": 0.698088526725769, "sampling/sampling_logp_difference/max": 0.48468494415283203, "sampling/sampling_logp_difference/mean": 0.010873785242438316, "step": 878 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 9.457403575652279e-05, "clip_ratio/low_min": 9.457403575652279e-05, "clip_ratio/region_mean": 9.457403575652279e-05, "completions/clipped_ratio": 0.34375, "completions/max_length": 768.0, "completions/max_terminated_length": 743.0, "completions/mean_length": 562.375, "completions/mean_terminated_length": 454.66668701171875, "completions/min_length": 274.0, "completions/min_terminated_length": 274.0, "entropy": 0.4696640409529209, "epoch": 0.47055674518201285, "frac_reward_zero_std": 0.25, "grad_norm": 0.00855310633778572, "learning_rate": 1e-05, "loss": 0.0386, "num_tokens": 17509793.0, "reward": 0.53125, "reward_std": 0.3198433816432953, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.7555736303329468, "sampling/importance_sampling_ratio/mean": 1.0000959634780884, "sampling/importance_sampling_ratio/min": 0.7506774663925171, "sampling/sampling_logp_difference/max": 0.5627956390380859, "sampling/sampling_logp_difference/mean": 0.015245474874973297, "step": 879 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00011901665493496694, "clip_ratio/low_min": 0.00011901665493496694, "clip_ratio/region_mean": 0.00011901665493496694, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 750.0, "completions/mean_length": 515.03125, "completions/mean_terminated_length": 468.1851806640625, "completions/min_length": 278.0, "completions/min_terminated_length": 278.0, "entropy": 0.39100852981209755, "epoch": 0.47109207708779444, "frac_reward_zero_std": 0.25, "grad_norm": 0.008996756747364998, "learning_rate": 1e-05, "loss": 0.0605, "num_tokens": 17530554.0, "reward": 0.59375, "reward_std": 0.3198433816432953, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.374659776687622, "sampling/importance_sampling_ratio/mean": 0.9998103976249695, "sampling/importance_sampling_ratio/min": 0.6815146803855896, "sampling/sampling_logp_difference/max": 0.38343748450279236, "sampling/sampling_logp_difference/mean": 0.013063844293355942, "step": 880 }, { "clip_ratio/high_max": 9.064539335668087e-05, "clip_ratio/high_mean": 9.064539335668087e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 9.064539335668087e-05, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 658.0, "completions/mean_length": 430.125, "completions/mean_terminated_length": 419.2257995605469, "completions/min_length": 235.0, "completions/min_terminated_length": 235.0, "entropy": 0.29076406359672546, "epoch": 0.471627408993576, "frac_reward_zero_std": 0.5, "grad_norm": 0.0038288207724690437, "learning_rate": 1e-05, "loss": 0.0471, "num_tokens": 17547958.0, "reward": 0.9375, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.9375, "rewards/accuracy_reward/std": 0.24593468010425568, "sampling/importance_sampling_ratio/max": 1.3932408094406128, "sampling/importance_sampling_ratio/mean": 1.000108242034912, "sampling/importance_sampling_ratio/min": 0.7467346787452698, "sampling/sampling_logp_difference/max": 0.3316326141357422, "sampling/sampling_logp_difference/mean": 0.01060293149203062, "step": 881 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00015880407954682596, "clip_ratio/low_min": 0.00015880407954682596, "clip_ratio/region_mean": 0.00015880407954682596, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 737.0, "completions/mean_length": 514.0, "completions/mean_terminated_length": 505.8064270019531, "completions/min_length": 265.0, "completions/min_terminated_length": 265.0, "entropy": 0.29901344515383244, "epoch": 0.4721627408993576, "frac_reward_zero_std": 0.5, "grad_norm": 0.03653116151690483, "learning_rate": 1e-05, "loss": -0.0376, "num_tokens": 17568518.0, "reward": 0.8125, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.572670340538025, "sampling/importance_sampling_ratio/mean": 0.9999311566352844, "sampling/importance_sampling_ratio/min": 0.7423138618469238, "sampling/sampling_logp_difference/max": 0.4527750015258789, "sampling/sampling_logp_difference/mean": 0.01066594012081623, "step": 882 }, { "clip_ratio/high_max": 0.00016806877101771533, "clip_ratio/high_mean": 0.00016806877101771533, "clip_ratio/low_mean": 5.952380888629705e-05, "clip_ratio/low_min": 5.952380888629705e-05, "clip_ratio/region_mean": 0.00022759257990401238, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 672.0, "completions/mean_length": 394.90625, "completions/mean_terminated_length": 382.8709411621094, "completions/min_length": 198.0, "completions/min_terminated_length": 198.0, "entropy": 0.40722164139151573, "epoch": 0.4726980728051392, "frac_reward_zero_std": 0.25, "grad_norm": 0.02149309031665325, "learning_rate": 1e-05, "loss": 0.0743, "num_tokens": 17584787.0, "reward": 0.65625, "reward_std": 0.378745436668396, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.514536738395691, "sampling/importance_sampling_ratio/mean": 1.0001946687698364, "sampling/importance_sampling_ratio/min": 0.7147672772407532, "sampling/sampling_logp_difference/max": 0.41510963439941406, "sampling/sampling_logp_difference/mean": 0.011294819414615631, "step": 883 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 550.0, "completions/max_terminated_length": 550.0, "completions/mean_length": 358.65625, "completions/mean_terminated_length": 358.65625, "completions/min_length": 224.0, "completions/min_terminated_length": 224.0, "entropy": 0.3161104694008827, "epoch": 0.4732334047109208, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 17599520.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.604664921760559, "sampling/importance_sampling_ratio/mean": 1.0000346899032593, "sampling/importance_sampling_ratio/min": 0.6983143091201782, "sampling/sampling_logp_difference/max": 0.4729149341583252, "sampling/sampling_logp_difference/mean": 0.012083711102604866, "step": 884 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 714.0, "completions/mean_length": 477.96875, "completions/mean_terminated_length": 424.25927734375, "completions/min_length": 261.0, "completions/min_terminated_length": 261.0, "entropy": 0.36180451698601246, "epoch": 0.4737687366167024, "frac_reward_zero_std": 0.25, "grad_norm": 0.006163413170725107, "learning_rate": 1e-05, "loss": 0.0661, "num_tokens": 17618311.0, "reward": 0.71875, "reward_std": 0.2651650309562683, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.5656629800796509, "sampling/importance_sampling_ratio/mean": 1.000460147857666, "sampling/importance_sampling_ratio/min": 0.6155635714530945, "sampling/sampling_logp_difference/max": 0.4852170944213867, "sampling/sampling_logp_difference/mean": 0.012288326397538185, "step": 885 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 689.0, "completions/mean_length": 505.59375, "completions/mean_terminated_length": 478.4482727050781, "completions/min_length": 313.0, "completions/min_terminated_length": 313.0, "entropy": 0.36343562975525856, "epoch": 0.4743040685224839, "frac_reward_zero_std": 0.25, "grad_norm": 0.010548599064350128, "learning_rate": 1e-05, "loss": 0.0216, "num_tokens": 17638890.0, "reward": 0.75, "reward_std": 0.3514062464237213, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.4335078001022339, "sampling/importance_sampling_ratio/mean": 0.9997308254241943, "sampling/importance_sampling_ratio/min": 0.7642113566398621, "sampling/sampling_logp_difference/max": 0.3601243495941162, "sampling/sampling_logp_difference/mean": 0.01170903630554676, "step": 886 }, { "clip_ratio/high_max": 0.00017748800019035116, "clip_ratio/high_mean": 0.00017748800019035116, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00017748800019035116, "completions/clipped_ratio": 0.0, "completions/max_length": 638.0, "completions/max_terminated_length": 638.0, "completions/mean_length": 415.375, "completions/mean_terminated_length": 415.375, "completions/min_length": 230.0, "completions/min_terminated_length": 230.0, "entropy": 0.284851860255003, "epoch": 0.4748394004282655, "frac_reward_zero_std": 0.0, "grad_norm": 0.010278324596583843, "learning_rate": 1e-05, "loss": -0.0258, "num_tokens": 17655742.0, "reward": 0.8125, "reward_std": 0.3945523500442505, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.4035282135009766, "sampling/importance_sampling_ratio/mean": 1.0001215934753418, "sampling/importance_sampling_ratio/min": 0.5479459166526794, "sampling/sampling_logp_difference/max": 0.6015787124633789, "sampling/sampling_logp_difference/mean": 0.011346030980348587, "step": 887 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.22200095676817e-05, "clip_ratio/low_min": 6.22200095676817e-05, "clip_ratio/region_mean": 6.22200095676817e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 709.0, "completions/max_terminated_length": 709.0, "completions/mean_length": 414.5625, "completions/mean_terminated_length": 414.5625, "completions/min_length": 228.0, "completions/min_terminated_length": 228.0, "entropy": 0.23697533085942268, "epoch": 0.4753747323340471, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": -0.0055, "num_tokens": 17672960.0, "reward": 0.78125, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.4025033712387085, "sampling/importance_sampling_ratio/mean": 1.0002896785736084, "sampling/importance_sampling_ratio/min": 0.6271695494651794, "sampling/sampling_logp_difference/max": 0.4665384292602539, "sampling/sampling_logp_difference/mean": 0.008857685141265392, "step": 888 }, { "clip_ratio/high_max": 4.8206711653620005e-05, "clip_ratio/high_mean": 4.8206711653620005e-05, "clip_ratio/low_mean": 5.339598283171654e-05, "clip_ratio/low_min": 5.339598283171654e-05, "clip_ratio/region_mean": 0.00010160269448533654, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 721.0, "completions/mean_length": 548.59375, "completions/mean_terminated_length": 533.9666748046875, "completions/min_length": 227.0, "completions/min_terminated_length": 227.0, "entropy": 0.31761903315782547, "epoch": 0.4759100642398287, "frac_reward_zero_std": 0.25, "grad_norm": 0.022894147783517838, "learning_rate": 1e-05, "loss": -0.0267, "num_tokens": 17694611.0, "reward": 0.6875, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.5240819454193115, "sampling/importance_sampling_ratio/mean": 0.9996128082275391, "sampling/importance_sampling_ratio/min": 0.6478152275085449, "sampling/sampling_logp_difference/max": 0.43414974212646484, "sampling/sampling_logp_difference/mean": 0.012083141133189201, "step": 889 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 717.0, "completions/mean_length": 515.78125, "completions/mean_terminated_length": 507.6451416015625, "completions/min_length": 324.0, "completions/min_terminated_length": 324.0, "entropy": 0.31561365723609924, "epoch": 0.47644539614561027, "frac_reward_zero_std": 0.25, "grad_norm": 0.008195150643587112, "learning_rate": 1e-05, "loss": 0.0728, "num_tokens": 17715268.0, "reward": 0.875, "reward_std": 0.292504221200943, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 1.435183048248291, "sampling/importance_sampling_ratio/mean": 1.00010085105896, "sampling/importance_sampling_ratio/min": 0.7544096112251282, "sampling/sampling_logp_difference/max": 0.36129236221313477, "sampling/sampling_logp_difference/mean": 0.010514057241380215, "step": 890 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 758.0, "completions/mean_length": 398.84375, "completions/mean_terminated_length": 386.93548583984375, "completions/min_length": 157.0, "completions/min_terminated_length": 157.0, "entropy": 0.29939622804522514, "epoch": 0.47698072805139186, "frac_reward_zero_std": 0.25, "grad_norm": 0.0051595428958535194, "learning_rate": 1e-05, "loss": 0.0278, "num_tokens": 17731695.0, "reward": 0.90625, "reward_std": 0.2651650309562683, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.3975964784622192, "sampling/importance_sampling_ratio/mean": 1.000022053718567, "sampling/importance_sampling_ratio/min": 0.7142203450202942, "sampling/sampling_logp_difference/max": 0.3365638256072998, "sampling/sampling_logp_difference/mean": 0.010699898935854435, "step": 891 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 681.0, "completions/max_terminated_length": 681.0, "completions/mean_length": 456.4375, "completions/mean_terminated_length": 456.4375, "completions/min_length": 260.0, "completions/min_terminated_length": 260.0, "entropy": 0.2786124423146248, "epoch": 0.47751605995717344, "frac_reward_zero_std": 0.0, "grad_norm": 0.018611766397953033, "learning_rate": 1e-05, "loss": -0.0686, "num_tokens": 17750061.0, "reward": 0.75, "reward_std": 0.4355512857437134, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.5237611532211304, "sampling/importance_sampling_ratio/mean": 1.0000518560409546, "sampling/importance_sampling_ratio/min": 0.6943444013595581, "sampling/sampling_logp_difference/max": 0.42118167877197266, "sampling/sampling_logp_difference/mean": 0.010647882707417011, "step": 892 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.374032843974419e-05, "clip_ratio/low_min": 5.374032843974419e-05, "clip_ratio/region_mean": 5.374032843974419e-05, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 767.0, "completions/mean_length": 502.75, "completions/mean_terminated_length": 485.0666809082031, "completions/min_length": 239.0, "completions/min_terminated_length": 239.0, "entropy": 0.2564187031239271, "epoch": 0.47805139186295503, "frac_reward_zero_std": 0.75, "grad_norm": 0.007111367769539356, "learning_rate": 1e-05, "loss": 0.0248, "num_tokens": 17770229.0, "reward": 0.9375, "reward_std": 0.1157275140285492, "rewards/accuracy_reward/mean": 0.9375, "rewards/accuracy_reward/std": 0.24593468010425568, "sampling/importance_sampling_ratio/max": 1.3762917518615723, "sampling/importance_sampling_ratio/mean": 1.0004146099090576, "sampling/importance_sampling_ratio/min": 0.655583381652832, "sampling/sampling_logp_difference/max": 0.4222297668457031, "sampling/sampling_logp_difference/mean": 0.009604319930076599, "step": 893 }, { "clip_ratio/high_max": 8.579272252973169e-05, "clip_ratio/high_mean": 8.579272252973169e-05, "clip_ratio/low_mean": 5.771006544819102e-05, "clip_ratio/low_min": 5.771006544819102e-05, "clip_ratio/region_mean": 0.0001435027879779227, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 685.0, "completions/mean_length": 462.34375, "completions/mean_terminated_length": 452.4838562011719, "completions/min_length": 182.0, "completions/min_terminated_length": 182.0, "entropy": 0.32170755974948406, "epoch": 0.4785867237687366, "frac_reward_zero_std": 0.25, "grad_norm": 0.01108202338218689, "learning_rate": 1e-05, "loss": 0.0361, "num_tokens": 17788640.0, "reward": 0.71875, "reward_std": 0.3608423173427582, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.4440582990646362, "sampling/importance_sampling_ratio/mean": 1.000043511390686, "sampling/importance_sampling_ratio/min": 0.7355419397354126, "sampling/sampling_logp_difference/max": 0.36745738983154297, "sampling/sampling_logp_difference/mean": 0.01109524816274643, "step": 894 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.312367284204811e-05, "clip_ratio/low_min": 5.312367284204811e-05, "clip_ratio/region_mean": 5.312367284204811e-05, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 521.0, "completions/mean_length": 419.21875, "completions/mean_terminated_length": 369.39288330078125, "completions/min_length": 185.0, "completions/min_terminated_length": 185.0, "entropy": 0.37245791032910347, "epoch": 0.4791220556745182, "frac_reward_zero_std": 0.25, "grad_norm": 0.026289524510502815, "learning_rate": 1e-05, "loss": 0.0392, "num_tokens": 17805663.0, "reward": 0.625, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.4213987588882446, "sampling/importance_sampling_ratio/mean": 1.0001524686813354, "sampling/importance_sampling_ratio/min": 0.7705093026161194, "sampling/sampling_logp_difference/max": 0.3516414165496826, "sampling/sampling_logp_difference/mean": 0.012494618073105812, "step": 895 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 7.339988223975524e-05, "clip_ratio/low_min": 7.339988223975524e-05, "clip_ratio/region_mean": 7.339988223975524e-05, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 746.0, "completions/mean_length": 436.0625, "completions/mean_terminated_length": 413.933349609375, "completions/min_length": 166.0, "completions/min_terminated_length": 166.0, "entropy": 0.3051671665161848, "epoch": 0.4796573875802998, "frac_reward_zero_std": 0.5, "grad_norm": 0.020245268940925598, "learning_rate": 1e-05, "loss": 0.0485, "num_tokens": 17823529.0, "reward": 0.875, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 1.3547430038452148, "sampling/importance_sampling_ratio/mean": 0.9999259114265442, "sampling/importance_sampling_ratio/min": 0.6948326230049133, "sampling/sampling_logp_difference/max": 0.36408424377441406, "sampling/sampling_logp_difference/mean": 0.011529726907610893, "step": 896 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 590.0, "completions/mean_length": 434.625, "completions/mean_terminated_length": 423.8709411621094, "completions/min_length": 307.0, "completions/min_terminated_length": 307.0, "entropy": 0.30700914561748505, "epoch": 0.4801927194860814, "frac_reward_zero_std": 0.75, "grad_norm": 0.004601879511028528, "learning_rate": 1e-05, "loss": 0.0446, "num_tokens": 17841141.0, "reward": 0.96875, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.96875, "rewards/accuracy_reward/std": 0.1767766922712326, "sampling/importance_sampling_ratio/max": 1.3869141340255737, "sampling/importance_sampling_ratio/mean": 0.9993304014205933, "sampling/importance_sampling_ratio/min": 0.6631650924682617, "sampling/sampling_logp_difference/max": 0.41073131561279297, "sampling/sampling_logp_difference/mean": 0.01103257853537798, "step": 897 }, { "clip_ratio/high_max": 0.00015404890291392803, "clip_ratio/high_mean": 0.00015404890291392803, "clip_ratio/low_mean": 6.423432932933792e-05, "clip_ratio/low_min": 6.423432932933792e-05, "clip_ratio/region_mean": 0.00021828323224326596, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 760.0, "completions/mean_length": 399.5625, "completions/mean_terminated_length": 387.6773986816406, "completions/min_length": 242.0, "completions/min_terminated_length": 242.0, "entropy": 0.39390813559293747, "epoch": 0.480728051391863, "frac_reward_zero_std": 0.25, "grad_norm": 0.01933855004608631, "learning_rate": 1e-05, "loss": 0.0304, "num_tokens": 17857607.0, "reward": 0.75, "reward_std": 0.3514062464237213, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.4763736724853516, "sampling/importance_sampling_ratio/mean": 0.9998878836631775, "sampling/importance_sampling_ratio/min": 0.5628733038902283, "sampling/sampling_logp_difference/max": 0.5747007131576538, "sampling/sampling_logp_difference/mean": 0.013597422279417515, "step": 898 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 680.0, "completions/mean_length": 491.21875, "completions/mean_terminated_length": 482.2903137207031, "completions/min_length": 282.0, "completions/min_terminated_length": 282.0, "entropy": 0.2863363791257143, "epoch": 0.48126338329764456, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 17877254.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.6229543685913086, "sampling/importance_sampling_ratio/mean": 1.0002821683883667, "sampling/importance_sampling_ratio/min": 0.6835976243019104, "sampling/sampling_logp_difference/max": 0.48424816131591797, "sampling/sampling_logp_difference/mean": 0.010139341466128826, "step": 899 }, { "clip_ratio/high_max": 7.413997809635475e-05, "clip_ratio/high_mean": 7.413997809635475e-05, "clip_ratio/low_mean": 0.00021566540090134367, "clip_ratio/low_min": 0.00021566540090134367, "clip_ratio/region_mean": 0.0002898053717217408, "completions/clipped_ratio": 0.0, "completions/max_length": 592.0, "completions/max_terminated_length": 592.0, "completions/mean_length": 398.625, "completions/mean_terminated_length": 398.625, "completions/min_length": 247.0, "completions/min_terminated_length": 247.0, "entropy": 0.31713657453656197, "epoch": 0.4817987152034261, "frac_reward_zero_std": 0.25, "grad_norm": 0.01953887566924095, "learning_rate": 1e-05, "loss": -0.0333, "num_tokens": 17893962.0, "reward": 0.78125, "reward_std": 0.3608423173427582, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.3457244634628296, "sampling/importance_sampling_ratio/mean": 1.0002864599227905, "sampling/importance_sampling_ratio/min": 0.660288393497467, "sampling/sampling_logp_difference/max": 0.41507863998413086, "sampling/sampling_logp_difference/mean": 0.011422082781791687, "step": 900 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 673.0, "completions/mean_length": 487.59375, "completions/mean_terminated_length": 478.5483703613281, "completions/min_length": 253.0, "completions/min_terminated_length": 253.0, "entropy": 0.2983247824013233, "epoch": 0.4823340471092077, "frac_reward_zero_std": 0.5, "grad_norm": 0.03428788483142853, "learning_rate": 1e-05, "loss": 0.002, "num_tokens": 17913477.0, "reward": 0.9375, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.9375, "rewards/accuracy_reward/std": 0.24593468010425568, "sampling/importance_sampling_ratio/max": 1.430167317390442, "sampling/importance_sampling_ratio/mean": 0.9998111724853516, "sampling/importance_sampling_ratio/min": 0.5547361373901367, "sampling/sampling_logp_difference/max": 0.5892627239227295, "sampling/sampling_logp_difference/mean": 0.011623206548392773, "step": 901 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 703.0, "completions/mean_length": 518.90625, "completions/mean_terminated_length": 461.423095703125, "completions/min_length": 283.0, "completions/min_terminated_length": 283.0, "entropy": 0.40202517434954643, "epoch": 0.4828693790149893, "frac_reward_zero_std": 0.25, "grad_norm": 0.006073471624404192, "learning_rate": 1e-05, "loss": 0.0523, "num_tokens": 17933850.0, "reward": 0.4375, "reward_std": 0.2925041913986206, "rewards/accuracy_reward/mean": 0.4375, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.000144600868225, "sampling/importance_sampling_ratio/min": 0.6256729960441589, "sampling/sampling_logp_difference/max": 0.8568992614746094, "sampling/sampling_logp_difference/mean": 0.014280413277447224, "step": 902 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 742.0, "completions/mean_length": 506.3125, "completions/mean_terminated_length": 445.923095703125, "completions/min_length": 202.0, "completions/min_terminated_length": 202.0, "entropy": 0.26994348876178265, "epoch": 0.48340471092077086, "frac_reward_zero_std": 0.25, "grad_norm": 0.007904134690761566, "learning_rate": 1e-05, "loss": 0.0765, "num_tokens": 17954708.0, "reward": 0.8125, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.4643410444259644, "sampling/importance_sampling_ratio/mean": 0.9998607635498047, "sampling/importance_sampling_ratio/min": 0.49734869599342346, "sampling/sampling_logp_difference/max": 0.6984639167785645, "sampling/sampling_logp_difference/mean": 0.010100830346345901, "step": 903 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 7.122506940504536e-05, "clip_ratio/low_min": 7.122506940504536e-05, "clip_ratio/region_mean": 7.122506940504536e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 733.0, "completions/max_terminated_length": 733.0, "completions/mean_length": 439.53125, "completions/mean_terminated_length": 439.53125, "completions/min_length": 191.0, "completions/min_terminated_length": 191.0, "entropy": 0.220725879073143, "epoch": 0.48394004282655245, "frac_reward_zero_std": 0.5, "grad_norm": 0.0029265962075442076, "learning_rate": 1e-05, "loss": -0.0214, "num_tokens": 17972157.0, "reward": 0.9375, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.9375, "rewards/accuracy_reward/std": 0.24593468010425568, "sampling/importance_sampling_ratio/max": 1.5479936599731445, "sampling/importance_sampling_ratio/mean": 0.9997026324272156, "sampling/importance_sampling_ratio/min": 0.6082855463027954, "sampling/sampling_logp_difference/max": 0.49711084365844727, "sampling/sampling_logp_difference/mean": 0.009222800843417645, "step": 904 }, { "clip_ratio/high_max": 0.00014070072211325169, "clip_ratio/high_mean": 0.00014070072211325169, "clip_ratio/low_mean": 6.42013328615576e-05, "clip_ratio/low_min": 6.42013328615576e-05, "clip_ratio/region_mean": 0.0002049020549748093, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 598.0, "completions/mean_length": 457.4375, "completions/mean_terminated_length": 425.3103332519531, "completions/min_length": 253.0, "completions/min_terminated_length": 253.0, "entropy": 0.41206288896501064, "epoch": 0.48447537473233404, "frac_reward_zero_std": 0.25, "grad_norm": 0.008017003536224365, "learning_rate": 1e-05, "loss": 0.1193, "num_tokens": 17990931.0, "reward": 0.75, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.3692039251327515, "sampling/importance_sampling_ratio/mean": 0.9999173879623413, "sampling/importance_sampling_ratio/min": 0.5028315782546997, "sampling/sampling_logp_difference/max": 0.6875, "sampling/sampling_logp_difference/mean": 0.013152733445167542, "step": 905 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00016133778262883425, "clip_ratio/low_min": 0.00016133778262883425, "clip_ratio/region_mean": 0.00016133778262883425, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 692.0, "completions/mean_length": 458.5625, "completions/mean_terminated_length": 426.5517272949219, "completions/min_length": 173.0, "completions/min_terminated_length": 173.0, "entropy": 0.3623971212655306, "epoch": 0.48501070663811563, "frac_reward_zero_std": 0.5, "grad_norm": 0.023558853194117546, "learning_rate": 1e-05, "loss": 0.0153, "num_tokens": 18009421.0, "reward": 0.71875, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.4311026334762573, "sampling/importance_sampling_ratio/mean": 0.9998683929443359, "sampling/importance_sampling_ratio/min": 0.6248417496681213, "sampling/sampling_logp_difference/max": 0.4702568054199219, "sampling/sampling_logp_difference/mean": 0.012686527334153652, "step": 906 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.623032120638527e-05, "clip_ratio/low_min": 5.623032120638527e-05, "clip_ratio/region_mean": 5.623032120638527e-05, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 674.0, "completions/mean_length": 474.46875, "completions/mean_terminated_length": 465.0, "completions/min_length": 302.0, "completions/min_terminated_length": 302.0, "entropy": 0.21107923239469528, "epoch": 0.4855460385438972, "frac_reward_zero_std": 0.75, "grad_norm": 0.014978027902543545, "learning_rate": 1e-05, "loss": 0.0238, "num_tokens": 18027932.0, "reward": 0.84375, "reward_std": 0.1293872892856598, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.41343355178833, "sampling/importance_sampling_ratio/mean": 1.0001240968704224, "sampling/importance_sampling_ratio/min": 0.7033680081367493, "sampling/sampling_logp_difference/max": 0.35187506675720215, "sampling/sampling_logp_difference/mean": 0.008225500583648682, "step": 907 }, { "clip_ratio/high_max": 0.0001752337957441341, "clip_ratio/high_mean": 0.0001752337957441341, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0001752337957441341, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 751.0, "completions/mean_length": 509.46875, "completions/mean_terminated_length": 492.2333679199219, "completions/min_length": 133.0, "completions/min_terminated_length": 133.0, "entropy": 0.3040708899497986, "epoch": 0.4860813704496788, "frac_reward_zero_std": 0.0, "grad_norm": 0.021471887826919556, "learning_rate": 1e-05, "loss": 0.0064, "num_tokens": 18047883.0, "reward": 0.46875, "reward_std": 0.47137707471847534, "rewards/accuracy_reward/mean": 0.46875, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.4772247076034546, "sampling/importance_sampling_ratio/mean": 0.9998332858085632, "sampling/importance_sampling_ratio/min": 0.595400869846344, "sampling/sampling_logp_difference/max": 0.5185203552246094, "sampling/sampling_logp_difference/mean": 0.011188496835529804, "step": 908 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.652474985457957e-05, "clip_ratio/low_min": 6.652474985457957e-05, "clip_ratio/region_mean": 6.652474985457957e-05, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 764.0, "completions/mean_length": 489.40625, "completions/mean_terminated_length": 470.8333435058594, "completions/min_length": 242.0, "completions/min_terminated_length": 242.0, "entropy": 0.39791784808039665, "epoch": 0.4866167023554604, "frac_reward_zero_std": 0.0, "grad_norm": 0.03481146693229675, "learning_rate": 1e-05, "loss": -0.0179, "num_tokens": 18068384.0, "reward": 0.59375, "reward_std": 0.4628904461860657, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.351954698562622, "sampling/importance_sampling_ratio/mean": 0.9997958540916443, "sampling/importance_sampling_ratio/min": 0.6212679743766785, "sampling/sampling_logp_difference/max": 0.4759927988052368, "sampling/sampling_logp_difference/mean": 0.013370419852435589, "step": 909 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 726.0, "completions/max_terminated_length": 726.0, "completions/mean_length": 468.90625, "completions/mean_terminated_length": 468.90625, "completions/min_length": 280.0, "completions/min_terminated_length": 280.0, "entropy": 0.26747251860797405, "epoch": 0.487152034261242, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 18087261.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.4127734899520874, "sampling/importance_sampling_ratio/mean": 1.0001856088638306, "sampling/importance_sampling_ratio/min": 0.6977643966674805, "sampling/sampling_logp_difference/max": 0.35987377166748047, "sampling/sampling_logp_difference/mean": 0.009961742907762527, "step": 910 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.76917193736881e-05, "clip_ratio/low_min": 4.76917193736881e-05, "clip_ratio/region_mean": 4.76917193736881e-05, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 699.0, "completions/mean_length": 516.3125, "completions/mean_terminated_length": 469.7037048339844, "completions/min_length": 286.0, "completions/min_terminated_length": 286.0, "entropy": 0.28392020240426064, "epoch": 0.4876873661670236, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0226, "num_tokens": 18107415.0, "reward": 0.84375, "reward_std": 0.1293872892856598, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.8648970127105713, "sampling/importance_sampling_ratio/mean": 1.000344157218933, "sampling/importance_sampling_ratio/min": 0.6863139867782593, "sampling/sampling_logp_difference/max": 0.6232057809829712, "sampling/sampling_logp_difference/mean": 0.01108316145837307, "step": 911 }, { "clip_ratio/high_max": 5.1845705456798896e-05, "clip_ratio/high_mean": 5.1845705456798896e-05, "clip_ratio/low_mean": 0.0001192373783851508, "clip_ratio/low_min": 0.0001192373783851508, "clip_ratio/region_mean": 0.0001710830838419497, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 745.0, "completions/mean_length": 534.21875, "completions/mean_terminated_length": 500.8214416503906, "completions/min_length": 340.0, "completions/min_terminated_length": 340.0, "entropy": 0.3648163303732872, "epoch": 0.48822269807280516, "frac_reward_zero_std": 0.0, "grad_norm": 0.010235407389700413, "learning_rate": 1e-05, "loss": 0.0583, "num_tokens": 18128574.0, "reward": 0.40625, "reward_std": 0.4534739851951599, "rewards/accuracy_reward/mean": 0.40625, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.448590636253357, "sampling/importance_sampling_ratio/mean": 1.0002288818359375, "sampling/importance_sampling_ratio/min": 0.7280926704406738, "sampling/sampling_logp_difference/max": 0.3705911636352539, "sampling/sampling_logp_difference/mean": 0.012508274056017399, "step": 912 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.8934463595505804e-05, "clip_ratio/low_min": 5.8934463595505804e-05, "clip_ratio/region_mean": 5.8934463595505804e-05, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 758.0, "completions/mean_length": 477.8125, "completions/mean_terminated_length": 458.4667053222656, "completions/min_length": 171.0, "completions/min_terminated_length": 171.0, "entropy": 0.3433845341205597, "epoch": 0.48875802997858675, "frac_reward_zero_std": 0.25, "grad_norm": 0.021932439878582954, "learning_rate": 1e-05, "loss": 0.0348, "num_tokens": 18148096.0, "reward": 0.78125, "reward_std": 0.3471629321575165, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.3927536010742188, "sampling/importance_sampling_ratio/mean": 0.9999805688858032, "sampling/importance_sampling_ratio/min": 0.5484291911125183, "sampling/sampling_logp_difference/max": 0.6006970405578613, "sampling/sampling_logp_difference/mean": 0.011226137168705463, "step": 913 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.856829713797197e-05, "clip_ratio/low_min": 6.856829713797197e-05, "clip_ratio/region_mean": 6.856829713797197e-05, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 742.0, "completions/mean_length": 476.875, "completions/mean_terminated_length": 457.4667053222656, "completions/min_length": 262.0, "completions/min_terminated_length": 262.0, "entropy": 0.3077407404780388, "epoch": 0.4892933618843683, "frac_reward_zero_std": 0.5, "grad_norm": 0.005473727826029062, "learning_rate": 1e-05, "loss": -0.0092, "num_tokens": 18167020.0, "reward": 0.78125, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.439332365989685, "sampling/importance_sampling_ratio/mean": 0.9999160766601562, "sampling/importance_sampling_ratio/min": 0.08567306399345398, "sampling/sampling_logp_difference/max": 2.457216739654541, "sampling/sampling_logp_difference/mean": 0.011770566925406456, "step": 914 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00011579434794839472, "clip_ratio/low_min": 0.00011579434794839472, "clip_ratio/region_mean": 0.00011579434794839472, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 658.0, "completions/mean_length": 466.4375, "completions/mean_terminated_length": 423.3571472167969, "completions/min_length": 279.0, "completions/min_terminated_length": 279.0, "entropy": 0.35018456168472767, "epoch": 0.48982869379014987, "frac_reward_zero_std": 0.5, "grad_norm": 0.029982883483171463, "learning_rate": 1e-05, "loss": 0.0987, "num_tokens": 18186130.0, "reward": 0.53125, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.6562178134918213, "sampling/importance_sampling_ratio/mean": 1.00028395652771, "sampling/importance_sampling_ratio/min": 0.6066259145736694, "sampling/sampling_logp_difference/max": 0.5045366287231445, "sampling/sampling_logp_difference/mean": 0.01201210543513298, "step": 915 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.994963769102469e-05, "clip_ratio/low_min": 6.994963769102469e-05, "clip_ratio/region_mean": 6.994963769102469e-05, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 631.0, "completions/mean_length": 430.71875, "completions/mean_terminated_length": 408.2333679199219, "completions/min_length": 217.0, "completions/min_terminated_length": 217.0, "entropy": 0.29074944369494915, "epoch": 0.49036402569593146, "frac_reward_zero_std": 0.5, "grad_norm": 0.03246884047985077, "learning_rate": 1e-05, "loss": 0.0344, "num_tokens": 18203801.0, "reward": 0.9375, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.9375, "rewards/accuracy_reward/std": 0.24593468010425568, "sampling/importance_sampling_ratio/max": 1.4905964136123657, "sampling/importance_sampling_ratio/mean": 1.0003291368484497, "sampling/importance_sampling_ratio/min": 0.6721236109733582, "sampling/sampling_logp_difference/max": 0.39917635917663574, "sampling/sampling_logp_difference/mean": 0.011309084482491016, "step": 916 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00011041528341593221, "clip_ratio/low_min": 0.00011041528341593221, "clip_ratio/region_mean": 0.00011041528341593221, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 737.0, "completions/mean_length": 548.53125, "completions/mean_terminated_length": 497.8846435546875, "completions/min_length": 202.0, "completions/min_terminated_length": 202.0, "entropy": 0.28678014129400253, "epoch": 0.49089935760171305, "frac_reward_zero_std": 0.25, "grad_norm": 0.01799173094332218, "learning_rate": 1e-05, "loss": 0.0943, "num_tokens": 18225154.0, "reward": 0.6875, "reward_std": 0.3745020925998688, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.6590856313705444, "sampling/importance_sampling_ratio/mean": 1.0002859830856323, "sampling/importance_sampling_ratio/min": 0.6085002422332764, "sampling/sampling_logp_difference/max": 0.5062665939331055, "sampling/sampling_logp_difference/mean": 0.010599044151604176, "step": 917 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00014400921645574272, "clip_ratio/low_min": 0.00014400921645574272, "clip_ratio/region_mean": 0.00014400921645574272, "completions/clipped_ratio": 0.0, "completions/max_length": 553.0, "completions/max_terminated_length": 553.0, "completions/mean_length": 398.28125, "completions/mean_terminated_length": 398.28125, "completions/min_length": 241.0, "completions/min_terminated_length": 241.0, "entropy": 0.30088853277266026, "epoch": 0.49143468950749464, "frac_reward_zero_std": 0.25, "grad_norm": 0.007649065461009741, "learning_rate": 1e-05, "loss": 0.0712, "num_tokens": 18241643.0, "reward": 0.84375, "reward_std": 0.3198433816432953, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.2989885807037354, "sampling/importance_sampling_ratio/mean": 1.0002886056900024, "sampling/importance_sampling_ratio/min": 0.66785728931427, "sampling/sampling_logp_difference/max": 0.40368080139160156, "sampling/sampling_logp_difference/mean": 0.011044279672205448, "step": 918 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 738.0, "completions/mean_length": 487.4375, "completions/mean_terminated_length": 458.4137878417969, "completions/min_length": 223.0, "completions/min_terminated_length": 223.0, "entropy": 0.3637304864823818, "epoch": 0.4919700214132762, "frac_reward_zero_std": 0.5, "grad_norm": 0.007687463890761137, "learning_rate": 1e-05, "loss": 0.0604, "num_tokens": 18261177.0, "reward": 0.5625, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.3802224397659302, "sampling/importance_sampling_ratio/mean": 0.9997232556343079, "sampling/importance_sampling_ratio/min": 0.6623309254646301, "sampling/sampling_logp_difference/max": 0.4119899272918701, "sampling/sampling_logp_difference/mean": 0.01202123798429966, "step": 919 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00010454619769006968, "clip_ratio/low_min": 0.00010454619769006968, "clip_ratio/region_mean": 0.00010454619769006968, "completions/clipped_ratio": 0.40625, "completions/max_length": 768.0, "completions/max_terminated_length": 738.0, "completions/mean_length": 611.59375, "completions/mean_terminated_length": 504.5789489746094, "completions/min_length": 335.0, "completions/min_terminated_length": 335.0, "entropy": 0.5391234718263149, "epoch": 0.4925053533190578, "frac_reward_zero_std": 0.5, "grad_norm": 0.006431222427636385, "learning_rate": 1e-05, "loss": 0.0252, "num_tokens": 18285100.0, "reward": 0.09375, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.09375, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.4450308084487915, "sampling/importance_sampling_ratio/mean": 0.9998129606246948, "sampling/importance_sampling_ratio/min": 0.6925428509712219, "sampling/sampling_logp_difference/max": 0.3681306838989258, "sampling/sampling_logp_difference/mean": 0.01693020947277546, "step": 920 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 736.0, "completions/max_terminated_length": 736.0, "completions/mean_length": 471.21875, "completions/mean_terminated_length": 471.21875, "completions/min_length": 263.0, "completions/min_terminated_length": 263.0, "entropy": 0.26552114076912403, "epoch": 0.4930406852248394, "frac_reward_zero_std": 0.5, "grad_norm": 0.02040373533964157, "learning_rate": 1e-05, "loss": 0.0075, "num_tokens": 18304203.0, "reward": 0.84375, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.4626009464263916, "sampling/importance_sampling_ratio/mean": 0.9998367428779602, "sampling/importance_sampling_ratio/min": 0.7014557123184204, "sampling/sampling_logp_difference/max": 0.3802163600921631, "sampling/sampling_logp_difference/mean": 0.009553835727274418, "step": 921 }, { "clip_ratio/high_max": 5.9045818488812074e-05, "clip_ratio/high_mean": 5.9045818488812074e-05, "clip_ratio/low_mean": 0.00010313531674910337, "clip_ratio/low_min": 0.00010313531674910337, "clip_ratio/region_mean": 0.00016218113523791544, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 764.0, "completions/mean_length": 515.28125, "completions/mean_terminated_length": 489.137939453125, "completions/min_length": 236.0, "completions/min_terminated_length": 236.0, "entropy": 0.35010990500450134, "epoch": 0.493576017130621, "frac_reward_zero_std": 0.0, "grad_norm": 0.01850818283855915, "learning_rate": 1e-05, "loss": 0.1123, "num_tokens": 18324716.0, "reward": 0.71875, "reward_std": 0.4534739851951599, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.5360453128814697, "sampling/importance_sampling_ratio/mean": 0.9998300671577454, "sampling/importance_sampling_ratio/min": 0.6004226207733154, "sampling/sampling_logp_difference/max": 0.5101215839385986, "sampling/sampling_logp_difference/mean": 0.012675362639129162, "step": 922 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 7.17565999366343e-05, "clip_ratio/low_min": 7.17565999366343e-05, "clip_ratio/region_mean": 7.17565999366343e-05, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 762.0, "completions/mean_length": 459.1875, "completions/mean_terminated_length": 427.2413635253906, "completions/min_length": 186.0, "completions/min_terminated_length": 186.0, "entropy": 0.3355816677212715, "epoch": 0.4941113490364026, "frac_reward_zero_std": 0.5, "grad_norm": 0.018069904297590256, "learning_rate": 1e-05, "loss": 0.0188, "num_tokens": 18343210.0, "reward": 0.78125, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.4299403429031372, "sampling/importance_sampling_ratio/mean": 1.000341773033142, "sampling/importance_sampling_ratio/min": 0.6935073137283325, "sampling/sampling_logp_difference/max": 0.3659934997558594, "sampling/sampling_logp_difference/mean": 0.011436311528086662, "step": 923 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.7129800552502275e-05, "clip_ratio/low_min": 5.7129800552502275e-05, "clip_ratio/region_mean": 5.7129800552502275e-05, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 745.0, "completions/mean_length": 533.4375, "completions/mean_terminated_length": 517.800048828125, "completions/min_length": 245.0, "completions/min_terminated_length": 245.0, "entropy": 0.32349422574043274, "epoch": 0.49464668094218417, "frac_reward_zero_std": 0.25, "grad_norm": 0.013042080216109753, "learning_rate": 1e-05, "loss": 0.0563, "num_tokens": 18364112.0, "reward": 0.625, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.4326272010803223, "sampling/importance_sampling_ratio/mean": 0.9996811151504517, "sampling/importance_sampling_ratio/min": 0.7073561549186707, "sampling/sampling_logp_difference/max": 0.3595099449157715, "sampling/sampling_logp_difference/mean": 0.011251432821154594, "step": 924 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 645.0, "completions/mean_length": 508.25, "completions/mean_terminated_length": 448.3077087402344, "completions/min_length": 273.0, "completions/min_terminated_length": 273.0, "entropy": 0.3550127577036619, "epoch": 0.49518201284796576, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.008, "num_tokens": 18384400.0, "reward": 0.65625, "reward_std": 0.1293872892856598, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.2908086776733398, "sampling/importance_sampling_ratio/mean": 0.999809741973877, "sampling/importance_sampling_ratio/min": 0.7054306268692017, "sampling/sampling_logp_difference/max": 0.34894680976867676, "sampling/sampling_logp_difference/mean": 0.012393838725984097, "step": 925 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.782419950468466e-05, "clip_ratio/low_min": 6.782419950468466e-05, "clip_ratio/region_mean": 6.782419950468466e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 763.0, "completions/max_terminated_length": 763.0, "completions/mean_length": 483.71875, "completions/mean_terminated_length": 483.71875, "completions/min_length": 269.0, "completions/min_terminated_length": 269.0, "entropy": 0.31992656737565994, "epoch": 0.49571734475374735, "frac_reward_zero_std": 0.25, "grad_norm": 0.01657484471797943, "learning_rate": 1e-05, "loss": 0.0573, "num_tokens": 18403991.0, "reward": 0.78125, "reward_std": 0.3377464711666107, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.4087289571762085, "sampling/importance_sampling_ratio/mean": 0.9999836087226868, "sampling/importance_sampling_ratio/min": 0.7234011888504028, "sampling/sampling_logp_difference/max": 0.34268784523010254, "sampling/sampling_logp_difference/mean": 0.011354964226484299, "step": 926 }, { "clip_ratio/high_max": 6.659563223365694e-05, "clip_ratio/high_mean": 6.659563223365694e-05, "clip_ratio/low_mean": 0.0001500398138887249, "clip_ratio/low_min": 0.0001500398138887249, "clip_ratio/region_mean": 0.00021663544612238184, "completions/clipped_ratio": 0.0, "completions/max_length": 682.0, "completions/max_terminated_length": 682.0, "completions/mean_length": 423.59375, "completions/mean_terminated_length": 423.59375, "completions/min_length": 176.0, "completions/min_terminated_length": 176.0, "entropy": 0.35049179941415787, "epoch": 0.49625267665952894, "frac_reward_zero_std": 0.25, "grad_norm": 0.026263821870088577, "learning_rate": 1e-05, "loss": -0.0016, "num_tokens": 18420602.0, "reward": 0.6875, "reward_std": 0.3471825420856476, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.4508293867111206, "sampling/importance_sampling_ratio/mean": 1.0002559423446655, "sampling/importance_sampling_ratio/min": 0.6940593123435974, "sampling/sampling_logp_difference/max": 0.3721354007720947, "sampling/sampling_logp_difference/mean": 0.011414631269872189, "step": 927 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.34375, "completions/max_length": 768.0, "completions/max_terminated_length": 766.0, "completions/mean_length": 527.8125, "completions/mean_terminated_length": 402.0, "completions/min_length": 282.0, "completions/min_terminated_length": 282.0, "entropy": 0.4881604462862015, "epoch": 0.49678800856531047, "frac_reward_zero_std": 0.75, "grad_norm": 0.008296244777739048, "learning_rate": 1e-05, "loss": 0.0004, "num_tokens": 18441876.0, "reward": 0.5625, "reward_std": 0.1157275140285492, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.5937150716781616, "sampling/importance_sampling_ratio/mean": 0.9997748732566833, "sampling/importance_sampling_ratio/min": 0.6914979815483093, "sampling/sampling_logp_difference/max": 0.4660677909851074, "sampling/sampling_logp_difference/mean": 0.015581191517412663, "step": 928 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 726.0, "completions/mean_length": 533.34375, "completions/mean_terminated_length": 489.8888854980469, "completions/min_length": 235.0, "completions/min_terminated_length": 235.0, "entropy": 0.3309678826481104, "epoch": 0.49732334047109206, "frac_reward_zero_std": 0.0, "grad_norm": 0.008973688818514347, "learning_rate": 1e-05, "loss": 0.057, "num_tokens": 18462743.0, "reward": 0.78125, "reward_std": 0.4218915104866028, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.398266315460205, "sampling/importance_sampling_ratio/mean": 0.9999146461486816, "sampling/importance_sampling_ratio/min": 0.6988242864608765, "sampling/sampling_logp_difference/max": 0.3583559989929199, "sampling/sampling_logp_difference/mean": 0.01103132776916027, "step": 929 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 633.0, "completions/max_terminated_length": 633.0, "completions/mean_length": 430.40625, "completions/mean_terminated_length": 430.40625, "completions/min_length": 236.0, "completions/min_terminated_length": 236.0, "entropy": 0.22674109786748886, "epoch": 0.49785867237687365, "frac_reward_zero_std": 0.25, "grad_norm": 0.032213736325502396, "learning_rate": 1e-05, "loss": -0.012, "num_tokens": 18479812.0, "reward": 0.59375, "reward_std": 0.3377464711666107, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.3437047004699707, "sampling/importance_sampling_ratio/mean": 1.0004483461380005, "sampling/importance_sampling_ratio/min": 0.7815067768096924, "sampling/sampling_logp_difference/max": 0.29543042182922363, "sampling/sampling_logp_difference/mean": 0.00871048029512167, "step": 930 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 751.0, "completions/mean_length": 484.125, "completions/mean_terminated_length": 454.75860595703125, "completions/min_length": 219.0, "completions/min_terminated_length": 219.0, "entropy": 0.35440368205308914, "epoch": 0.49839400428265523, "frac_reward_zero_std": 0.25, "grad_norm": 0.01829420030117035, "learning_rate": 1e-05, "loss": 0.0341, "num_tokens": 18499112.0, "reward": 0.3125, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.3125, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.365425705909729, "sampling/importance_sampling_ratio/mean": 0.9999644160270691, "sampling/importance_sampling_ratio/min": 0.1889118105173111, "sampling/sampling_logp_difference/max": 1.6664749383926392, "sampling/sampling_logp_difference/mean": 0.012364272028207779, "step": 931 }, { "clip_ratio/high_max": 6.124448555056006e-05, "clip_ratio/high_mean": 6.124448555056006e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 6.124448555056006e-05, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 575.0, "completions/mean_length": 439.5625, "completions/mean_terminated_length": 405.5862121582031, "completions/min_length": 190.0, "completions/min_terminated_length": 190.0, "entropy": 0.40438541769981384, "epoch": 0.4989293361884368, "frac_reward_zero_std": 0.25, "grad_norm": 0.014041540212929249, "learning_rate": 1e-05, "loss": -0.025, "num_tokens": 18517426.0, "reward": 0.71875, "reward_std": 0.3787454068660736, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.5562971830368042, "sampling/importance_sampling_ratio/mean": 1.0000966787338257, "sampling/importance_sampling_ratio/min": 0.6646143198013306, "sampling/sampling_logp_difference/max": 0.4423093795776367, "sampling/sampling_logp_difference/mean": 0.01447139959782362, "step": 932 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.191183638293296e-05, "clip_ratio/low_min": 6.191183638293296e-05, "clip_ratio/region_mean": 6.191183638293296e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 612.0, "completions/max_terminated_length": 612.0, "completions/mean_length": 467.59375, "completions/mean_terminated_length": 467.59375, "completions/min_length": 267.0, "completions/min_terminated_length": 267.0, "entropy": 0.2528857309371233, "epoch": 0.4994646680942184, "frac_reward_zero_std": 0.25, "grad_norm": 0.023867884650826454, "learning_rate": 1e-05, "loss": -0.0065, "num_tokens": 18535997.0, "reward": 0.6875, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.3603086471557617, "sampling/importance_sampling_ratio/mean": 0.9998392462730408, "sampling/importance_sampling_ratio/min": 0.681882917881012, "sampling/sampling_logp_difference/max": 0.38289737701416016, "sampling/sampling_logp_difference/mean": 0.009321975521743298, "step": 933 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 9.619084448786452e-05, "clip_ratio/low_min": 9.619084448786452e-05, "clip_ratio/region_mean": 9.619084448786452e-05, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 718.0, "completions/mean_length": 463.0625, "completions/mean_terminated_length": 442.7333679199219, "completions/min_length": 249.0, "completions/min_terminated_length": 249.0, "entropy": 0.3133617974817753, "epoch": 0.5, "frac_reward_zero_std": 0.25, "grad_norm": 0.011186545714735985, "learning_rate": 1e-05, "loss": 0.0664, "num_tokens": 18554479.0, "reward": 0.65625, "reward_std": 0.3198433816432953, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.3334717750549316, "sampling/importance_sampling_ratio/mean": 1.0001076459884644, "sampling/importance_sampling_ratio/min": 0.7463027238845825, "sampling/sampling_logp_difference/max": 0.29262399673461914, "sampling/sampling_logp_difference/mean": 0.011498593725264072, "step": 934 }, { "clip_ratio/high_max": 7.11845132173039e-05, "clip_ratio/high_mean": 7.11845132173039e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 7.11845132173039e-05, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 687.0, "completions/mean_length": 445.0625, "completions/mean_terminated_length": 411.6551818847656, "completions/min_length": 211.0, "completions/min_terminated_length": 211.0, "entropy": 0.2541938256472349, "epoch": 0.5005353319057816, "frac_reward_zero_std": 0.25, "grad_norm": 0.011226793751120567, "learning_rate": 1e-05, "loss": 0.0045, "num_tokens": 18572641.0, "reward": 0.78125, "reward_std": 0.3061639666557312, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.9631344079971313, "sampling/importance_sampling_ratio/mean": 1.000319242477417, "sampling/importance_sampling_ratio/min": 0.7012016773223877, "sampling/sampling_logp_difference/max": 0.6745424270629883, "sampling/sampling_logp_difference/mean": 0.009384569711983204, "step": 935 }, { "clip_ratio/high_max": 0.00012135922588640824, "clip_ratio/high_mean": 0.00012135922588640824, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00012135922588640824, "completions/clipped_ratio": 0.0, "completions/max_length": 715.0, "completions/max_terminated_length": 715.0, "completions/mean_length": 403.5625, "completions/mean_terminated_length": 403.5625, "completions/min_length": 169.0, "completions/min_terminated_length": 169.0, "entropy": 0.2944677174091339, "epoch": 0.5010706638115632, "frac_reward_zero_std": 0.25, "grad_norm": 0.032058071345090866, "learning_rate": 1e-05, "loss": -0.0307, "num_tokens": 18588859.0, "reward": 0.5625, "reward_std": 0.3471825420856476, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.7584322690963745, "sampling/importance_sampling_ratio/mean": 0.9996654391288757, "sampling/importance_sampling_ratio/min": 0.7382290363311768, "sampling/sampling_logp_difference/max": 0.564422607421875, "sampling/sampling_logp_difference/mean": 0.01127475406974554, "step": 936 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.76834318053443e-05, "clip_ratio/low_min": 5.76834318053443e-05, "clip_ratio/region_mean": 5.76834318053443e-05, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 660.0, "completions/mean_length": 502.21875, "completions/mean_terminated_length": 484.5000305175781, "completions/min_length": 316.0, "completions/min_terminated_length": 316.0, "entropy": 0.3367830812931061, "epoch": 0.5016059957173448, "frac_reward_zero_std": 0.25, "grad_norm": 0.015154167078435421, "learning_rate": 1e-05, "loss": -0.0086, "num_tokens": 18608762.0, "reward": 0.4375, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.4375, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.4711087942123413, "sampling/importance_sampling_ratio/mean": 1.0000927448272705, "sampling/importance_sampling_ratio/min": 0.6639459729194641, "sampling/sampling_logp_difference/max": 0.40955448150634766, "sampling/sampling_logp_difference/mean": 0.011766834184527397, "step": 937 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 7.575757626909763e-05, "clip_ratio/low_min": 7.575757626909763e-05, "clip_ratio/region_mean": 7.575757626909763e-05, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 493.0, "completions/mean_length": 353.65625, "completions/mean_terminated_length": 340.2903137207031, "completions/min_length": 215.0, "completions/min_terminated_length": 215.0, "entropy": 0.3739020489156246, "epoch": 0.5021413276231264, "frac_reward_zero_std": 0.5, "grad_norm": 0.03872900456190109, "learning_rate": 1e-05, "loss": 0.1192, "num_tokens": 18624191.0, "reward": 0.9375, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.9375, "rewards/accuracy_reward/std": 0.24593468010425568, "sampling/importance_sampling_ratio/max": 1.3281844854354858, "sampling/importance_sampling_ratio/mean": 0.9997400045394897, "sampling/importance_sampling_ratio/min": 0.5952030420303345, "sampling/sampling_logp_difference/max": 0.518852710723877, "sampling/sampling_logp_difference/mean": 0.01244304608553648, "step": 938 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 613.0, "completions/mean_length": 435.8125, "completions/mean_terminated_length": 425.0967712402344, "completions/min_length": 245.0, "completions/min_terminated_length": 245.0, "entropy": 0.2987457551062107, "epoch": 0.5026766595289079, "frac_reward_zero_std": 0.5, "grad_norm": 0.007831594906747341, "learning_rate": 1e-05, "loss": 0.0256, "num_tokens": 18641537.0, "reward": 0.75, "reward_std": 0.2314550280570984, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.4131138324737549, "sampling/importance_sampling_ratio/mean": 1.0005991458892822, "sampling/importance_sampling_ratio/min": 0.6951550841331482, "sampling/sampling_logp_difference/max": 0.3636202812194824, "sampling/sampling_logp_difference/mean": 0.010999714024364948, "step": 939 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.6306307669728994e-05, "clip_ratio/low_min": 5.6306307669728994e-05, "clip_ratio/region_mean": 5.6306307669728994e-05, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 717.0, "completions/mean_length": 521.25, "completions/mean_terminated_length": 513.290283203125, "completions/min_length": 332.0, "completions/min_terminated_length": 332.0, "entropy": 0.31170865148305893, "epoch": 0.5032119914346895, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0044, "num_tokens": 18661945.0, "reward": 0.71875, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.5880764722824097, "sampling/importance_sampling_ratio/mean": 1.000365138053894, "sampling/importance_sampling_ratio/min": 0.585188627243042, "sampling/sampling_logp_difference/max": 0.5358209609985352, "sampling/sampling_logp_difference/mean": 0.011812165379524231, "step": 940 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00012867815530626103, "clip_ratio/low_min": 0.00012867815530626103, "clip_ratio/region_mean": 0.00012867815530626103, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 563.0, "completions/mean_length": 446.3125, "completions/mean_terminated_length": 435.93548583984375, "completions/min_length": 265.0, "completions/min_terminated_length": 265.0, "entropy": 0.2936486452817917, "epoch": 0.5037473233404711, "frac_reward_zero_std": 0.5, "grad_norm": 0.02061215043067932, "learning_rate": 1e-05, "loss": 0.0116, "num_tokens": 18680363.0, "reward": 0.71875, "reward_std": 0.2630178928375244, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.485531210899353, "sampling/importance_sampling_ratio/mean": 1.0001529455184937, "sampling/importance_sampling_ratio/min": 0.6224051713943481, "sampling/sampling_logp_difference/max": 0.4741640090942383, "sampling/sampling_logp_difference/mean": 0.010788899846374989, "step": 941 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.003842281643301e-05, "clip_ratio/low_min": 6.003842281643301e-05, "clip_ratio/region_mean": 6.003842281643301e-05, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 747.0, "completions/mean_length": 473.40625, "completions/mean_terminated_length": 442.9310302734375, "completions/min_length": 249.0, "completions/min_terminated_length": 249.0, "entropy": 0.2970592677593231, "epoch": 0.5042826552462527, "frac_reward_zero_std": 0.25, "grad_norm": 0.02228197641670704, "learning_rate": 1e-05, "loss": 0.0424, "num_tokens": 18699296.0, "reward": 0.6875, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.9042567014694214, "sampling/importance_sampling_ratio/mean": 0.9999818205833435, "sampling/importance_sampling_ratio/min": 0.6825937032699585, "sampling/sampling_logp_difference/max": 0.6440917253494263, "sampling/sampling_logp_difference/mean": 0.01097430195659399, "step": 942 }, { "clip_ratio/high_max": 5.857544601894915e-05, "clip_ratio/high_mean": 5.857544601894915e-05, "clip_ratio/low_mean": 0.00013716623652726412, "clip_ratio/low_min": 0.00013716623652726412, "clip_ratio/region_mean": 0.00019574168254621327, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 681.0, "completions/mean_length": 463.3125, "completions/mean_terminated_length": 453.4838562011719, "completions/min_length": 230.0, "completions/min_terminated_length": 230.0, "entropy": 0.32913353852927685, "epoch": 0.5048179871520343, "frac_reward_zero_std": 0.25, "grad_norm": 0.010207954794168472, "learning_rate": 1e-05, "loss": 0.0461, "num_tokens": 18717842.0, "reward": 0.8125, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.385231375694275, "sampling/importance_sampling_ratio/mean": 1.0003337860107422, "sampling/importance_sampling_ratio/min": 0.6972249150276184, "sampling/sampling_logp_difference/max": 0.36064720153808594, "sampling/sampling_logp_difference/mean": 0.011917158961296082, "step": 943 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.163708167150617e-05, "clip_ratio/low_min": 6.163708167150617e-05, "clip_ratio/region_mean": 6.163708167150617e-05, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 613.0, "completions/mean_length": 428.03125, "completions/mean_terminated_length": 417.06451416015625, "completions/min_length": 197.0, "completions/min_terminated_length": 197.0, "entropy": 0.2715218998491764, "epoch": 0.5053533190578159, "frac_reward_zero_std": 0.5, "grad_norm": 0.007468083873391151, "learning_rate": 1e-05, "loss": 0.0921, "num_tokens": 18734731.0, "reward": 0.65625, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.309759497642517, "sampling/importance_sampling_ratio/mean": 1.0000685453414917, "sampling/importance_sampling_ratio/min": 0.6160060167312622, "sampling/sampling_logp_difference/max": 0.4844985008239746, "sampling/sampling_logp_difference/mean": 0.010679789818823338, "step": 944 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 714.0, "completions/mean_length": 488.09375, "completions/mean_terminated_length": 448.107177734375, "completions/min_length": 225.0, "completions/min_terminated_length": 225.0, "entropy": 0.2994806170463562, "epoch": 0.5058886509635975, "frac_reward_zero_std": 0.25, "grad_norm": 0.024860825389623642, "learning_rate": 1e-05, "loss": 0.0204, "num_tokens": 18753590.0, "reward": 0.75, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.4279488325119019, "sampling/importance_sampling_ratio/mean": 0.9999743700027466, "sampling/importance_sampling_ratio/min": 0.7017055749893188, "sampling/sampling_logp_difference/max": 0.35623908042907715, "sampling/sampling_logp_difference/mean": 0.009921894408762455, "step": 945 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.56537636637222e-05, "clip_ratio/low_min": 4.56537636637222e-05, "clip_ratio/region_mean": 4.56537636637222e-05, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 758.0, "completions/mean_length": 539.40625, "completions/mean_terminated_length": 497.0740661621094, "completions/min_length": 188.0, "completions/min_terminated_length": 188.0, "entropy": 0.48707421869039536, "epoch": 0.5064239828693791, "frac_reward_zero_std": 0.25, "grad_norm": 0.01256539486348629, "learning_rate": 1e-05, "loss": 0.0928, "num_tokens": 18774515.0, "reward": 0.75, "reward_std": 0.3745020925998688, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.5281684398651123, "sampling/importance_sampling_ratio/mean": 1.0002484321594238, "sampling/importance_sampling_ratio/min": 0.6692571640014648, "sampling/sampling_logp_difference/max": 0.424069881439209, "sampling/sampling_logp_difference/mean": 0.015065666288137436, "step": 946 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.887052586534992e-05, "clip_ratio/low_min": 6.887052586534992e-05, "clip_ratio/region_mean": 6.887052586534992e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 590.0, "completions/max_terminated_length": 590.0, "completions/mean_length": 388.75, "completions/mean_terminated_length": 388.75, "completions/min_length": 138.0, "completions/min_terminated_length": 138.0, "entropy": 0.28587933629751205, "epoch": 0.5069593147751607, "frac_reward_zero_std": 0.75, "grad_norm": 0.035891104489564896, "learning_rate": 1e-05, "loss": -0.0064, "num_tokens": 18790155.0, "reward": 0.8125, "reward_std": 0.1157275140285492, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.4828362464904785, "sampling/importance_sampling_ratio/mean": 1.0002168416976929, "sampling/importance_sampling_ratio/min": 0.6562126278877258, "sampling/sampling_logp_difference/max": 0.42127037048339844, "sampling/sampling_logp_difference/mean": 0.010166658088564873, "step": 947 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00014253899280447513, "clip_ratio/low_min": 0.00014253899280447513, "clip_ratio/region_mean": 0.00014253899280447513, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 614.0, "completions/mean_length": 452.1875, "completions/mean_terminated_length": 442.0, "completions/min_length": 337.0, "completions/min_terminated_length": 337.0, "entropy": 0.2540544904768467, "epoch": 0.5074946466809421, "frac_reward_zero_std": 0.5, "grad_norm": 0.00658551836386323, "learning_rate": 1e-05, "loss": 0.0424, "num_tokens": 18808257.0, "reward": 0.875, "reward_std": 0.2314550280570984, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9995406866073608, "sampling/importance_sampling_ratio/min": 0.6581525802612305, "sampling/sampling_logp_difference/max": 1.0281407833099365, "sampling/sampling_logp_difference/mean": 0.010223720222711563, "step": 948 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00023598983898409642, "clip_ratio/low_min": 0.00023598983898409642, "clip_ratio/region_mean": 0.00023598983898409642, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 744.0, "completions/mean_length": 504.0, "completions/mean_terminated_length": 466.2857360839844, "completions/min_length": 285.0, "completions/min_terminated_length": 285.0, "entropy": 0.3444122467190027, "epoch": 0.5080299785867237, "frac_reward_zero_std": 0.25, "grad_norm": 0.01840297132730484, "learning_rate": 1e-05, "loss": 0.0339, "num_tokens": 18828345.0, "reward": 0.625, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.636265754699707, "sampling/importance_sampling_ratio/mean": 1.0002254247665405, "sampling/importance_sampling_ratio/min": 0.6133067011833191, "sampling/sampling_logp_difference/max": 0.4924166202545166, "sampling/sampling_logp_difference/mean": 0.012093781493604183, "step": 949 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 756.0, "completions/max_terminated_length": 756.0, "completions/mean_length": 494.3125, "completions/mean_terminated_length": 494.3125, "completions/min_length": 249.0, "completions/min_terminated_length": 249.0, "entropy": 0.27808588184416294, "epoch": 0.5085653104925053, "frac_reward_zero_std": 0.5, "grad_norm": 0.013713176362216473, "learning_rate": 1e-05, "loss": -0.0167, "num_tokens": 18847931.0, "reward": 0.8125, "reward_std": 0.2587745785713196, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.5939327478408813, "sampling/importance_sampling_ratio/mean": 0.9996863007545471, "sampling/importance_sampling_ratio/min": 0.6843679547309875, "sampling/sampling_logp_difference/max": 0.4662044048309326, "sampling/sampling_logp_difference/mean": 0.010790374130010605, "step": 950 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 716.0, "completions/max_terminated_length": 716.0, "completions/mean_length": 405.1875, "completions/mean_terminated_length": 405.1875, "completions/min_length": 207.0, "completions/min_terminated_length": 207.0, "entropy": 0.34625665098428726, "epoch": 0.5091006423982869, "frac_reward_zero_std": 0.25, "grad_norm": 0.019684698432683945, "learning_rate": 1e-05, "loss": -0.0817, "num_tokens": 18864417.0, "reward": 0.875, "reward_std": 0.2925041913986206, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 1.4535462856292725, "sampling/importance_sampling_ratio/mean": 0.9996865391731262, "sampling/importance_sampling_ratio/min": 0.7225186824798584, "sampling/sampling_logp_difference/max": 0.3740062713623047, "sampling/sampling_logp_difference/mean": 0.01191488653421402, "step": 951 }, { "clip_ratio/high_max": 4.736642586067319e-05, "clip_ratio/high_mean": 4.736642586067319e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 4.736642586067319e-05, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 764.0, "completions/mean_length": 487.3125, "completions/mean_terminated_length": 478.258056640625, "completions/min_length": 212.0, "completions/min_terminated_length": 212.0, "entropy": 0.2598949037492275, "epoch": 0.5096359743040685, "frac_reward_zero_std": 0.5, "grad_norm": 0.028428837656974792, "learning_rate": 1e-05, "loss": 0.0199, "num_tokens": 18883467.0, "reward": 0.9375, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.9375, "rewards/accuracy_reward/std": 0.24593468010425568, "sampling/importance_sampling_ratio/max": 1.8784284591674805, "sampling/importance_sampling_ratio/mean": 1.0001908540725708, "sampling/importance_sampling_ratio/min": 0.640715479850769, "sampling/sampling_logp_difference/max": 0.6304354667663574, "sampling/sampling_logp_difference/mean": 0.009616902098059654, "step": 952 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00011550290582817979, "clip_ratio/low_min": 0.00011550290582817979, "clip_ratio/region_mean": 0.00011550290582817979, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 718.0, "completions/mean_length": 461.84375, "completions/mean_terminated_length": 441.433349609375, "completions/min_length": 251.0, "completions/min_terminated_length": 251.0, "entropy": 0.3623839858919382, "epoch": 0.5101713062098501, "frac_reward_zero_std": 0.5, "grad_norm": 0.009525010362267494, "learning_rate": 1e-05, "loss": 0.0311, "num_tokens": 18902030.0, "reward": 0.625, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.32841157913208, "sampling/importance_sampling_ratio/mean": 1.0002055168151855, "sampling/importance_sampling_ratio/min": 0.4049910306930542, "sampling/sampling_logp_difference/max": 0.9038903713226318, "sampling/sampling_logp_difference/mean": 0.011523718945682049, "step": 953 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.225752465776168e-05, "clip_ratio/low_min": 5.225752465776168e-05, "clip_ratio/region_mean": 5.225752465776168e-05, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 694.0, "completions/mean_length": 500.5, "completions/mean_terminated_length": 462.2857360839844, "completions/min_length": 301.0, "completions/min_terminated_length": 301.0, "entropy": 0.35181020572781563, "epoch": 0.5107066381156317, "frac_reward_zero_std": 0.5, "grad_norm": 0.013160860165953636, "learning_rate": 1e-05, "loss": -0.0162, "num_tokens": 18922510.0, "reward": 0.71875, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.6601618528366089, "sampling/importance_sampling_ratio/mean": 1.0003364086151123, "sampling/importance_sampling_ratio/min": 0.7017132639884949, "sampling/sampling_logp_difference/max": 0.5069150924682617, "sampling/sampling_logp_difference/mean": 0.011754768900573254, "step": 954 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 754.0, "completions/max_terminated_length": 754.0, "completions/mean_length": 443.6875, "completions/mean_terminated_length": 443.6875, "completions/min_length": 154.0, "completions/min_terminated_length": 154.0, "entropy": 0.2924421951174736, "epoch": 0.5112419700214133, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 18940556.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.4143191576004028, "sampling/importance_sampling_ratio/mean": 0.9999143481254578, "sampling/importance_sampling_ratio/min": 0.7306100130081177, "sampling/sampling_logp_difference/max": 0.3466482162475586, "sampling/sampling_logp_difference/mean": 0.01069424208253622, "step": 955 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.955216693109833e-05, "clip_ratio/low_min": 5.955216693109833e-05, "clip_ratio/region_mean": 5.955216693109833e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 699.0, "completions/max_terminated_length": 699.0, "completions/mean_length": 442.28125, "completions/mean_terminated_length": 442.28125, "completions/min_length": 166.0, "completions/min_terminated_length": 166.0, "entropy": 0.34365570545196533, "epoch": 0.5117773019271948, "frac_reward_zero_std": 0.5, "grad_norm": 0.013136308640241623, "learning_rate": 1e-05, "loss": 0.0678, "num_tokens": 18958221.0, "reward": 0.84375, "reward_std": 0.22201895713806152, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.5441542863845825, "sampling/importance_sampling_ratio/mean": 0.9999651908874512, "sampling/importance_sampling_ratio/min": 0.7165300250053406, "sampling/sampling_logp_difference/max": 0.434476375579834, "sampling/sampling_logp_difference/mean": 0.011885290965437889, "step": 956 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00012048192729707807, "clip_ratio/low_min": 0.00012048192729707807, "clip_ratio/region_mean": 0.00012048192729707807, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 736.0, "completions/mean_length": 458.625, "completions/mean_terminated_length": 448.6451416015625, "completions/min_length": 280.0, "completions/min_terminated_length": 280.0, "entropy": 0.24939272366464138, "epoch": 0.5123126338329764, "frac_reward_zero_std": 0.5, "grad_norm": 0.021936535835266113, "learning_rate": 1e-05, "loss": 0.0042, "num_tokens": 18976265.0, "reward": 0.90625, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.4704183340072632, "sampling/importance_sampling_ratio/mean": 0.9997578859329224, "sampling/importance_sampling_ratio/min": 0.6308900117874146, "sampling/sampling_logp_difference/max": 0.46062374114990234, "sampling/sampling_logp_difference/mean": 0.009267176501452923, "step": 957 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 698.0, "completions/mean_length": 489.09375, "completions/mean_terminated_length": 470.5000305175781, "completions/min_length": 265.0, "completions/min_terminated_length": 265.0, "entropy": 0.29304561018943787, "epoch": 0.512847965738758, "frac_reward_zero_std": 0.75, "grad_norm": 0.003565209684893489, "learning_rate": 1e-05, "loss": 0.0305, "num_tokens": 18995868.0, "reward": 0.96875, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.96875, "rewards/accuracy_reward/std": 0.1767766922712326, "sampling/importance_sampling_ratio/max": 1.336312174797058, "sampling/importance_sampling_ratio/mean": 0.9999875426292419, "sampling/importance_sampling_ratio/min": 0.6728830933570862, "sampling/sampling_logp_difference/max": 0.39618372917175293, "sampling/sampling_logp_difference/mean": 0.010725563392043114, "step": 958 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.364562432281673e-05, "clip_ratio/low_min": 6.364562432281673e-05, "clip_ratio/region_mean": 6.364562432281673e-05, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 701.0, "completions/mean_length": 473.90625, "completions/mean_terminated_length": 464.4193420410156, "completions/min_length": 88.0, "completions/min_terminated_length": 88.0, "entropy": 0.24543077498674393, "epoch": 0.5133832976445396, "frac_reward_zero_std": 0.5, "grad_norm": 0.015022432431578636, "learning_rate": 1e-05, "loss": 0.0147, "num_tokens": 19014689.0, "reward": 0.875, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 1.8724751472473145, "sampling/importance_sampling_ratio/mean": 1.000110387802124, "sampling/importance_sampling_ratio/min": 0.7357137799263, "sampling/sampling_logp_difference/max": 0.6272611618041992, "sampling/sampling_logp_difference/mean": 0.00897144339978695, "step": 959 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.49688154226169e-05, "clip_ratio/low_min": 6.49688154226169e-05, "clip_ratio/region_mean": 6.49688154226169e-05, "completions/clipped_ratio": 0.28125, "completions/max_length": 768.0, "completions/max_terminated_length": 696.0, "completions/mean_length": 582.5625, "completions/mean_terminated_length": 510.0, "completions/min_length": 216.0, "completions/min_terminated_length": 216.0, "entropy": 0.4166363328695297, "epoch": 0.5139186295503212, "frac_reward_zero_std": 0.5, "grad_norm": 0.016490228474140167, "learning_rate": 1e-05, "loss": -0.014, "num_tokens": 19037267.0, "reward": 0.125, "reward_std": 0.2314550280570984, "rewards/accuracy_reward/mean": 0.125, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 1.3851897716522217, "sampling/importance_sampling_ratio/mean": 1.0000791549682617, "sampling/importance_sampling_ratio/min": 0.7044484615325928, "sampling/sampling_logp_difference/max": 0.3503401279449463, "sampling/sampling_logp_difference/mean": 0.013296709395945072, "step": 960 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 726.0, "completions/max_terminated_length": 726.0, "completions/mean_length": 448.21875, "completions/mean_terminated_length": 448.21875, "completions/min_length": 258.0, "completions/min_terminated_length": 258.0, "entropy": 0.2712840549647808, "epoch": 0.5144539614561028, "frac_reward_zero_std": 0.5, "grad_norm": 0.0034663125406950712, "learning_rate": 1e-05, "loss": 0.061, "num_tokens": 19055258.0, "reward": 0.6875, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.6785802841186523, "sampling/importance_sampling_ratio/mean": 0.9998134970664978, "sampling/importance_sampling_ratio/min": 0.6850326657295227, "sampling/sampling_logp_difference/max": 0.5179483890533447, "sampling/sampling_logp_difference/mean": 0.010201629251241684, "step": 961 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00017628124260227196, "clip_ratio/low_min": 0.00017628124260227196, "clip_ratio/region_mean": 0.00017628124260227196, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 660.0, "completions/mean_length": 519.25, "completions/mean_terminated_length": 461.8461608886719, "completions/min_length": 298.0, "completions/min_terminated_length": 298.0, "entropy": 0.38563186675310135, "epoch": 0.5149892933618844, "frac_reward_zero_std": 0.5, "grad_norm": 0.008774523623287678, "learning_rate": 1e-05, "loss": -0.0085, "num_tokens": 19075802.0, "reward": 0.625, "reward_std": 0.2314550280570984, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.4578242301940918, "sampling/importance_sampling_ratio/mean": 0.999571681022644, "sampling/importance_sampling_ratio/min": 0.626001238822937, "sampling/sampling_logp_difference/max": 0.4684029817581177, "sampling/sampling_logp_difference/mean": 0.012711354531347752, "step": 962 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 657.0, "completions/mean_length": 441.875, "completions/mean_terminated_length": 431.3548278808594, "completions/min_length": 229.0, "completions/min_terminated_length": 229.0, "entropy": 0.30633309856057167, "epoch": 0.515524625267666, "frac_reward_zero_std": 0.5, "grad_norm": 0.02011452428996563, "learning_rate": 1e-05, "loss": 0.006, "num_tokens": 19093990.0, "reward": 0.90625, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.4361798763275146, "sampling/importance_sampling_ratio/mean": 0.999936580657959, "sampling/importance_sampling_ratio/min": 0.6223956346511841, "sampling/sampling_logp_difference/max": 0.4741792678833008, "sampling/sampling_logp_difference/mean": 0.011413052678108215, "step": 963 }, { "clip_ratio/high_max": 6.513809057651088e-05, "clip_ratio/high_mean": 6.513809057651088e-05, "clip_ratio/low_mean": 6.094588025007397e-05, "clip_ratio/low_min": 6.094588025007397e-05, "clip_ratio/region_mean": 0.00012608397082658485, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 636.0, "completions/mean_length": 460.90625, "completions/mean_terminated_length": 440.433349609375, "completions/min_length": 306.0, "completions/min_terminated_length": 306.0, "entropy": 0.2541094161570072, "epoch": 0.5160599571734475, "frac_reward_zero_std": 0.25, "grad_norm": 0.015052350237965584, "learning_rate": 1e-05, "loss": 0.0081, "num_tokens": 19112347.0, "reward": 0.71875, "reward_std": 0.35564959049224854, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.4809685945510864, "sampling/importance_sampling_ratio/mean": 1.000511646270752, "sampling/importance_sampling_ratio/min": 0.6855025291442871, "sampling/sampling_logp_difference/max": 0.3926963806152344, "sampling/sampling_logp_difference/mean": 0.009388880804181099, "step": 964 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 765.0, "completions/mean_length": 492.84375, "completions/mean_terminated_length": 464.3793029785156, "completions/min_length": 279.0, "completions/min_terminated_length": 279.0, "entropy": 0.31386849097907543, "epoch": 0.5165952890792291, "frac_reward_zero_std": 0.25, "grad_norm": 0.006239220499992371, "learning_rate": 1e-05, "loss": 0.0961, "num_tokens": 19131846.0, "reward": 0.78125, "reward_std": 0.3061639666557312, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.675916314125061, "sampling/importance_sampling_ratio/mean": 1.0001627206802368, "sampling/importance_sampling_ratio/min": 0.7222133278846741, "sampling/sampling_logp_difference/max": 0.5163600444793701, "sampling/sampling_logp_difference/mean": 0.010937148705124855, "step": 965 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.329113966785371e-05, "clip_ratio/low_min": 6.329113966785371e-05, "clip_ratio/region_mean": 6.329113966785371e-05, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 737.0, "completions/mean_length": 478.78125, "completions/mean_terminated_length": 469.45159912109375, "completions/min_length": 247.0, "completions/min_terminated_length": 247.0, "entropy": 0.277352349832654, "epoch": 0.5171306209850107, "frac_reward_zero_std": 0.5, "grad_norm": 0.013667028397321701, "learning_rate": 1e-05, "loss": -0.0055, "num_tokens": 19151223.0, "reward": 0.78125, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.4211808443069458, "sampling/importance_sampling_ratio/mean": 0.9998354911804199, "sampling/importance_sampling_ratio/min": 0.7232437133789062, "sampling/sampling_logp_difference/max": 0.3514881134033203, "sampling/sampling_logp_difference/mean": 0.010061788372695446, "step": 966 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 716.0, "completions/mean_length": 484.71875, "completions/mean_terminated_length": 475.58062744140625, "completions/min_length": 124.0, "completions/min_terminated_length": 124.0, "entropy": 0.24185454845428467, "epoch": 0.5176659528907923, "frac_reward_zero_std": 0.75, "grad_norm": 0.006015704944729805, "learning_rate": 1e-05, "loss": -0.0032, "num_tokens": 19170782.0, "reward": 0.78125, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.3794660568237305, "sampling/importance_sampling_ratio/mean": 0.9993918538093567, "sampling/importance_sampling_ratio/min": 0.6304227113723755, "sampling/sampling_logp_difference/max": 0.46136474609375, "sampling/sampling_logp_difference/mean": 0.009323207661509514, "step": 967 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0001298701245104894, "clip_ratio/low_min": 0.0001298701245104894, "clip_ratio/region_mean": 0.0001298701245104894, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 682.0, "completions/mean_length": 503.15625, "completions/mean_terminated_length": 485.5000305175781, "completions/min_length": 304.0, "completions/min_terminated_length": 304.0, "entropy": 0.3035869039595127, "epoch": 0.5182012847965739, "frac_reward_zero_std": 0.25, "grad_norm": 0.01092250645160675, "learning_rate": 1e-05, "loss": -0.0309, "num_tokens": 19191171.0, "reward": 0.59375, "reward_std": 0.378745436668396, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.4265621900558472, "sampling/importance_sampling_ratio/mean": 0.9999486804008484, "sampling/importance_sampling_ratio/min": 0.6853776574134827, "sampling/sampling_logp_difference/max": 0.37778520584106445, "sampling/sampling_logp_difference/mean": 0.01078245509415865, "step": 968 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.375, "completions/max_length": 768.0, "completions/max_terminated_length": 760.0, "completions/mean_length": 619.96875, "completions/mean_terminated_length": 531.1500244140625, "completions/min_length": 290.0, "completions/min_terminated_length": 290.0, "entropy": 0.43357282504439354, "epoch": 0.5187366167023555, "frac_reward_zero_std": 0.5, "grad_norm": 0.013624757528305054, "learning_rate": 1e-05, "loss": 0.0109, "num_tokens": 19215442.0, "reward": 0.46875, "reward_std": 0.2630178928375244, "rewards/accuracy_reward/mean": 0.46875, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.3855798244476318, "sampling/importance_sampling_ratio/mean": 1.0001927614212036, "sampling/importance_sampling_ratio/min": 0.34400755167007446, "sampling/sampling_logp_difference/max": 1.067091703414917, "sampling/sampling_logp_difference/mean": 0.012979980558156967, "step": 969 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 567.0, "completions/max_terminated_length": 567.0, "completions/mean_length": 344.6875, "completions/mean_terminated_length": 344.6875, "completions/min_length": 218.0, "completions/min_terminated_length": 218.0, "entropy": 0.2605263330042362, "epoch": 0.5192719486081371, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 19230104.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.431227445602417, "sampling/importance_sampling_ratio/mean": 1.0000169277191162, "sampling/importance_sampling_ratio/min": 0.6947574019432068, "sampling/sampling_logp_difference/max": 0.36419248580932617, "sampling/sampling_logp_difference/mean": 0.00965052843093872, "step": 970 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00011839962462545373, "clip_ratio/low_min": 0.00011839962462545373, "clip_ratio/region_mean": 0.00011839962462545373, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 765.0, "completions/mean_length": 551.09375, "completions/mean_terminated_length": 528.6551513671875, "completions/min_length": 270.0, "completions/min_terminated_length": 270.0, "entropy": 0.3060551155358553, "epoch": 0.5198072805139187, "frac_reward_zero_std": 0.25, "grad_norm": 0.016152273863554, "learning_rate": 1e-05, "loss": 0.0353, "num_tokens": 19251691.0, "reward": 0.6875, "reward_std": 0.3514062464237213, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.365064024925232, "sampling/importance_sampling_ratio/mean": 0.999962568283081, "sampling/importance_sampling_ratio/min": 0.5747963190078735, "sampling/sampling_logp_difference/max": 0.5537395477294922, "sampling/sampling_logp_difference/mean": 0.010949786752462387, "step": 971 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00021914325043326244, "clip_ratio/low_min": 0.00021914325043326244, "clip_ratio/region_mean": 0.00021914325043326244, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 745.0, "completions/mean_length": 506.53125, "completions/mean_terminated_length": 446.19232177734375, "completions/min_length": 250.0, "completions/min_terminated_length": 250.0, "entropy": 0.3677872810512781, "epoch": 0.5203426124197003, "frac_reward_zero_std": 0.25, "grad_norm": 0.012668418698012829, "learning_rate": 1e-05, "loss": 0.1049, "num_tokens": 19271316.0, "reward": 0.625, "reward_std": 0.3924052119255066, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.3050062656402588, "sampling/importance_sampling_ratio/mean": 0.9999018907546997, "sampling/importance_sampling_ratio/min": 0.6798360347747803, "sampling/sampling_logp_difference/max": 0.38590359687805176, "sampling/sampling_logp_difference/mean": 0.012253649532794952, "step": 972 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.791107858181931e-05, "clip_ratio/low_min": 4.791107858181931e-05, "clip_ratio/region_mean": 4.791107858181931e-05, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 657.0, "completions/mean_length": 490.28125, "completions/mean_terminated_length": 438.85186767578125, "completions/min_length": 181.0, "completions/min_terminated_length": 181.0, "entropy": 0.3771228492259979, "epoch": 0.5208779443254818, "frac_reward_zero_std": 0.5, "grad_norm": 0.008566985838115215, "learning_rate": 1e-05, "loss": 0.0809, "num_tokens": 19291005.0, "reward": 0.8125, "reward_std": 0.2587745785713196, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.5105116367340088, "sampling/importance_sampling_ratio/mean": 1.0001238584518433, "sampling/importance_sampling_ratio/min": 0.6620843410491943, "sampling/sampling_logp_difference/max": 0.4124484062194824, "sampling/sampling_logp_difference/mean": 0.01264272816479206, "step": 973 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 534.0, "completions/max_terminated_length": 534.0, "completions/mean_length": 364.59375, "completions/mean_terminated_length": 364.59375, "completions/min_length": 217.0, "completions/min_terminated_length": 217.0, "entropy": 0.2749151811003685, "epoch": 0.5214132762312634, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 19306368.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.333916425704956, "sampling/importance_sampling_ratio/mean": 1.0003809928894043, "sampling/importance_sampling_ratio/min": 0.6982260346412659, "sampling/sampling_logp_difference/max": 0.35921239852905273, "sampling/sampling_logp_difference/mean": 0.010014554485678673, "step": 974 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.3717230912297964e-05, "clip_ratio/low_min": 5.3717230912297964e-05, "clip_ratio/region_mean": 5.3717230912297964e-05, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 748.0, "completions/mean_length": 506.9375, "completions/mean_terminated_length": 489.5333557128906, "completions/min_length": 215.0, "completions/min_terminated_length": 215.0, "entropy": 0.2950927261263132, "epoch": 0.521948608137045, "frac_reward_zero_std": 0.0, "grad_norm": 0.009787298738956451, "learning_rate": 1e-05, "loss": -0.0156, "num_tokens": 19326622.0, "reward": 0.3125, "reward_std": 0.4082317352294922, "rewards/accuracy_reward/mean": 0.3125, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.5575547218322754, "sampling/importance_sampling_ratio/mean": 0.9995861649513245, "sampling/importance_sampling_ratio/min": 0.6320212483406067, "sampling/sampling_logp_difference/max": 0.4588322639465332, "sampling/sampling_logp_difference/mean": 0.011625087819993496, "step": 975 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 637.0, "completions/max_terminated_length": 637.0, "completions/mean_length": 432.1875, "completions/mean_terminated_length": 432.1875, "completions/min_length": 192.0, "completions/min_terminated_length": 192.0, "entropy": 0.31722994707524776, "epoch": 0.5224839400428265, "frac_reward_zero_std": 0.25, "grad_norm": 0.019412700086832047, "learning_rate": 1e-05, "loss": -0.0215, "num_tokens": 19344028.0, "reward": 0.75, "reward_std": 0.3514062464237213, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.5474977493286133, "sampling/importance_sampling_ratio/mean": 0.9997346997261047, "sampling/importance_sampling_ratio/min": 0.6909081935882568, "sampling/sampling_logp_difference/max": 0.43663930892944336, "sampling/sampling_logp_difference/mean": 0.011539718136191368, "step": 976 }, { "clip_ratio/high_max": 5.064829747425392e-05, "clip_ratio/high_mean": 5.064829747425392e-05, "clip_ratio/low_mean": 9.850141213973984e-05, "clip_ratio/low_min": 9.850141213973984e-05, "clip_ratio/region_mean": 0.00014914970961399376, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 744.0, "completions/mean_length": 647.6875, "completions/mean_terminated_length": 619.923095703125, "completions/min_length": 440.0, "completions/min_terminated_length": 440.0, "entropy": 0.2991927396506071, "epoch": 0.5230192719486081, "frac_reward_zero_std": 0.25, "grad_norm": 0.011010593734681606, "learning_rate": 1e-05, "loss": 0.0353, "num_tokens": 19369434.0, "reward": 0.625, "reward_std": 0.3745020925998688, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.7090359926223755, "sampling/importance_sampling_ratio/mean": 0.9997743368148804, "sampling/importance_sampling_ratio/min": 0.6227453947067261, "sampling/sampling_logp_difference/max": 0.5359294414520264, "sampling/sampling_logp_difference/mean": 0.010291263461112976, "step": 977 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 703.0, "completions/mean_length": 476.96875, "completions/mean_terminated_length": 467.58062744140625, "completions/min_length": 265.0, "completions/min_terminated_length": 265.0, "entropy": 0.27222805842757225, "epoch": 0.5235546038543897, "frac_reward_zero_std": 0.25, "grad_norm": 0.0072904909029603004, "learning_rate": 1e-05, "loss": 0.0522, "num_tokens": 19387937.0, "reward": 0.90625, "reward_std": 0.2651650309562683, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.6329829692840576, "sampling/importance_sampling_ratio/mean": 1.000044345855713, "sampling/importance_sampling_ratio/min": 0.6657766103744507, "sampling/sampling_logp_difference/max": 0.49040842056274414, "sampling/sampling_logp_difference/mean": 0.009205597452819347, "step": 978 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 710.0, "completions/mean_length": 535.53125, "completions/mean_terminated_length": 502.3214416503906, "completions/min_length": 206.0, "completions/min_terminated_length": 206.0, "entropy": 0.38416952081024647, "epoch": 0.5240899357601713, "frac_reward_zero_std": 0.0, "grad_norm": 0.010845184326171875, "learning_rate": 1e-05, "loss": 0.0512, "num_tokens": 19409034.0, "reward": 0.75, "reward_std": 0.4261348247528076, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.4432660341262817, "sampling/importance_sampling_ratio/mean": 1.000100016593933, "sampling/importance_sampling_ratio/min": 0.5611334443092346, "sampling/sampling_logp_difference/max": 0.577796459197998, "sampling/sampling_logp_difference/mean": 0.012798493728041649, "step": 979 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 8.074935612967238e-05, "clip_ratio/low_min": 8.074935612967238e-05, "clip_ratio/region_mean": 8.074935612967238e-05, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 685.0, "completions/mean_length": 523.0625, "completions/mean_terminated_length": 488.0714416503906, "completions/min_length": 327.0, "completions/min_terminated_length": 327.0, "entropy": 0.30425986647605896, "epoch": 0.5246252676659529, "frac_reward_zero_std": 0.5, "grad_norm": 0.018338380381464958, "learning_rate": 1e-05, "loss": 0.0305, "num_tokens": 19429740.0, "reward": 0.75, "reward_std": 0.26726123690605164, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.2874058485031128, "sampling/importance_sampling_ratio/mean": 1.0000114440917969, "sampling/importance_sampling_ratio/min": 0.6871376037597656, "sampling/sampling_logp_difference/max": 0.37522077560424805, "sampling/sampling_logp_difference/mean": 0.01055125892162323, "step": 980 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.624271190958098e-05, "clip_ratio/low_min": 6.624271190958098e-05, "clip_ratio/region_mean": 6.624271190958098e-05, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 705.0, "completions/mean_length": 512.28125, "completions/mean_terminated_length": 495.2333679199219, "completions/min_length": 339.0, "completions/min_terminated_length": 339.0, "entropy": 0.28203972429037094, "epoch": 0.5251605995717344, "frac_reward_zero_std": 0.5, "grad_norm": 0.00662532402202487, "learning_rate": 1e-05, "loss": 0.016, "num_tokens": 19449669.0, "reward": 0.78125, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.6777180433273315, "sampling/importance_sampling_ratio/mean": 0.9999814033508301, "sampling/importance_sampling_ratio/min": 0.7238746881484985, "sampling/sampling_logp_difference/max": 0.5174345970153809, "sampling/sampling_logp_difference/mean": 0.011040785349905491, "step": 981 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 751.0, "completions/max_terminated_length": 751.0, "completions/mean_length": 429.375, "completions/mean_terminated_length": 429.375, "completions/min_length": 186.0, "completions/min_terminated_length": 186.0, "entropy": 0.33899323269724846, "epoch": 0.525695931477516, "frac_reward_zero_std": 0.75, "grad_norm": 0.016133377328515053, "learning_rate": 1e-05, "loss": -0.025, "num_tokens": 19466689.0, "reward": 0.84375, "reward_std": 0.1293872892856598, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.5335289239883423, "sampling/importance_sampling_ratio/mean": 1.0002915859222412, "sampling/importance_sampling_ratio/min": 0.4264197051525116, "sampling/sampling_logp_difference/max": 0.8523311614990234, "sampling/sampling_logp_difference/mean": 0.011671149171888828, "step": 982 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00024546719578211196, "clip_ratio/low_min": 0.00024546719578211196, "clip_ratio/region_mean": 0.00024546719578211196, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 733.0, "completions/mean_length": 565.09375, "completions/mean_terminated_length": 508.2799987792969, "completions/min_length": 308.0, "completions/min_terminated_length": 308.0, "entropy": 0.34883488342165947, "epoch": 0.5262312633832976, "frac_reward_zero_std": 0.25, "grad_norm": 0.01370038092136383, "learning_rate": 1e-05, "loss": 0.063, "num_tokens": 19488956.0, "reward": 0.5625, "reward_std": 0.3514062762260437, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.4622024297714233, "sampling/importance_sampling_ratio/mean": 1.0004162788391113, "sampling/importance_sampling_ratio/min": 0.6158801913261414, "sampling/sampling_logp_difference/max": 0.48470282554626465, "sampling/sampling_logp_difference/mean": 0.01206190139055252, "step": 983 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00010218562238151208, "clip_ratio/low_min": 0.00010218562238151208, "clip_ratio/region_mean": 0.00010218562238151208, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 716.0, "completions/mean_length": 546.6875, "completions/mean_terminated_length": 495.61541748046875, "completions/min_length": 340.0, "completions/min_terminated_length": 340.0, "entropy": 0.38961392268538475, "epoch": 0.5267665952890792, "frac_reward_zero_std": 0.0, "grad_norm": 0.019659826532006264, "learning_rate": 1e-05, "loss": 0.1077, "num_tokens": 19510434.0, "reward": 0.625, "reward_std": 0.5081326961517334, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.4071341753005981, "sampling/importance_sampling_ratio/mean": 0.9999825954437256, "sampling/importance_sampling_ratio/min": 0.5802328586578369, "sampling/sampling_logp_difference/max": 0.5443258285522461, "sampling/sampling_logp_difference/mean": 0.012630732730031013, "step": 984 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 7.701786671532318e-05, "clip_ratio/low_min": 7.701786671532318e-05, "clip_ratio/region_mean": 7.701786671532318e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 722.0, "completions/max_terminated_length": 722.0, "completions/mean_length": 429.28125, "completions/mean_terminated_length": 429.28125, "completions/min_length": 218.0, "completions/min_terminated_length": 218.0, "entropy": 0.2875707168132067, "epoch": 0.5273019271948608, "frac_reward_zero_std": 0.75, "grad_norm": 0.00487469881772995, "learning_rate": 1e-05, "loss": -0.0164, "num_tokens": 19527747.0, "reward": 0.78125, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.2798290252685547, "sampling/importance_sampling_ratio/mean": 0.9999646544456482, "sampling/importance_sampling_ratio/min": 0.6921064257621765, "sampling/sampling_logp_difference/max": 0.3680155277252197, "sampling/sampling_logp_difference/mean": 0.010059557855129242, "step": 985 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 660.0, "completions/mean_length": 396.40625, "completions/mean_terminated_length": 384.4193420410156, "completions/min_length": 218.0, "completions/min_terminated_length": 218.0, "entropy": 0.2835698686540127, "epoch": 0.5278372591006424, "frac_reward_zero_std": 0.5, "grad_norm": 0.019992968067526817, "learning_rate": 1e-05, "loss": 0.0428, "num_tokens": 19544768.0, "reward": 0.875, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 1.42948579788208, "sampling/importance_sampling_ratio/mean": 0.9999502301216125, "sampling/importance_sampling_ratio/min": 0.6881143450737, "sampling/sampling_logp_difference/max": 0.37380027770996094, "sampling/sampling_logp_difference/mean": 0.01113945059478283, "step": 986 }, { "clip_ratio/high_max": 0.0001092864986276254, "clip_ratio/high_mean": 0.0001092864986276254, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0001092864986276254, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 761.0, "completions/mean_length": 560.78125, "completions/mean_terminated_length": 491.7083435058594, "completions/min_length": 249.0, "completions/min_terminated_length": 249.0, "entropy": 0.36560990288853645, "epoch": 0.528372591006424, "frac_reward_zero_std": 0.0, "grad_norm": 0.015714678913354874, "learning_rate": 1e-05, "loss": 0.0451, "num_tokens": 19566945.0, "reward": 0.40625, "reward_std": 0.4534739851951599, "rewards/accuracy_reward/mean": 0.40625, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.39082670211792, "sampling/importance_sampling_ratio/mean": 0.9996116757392883, "sampling/importance_sampling_ratio/min": 0.6515651941299438, "sampling/sampling_logp_difference/max": 0.4283778667449951, "sampling/sampling_logp_difference/mean": 0.012630755081772804, "step": 987 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 695.0, "completions/mean_length": 454.21875, "completions/mean_terminated_length": 444.0967712402344, "completions/min_length": 252.0, "completions/min_terminated_length": 252.0, "entropy": 0.2823651097714901, "epoch": 0.5289079229122056, "frac_reward_zero_std": 0.5, "grad_norm": 0.013048914261162281, "learning_rate": 1e-05, "loss": 0.0101, "num_tokens": 19585160.0, "reward": 0.875, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 1.5002124309539795, "sampling/importance_sampling_ratio/mean": 0.9995115399360657, "sampling/importance_sampling_ratio/min": 0.5749893188476562, "sampling/sampling_logp_difference/max": 0.5534038543701172, "sampling/sampling_logp_difference/mean": 0.010700213722884655, "step": 988 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 560.0, "completions/max_terminated_length": 560.0, "completions/mean_length": 392.9375, "completions/mean_terminated_length": 392.9375, "completions/min_length": 237.0, "completions/min_terminated_length": 237.0, "entropy": 0.2659153249114752, "epoch": 0.5294432548179872, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0201, "num_tokens": 19601806.0, "reward": 0.96875, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.96875, "rewards/accuracy_reward/std": 0.1767766922712326, "sampling/importance_sampling_ratio/max": 1.5709412097930908, "sampling/importance_sampling_ratio/mean": 1.0001845359802246, "sampling/importance_sampling_ratio/min": 0.5926231145858765, "sampling/sampling_logp_difference/max": 0.5231966972351074, "sampling/sampling_logp_difference/mean": 0.010107295587658882, "step": 989 }, { "clip_ratio/high_max": 5.955216693109833e-05, "clip_ratio/high_mean": 5.955216693109833e-05, "clip_ratio/low_mean": 7.318500865949318e-05, "clip_ratio/low_min": 7.318500865949318e-05, "clip_ratio/region_mean": 0.0001327371755905915, "completions/clipped_ratio": 0.0, "completions/max_length": 674.0, "completions/max_terminated_length": 674.0, "completions/mean_length": 461.78125, "completions/mean_terminated_length": 461.78125, "completions/min_length": 159.0, "completions/min_terminated_length": 159.0, "entropy": 0.3579945210367441, "epoch": 0.5299785867237687, "frac_reward_zero_std": 0.25, "grad_norm": 0.014617021195590496, "learning_rate": 1e-05, "loss": -0.0012, "num_tokens": 19620383.0, "reward": 0.71875, "reward_std": 0.35564959049224854, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.4609228372573853, "sampling/importance_sampling_ratio/mean": 0.9997321963310242, "sampling/importance_sampling_ratio/min": 0.7005718946456909, "sampling/sampling_logp_difference/max": 0.37906837463378906, "sampling/sampling_logp_difference/mean": 0.01290670782327652, "step": 990 }, { "clip_ratio/high_max": 5.526083259610459e-05, "clip_ratio/high_mean": 5.526083259610459e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 5.526083259610459e-05, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 757.0, "completions/mean_length": 539.75, "completions/mean_terminated_length": 532.3870849609375, "completions/min_length": 311.0, "completions/min_terminated_length": 311.0, "entropy": 0.2842806428670883, "epoch": 0.5305139186295503, "frac_reward_zero_std": 0.25, "grad_norm": 0.006144804880023003, "learning_rate": 1e-05, "loss": 0.0508, "num_tokens": 19641103.0, "reward": 0.90625, "reward_std": 0.2651650309562683, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.5386121273040771, "sampling/importance_sampling_ratio/mean": 1.0001035928726196, "sampling/importance_sampling_ratio/min": 0.6881845593452454, "sampling/sampling_logp_difference/max": 0.4308807849884033, "sampling/sampling_logp_difference/mean": 0.010615945793688297, "step": 991 }, { "clip_ratio/high_max": 8.411843737121671e-05, "clip_ratio/high_mean": 8.411843737121671e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 8.411843737121671e-05, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 525.0, "completions/mean_length": 390.0625, "completions/mean_terminated_length": 364.86669921875, "completions/min_length": 166.0, "completions/min_terminated_length": 166.0, "entropy": 0.31818179227411747, "epoch": 0.5310492505353319, "frac_reward_zero_std": 0.5, "grad_norm": 0.007501174695789814, "learning_rate": 1e-05, "loss": 0.0988, "num_tokens": 19657033.0, "reward": 0.75, "reward_std": 0.2314550280570984, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.7083754539489746, "sampling/importance_sampling_ratio/mean": 1.0004456043243408, "sampling/importance_sampling_ratio/min": 0.669009804725647, "sampling/sampling_logp_difference/max": 0.5355429649353027, "sampling/sampling_logp_difference/mean": 0.011256661266088486, "step": 992 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 614.0, "completions/max_terminated_length": 614.0, "completions/mean_length": 426.125, "completions/mean_terminated_length": 426.125, "completions/min_length": 244.0, "completions/min_terminated_length": 244.0, "entropy": 0.24906635843217373, "epoch": 0.5315845824411135, "frac_reward_zero_std": 0.75, "grad_norm": 0.003510852111503482, "learning_rate": 1e-05, "loss": -0.0206, "num_tokens": 19674205.0, "reward": 0.96875, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.96875, "rewards/accuracy_reward/std": 0.1767766922712326, "sampling/importance_sampling_ratio/max": 1.5225409269332886, "sampling/importance_sampling_ratio/mean": 0.9998700022697449, "sampling/importance_sampling_ratio/min": 0.5216495990753174, "sampling/sampling_logp_difference/max": 0.650759220123291, "sampling/sampling_logp_difference/mean": 0.010037382133305073, "step": 993 }, { "clip_ratio/high_max": 6.860592839075252e-05, "clip_ratio/high_mean": 6.860592839075252e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 6.860592839075252e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 732.0, "completions/max_terminated_length": 732.0, "completions/mean_length": 395.40625, "completions/mean_terminated_length": 395.40625, "completions/min_length": 255.0, "completions/min_terminated_length": 255.0, "entropy": 0.24751000106334686, "epoch": 0.5321199143468951, "frac_reward_zero_std": 0.75, "grad_norm": 0.019045062363147736, "learning_rate": 1e-05, "loss": -0.0267, "num_tokens": 19690306.0, "reward": 0.9375, "reward_std": 0.1157275140285492, "rewards/accuracy_reward/mean": 0.9375, "rewards/accuracy_reward/std": 0.24593468010425568, "sampling/importance_sampling_ratio/max": 1.572339653968811, "sampling/importance_sampling_ratio/mean": 1.0002657175064087, "sampling/importance_sampling_ratio/min": 0.6859140396118164, "sampling/sampling_logp_difference/max": 0.45256471633911133, "sampling/sampling_logp_difference/mean": 0.010092799551784992, "step": 994 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.148270065546967e-05, "clip_ratio/low_min": 5.148270065546967e-05, "clip_ratio/region_mean": 5.148270065546967e-05, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 738.0, "completions/mean_length": 482.0625, "completions/mean_terminated_length": 463.0000305175781, "completions/min_length": 262.0, "completions/min_terminated_length": 262.0, "entropy": 0.2893268521875143, "epoch": 0.5326552462526767, "frac_reward_zero_std": 0.75, "grad_norm": 0.02204366959631443, "learning_rate": 1e-05, "loss": 0.0335, "num_tokens": 19709484.0, "reward": 0.9375, "reward_std": 0.1157275140285492, "rewards/accuracy_reward/mean": 0.9375, "rewards/accuracy_reward/std": 0.24593468010425568, "sampling/importance_sampling_ratio/max": 1.4051787853240967, "sampling/importance_sampling_ratio/mean": 1.0002083778381348, "sampling/importance_sampling_ratio/min": 0.7070884108543396, "sampling/sampling_logp_difference/max": 0.3465995788574219, "sampling/sampling_logp_difference/mean": 0.010966657660901546, "step": 995 }, { "clip_ratio/high_max": 6.551362457685173e-05, "clip_ratio/high_mean": 6.551362457685173e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 6.551362457685173e-05, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 710.0, "completions/mean_length": 462.21875, "completions/mean_terminated_length": 376.6000061035156, "completions/min_length": 205.0, "completions/min_terminated_length": 205.0, "entropy": 0.5366212762892246, "epoch": 0.5331905781584583, "frac_reward_zero_std": 0.25, "grad_norm": 0.007105550728738308, "learning_rate": 1e-05, "loss": 0.0335, "num_tokens": 19728091.0, "reward": 0.625, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.8256914615631104, "sampling/importance_sampling_ratio/mean": 0.9998644590377808, "sampling/importance_sampling_ratio/min": 0.6105164289474487, "sampling/sampling_logp_difference/max": 0.6019587516784668, "sampling/sampling_logp_difference/mean": 0.013786759227514267, "step": 996 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 732.0, "completions/mean_length": 568.53125, "completions/mean_terminated_length": 562.0967407226562, "completions/min_length": 304.0, "completions/min_terminated_length": 304.0, "entropy": 0.24387717805802822, "epoch": 0.5337259100642399, "frac_reward_zero_std": 0.5, "grad_norm": 0.004168031737208366, "learning_rate": 1e-05, "loss": 0.0362, "num_tokens": 19750332.0, "reward": 0.6875, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.4168020486831665, "sampling/importance_sampling_ratio/mean": 1.0003665685653687, "sampling/importance_sampling_ratio/min": 0.6397484540939331, "sampling/sampling_logp_difference/max": 0.44668030738830566, "sampling/sampling_logp_difference/mean": 0.008794558234512806, "step": 997 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 693.0, "completions/max_terminated_length": 693.0, "completions/mean_length": 480.1875, "completions/mean_terminated_length": 480.1875, "completions/min_length": 289.0, "completions/min_terminated_length": 289.0, "entropy": 0.23125405795872211, "epoch": 0.5342612419700214, "frac_reward_zero_std": 0.75, "grad_norm": 0.01607550121843815, "learning_rate": 1e-05, "loss": 0.0423, "num_tokens": 19769498.0, "reward": 0.9375, "reward_std": 0.1157275140285492, "rewards/accuracy_reward/mean": 0.9375, "rewards/accuracy_reward/std": 0.24593468010425568, "sampling/importance_sampling_ratio/max": 1.3502311706542969, "sampling/importance_sampling_ratio/mean": 0.999817430973053, "sampling/importance_sampling_ratio/min": 0.5263004899024963, "sampling/sampling_logp_difference/max": 0.6418828964233398, "sampling/sampling_logp_difference/mean": 0.009172629565000534, "step": 998 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 508.0, "completions/max_terminated_length": 508.0, "completions/mean_length": 359.0625, "completions/mean_terminated_length": 359.0625, "completions/min_length": 259.0, "completions/min_terminated_length": 259.0, "entropy": 0.2785221226513386, "epoch": 0.534796573875803, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 19784412.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.5911743640899658, "sampling/importance_sampling_ratio/mean": 1.0001888275146484, "sampling/importance_sampling_ratio/min": 0.6769352555274963, "sampling/sampling_logp_difference/max": 0.46447229385375977, "sampling/sampling_logp_difference/mean": 0.010836697183549404, "step": 999 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.385609620134346e-05, "clip_ratio/low_min": 5.385609620134346e-05, "clip_ratio/region_mean": 5.385609620134346e-05, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 746.0, "completions/mean_length": 545.75, "completions/mean_terminated_length": 483.5199890136719, "completions/min_length": 264.0, "completions/min_terminated_length": 264.0, "entropy": 0.38301945105195045, "epoch": 0.5353319057815846, "frac_reward_zero_std": 0.0, "grad_norm": 0.01854352094233036, "learning_rate": 1e-05, "loss": 0.1301, "num_tokens": 19805500.0, "reward": 0.5, "reward_std": 0.5081326961517334, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5080004930496216, "sampling/importance_sampling_ratio/max": 1.3488013744354248, "sampling/importance_sampling_ratio/mean": 0.9999552965164185, "sampling/importance_sampling_ratio/min": 0.7033720016479492, "sampling/sampling_logp_difference/max": 0.3518693447113037, "sampling/sampling_logp_difference/mean": 0.012176279909908772, "step": 1000 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 628.0, "completions/mean_length": 449.1875, "completions/mean_terminated_length": 438.9031982421875, "completions/min_length": 312.0, "completions/min_terminated_length": 312.0, "entropy": 0.2753011118620634, "epoch": 0.5358672376873662, "frac_reward_zero_std": 0.25, "grad_norm": 0.02747787907719612, "learning_rate": 1e-05, "loss": -0.0111, "num_tokens": 19824154.0, "reward": 0.90625, "reward_std": 0.2651650309562683, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.5519434213638306, "sampling/importance_sampling_ratio/mean": 1.000260353088379, "sampling/importance_sampling_ratio/min": 0.7156000733375549, "sampling/sampling_logp_difference/max": 0.43950796127319336, "sampling/sampling_logp_difference/mean": 0.010242156684398651, "step": 1001 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 716.0, "completions/mean_length": 482.125, "completions/mean_terminated_length": 463.0666809082031, "completions/min_length": 266.0, "completions/min_terminated_length": 266.0, "entropy": 0.29971857368946075, "epoch": 0.5364025695931478, "frac_reward_zero_std": 0.25, "grad_norm": 0.009986001998186111, "learning_rate": 1e-05, "loss": 0.0325, "num_tokens": 19843542.0, "reward": 0.8125, "reward_std": 0.3471825420856476, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.4225226640701294, "sampling/importance_sampling_ratio/mean": 1.0002081394195557, "sampling/importance_sampling_ratio/min": 0.7357439398765564, "sampling/sampling_logp_difference/max": 0.3524317741394043, "sampling/sampling_logp_difference/mean": 0.010664070025086403, "step": 1002 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 642.0, "completions/max_terminated_length": 642.0, "completions/mean_length": 411.09375, "completions/mean_terminated_length": 411.09375, "completions/min_length": 261.0, "completions/min_terminated_length": 261.0, "entropy": 0.30399791710078716, "epoch": 0.5369379014989293, "frac_reward_zero_std": 0.75, "grad_norm": 0.007462628651410341, "learning_rate": 1e-05, "loss": 0.0056, "num_tokens": 19860617.0, "reward": 0.90625, "reward_std": 0.1293872892856598, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.4144448041915894, "sampling/importance_sampling_ratio/mean": 0.9999246001243591, "sampling/importance_sampling_ratio/min": 0.5888384580612183, "sampling/sampling_logp_difference/max": 0.5296034812927246, "sampling/sampling_logp_difference/mean": 0.01047993078827858, "step": 1003 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00014293882122728974, "clip_ratio/low_min": 0.00014293882122728974, "clip_ratio/region_mean": 0.00014293882122728974, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 698.0, "completions/mean_length": 525.46875, "completions/mean_terminated_length": 490.8214416503906, "completions/min_length": 241.0, "completions/min_terminated_length": 241.0, "entropy": 0.4285593256354332, "epoch": 0.5374732334047109, "frac_reward_zero_std": 0.25, "grad_norm": 0.02118309959769249, "learning_rate": 1e-05, "loss": 0.0742, "num_tokens": 19881176.0, "reward": 0.59375, "reward_std": 0.3787454068660736, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.6302584409713745, "sampling/importance_sampling_ratio/mean": 1.0003572702407837, "sampling/importance_sampling_ratio/min": 0.4600124657154083, "sampling/sampling_logp_difference/max": 0.7765016555786133, "sampling/sampling_logp_difference/mean": 0.014100081287324429, "step": 1004 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 638.0, "completions/mean_length": 381.25, "completions/mean_terminated_length": 368.774169921875, "completions/min_length": 149.0, "completions/min_terminated_length": 149.0, "entropy": 0.28941107355058193, "epoch": 0.5380085653104925, "frac_reward_zero_std": 0.75, "grad_norm": 0.005712924525141716, "learning_rate": 1e-05, "loss": 0.0168, "num_tokens": 19896384.0, "reward": 0.9375, "reward_std": 0.1157275140285492, "rewards/accuracy_reward/mean": 0.9375, "rewards/accuracy_reward/std": 0.24593468010425568, "sampling/importance_sampling_ratio/max": 1.4629381895065308, "sampling/importance_sampling_ratio/mean": 1.0005462169647217, "sampling/importance_sampling_ratio/min": 0.7008655667304993, "sampling/sampling_logp_difference/max": 0.3804469108581543, "sampling/sampling_logp_difference/mean": 0.01073462050408125, "step": 1005 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.056634472566657e-05, "clip_ratio/low_min": 5.056634472566657e-05, "clip_ratio/region_mean": 5.056634472566657e-05, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 677.0, "completions/mean_length": 457.28125, "completions/mean_terminated_length": 436.5666809082031, "completions/min_length": 274.0, "completions/min_terminated_length": 274.0, "entropy": 0.33258638344705105, "epoch": 0.538543897216274, "frac_reward_zero_std": 0.25, "grad_norm": 0.01634703204035759, "learning_rate": 1e-05, "loss": 0.0405, "num_tokens": 19914609.0, "reward": 0.78125, "reward_std": 0.3377464711666107, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.3282477855682373, "sampling/importance_sampling_ratio/mean": 0.9999423623085022, "sampling/importance_sampling_ratio/min": 0.6146914958953857, "sampling/sampling_logp_difference/max": 0.4866347312927246, "sampling/sampling_logp_difference/mean": 0.011314889416098595, "step": 1006 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00018642631766851991, "clip_ratio/low_min": 0.00018642631766851991, "clip_ratio/region_mean": 0.00018642631766851991, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 762.0, "completions/mean_length": 489.1875, "completions/mean_terminated_length": 480.19354248046875, "completions/min_length": 256.0, "completions/min_terminated_length": 256.0, "entropy": 0.2814666237682104, "epoch": 0.5390792291220556, "frac_reward_zero_std": 0.25, "grad_norm": 0.014355567283928394, "learning_rate": 1e-05, "loss": 0.0065, "num_tokens": 19934447.0, "reward": 0.65625, "reward_std": 0.3966485261917114, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.4836410284042358, "sampling/importance_sampling_ratio/mean": 0.9998093843460083, "sampling/importance_sampling_ratio/min": 0.6395634412765503, "sampling/sampling_logp_difference/max": 0.44696950912475586, "sampling/sampling_logp_difference/mean": 0.010976734571158886, "step": 1007 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 755.0, "completions/max_terminated_length": 755.0, "completions/mean_length": 490.125, "completions/mean_terminated_length": 490.125, "completions/min_length": 270.0, "completions/min_terminated_length": 270.0, "entropy": 0.3004448153078556, "epoch": 0.5396145610278372, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 19953699.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.6653494834899902, "sampling/importance_sampling_ratio/mean": 1.0000314712524414, "sampling/importance_sampling_ratio/min": 0.6230331659317017, "sampling/sampling_logp_difference/max": 0.5100350379943848, "sampling/sampling_logp_difference/mean": 0.01125288661569357, "step": 1008 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.8449612222611904e-05, "clip_ratio/low_min": 4.8449612222611904e-05, "clip_ratio/region_mean": 4.8449612222611904e-05, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 733.0, "completions/mean_length": 503.6875, "completions/mean_terminated_length": 429.67999267578125, "completions/min_length": 217.0, "completions/min_terminated_length": 217.0, "entropy": 0.3737986646592617, "epoch": 0.5401498929336188, "frac_reward_zero_std": 0.5, "grad_norm": 0.016752269119024277, "learning_rate": 1e-05, "loss": 0.0322, "num_tokens": 19973993.0, "reward": 0.71875, "reward_std": 0.2630178928375244, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.435368537902832, "sampling/importance_sampling_ratio/mean": 0.9996938705444336, "sampling/importance_sampling_ratio/min": 0.6979666948318481, "sampling/sampling_logp_difference/max": 0.3614215850830078, "sampling/sampling_logp_difference/mean": 0.012252227403223515, "step": 1009 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00029390008421614766, "clip_ratio/low_min": 0.00029390008421614766, "clip_ratio/region_mean": 0.00029390008421614766, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 649.0, "completions/mean_length": 497.3125, "completions/mean_terminated_length": 434.8461608886719, "completions/min_length": 238.0, "completions/min_terminated_length": 238.0, "entropy": 0.40621403232216835, "epoch": 0.5406852248394004, "frac_reward_zero_std": 0.25, "grad_norm": 0.008822666481137276, "learning_rate": 1e-05, "loss": 0.0064, "num_tokens": 19993595.0, "reward": 0.375, "reward_std": 0.292504221200943, "rewards/accuracy_reward/mean": 0.375, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.4496333599090576, "sampling/importance_sampling_ratio/mean": 0.9998958706855774, "sampling/importance_sampling_ratio/min": 0.6977450847625732, "sampling/sampling_logp_difference/max": 0.3713107109069824, "sampling/sampling_logp_difference/mean": 0.013254624791443348, "step": 1010 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 595.0, "completions/max_terminated_length": 595.0, "completions/mean_length": 378.71875, "completions/mean_terminated_length": 378.71875, "completions/min_length": 168.0, "completions/min_terminated_length": 168.0, "entropy": 0.256598187610507, "epoch": 0.541220556745182, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": -0.0133, "num_tokens": 20009186.0, "reward": 0.96875, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.96875, "rewards/accuracy_reward/std": 0.1767766922712326, "sampling/importance_sampling_ratio/max": 1.4466054439544678, "sampling/importance_sampling_ratio/mean": 1.0001771450042725, "sampling/importance_sampling_ratio/min": 0.7326815128326416, "sampling/sampling_logp_difference/max": 0.3692197799682617, "sampling/sampling_logp_difference/mean": 0.009641504846513271, "step": 1011 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00011813741366495378, "clip_ratio/low_min": 0.00011813741366495378, "clip_ratio/region_mean": 0.00011813741366495378, "completions/clipped_ratio": 0.375, "completions/max_length": 768.0, "completions/max_terminated_length": 768.0, "completions/mean_length": 563.40625, "completions/mean_terminated_length": 440.6499938964844, "completions/min_length": 278.0, "completions/min_terminated_length": 278.0, "entropy": 0.41633135825395584, "epoch": 0.5417558886509636, "frac_reward_zero_std": 0.5, "grad_norm": 0.007999341003596783, "learning_rate": 1e-05, "loss": 0.0028, "num_tokens": 20031607.0, "reward": 0.5625, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.4557040929794312, "sampling/importance_sampling_ratio/mean": 1.000044584274292, "sampling/importance_sampling_ratio/min": 0.7197877168655396, "sampling/sampling_logp_difference/max": 0.3754897117614746, "sampling/sampling_logp_difference/mean": 0.012992034666240215, "step": 1012 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 683.0, "completions/max_terminated_length": 683.0, "completions/mean_length": 381.0625, "completions/mean_terminated_length": 381.0625, "completions/min_length": 234.0, "completions/min_terminated_length": 234.0, "entropy": 0.2639304231852293, "epoch": 0.5422912205567452, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0097, "num_tokens": 20047625.0, "reward": 0.96875, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.96875, "rewards/accuracy_reward/std": 0.1767766922712326, "sampling/importance_sampling_ratio/max": 1.6141644716262817, "sampling/importance_sampling_ratio/mean": 0.9996117949485779, "sampling/importance_sampling_ratio/min": 0.6979535818099976, "sampling/sampling_logp_difference/max": 0.4788174629211426, "sampling/sampling_logp_difference/mean": 0.009938123635947704, "step": 1013 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.416838004952297e-05, "clip_ratio/low_min": 6.416838004952297e-05, "clip_ratio/region_mean": 6.416838004952297e-05, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 531.0, "completions/mean_length": 422.71875, "completions/mean_terminated_length": 411.58062744140625, "completions/min_length": 240.0, "completions/min_terminated_length": 240.0, "entropy": 0.22711737640202045, "epoch": 0.5428265524625268, "frac_reward_zero_std": 0.75, "grad_norm": 0.005366690922528505, "learning_rate": 1e-05, "loss": 0.0511, "num_tokens": 20065200.0, "reward": 0.96875, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.96875, "rewards/accuracy_reward/std": 0.1767766922712326, "sampling/importance_sampling_ratio/max": 1.4140678644180298, "sampling/importance_sampling_ratio/mean": 0.9999990463256836, "sampling/importance_sampling_ratio/min": 0.692293107509613, "sampling/sampling_logp_difference/max": 0.36774587631225586, "sampling/sampling_logp_difference/mean": 0.009445932693779469, "step": 1014 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 716.0, "completions/max_terminated_length": 716.0, "completions/mean_length": 428.0625, "completions/mean_terminated_length": 428.0625, "completions/min_length": 240.0, "completions/min_terminated_length": 240.0, "entropy": 0.2568762693554163, "epoch": 0.5433618843683083, "frac_reward_zero_std": 0.5, "grad_norm": 0.01963024213910103, "learning_rate": 1e-05, "loss": -0.0212, "num_tokens": 20082386.0, "reward": 0.90625, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.4306024312973022, "sampling/importance_sampling_ratio/mean": 0.9998763799667358, "sampling/importance_sampling_ratio/min": 0.697256863117218, "sampling/sampling_logp_difference/max": 0.36060142517089844, "sampling/sampling_logp_difference/mean": 0.009530318900942802, "step": 1015 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 642.0, "completions/mean_length": 448.71875, "completions/mean_terminated_length": 403.1071472167969, "completions/min_length": 223.0, "completions/min_terminated_length": 223.0, "entropy": 0.36373442970216274, "epoch": 0.5438972162740899, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 20100249.0, "reward": 0.75, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.4357532262802124, "sampling/importance_sampling_ratio/mean": 0.9998073577880859, "sampling/importance_sampling_ratio/min": 0.7113254070281982, "sampling/sampling_logp_difference/max": 0.36168956756591797, "sampling/sampling_logp_difference/mean": 0.011635943315923214, "step": 1016 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 641.0, "completions/mean_length": 428.0625, "completions/mean_terminated_length": 417.0967712402344, "completions/min_length": 190.0, "completions/min_terminated_length": 190.0, "entropy": 0.25626081973314285, "epoch": 0.5444325481798715, "frac_reward_zero_std": 0.5, "grad_norm": 0.011524244211614132, "learning_rate": 1e-05, "loss": 0.0695, "num_tokens": 20117715.0, "reward": 0.8125, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.5319937467575073, "sampling/importance_sampling_ratio/mean": 1.0003708600997925, "sampling/importance_sampling_ratio/min": 0.7119307518005371, "sampling/sampling_logp_difference/max": 0.42656993865966797, "sampling/sampling_logp_difference/mean": 0.009544669650495052, "step": 1017 }, { "clip_ratio/high_max": 6.231306178960949e-05, "clip_ratio/high_mean": 6.231306178960949e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 6.231306178960949e-05, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 744.0, "completions/mean_length": 538.875, "completions/mean_terminated_length": 515.1724243164062, "completions/min_length": 223.0, "completions/min_terminated_length": 223.0, "entropy": 0.2870079483836889, "epoch": 0.5449678800856531, "frac_reward_zero_std": 0.0, "grad_norm": 0.027886562049388885, "learning_rate": 1e-05, "loss": 0.0037, "num_tokens": 20138975.0, "reward": 0.53125, "reward_std": 0.4534739851951599, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.624444842338562, "sampling/importance_sampling_ratio/mean": 1.0002611875534058, "sampling/importance_sampling_ratio/min": 0.5770533084869385, "sampling/sampling_logp_difference/max": 0.5498206615447998, "sampling/sampling_logp_difference/mean": 0.010416710749268532, "step": 1018 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0001102035676012747, "clip_ratio/low_min": 0.0001102035676012747, "clip_ratio/region_mean": 0.0001102035676012747, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 610.0, "completions/mean_length": 492.03125, "completions/mean_terminated_length": 428.3461608886719, "completions/min_length": 265.0, "completions/min_terminated_length": 265.0, "entropy": 0.39040610007941723, "epoch": 0.5455032119914347, "frac_reward_zero_std": 0.25, "grad_norm": 0.018447870388627052, "learning_rate": 1e-05, "loss": 0.0321, "num_tokens": 20158560.0, "reward": 0.53125, "reward_std": 0.3471629321575165, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.409449577331543, "sampling/importance_sampling_ratio/mean": 1.0000241994857788, "sampling/importance_sampling_ratio/min": 0.5752276182174683, "sampling/sampling_logp_difference/max": 0.5529894828796387, "sampling/sampling_logp_difference/mean": 0.011754137463867664, "step": 1019 }, { "clip_ratio/high_max": 8.327781688421965e-05, "clip_ratio/high_mean": 8.327781688421965e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 8.327781688421965e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 639.0, "completions/max_terminated_length": 639.0, "completions/mean_length": 420.25, "completions/mean_terminated_length": 420.25, "completions/min_length": 211.0, "completions/min_terminated_length": 211.0, "entropy": 0.25564243644475937, "epoch": 0.5460385438972163, "frac_reward_zero_std": 0.5, "grad_norm": 0.007572890259325504, "learning_rate": 1e-05, "loss": 0.035, "num_tokens": 20175368.0, "reward": 0.9375, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.9375, "rewards/accuracy_reward/std": 0.24593468010425568, "sampling/importance_sampling_ratio/max": 1.5367435216903687, "sampling/importance_sampling_ratio/mean": 0.9999706149101257, "sampling/importance_sampling_ratio/min": 0.7095188498497009, "sampling/sampling_logp_difference/max": 0.42966556549072266, "sampling/sampling_logp_difference/mean": 0.010025544092059135, "step": 1020 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 719.0, "completions/mean_length": 547.5625, "completions/mean_terminated_length": 506.7407531738281, "completions/min_length": 275.0, "completions/min_terminated_length": 275.0, "entropy": 0.40882474556565285, "epoch": 0.5465738758029979, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": -0.0241, "num_tokens": 20196762.0, "reward": 0.71875, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.4408026933670044, "sampling/importance_sampling_ratio/mean": 1.0000535249710083, "sampling/importance_sampling_ratio/min": 0.6971439719200134, "sampling/sampling_logp_difference/max": 0.3652002811431885, "sampling/sampling_logp_difference/mean": 0.013723382726311684, "step": 1021 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00014731880219187587, "clip_ratio/low_min": 0.00014731880219187587, "clip_ratio/region_mean": 0.00014731880219187587, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 763.0, "completions/mean_length": 427.78125, "completions/mean_terminated_length": 416.8064270019531, "completions/min_length": 172.0, "completions/min_terminated_length": 172.0, "entropy": 0.3384027648717165, "epoch": 0.5471092077087795, "frac_reward_zero_std": 0.5, "grad_norm": 0.026471413671970367, "learning_rate": 1e-05, "loss": 0.051, "num_tokens": 20214123.0, "reward": 0.78125, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.3845716714859009, "sampling/importance_sampling_ratio/mean": 1.0003368854522705, "sampling/importance_sampling_ratio/min": 0.37910571694374084, "sampling/sampling_logp_difference/max": 0.969940185546875, "sampling/sampling_logp_difference/mean": 0.01268053986132145, "step": 1022 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 653.0, "completions/max_terminated_length": 653.0, "completions/mean_length": 431.59375, "completions/mean_terminated_length": 431.59375, "completions/min_length": 316.0, "completions/min_terminated_length": 316.0, "entropy": 0.24964939430356026, "epoch": 0.547644539614561, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 20232214.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.4343326091766357, "sampling/importance_sampling_ratio/mean": 0.9999062418937683, "sampling/importance_sampling_ratio/min": 0.6914226412773132, "sampling/sampling_logp_difference/max": 0.3690040111541748, "sampling/sampling_logp_difference/mean": 0.01000040490180254, "step": 1023 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 7.301401637960225e-05, "clip_ratio/low_min": 7.301401637960225e-05, "clip_ratio/region_mean": 7.301401637960225e-05, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 656.0, "completions/mean_length": 522.53125, "completions/mean_terminated_length": 506.16668701171875, "completions/min_length": 326.0, "completions/min_terminated_length": 326.0, "entropy": 0.2866067998111248, "epoch": 0.5481798715203426, "frac_reward_zero_std": 0.0, "grad_norm": 0.011995550245046616, "learning_rate": 1e-05, "loss": -0.0003, "num_tokens": 20252823.0, "reward": 0.5, "reward_std": 0.4671337604522705, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5080004930496216, "sampling/importance_sampling_ratio/max": 1.8201465606689453, "sampling/importance_sampling_ratio/mean": 1.000035285949707, "sampling/importance_sampling_ratio/min": 0.6696924567222595, "sampling/sampling_logp_difference/max": 0.5989170074462891, "sampling/sampling_logp_difference/mean": 0.010623245500028133, "step": 1024 } ], "logging_steps": 1, "max_steps": 1024, "num_input_tokens_seen": 20252823, "num_train_epochs": 1, "save_steps": 64, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": true, "should_training_stop": true }, "attributes": {} } }, "total_flos": 0.0, "train_batch_size": 4, "trial_name": null, "trial_params": null }