diff --git "a/trainer_state.json" "b/trainer_state.json" new file mode 100644--- /dev/null +++ "b/trainer_state.json" @@ -0,0 +1,31778 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.5481798715203426, + "eval_steps": 500, + "global_step": 1024, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.5, + "completions/max_length": 768.0, + "completions/max_terminated_length": 753.0, + "completions/mean_length": 664.84375, + "completions/mean_terminated_length": 561.6875, + "completions/min_length": 322.0, + "completions/min_terminated_length": 322.0, + "entropy": 0.48598330840468407, + "epoch": 0.0005353319057815846, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.0032464242540299892, + "learning_rate": 1e-05, + "loss": 0.0473, + "num_tokens": 25147.0, + "reward": 0.46875, + "reward_std": 0.2651650309562683, + "rewards/accuracy_reward/mean": 0.46875, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.2930827140808105, + "sampling/importance_sampling_ratio/mean": 0.9998305439949036, + "sampling/importance_sampling_ratio/min": 0.6979114413261414, + "sampling/sampling_logp_difference/max": 0.3596630096435547, + "sampling/sampling_logp_difference/mean": 0.012516415677964687, + "step": 1 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 743.0, + "completions/mean_length": 605.75, + "completions/mean_terminated_length": 560.3200073242188, + "completions/min_length": 274.0, + "completions/min_terminated_length": 274.0, + "entropy": 0.4345819912850857, + "epoch": 0.0010706638115631692, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.004117041360586882, + "learning_rate": 1e-05, + "loss": 0.076, + "num_tokens": 48379.0, + "reward": 0.59375, + "reward_std": 0.3198433816432953, + "rewards/accuracy_reward/mean": 0.59375, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.8063730001449585, + "sampling/importance_sampling_ratio/mean": 1.0002689361572266, + "sampling/importance_sampling_ratio/min": 0.5230656266212463, + "sampling/sampling_logp_difference/max": 0.6480484008789062, + "sampling/sampling_logp_difference/mean": 0.011621036566793919, + "step": 2 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 698.0, + "completions/mean_length": 583.4375, + "completions/mean_terminated_length": 511.2174072265625, + "completions/min_length": 204.0, + "completions/min_terminated_length": 204.0, + "entropy": 0.5384182259440422, + "epoch": 0.0016059957173447537, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.01117366086691618, + "learning_rate": 1e-05, + "loss": 0.1108, + "num_tokens": 70521.0, + "reward": 0.65625, + "reward_std": 0.47137710452079773, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.4263523817062378, + "sampling/importance_sampling_ratio/mean": 1.0000293254852295, + "sampling/importance_sampling_ratio/min": 0.7268283367156982, + "sampling/sampling_logp_difference/max": 0.3551204204559326, + "sampling/sampling_logp_difference/mean": 0.013133048079907894, + "step": 3 + }, + { + "clip_ratio/high_max": 4.39831092080567e-05, + "clip_ratio/high_mean": 4.39831092080567e-05, + "clip_ratio/low_mean": 0.00010378849401604384, + "clip_ratio/low_min": 0.00010378849401604384, + "clip_ratio/region_mean": 0.00014777160322410055, + "completions/clipped_ratio": 0.40625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 734.0, + "completions/mean_length": 613.5625, + "completions/mean_terminated_length": 507.8947448730469, + "completions/min_length": 217.0, + "completions/min_terminated_length": 217.0, + "entropy": 0.47148050367832184, + "epoch": 0.0021413276231263384, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.013791498728096485, + "learning_rate": 1e-05, + "loss": -0.0218, + "num_tokens": 94467.0, + "reward": 0.34375, + "reward_std": 0.2041158676147461, + "rewards/accuracy_reward/mean": 0.34375, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.3169457912445068, + "sampling/importance_sampling_ratio/mean": 1.0000814199447632, + "sampling/importance_sampling_ratio/min": 0.7270077466964722, + "sampling/sampling_logp_difference/max": 0.3188180923461914, + "sampling/sampling_logp_difference/mean": 0.012500625103712082, + "step": 4 + }, + { + "clip_ratio/high_max": 4.451566928764805e-05, + "clip_ratio/high_mean": 4.451566928764805e-05, + "clip_ratio/low_mean": 0.000116039089334663, + "clip_ratio/low_min": 0.000116039089334663, + "clip_ratio/region_mean": 0.00016055475862231106, + "completions/clipped_ratio": 0.34375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 744.0, + "completions/mean_length": 626.59375, + "completions/mean_terminated_length": 552.5238037109375, + "completions/min_length": 356.0, + "completions/min_terminated_length": 356.0, + "entropy": 0.47515081241726875, + "epoch": 0.0026766595289079227, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.012919590808451176, + "learning_rate": 1e-05, + "loss": 0.0341, + "num_tokens": 118310.0, + "reward": 0.5, + "reward_std": 0.5175491571426392, + "rewards/accuracy_reward/mean": 0.5, + "rewards/accuracy_reward/std": 0.5080004930496216, + "sampling/importance_sampling_ratio/max": 1.4363598823547363, + "sampling/importance_sampling_ratio/mean": 1.0000596046447754, + "sampling/importance_sampling_ratio/min": 0.7076445817947388, + "sampling/sampling_logp_difference/max": 0.36211204528808594, + "sampling/sampling_logp_difference/mean": 0.013171524740755558, + "step": 5 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0003781330888159573, + "clip_ratio/low_min": 0.0003781330888159573, + "clip_ratio/region_mean": 0.0003781330888159573, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 763.0, + "completions/mean_length": 558.78125, + "completions/mean_terminated_length": 520.0370483398438, + "completions/min_length": 160.0, + "completions/min_terminated_length": 160.0, + "entropy": 0.5127999149262905, + "epoch": 0.0032119914346895075, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.011963306926190853, + "learning_rate": 1e-05, + "loss": 0.0437, + "num_tokens": 140095.0, + "reward": 0.78125, + "reward_std": 0.2041158676147461, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.5415970087051392, + "sampling/importance_sampling_ratio/mean": 1.0000983476638794, + "sampling/importance_sampling_ratio/min": 0.6230328679084778, + "sampling/sampling_logp_difference/max": 0.4731559753417969, + "sampling/sampling_logp_difference/mean": 0.013560203835368156, + "step": 6 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00012046683696098626, + "clip_ratio/low_min": 0.00012046683696098626, + "clip_ratio/region_mean": 0.00012046683696098626, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 762.0, + "completions/mean_length": 565.0, + "completions/mean_terminated_length": 544.0, + "completions/min_length": 304.0, + "completions/min_terminated_length": 304.0, + "entropy": 0.4087902195751667, + "epoch": 0.003747323340471092, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.010129212401807308, + "learning_rate": 1e-05, + "loss": 0.0421, + "num_tokens": 161799.0, + "reward": 0.71875, + "reward_std": 0.3787454068660736, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.552953839302063, + "sampling/importance_sampling_ratio/mean": 0.9997699856758118, + "sampling/importance_sampling_ratio/min": 0.7039182186126709, + "sampling/sampling_logp_difference/max": 0.4401588439941406, + "sampling/sampling_logp_difference/mean": 0.011312289163470268, + "step": 7 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 752.0, + "completions/mean_length": 670.0, + "completions/mean_terminated_length": 611.2000122070312, + "completions/min_length": 372.0, + "completions/min_terminated_length": 372.0, + "entropy": 0.40500492975115776, + "epoch": 0.004282655246252677, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.005402647890150547, + "learning_rate": 1e-05, + "loss": 0.0368, + "num_tokens": 186759.0, + "reward": 0.53125, + "reward_std": 0.24511480331420898, + "rewards/accuracy_reward/mean": 0.53125, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.4619346857070923, + "sampling/importance_sampling_ratio/mean": 0.999567449092865, + "sampling/importance_sampling_ratio/min": 0.3118903934955597, + "sampling/sampling_logp_difference/max": 1.1651034355163574, + "sampling/sampling_logp_difference/mean": 0.010809365659952164, + "step": 8 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 766.0, + "completions/mean_length": 647.40625, + "completions/mean_terminated_length": 575.0499877929688, + "completions/min_length": 280.0, + "completions/min_terminated_length": 280.0, + "entropy": 0.36652812361717224, + "epoch": 0.004817987152034261, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": 0.0301, + "num_tokens": 211412.0, + "reward": 0.65625, + "reward_std": 0.2041158676147461, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.2698006629943848, + "sampling/importance_sampling_ratio/mean": 1.0001059770584106, + "sampling/importance_sampling_ratio/min": 0.7396385073661804, + "sampling/sampling_logp_difference/max": 0.3015937805175781, + "sampling/sampling_logp_difference/mean": 0.010424943640828133, + "step": 9 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 6.49688154226169e-05, + "clip_ratio/low_min": 6.49688154226169e-05, + "clip_ratio/region_mean": 6.49688154226169e-05, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 726.0, + "completions/mean_length": 525.96875, + "completions/mean_terminated_length": 518.1612548828125, + "completions/min_length": 267.0, + "completions/min_terminated_length": 267.0, + "entropy": 0.36391472816467285, + "epoch": 0.0053533190578158455, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": 0.0078, + "num_tokens": 231275.0, + "reward": 0.90625, + "reward_std": 0.1293872892856598, + "rewards/accuracy_reward/mean": 0.90625, + "rewards/accuracy_reward/std": 0.2961445748806, + "sampling/importance_sampling_ratio/max": 1.32644522190094, + "sampling/importance_sampling_ratio/mean": 1.0003172159194946, + "sampling/importance_sampling_ratio/min": 0.7914782762527466, + "sampling/sampling_logp_difference/max": 0.2825026512145996, + "sampling/sampling_logp_difference/mean": 0.0099624739959836, + "step": 10 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 4.069010537932627e-05, + "clip_ratio/low_min": 4.069010537932627e-05, + "clip_ratio/region_mean": 4.069010537932627e-05, + "completions/clipped_ratio": 0.65625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 719.0, + "completions/mean_length": 708.84375, + "completions/mean_terminated_length": 595.9091186523438, + "completions/min_length": 324.0, + "completions/min_terminated_length": 324.0, + "entropy": 0.6045695021748543, + "epoch": 0.005888650963597431, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.010288149118423462, + "learning_rate": 1e-05, + "loss": 0.0168, + "num_tokens": 257926.0, + "reward": 0.1875, + "reward_std": 0.249358132481575, + "rewards/accuracy_reward/mean": 0.1875, + "rewards/accuracy_reward/std": 0.3965577781200409, + "sampling/importance_sampling_ratio/max": 1.4171395301818848, + "sampling/importance_sampling_ratio/mean": 0.9999479651451111, + "sampling/importance_sampling_ratio/min": 0.7186304330825806, + "sampling/sampling_logp_difference/max": 0.34864044189453125, + "sampling/sampling_logp_difference/mean": 0.015258465893566608, + "step": 11 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 760.0, + "completions/mean_length": 551.21875, + "completions/mean_terminated_length": 490.5199890136719, + "completions/min_length": 295.0, + "completions/min_terminated_length": 295.0, + "entropy": 0.40489397943019867, + "epoch": 0.006423982869379015, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.012296498753130436, + "learning_rate": 1e-05, + "loss": 0.0895, + "num_tokens": 279069.0, + "reward": 0.75, + "reward_std": 0.3335031569004059, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.4719487428665161, + "sampling/importance_sampling_ratio/mean": 0.9999110698699951, + "sampling/importance_sampling_ratio/min": 0.704704761505127, + "sampling/sampling_logp_difference/max": 0.38658714294433594, + "sampling/sampling_logp_difference/mean": 0.011324138380587101, + "step": 12 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00010533543172641657, + "clip_ratio/low_min": 0.00010533543172641657, + "clip_ratio/region_mean": 0.00010533543172641657, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 766.0, + "completions/mean_length": 620.625, + "completions/mean_terminated_length": 562.95654296875, + "completions/min_length": 371.0, + "completions/min_terminated_length": 371.0, + "entropy": 0.6603880785405636, + "epoch": 0.006959314775160599, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.012373446486890316, + "learning_rate": 1e-05, + "loss": 0.0984, + "num_tokens": 302353.0, + "reward": 0.5625, + "reward_std": 0.5260357856750488, + "rewards/accuracy_reward/mean": 0.5625, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 1.3349218368530273, + "sampling/importance_sampling_ratio/mean": 1.0000901222229004, + "sampling/importance_sampling_ratio/min": 0.5918812155723572, + "sampling/sampling_logp_difference/max": 0.524449348449707, + "sampling/sampling_logp_difference/mean": 0.013712276704609394, + "step": 13 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 9.522695108898915e-05, + "clip_ratio/low_min": 9.522695108898915e-05, + "clip_ratio/region_mean": 9.522695108898915e-05, + "completions/clipped_ratio": 0.34375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 741.0, + "completions/mean_length": 620.0, + "completions/mean_terminated_length": 542.4761962890625, + "completions/min_length": 221.0, + "completions/min_terminated_length": 221.0, + "entropy": 0.39660315588116646, + "epoch": 0.007494646680942184, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.007810923270881176, + "learning_rate": 1e-05, + "loss": 0.0522, + "num_tokens": 325745.0, + "reward": 0.625, + "reward_std": 0.3104073107242584, + "rewards/accuracy_reward/mean": 0.625, + "rewards/accuracy_reward/std": 0.49186936020851135, + "sampling/importance_sampling_ratio/max": 1.463096261024475, + "sampling/importance_sampling_ratio/mean": 0.9998887181282043, + "sampling/importance_sampling_ratio/min": 0.7350841164588928, + "sampling/sampling_logp_difference/max": 0.3805549144744873, + "sampling/sampling_logp_difference/mean": 0.011892049573361874, + "step": 14 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.369415885070339e-05, + "clip_ratio/low_min": 5.369415885070339e-05, + "clip_ratio/region_mean": 5.369415885070339e-05, + "completions/clipped_ratio": 0.40625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 704.0, + "completions/mean_length": 623.4375, + "completions/mean_terminated_length": 524.5263061523438, + "completions/min_length": 184.0, + "completions/min_terminated_length": 184.0, + "entropy": 0.38310953974723816, + "epoch": 0.008029978586723769, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.007060438394546509, + "learning_rate": 1e-05, + "loss": -0.0022, + "num_tokens": 349143.0, + "reward": 0.40625, + "reward_std": 0.4765698313713074, + "rewards/accuracy_reward/mean": 0.40625, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.4384175539016724, + "sampling/importance_sampling_ratio/mean": 0.9999220371246338, + "sampling/importance_sampling_ratio/min": 0.729432225227356, + "sampling/sampling_logp_difference/max": 0.3635435104370117, + "sampling/sampling_logp_difference/mean": 0.010586755350232124, + "step": 15 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 4.657228055293672e-05, + "clip_ratio/low_min": 4.657228055293672e-05, + "clip_ratio/region_mean": 4.657228055293672e-05, + "completions/clipped_ratio": 0.375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 720.0, + "completions/mean_length": 649.0625, + "completions/mean_terminated_length": 577.7000122070312, + "completions/min_length": 429.0, + "completions/min_terminated_length": 429.0, + "entropy": 0.7524018473923206, + "epoch": 0.008565310492505354, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.023527327924966812, + "learning_rate": 1e-05, + "loss": 0.0716, + "num_tokens": 374665.0, + "reward": 0.5, + "reward_std": 0.4492306709289551, + "rewards/accuracy_reward/mean": 0.5, + "rewards/accuracy_reward/std": 0.5080004930496216, + "sampling/importance_sampling_ratio/max": 1.5210533142089844, + "sampling/importance_sampling_ratio/mean": 1.000063419342041, + "sampling/importance_sampling_ratio/min": 0.637096107006073, + "sampling/sampling_logp_difference/max": 0.4508347511291504, + "sampling/sampling_logp_difference/mean": 0.01580382138490677, + "step": 16 + }, + { + "clip_ratio/high_max": 9.996496009989642e-05, + "clip_ratio/high_mean": 9.996496009989642e-05, + "clip_ratio/low_mean": 0.00015877384066698141, + "clip_ratio/low_min": 0.00015877384066698141, + "clip_ratio/region_mean": 0.00025873880076687783, + "completions/clipped_ratio": 0.34375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 736.0, + "completions/mean_length": 617.75, + "completions/mean_terminated_length": 539.047607421875, + "completions/min_length": 299.0, + "completions/min_terminated_length": 299.0, + "entropy": 0.5123349986970425, + "epoch": 0.009100642398286937, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.016295379027724266, + "learning_rate": 1e-05, + "loss": 0.0207, + "num_tokens": 398241.0, + "reward": 0.25, + "reward_std": 0.4629100561141968, + "rewards/accuracy_reward/mean": 0.25, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.3084748983383179, + "sampling/importance_sampling_ratio/mean": 0.9996823668479919, + "sampling/importance_sampling_ratio/min": 0.7431430816650391, + "sampling/sampling_logp_difference/max": 0.29686665534973145, + "sampling/sampling_logp_difference/mean": 0.012664510868489742, + "step": 17 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0001043877964548301, + "clip_ratio/low_min": 0.0001043877964548301, + "clip_ratio/region_mean": 0.0001043877964548301, + "completions/clipped_ratio": 0.375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 749.0, + "completions/mean_length": 631.15625, + "completions/mean_terminated_length": 549.0499877929688, + "completions/min_length": 342.0, + "completions/min_terminated_length": 342.0, + "entropy": 0.42930785194039345, + "epoch": 0.009635974304068522, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.004890018608421087, + "learning_rate": 1e-05, + "loss": 0.0594, + "num_tokens": 422318.0, + "reward": 0.65625, + "reward_std": 0.3608423173427582, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.3816895484924316, + "sampling/importance_sampling_ratio/mean": 1.0002059936523438, + "sampling/importance_sampling_ratio/min": 0.7530749440193176, + "sampling/sampling_logp_difference/max": 0.3233070373535156, + "sampling/sampling_logp_difference/mean": 0.011523136869072914, + "step": 18 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.9780009905807674e-05, + "clip_ratio/low_min": 5.9780009905807674e-05, + "clip_ratio/region_mean": 5.9780009905807674e-05, + "completions/clipped_ratio": 0.53125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 716.0, + "completions/mean_length": 647.40625, + "completions/mean_terminated_length": 510.7333679199219, + "completions/min_length": 260.0, + "completions/min_terminated_length": 260.0, + "entropy": 0.44347527250647545, + "epoch": 0.010171306209850108, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.010858778841793537, + "learning_rate": 1e-05, + "loss": 0.0795, + "num_tokens": 447203.0, + "reward": 0.4375, + "reward_std": 0.3514062762260437, + "rewards/accuracy_reward/mean": 0.4375, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 1.3750196695327759, + "sampling/importance_sampling_ratio/mean": 0.9996398687362671, + "sampling/importance_sampling_ratio/min": 0.7026919722557068, + "sampling/sampling_logp_difference/max": 0.35283660888671875, + "sampling/sampling_logp_difference/mean": 0.01247062161564827, + "step": 19 + }, + { + "clip_ratio/high_max": 6.782419950468466e-05, + "clip_ratio/high_mean": 6.782419950468466e-05, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 6.782419950468466e-05, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 707.0, + "completions/mean_length": 496.125, + "completions/mean_terminated_length": 445.77777099609375, + "completions/min_length": 189.0, + "completions/min_terminated_length": 189.0, + "entropy": 0.49348969385027885, + "epoch": 0.010706638115631691, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.016775470227003098, + "learning_rate": 1e-05, + "loss": 0.0511, + "num_tokens": 466759.0, + "reward": 0.65625, + "reward_std": 0.3377465009689331, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.519389271736145, + "sampling/importance_sampling_ratio/mean": 0.9998260736465454, + "sampling/importance_sampling_ratio/min": 0.7127357721328735, + "sampling/sampling_logp_difference/max": 0.4183084964752197, + "sampling/sampling_logp_difference/mean": 0.01263282261788845, + "step": 20 + }, + { + "clip_ratio/high_max": 5.5506217904621735e-05, + "clip_ratio/high_mean": 5.5506217904621735e-05, + "clip_ratio/low_mean": 4.98802874062676e-05, + "clip_ratio/low_min": 4.98802874062676e-05, + "clip_ratio/region_mean": 0.00010538650531088933, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 757.0, + "completions/mean_length": 526.78125, + "completions/mean_terminated_length": 482.1111145019531, + "completions/min_length": 197.0, + "completions/min_terminated_length": 197.0, + "entropy": 0.4062811993062496, + "epoch": 0.011241970021413276, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.006292411591857672, + "learning_rate": 1e-05, + "loss": 0.0678, + "num_tokens": 487648.0, + "reward": 0.65625, + "reward_std": 0.3198433816432953, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.2623493671417236, + "sampling/importance_sampling_ratio/mean": 0.9999346733093262, + "sampling/importance_sampling_ratio/min": 0.6255134344100952, + "sampling/sampling_logp_difference/max": 0.46918249130249023, + "sampling/sampling_logp_difference/mean": 0.011023581959307194, + "step": 21 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 4.1377028537681326e-05, + "clip_ratio/low_min": 4.1377028537681326e-05, + "clip_ratio/region_mean": 4.1377028537681326e-05, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 756.0, + "completions/mean_length": 599.15625, + "completions/mean_terminated_length": 533.0869750976562, + "completions/min_length": 280.0, + "completions/min_terminated_length": 280.0, + "entropy": 0.4398523196578026, + "epoch": 0.011777301927194861, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.0055059464648365974, + "learning_rate": 1e-05, + "loss": -0.0123, + "num_tokens": 510197.0, + "reward": 0.71875, + "reward_std": 0.4534739851951599, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.4150367975234985, + "sampling/importance_sampling_ratio/mean": 0.999511182308197, + "sampling/importance_sampling_ratio/min": 0.6897582411766052, + "sampling/sampling_logp_difference/max": 0.3714141845703125, + "sampling/sampling_logp_difference/mean": 0.012775862589478493, + "step": 22 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00016042274728533812, + "clip_ratio/low_min": 0.00016042274728533812, + "clip_ratio/region_mean": 0.00016042274728533812, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 717.0, + "completions/mean_length": 589.46875, + "completions/mean_terminated_length": 548.2692260742188, + "completions/min_length": 303.0, + "completions/min_terminated_length": 303.0, + "entropy": 0.4353533983230591, + "epoch": 0.012312633832976445, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.020144155248999596, + "learning_rate": 1e-05, + "loss": 0.0159, + "num_tokens": 533172.0, + "reward": 0.71875, + "reward_std": 0.3198433816432953, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.5432878732681274, + "sampling/importance_sampling_ratio/mean": 1.0002663135528564, + "sampling/importance_sampling_ratio/min": 0.69236159324646, + "sampling/sampling_logp_difference/max": 0.4339151382446289, + "sampling/sampling_logp_difference/mean": 0.01157721970230341, + "step": 23 + }, + { + "clip_ratio/high_max": 4.7819434257689863e-05, + "clip_ratio/high_mean": 4.7819434257689863e-05, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 4.7819434257689863e-05, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 739.0, + "completions/mean_length": 628.625, + "completions/mean_terminated_length": 574.0869750976562, + "completions/min_length": 341.0, + "completions/min_terminated_length": 341.0, + "entropy": 0.511908870190382, + "epoch": 0.01284796573875803, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.00383503595367074, + "learning_rate": 1e-05, + "loss": 0.0682, + "num_tokens": 557360.0, + "reward": 0.65625, + "reward_std": 0.3808925747871399, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.4700541496276855, + "sampling/importance_sampling_ratio/mean": 0.99983811378479, + "sampling/importance_sampling_ratio/min": 0.36265435814857483, + "sampling/sampling_logp_difference/max": 1.0143051147460938, + "sampling/sampling_logp_difference/mean": 0.013304595835506916, + "step": 24 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00023704511841060594, + "clip_ratio/low_min": 0.00023704511841060594, + "clip_ratio/region_mean": 0.00023704511841060594, + "completions/clipped_ratio": 0.3125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 742.0, + "completions/mean_length": 619.96875, + "completions/mean_terminated_length": 552.6818237304688, + "completions/min_length": 317.0, + "completions/min_terminated_length": 317.0, + "entropy": 0.5788429118692875, + "epoch": 0.013383297644539615, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.016527721658349037, + "learning_rate": 1e-05, + "loss": 0.1029, + "num_tokens": 580695.0, + "reward": 0.65625, + "reward_std": 0.4628904461860657, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.442878246307373, + "sampling/importance_sampling_ratio/mean": 0.9998840689659119, + "sampling/importance_sampling_ratio/min": 0.7391331791877747, + "sampling/sampling_logp_difference/max": 0.3666398525238037, + "sampling/sampling_logp_difference/mean": 0.014603289775550365, + "step": 25 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 8.138021075865254e-05, + "clip_ratio/low_min": 8.138021075865254e-05, + "clip_ratio/region_mean": 8.138021075865254e-05, + "completions/clipped_ratio": 0.53125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 708.0, + "completions/mean_length": 666.21875, + "completions/mean_terminated_length": 550.86669921875, + "completions/min_length": 226.0, + "completions/min_terminated_length": 226.0, + "entropy": 0.48395277187228203, + "epoch": 0.013918629550321198, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.004640842787921429, + "learning_rate": 1e-05, + "loss": 0.0397, + "num_tokens": 606054.0, + "reward": 0.28125, + "reward_std": 0.3471629321575165, + "rewards/accuracy_reward/mean": 0.28125, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.3310222625732422, + "sampling/importance_sampling_ratio/mean": 1.0000438690185547, + "sampling/importance_sampling_ratio/min": 0.2625313699245453, + "sampling/sampling_logp_difference/max": 1.3373847007751465, + "sampling/sampling_logp_difference/mean": 0.01332576759159565, + "step": 26 + }, + { + "clip_ratio/high_max": 5.9073725424241275e-05, + "clip_ratio/high_mean": 5.9073725424241275e-05, + "clip_ratio/low_mean": 9.77333947957959e-05, + "clip_ratio/low_min": 9.77333947957959e-05, + "clip_ratio/region_mean": 0.00015680712022003718, + "completions/clipped_ratio": 0.5625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 670.0, + "completions/mean_length": 666.40625, + "completions/mean_terminated_length": 535.7857666015625, + "completions/min_length": 272.0, + "completions/min_terminated_length": 272.0, + "entropy": 0.5612205415964127, + "epoch": 0.014453961456102784, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.021868634968996048, + "learning_rate": 1e-05, + "loss": 0.0633, + "num_tokens": 631235.0, + "reward": 0.34375, + "reward_std": 0.4218915104866028, + "rewards/accuracy_reward/mean": 0.34375, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.4124186038970947, + "sampling/importance_sampling_ratio/mean": 0.9998308420181274, + "sampling/importance_sampling_ratio/min": 0.7335000038146973, + "sampling/sampling_logp_difference/max": 0.3453035354614258, + "sampling/sampling_logp_difference/mean": 0.013546819798648357, + "step": 27 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 4.6921923058107495e-05, + "clip_ratio/low_min": 4.6921923058107495e-05, + "clip_ratio/region_mean": 4.6921923058107495e-05, + "completions/clipped_ratio": 0.375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 768.0, + "completions/mean_length": 681.625, + "completions/mean_terminated_length": 629.7999877929688, + "completions/min_length": 450.0, + "completions/min_terminated_length": 450.0, + "entropy": 0.4054417908191681, + "epoch": 0.014989293361884369, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.00730283185839653, + "learning_rate": 1e-05, + "loss": 0.0291, + "num_tokens": 657327.0, + "reward": 0.375, + "reward_std": 0.3924051821231842, + "rewards/accuracy_reward/mean": 0.375, + "rewards/accuracy_reward/std": 0.49186936020851135, + "sampling/importance_sampling_ratio/max": 1.3310648202896118, + "sampling/importance_sampling_ratio/mean": 0.999858558177948, + "sampling/importance_sampling_ratio/min": 0.7653622627258301, + "sampling/sampling_logp_difference/max": 0.2859792709350586, + "sampling/sampling_logp_difference/mean": 0.011034416034817696, + "step": 28 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.615453846985474e-05, + "clip_ratio/low_min": 5.615453846985474e-05, + "clip_ratio/region_mean": 5.615453846985474e-05, + "completions/clipped_ratio": 0.25, + "completions/max_length": 768.0, + "completions/max_terminated_length": 745.0, + "completions/mean_length": 635.21875, + "completions/mean_terminated_length": 590.9583740234375, + "completions/min_length": 393.0, + "completions/min_terminated_length": 393.0, + "entropy": 0.3583945333957672, + "epoch": 0.015524625267665952, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.021860741078853607, + "learning_rate": 1e-05, + "loss": 0.0738, + "num_tokens": 681894.0, + "reward": 0.65625, + "reward_std": 0.4397946000099182, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 2.0, + "sampling/importance_sampling_ratio/mean": 1.0001440048217773, + "sampling/importance_sampling_ratio/min": 0.6967951655387878, + "sampling/sampling_logp_difference/max": 0.967193603515625, + "sampling/sampling_logp_difference/mean": 0.010393482632935047, + "step": 29 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.53125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 755.0, + "completions/mean_length": 693.8125, + "completions/mean_terminated_length": 609.7333374023438, + "completions/min_length": 354.0, + "completions/min_terminated_length": 354.0, + "entropy": 0.7556599229574203, + "epoch": 0.016059957173447537, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.005575866438448429, + "learning_rate": 1e-05, + "loss": 0.0234, + "num_tokens": 708392.0, + "reward": 0.3125, + "reward_std": 0.2587745785713196, + "rewards/accuracy_reward/mean": 0.3125, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.5003604888916016, + "sampling/importance_sampling_ratio/mean": 0.9996361136436462, + "sampling/importance_sampling_ratio/min": 0.6749789714813232, + "sampling/sampling_logp_difference/max": 0.40570545196533203, + "sampling/sampling_logp_difference/mean": 0.01721298322081566, + "step": 30 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 9.409141421201639e-05, + "clip_ratio/low_min": 9.409141421201639e-05, + "clip_ratio/region_mean": 9.409141421201639e-05, + "completions/clipped_ratio": 0.3125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 690.0, + "completions/mean_length": 607.5, + "completions/mean_terminated_length": 534.5454711914062, + "completions/min_length": 351.0, + "completions/min_terminated_length": 351.0, + "entropy": 0.3454369604587555, + "epoch": 0.016595289079229122, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.01821541041135788, + "learning_rate": 1e-05, + "loss": 0.0106, + "num_tokens": 731512.0, + "reward": 0.6875, + "reward_std": 0.292504221200943, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.6597185134887695, + "sampling/importance_sampling_ratio/mean": 1.0002830028533936, + "sampling/importance_sampling_ratio/min": 0.7118902206420898, + "sampling/sampling_logp_difference/max": 0.506648063659668, + "sampling/sampling_logp_difference/mean": 0.00990574061870575, + "step": 31 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 9.516424688627012e-05, + "clip_ratio/low_min": 9.516424688627012e-05, + "clip_ratio/region_mean": 9.516424688627012e-05, + "completions/clipped_ratio": 0.375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 731.0, + "completions/mean_length": 613.40625, + "completions/mean_terminated_length": 520.6500244140625, + "completions/min_length": 179.0, + "completions/min_terminated_length": 179.0, + "entropy": 0.3875400647521019, + "epoch": 0.017130620985010708, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.013307609595358372, + "learning_rate": 1e-05, + "loss": 0.0834, + "num_tokens": 754885.0, + "reward": 0.46875, + "reward_std": 0.4534739851951599, + "rewards/accuracy_reward/mean": 0.46875, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 2.0, + "sampling/importance_sampling_ratio/mean": 1.0003117322921753, + "sampling/importance_sampling_ratio/min": 0.687193751335144, + "sampling/sampling_logp_difference/max": 1.1515445709228516, + "sampling/sampling_logp_difference/mean": 0.011079014278948307, + "step": 32 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 6.041565939085558e-05, + "clip_ratio/low_min": 6.041565939085558e-05, + "clip_ratio/region_mean": 6.041565939085558e-05, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 761.0, + "completions/mean_length": 566.3125, + "completions/mean_terminated_length": 537.5, + "completions/min_length": 279.0, + "completions/min_terminated_length": 279.0, + "entropy": 0.4358655549585819, + "epoch": 0.017665952890792293, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.00740576907992363, + "learning_rate": 1e-05, + "loss": 0.0131, + "num_tokens": 776631.0, + "reward": 0.625, + "reward_std": 0.2177756428718567, + "rewards/accuracy_reward/mean": 0.625, + "rewards/accuracy_reward/std": 0.49186936020851135, + "sampling/importance_sampling_ratio/max": 1.449811339378357, + "sampling/importance_sampling_ratio/mean": 1.0000813007354736, + "sampling/importance_sampling_ratio/min": 0.7329732775688171, + "sampling/sampling_logp_difference/max": 0.3714334964752197, + "sampling/sampling_logp_difference/mean": 0.012371337972581387, + "step": 33 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.46875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 768.0, + "completions/mean_length": 663.46875, + "completions/mean_terminated_length": 571.2352905273438, + "completions/min_length": 199.0, + "completions/min_terminated_length": 199.0, + "entropy": 0.5534002110362053, + "epoch": 0.018201284796573874, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.008740512654185295, + "learning_rate": 1e-05, + "loss": 0.0472, + "num_tokens": 801750.0, + "reward": 0.46875, + "reward_std": 0.2630178928375244, + "rewards/accuracy_reward/mean": 0.46875, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.5338393449783325, + "sampling/importance_sampling_ratio/mean": 1.0004141330718994, + "sampling/importance_sampling_ratio/min": 0.5497848391532898, + "sampling/sampling_logp_difference/max": 0.5982282161712646, + "sampling/sampling_logp_difference/mean": 0.013347679749131203, + "step": 34 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 4.1848008550005034e-05, + "clip_ratio/low_min": 4.1848008550005034e-05, + "clip_ratio/region_mean": 4.1848008550005034e-05, + "completions/clipped_ratio": 0.34375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 755.0, + "completions/mean_length": 625.46875, + "completions/mean_terminated_length": 550.8095092773438, + "completions/min_length": 296.0, + "completions/min_terminated_length": 296.0, + "entropy": 0.5512795709073544, + "epoch": 0.01873661670235546, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.007339011412113905, + "learning_rate": 1e-05, + "loss": 0.0789, + "num_tokens": 825661.0, + "reward": 0.59375, + "reward_std": 0.3608423173427582, + "rewards/accuracy_reward/mean": 0.59375, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.2961760759353638, + "sampling/importance_sampling_ratio/mean": 1.0002658367156982, + "sampling/importance_sampling_ratio/min": 0.7105289697647095, + "sampling/sampling_logp_difference/max": 0.34174561500549316, + "sampling/sampling_logp_difference/mean": 0.013928350992500782, + "step": 35 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00014985193411121145, + "clip_ratio/low_min": 0.00014985193411121145, + "clip_ratio/region_mean": 0.00014985193411121145, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 733.0, + "completions/mean_length": 564.9375, + "completions/mean_terminated_length": 518.0769653320312, + "completions/min_length": 194.0, + "completions/min_terminated_length": 194.0, + "entropy": 0.3453544117510319, + "epoch": 0.019271948608137045, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.005400785245001316, + "learning_rate": 1e-05, + "loss": -0.0211, + "num_tokens": 847395.0, + "reward": 0.59375, + "reward_std": 0.3377465009689331, + "rewards/accuracy_reward/mean": 0.59375, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.2773507833480835, + "sampling/importance_sampling_ratio/mean": 1.000115156173706, + "sampling/importance_sampling_ratio/min": 0.6307759881019592, + "sampling/sampling_logp_difference/max": 0.46080446243286133, + "sampling/sampling_logp_difference/mean": 0.009833919815719128, + "step": 36 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 4.5126354962121695e-05, + "clip_ratio/low_min": 4.5126354962121695e-05, + "clip_ratio/region_mean": 4.5126354962121695e-05, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 737.0, + "completions/mean_length": 590.5, + "completions/mean_terminated_length": 549.5384521484375, + "completions/min_length": 369.0, + "completions/min_terminated_length": 369.0, + "entropy": 0.4163068048655987, + "epoch": 0.01980728051391863, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.019713765010237694, + "learning_rate": 1e-05, + "loss": 0.0623, + "num_tokens": 870251.0, + "reward": 0.71875, + "reward_std": 0.2651650309562683, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.416803002357483, + "sampling/importance_sampling_ratio/mean": 1.0000214576721191, + "sampling/importance_sampling_ratio/min": 0.6959530711174011, + "sampling/sampling_logp_difference/max": 0.3624730110168457, + "sampling/sampling_logp_difference/mean": 0.011494419537484646, + "step": 37 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 9.050188964465633e-05, + "clip_ratio/low_min": 9.050188964465633e-05, + "clip_ratio/region_mean": 9.050188964465633e-05, + "completions/clipped_ratio": 0.3125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 755.0, + "completions/mean_length": 635.71875, + "completions/mean_terminated_length": 575.5909423828125, + "completions/min_length": 368.0, + "completions/min_terminated_length": 368.0, + "entropy": 0.41894080862402916, + "epoch": 0.020342612419700215, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.00975870992988348, + "learning_rate": 1e-05, + "loss": 0.0046, + "num_tokens": 894474.0, + "reward": 0.46875, + "reward_std": 0.4628904461860657, + "rewards/accuracy_reward/mean": 0.46875, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.442466139793396, + "sampling/importance_sampling_ratio/mean": 1.0000383853912354, + "sampling/importance_sampling_ratio/min": 0.6907856464385986, + "sampling/sampling_logp_difference/max": 0.36992573738098145, + "sampling/sampling_logp_difference/mean": 0.012097334489226341, + "step": 38 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 4.9682035751175135e-05, + "clip_ratio/low_min": 4.9682035751175135e-05, + "clip_ratio/region_mean": 4.9682035751175135e-05, + "completions/clipped_ratio": 0.375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 768.0, + "completions/mean_length": 631.65625, + "completions/mean_terminated_length": 549.8500366210938, + "completions/min_length": 230.0, + "completions/min_terminated_length": 230.0, + "entropy": 0.40683867037296295, + "epoch": 0.0208779443254818, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.007603779900819063, + "learning_rate": 1e-05, + "loss": 0.0653, + "num_tokens": 918247.0, + "reward": 0.5, + "reward_std": 0.3514062464237213, + "rewards/accuracy_reward/mean": 0.5, + "rewards/accuracy_reward/std": 0.5080004930496216, + "sampling/importance_sampling_ratio/max": 1.395285964012146, + "sampling/importance_sampling_ratio/mean": 0.9997595548629761, + "sampling/importance_sampling_ratio/min": 0.6868053674697876, + "sampling/sampling_logp_difference/max": 0.375704288482666, + "sampling/sampling_logp_difference/mean": 0.010730033740401268, + "step": 39 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.225752465776168e-05, + "clip_ratio/low_min": 5.225752465776168e-05, + "clip_ratio/region_mean": 5.225752465776168e-05, + "completions/clipped_ratio": 0.34375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 746.0, + "completions/mean_length": 612.875, + "completions/mean_terminated_length": 531.6190795898438, + "completions/min_length": 380.0, + "completions/min_terminated_length": 380.0, + "entropy": 0.6648869961500168, + "epoch": 0.021413276231263382, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.0019930468406528234, + "learning_rate": 1e-05, + "loss": 0.0421, + "num_tokens": 941723.0, + "reward": 0.5, + "reward_std": 0.1767766922712326, + "rewards/accuracy_reward/mean": 0.5, + "rewards/accuracy_reward/std": 0.5080004930496216, + "sampling/importance_sampling_ratio/max": 1.2759915590286255, + "sampling/importance_sampling_ratio/mean": 1.0007076263427734, + "sampling/importance_sampling_ratio/min": 0.7806393504142761, + "sampling/sampling_logp_difference/max": 0.24764204025268555, + "sampling/sampling_logp_difference/mean": 0.01612691953778267, + "step": 40 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.137690095580183e-05, + "clip_ratio/low_min": 5.137690095580183e-05, + "clip_ratio/region_mean": 5.137690095580183e-05, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 685.0, + "completions/mean_length": 542.25, + "completions/mean_terminated_length": 479.03997802734375, + "completions/min_length": 282.0, + "completions/min_terminated_length": 282.0, + "entropy": 0.40131812915205956, + "epoch": 0.021948608137044967, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.0050626760348677635, + "learning_rate": 1e-05, + "loss": 0.0117, + "num_tokens": 962587.0, + "reward": 0.5, + "reward_std": 0.292504221200943, + "rewards/accuracy_reward/mean": 0.5, + "rewards/accuracy_reward/std": 0.5080004930496216, + "sampling/importance_sampling_ratio/max": 1.278491497039795, + "sampling/importance_sampling_ratio/mean": 0.9998194575309753, + "sampling/importance_sampling_ratio/min": 0.7864078879356384, + "sampling/sampling_logp_difference/max": 0.2456808090209961, + "sampling/sampling_logp_difference/mean": 0.011833418160676956, + "step": 41 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.000168136422871612, + "clip_ratio/low_min": 0.000168136422871612, + "clip_ratio/region_mean": 0.000168136422871612, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 753.0, + "completions/mean_length": 565.09375, + "completions/mean_terminated_length": 527.5184936523438, + "completions/min_length": 215.0, + "completions/min_terminated_length": 215.0, + "entropy": 0.3964141681790352, + "epoch": 0.022483940042826552, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.017772329971194267, + "learning_rate": 1e-05, + "loss": 0.0801, + "num_tokens": 984094.0, + "reward": 0.59375, + "reward_std": 0.4628904461860657, + "rewards/accuracy_reward/mean": 0.59375, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.447232723236084, + "sampling/importance_sampling_ratio/mean": 0.9995170831680298, + "sampling/importance_sampling_ratio/min": 0.6852384805679321, + "sampling/sampling_logp_difference/max": 0.377988338470459, + "sampling/sampling_logp_difference/mean": 0.011716882698237896, + "step": 42 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 9.637625043978915e-05, + "clip_ratio/low_min": 9.637625043978915e-05, + "clip_ratio/region_mean": 9.637625043978915e-05, + "completions/clipped_ratio": 0.3125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 758.0, + "completions/mean_length": 568.65625, + "completions/mean_terminated_length": 478.04547119140625, + "completions/min_length": 233.0, + "completions/min_terminated_length": 233.0, + "entropy": 0.39531830325722694, + "epoch": 0.023019271948608137, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.0030931883957237005, + "learning_rate": 1e-05, + "loss": 0.0374, + "num_tokens": 1006171.0, + "reward": 0.5, + "reward_std": 0.1767766922712326, + "rewards/accuracy_reward/mean": 0.5, + "rewards/accuracy_reward/std": 0.5080004930496216, + "sampling/importance_sampling_ratio/max": 1.4095613956451416, + "sampling/importance_sampling_ratio/mean": 1.0000072717666626, + "sampling/importance_sampling_ratio/min": 0.7386751770973206, + "sampling/sampling_logp_difference/max": 0.3432786464691162, + "sampling/sampling_logp_difference/mean": 0.011342217214405537, + "step": 43 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.0220973207615316e-05, + "clip_ratio/low_min": 5.0220973207615316e-05, + "clip_ratio/region_mean": 5.0220973207615316e-05, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 767.0, + "completions/mean_length": 591.84375, + "completions/mean_terminated_length": 522.9130249023438, + "completions/min_length": 268.0, + "completions/min_terminated_length": 268.0, + "entropy": 0.39927684888243675, + "epoch": 0.023554603854389723, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.005198259372264147, + "learning_rate": 1e-05, + "loss": 0.0131, + "num_tokens": 1028838.0, + "reward": 0.75, + "reward_std": 0.292504221200943, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.3477733135223389, + "sampling/importance_sampling_ratio/mean": 0.9998369216918945, + "sampling/importance_sampling_ratio/min": 0.6835851669311523, + "sampling/sampling_logp_difference/max": 0.380403995513916, + "sampling/sampling_logp_difference/mean": 0.00997208058834076, + "step": 44 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 4.4658805563813075e-05, + "clip_ratio/low_min": 4.4658805563813075e-05, + "clip_ratio/region_mean": 4.4658805563813075e-05, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 764.0, + "completions/mean_length": 632.125, + "completions/mean_terminated_length": 578.95654296875, + "completions/min_length": 437.0, + "completions/min_terminated_length": 437.0, + "entropy": 0.4036395661532879, + "epoch": 0.024089935760171308, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.007570364512503147, + "learning_rate": 1e-05, + "loss": 0.0508, + "num_tokens": 1053194.0, + "reward": 0.6875, + "reward_std": 0.3514062464237213, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.365864872932434, + "sampling/importance_sampling_ratio/mean": 1.0003732442855835, + "sampling/importance_sampling_ratio/min": 0.7070538401603699, + "sampling/sampling_logp_difference/max": 0.3466484546661377, + "sampling/sampling_logp_difference/mean": 0.011278525926172733, + "step": 45 + }, + { + "clip_ratio/high_max": 4.98802874062676e-05, + "clip_ratio/high_mean": 4.98802874062676e-05, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 4.98802874062676e-05, + "completions/clipped_ratio": 0.40625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 765.0, + "completions/mean_length": 636.6875, + "completions/mean_terminated_length": 546.8421020507812, + "completions/min_length": 301.0, + "completions/min_terminated_length": 301.0, + "entropy": 0.40303152799606323, + "epoch": 0.02462526766595289, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.0037608416751027107, + "learning_rate": 1e-05, + "loss": 0.0215, + "num_tokens": 1077584.0, + "reward": 0.65625, + "reward_std": 0.3061639666557312, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.2744766473770142, + "sampling/importance_sampling_ratio/mean": 0.999945342540741, + "sampling/importance_sampling_ratio/min": 0.7364591360092163, + "sampling/sampling_logp_difference/max": 0.30590152740478516, + "sampling/sampling_logp_difference/mean": 0.01124804001301527, + "step": 46 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 752.0, + "completions/mean_length": 557.90625, + "completions/mean_terminated_length": 527.8928833007812, + "completions/min_length": 315.0, + "completions/min_terminated_length": 315.0, + "entropy": 0.5076882019639015, + "epoch": 0.025160599571734475, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.010317984037101269, + "learning_rate": 1e-05, + "loss": 0.048, + "num_tokens": 1099197.0, + "reward": 0.65625, + "reward_std": 0.4355708956718445, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.2960543632507324, + "sampling/importance_sampling_ratio/mean": 0.999902069568634, + "sampling/importance_sampling_ratio/min": 0.627313494682312, + "sampling/sampling_logp_difference/max": 0.4663088321685791, + "sampling/sampling_logp_difference/mean": 0.013130159117281437, + "step": 47 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 751.0, + "completions/mean_length": 497.90625, + "completions/mean_terminated_length": 469.96551513671875, + "completions/min_length": 329.0, + "completions/min_terminated_length": 329.0, + "entropy": 0.4591529108583927, + "epoch": 0.02569593147751606, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.00704563595354557, + "learning_rate": 1e-05, + "loss": 0.0141, + "num_tokens": 1118266.0, + "reward": 0.78125, + "reward_std": 0.24511480331420898, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.2926690578460693, + "sampling/importance_sampling_ratio/mean": 0.9999036192893982, + "sampling/importance_sampling_ratio/min": 0.6051042079925537, + "sampling/sampling_logp_difference/max": 0.502354621887207, + "sampling/sampling_logp_difference/mean": 0.011426439508795738, + "step": 48 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00020978445900254883, + "clip_ratio/low_min": 0.00020978445900254883, + "clip_ratio/region_mean": 0.00020978445900254883, + "completions/clipped_ratio": 0.3125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 731.0, + "completions/mean_length": 565.28125, + "completions/mean_terminated_length": 473.1363830566406, + "completions/min_length": 257.0, + "completions/min_terminated_length": 257.0, + "entropy": 0.5920190773904324, + "epoch": 0.026231263383297645, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.025336306542158127, + "learning_rate": 1e-05, + "loss": 0.0812, + "num_tokens": 1140291.0, + "reward": 0.59375, + "reward_std": 0.4218915104866028, + "rewards/accuracy_reward/mean": 0.59375, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.2873826026916504, + "sampling/importance_sampling_ratio/mean": 0.9996625781059265, + "sampling/importance_sampling_ratio/min": 0.7075410485267639, + "sampling/sampling_logp_difference/max": 0.3459596633911133, + "sampling/sampling_logp_difference/mean": 0.015037329867482185, + "step": 49 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 768.0, + "completions/max_terminated_length": 768.0, + "completions/mean_length": 567.90625, + "completions/mean_terminated_length": 501.2083435058594, + "completions/min_length": 245.0, + "completions/min_terminated_length": 245.0, + "entropy": 0.37092962861061096, + "epoch": 0.02676659528907923, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.006071723531931639, + "learning_rate": 1e-05, + "loss": 0.0158, + "num_tokens": 1162016.0, + "reward": 0.5625, + "reward_std": 0.1157275140285492, + "rewards/accuracy_reward/mean": 0.5625, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 1.3203513622283936, + "sampling/importance_sampling_ratio/mean": 1.0002940893173218, + "sampling/importance_sampling_ratio/min": 0.7036080360412598, + "sampling/sampling_logp_difference/max": 0.3515338897705078, + "sampling/sampling_logp_difference/mean": 0.010587909258902073, + "step": 50 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 738.0, + "completions/mean_length": 604.125, + "completions/mean_terminated_length": 573.7777709960938, + "completions/min_length": 364.0, + "completions/min_terminated_length": 364.0, + "entropy": 0.37366531416773796, + "epoch": 0.027301927194860815, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.006548416335135698, + "learning_rate": 1e-05, + "loss": 0.0512, + "num_tokens": 1185244.0, + "reward": 0.65625, + "reward_std": 0.3471629321575165, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.4837409257888794, + "sampling/importance_sampling_ratio/mean": 1.0000320672988892, + "sampling/importance_sampling_ratio/min": 0.7552201747894287, + "sampling/sampling_logp_difference/max": 0.39456653594970703, + "sampling/sampling_logp_difference/mean": 0.0101691335439682, + "step": 51 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00010076310354634188, + "clip_ratio/low_min": 0.00010076310354634188, + "clip_ratio/region_mean": 0.00010076310354634188, + "completions/clipped_ratio": 0.25, + "completions/max_length": 768.0, + "completions/max_terminated_length": 768.0, + "completions/mean_length": 580.0, + "completions/mean_terminated_length": 517.3333740234375, + "completions/min_length": 247.0, + "completions/min_terminated_length": 247.0, + "entropy": 0.43295803666114807, + "epoch": 0.027837259100642397, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.011636517941951752, + "learning_rate": 1e-05, + "loss": 0.0631, + "num_tokens": 1207460.0, + "reward": 0.4375, + "reward_std": 0.2925041913986206, + "rewards/accuracy_reward/mean": 0.4375, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 1.3694878816604614, + "sampling/importance_sampling_ratio/mean": 0.9999204277992249, + "sampling/importance_sampling_ratio/min": 0.7504498958587646, + "sampling/sampling_logp_difference/max": 0.3144369125366211, + "sampling/sampling_logp_difference/mean": 0.012383855879306793, + "step": 52 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.8823530707741156e-05, + "clip_ratio/low_min": 5.8823530707741156e-05, + "clip_ratio/region_mean": 5.8823530707741156e-05, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 753.0, + "completions/mean_length": 640.71875, + "completions/mean_terminated_length": 590.9130249023438, + "completions/min_length": 338.0, + "completions/min_terminated_length": 338.0, + "entropy": 0.42586465552449226, + "epoch": 0.028372591006423982, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.01022303570061922, + "learning_rate": 1e-05, + "loss": 0.039, + "num_tokens": 1231555.0, + "reward": 0.6875, + "reward_std": 0.4671337604522705, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.3079744577407837, + "sampling/importance_sampling_ratio/mean": 1.0001554489135742, + "sampling/importance_sampling_ratio/min": 0.6923260688781738, + "sampling/sampling_logp_difference/max": 0.36769819259643555, + "sampling/sampling_logp_difference/mean": 0.011949202977120876, + "step": 53 + }, + { + "clip_ratio/high_max": 5.577866977546364e-05, + "clip_ratio/high_mean": 5.577866977546364e-05, + "clip_ratio/low_mean": 0.00014533525245497003, + "clip_ratio/low_min": 0.00014533525245497003, + "clip_ratio/region_mean": 0.00020111392223043367, + "completions/clipped_ratio": 0.40625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 732.0, + "completions/mean_length": 623.90625, + "completions/mean_terminated_length": 525.3157958984375, + "completions/min_length": 348.0, + "completions/min_terminated_length": 348.0, + "entropy": 0.554719552397728, + "epoch": 0.028907922912205567, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.011497048661112785, + "learning_rate": 1e-05, + "loss": 0.069, + "num_tokens": 1255304.0, + "reward": 0.5, + "reward_std": 0.3514062464237213, + "rewards/accuracy_reward/mean": 0.5, + "rewards/accuracy_reward/std": 0.5080004930496216, + "sampling/importance_sampling_ratio/max": 1.4615116119384766, + "sampling/importance_sampling_ratio/mean": 0.9998668432235718, + "sampling/importance_sampling_ratio/min": 0.6180641651153564, + "sampling/sampling_logp_difference/max": 0.48116302490234375, + "sampling/sampling_logp_difference/mean": 0.013421766459941864, + "step": 54 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 759.0, + "completions/mean_length": 639.9375, + "completions/mean_terminated_length": 589.8261108398438, + "completions/min_length": 411.0, + "completions/min_terminated_length": 411.0, + "entropy": 0.48903779312968254, + "epoch": 0.029443254817987152, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.02093012072145939, + "learning_rate": 1e-05, + "loss": 0.0902, + "num_tokens": 1279526.0, + "reward": 0.75, + "reward_std": 0.4355512857437134, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.558075189590454, + "sampling/importance_sampling_ratio/mean": 1.0000821352005005, + "sampling/importance_sampling_ratio/min": 0.7071018815040588, + "sampling/sampling_logp_difference/max": 0.4434511661529541, + "sampling/sampling_logp_difference/mean": 0.012461931444704533, + "step": 55 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00010899157132371329, + "clip_ratio/low_min": 0.00010899157132371329, + "clip_ratio/region_mean": 0.00010899157132371329, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 757.0, + "completions/mean_length": 573.6875, + "completions/mean_terminated_length": 519.2799682617188, + "completions/min_length": 228.0, + "completions/min_terminated_length": 228.0, + "entropy": 0.5139472596347332, + "epoch": 0.029978586723768737, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.012574407272040844, + "learning_rate": 1e-05, + "loss": -0.0136, + "num_tokens": 1301644.0, + "reward": 0.65625, + "reward_std": 0.24511480331420898, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.303346872329712, + "sampling/importance_sampling_ratio/mean": 1.0001070499420166, + "sampling/importance_sampling_ratio/min": 0.7205821871757507, + "sampling/sampling_logp_difference/max": 0.3276958465576172, + "sampling/sampling_logp_difference/mean": 0.013474510982632637, + "step": 56 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.1759834605036303e-05, + "clip_ratio/low_min": 5.1759834605036303e-05, + "clip_ratio/region_mean": 5.1759834605036303e-05, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 662.0, + "completions/mean_length": 491.625, + "completions/mean_terminated_length": 427.8461608886719, + "completions/min_length": 263.0, + "completions/min_terminated_length": 263.0, + "entropy": 0.4833686873316765, + "epoch": 0.030513918629550323, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.004445187747478485, + "learning_rate": 1e-05, + "loss": 0.0981, + "num_tokens": 1320616.0, + "reward": 0.78125, + "reward_std": 0.3061639964580536, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.339613914489746, + "sampling/importance_sampling_ratio/mean": 0.9995917677879333, + "sampling/importance_sampling_ratio/min": 0.7019649744033813, + "sampling/sampling_logp_difference/max": 0.35387176275253296, + "sampling/sampling_logp_difference/mean": 0.01272429060190916, + "step": 57 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 4.459507545107044e-05, + "clip_ratio/low_min": 4.459507545107044e-05, + "clip_ratio/region_mean": 4.459507545107044e-05, + "completions/clipped_ratio": 0.40625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 733.0, + "completions/mean_length": 648.4375, + "completions/mean_terminated_length": 566.631591796875, + "completions/min_length": 399.0, + "completions/min_terminated_length": 399.0, + "entropy": 0.4782155305147171, + "epoch": 0.031049250535331904, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.011375176720321178, + "learning_rate": 1e-05, + "loss": 0.0096, + "num_tokens": 1345430.0, + "reward": 0.5, + "reward_std": 0.2314550280570984, + "rewards/accuracy_reward/mean": 0.5, + "rewards/accuracy_reward/std": 0.5080004930496216, + "sampling/importance_sampling_ratio/max": 1.4411849975585938, + "sampling/importance_sampling_ratio/mean": 1.000040054321289, + "sampling/importance_sampling_ratio/min": 0.7014603614807129, + "sampling/sampling_logp_difference/max": 0.3654656410217285, + "sampling/sampling_logp_difference/mean": 0.012318131513893604, + "step": 58 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0003555068797140848, + "clip_ratio/low_min": 0.0003555068797140848, + "clip_ratio/region_mean": 0.0003555068797140848, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 759.0, + "completions/mean_length": 510.15625, + "completions/mean_terminated_length": 492.9667053222656, + "completions/min_length": 297.0, + "completions/min_terminated_length": 297.0, + "entropy": 0.562933512032032, + "epoch": 0.03158458244111349, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.00894081499427557, + "learning_rate": 1e-05, + "loss": -0.0186, + "num_tokens": 1365619.0, + "reward": 0.625, + "reward_std": 0.3650856614112854, + "rewards/accuracy_reward/mean": 0.625, + "rewards/accuracy_reward/std": 0.49186936020851135, + "sampling/importance_sampling_ratio/max": 1.313398838043213, + "sampling/importance_sampling_ratio/mean": 0.9997080564498901, + "sampling/importance_sampling_ratio/min": 0.5310248136520386, + "sampling/sampling_logp_difference/max": 0.6329464912414551, + "sampling/sampling_logp_difference/mean": 0.013425687327980995, + "step": 59 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.34375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 642.0, + "completions/mean_length": 583.03125, + "completions/mean_terminated_length": 486.1428527832031, + "completions/min_length": 339.0, + "completions/min_terminated_length": 339.0, + "entropy": 0.5129562467336655, + "epoch": 0.032119914346895075, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.005703146103769541, + "learning_rate": 1e-05, + "loss": 0.0481, + "num_tokens": 1388652.0, + "reward": 0.625, + "reward_std": 0.2177756428718567, + "rewards/accuracy_reward/mean": 0.625, + "rewards/accuracy_reward/std": 0.49186936020851135, + "sampling/importance_sampling_ratio/max": 1.3739655017852783, + "sampling/importance_sampling_ratio/mean": 0.9998830556869507, + "sampling/importance_sampling_ratio/min": 0.45569929480552673, + "sampling/sampling_logp_difference/max": 0.7859221696853638, + "sampling/sampling_logp_difference/mean": 0.014459557831287384, + "step": 60 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00010529793144087307, + "clip_ratio/low_min": 0.00010529793144087307, + "clip_ratio/region_mean": 0.00010529793144087307, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 752.0, + "completions/mean_length": 596.875, + "completions/mean_terminated_length": 529.9130249023438, + "completions/min_length": 328.0, + "completions/min_terminated_length": 328.0, + "entropy": 0.45681414380669594, + "epoch": 0.032655246252676656, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.013348175212740898, + "learning_rate": 1e-05, + "loss": -0.0109, + "num_tokens": 1411592.0, + "reward": 0.5, + "reward_std": 0.2314550280570984, + "rewards/accuracy_reward/mean": 0.5, + "rewards/accuracy_reward/std": 0.5080004930496216, + "sampling/importance_sampling_ratio/max": 1.4582934379577637, + "sampling/importance_sampling_ratio/mean": 0.9998890161514282, + "sampling/importance_sampling_ratio/min": 0.7152434587478638, + "sampling/sampling_logp_difference/max": 0.37726688385009766, + "sampling/sampling_logp_difference/mean": 0.012740189209580421, + "step": 61 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 768.0, + "completions/mean_length": 509.625, + "completions/mean_terminated_length": 450.0000305175781, + "completions/min_length": 263.0, + "completions/min_terminated_length": 263.0, + "entropy": 0.5020339004695415, + "epoch": 0.033190578158458245, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.006378734949976206, + "learning_rate": 1e-05, + "loss": 0.0418, + "num_tokens": 1431348.0, + "reward": 0.53125, + "reward_std": 0.24511480331420898, + "rewards/accuracy_reward/mean": 0.53125, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.4372516870498657, + "sampling/importance_sampling_ratio/mean": 1.000119924545288, + "sampling/importance_sampling_ratio/min": 0.509554922580719, + "sampling/sampling_logp_difference/max": 0.674217700958252, + "sampling/sampling_logp_difference/mean": 0.014347701333463192, + "step": 62 + }, + { + "clip_ratio/high_max": 5.548158151214011e-05, + "clip_ratio/high_mean": 5.548158151214011e-05, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 5.548158151214011e-05, + "completions/clipped_ratio": 0.46875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 616.0, + "completions/mean_length": 619.4375, + "completions/mean_terminated_length": 488.3529357910156, + "completions/min_length": 359.0, + "completions/min_terminated_length": 359.0, + "entropy": 0.5028683356940746, + "epoch": 0.03372591006423983, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.008214807137846947, + "learning_rate": 1e-05, + "loss": 0.047, + "num_tokens": 1454962.0, + "reward": 0.34375, + "reward_std": 0.1293872892856598, + "rewards/accuracy_reward/mean": 0.34375, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.3948371410369873, + "sampling/importance_sampling_ratio/mean": 0.9998055696487427, + "sampling/importance_sampling_ratio/min": 0.6906365752220154, + "sampling/sampling_logp_difference/max": 0.3701416254043579, + "sampling/sampling_logp_difference/mean": 0.013865773566067219, + "step": 63 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00010791813110699877, + "clip_ratio/low_min": 0.00010791813110699877, + "clip_ratio/region_mean": 0.00010791813110699877, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 749.0, + "completions/mean_length": 579.0, + "completions/mean_terminated_length": 505.0434875488281, + "completions/min_length": 194.0, + "completions/min_terminated_length": 194.0, + "entropy": 0.5049542114138603, + "epoch": 0.034261241970021415, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.011847138404846191, + "learning_rate": 1e-05, + "loss": 0.0918, + "num_tokens": 1477394.0, + "reward": 0.5, + "reward_std": 0.3745020925998688, + "rewards/accuracy_reward/mean": 0.5, + "rewards/accuracy_reward/std": 0.5080004930496216, + "sampling/importance_sampling_ratio/max": 1.4061970710754395, + "sampling/importance_sampling_ratio/mean": 1.000219702720642, + "sampling/importance_sampling_ratio/min": 0.39653947949409485, + "sampling/sampling_logp_difference/max": 0.9249796867370605, + "sampling/sampling_logp_difference/mean": 0.013348663225769997, + "step": 64 + }, + { + "clip_ratio/high_max": 5.86579080845695e-05, + "clip_ratio/high_mean": 5.86579080845695e-05, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 5.86579080845695e-05, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 746.0, + "completions/mean_length": 606.375, + "completions/mean_terminated_length": 576.4444580078125, + "completions/min_length": 351.0, + "completions/min_terminated_length": 351.0, + "entropy": 0.3726109452545643, + "epoch": 0.034796573875803, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.012685705907642841, + "learning_rate": 1e-05, + "loss": 0.0339, + "num_tokens": 1500550.0, + "reward": 0.8125, + "reward_std": 0.249358132481575, + "rewards/accuracy_reward/mean": 0.8125, + "rewards/accuracy_reward/std": 0.3965577781200409, + "sampling/importance_sampling_ratio/max": 1.3059941530227661, + "sampling/importance_sampling_ratio/mean": 1.0000492334365845, + "sampling/importance_sampling_ratio/min": 0.6384655833244324, + "sampling/sampling_logp_difference/max": 0.4486875534057617, + "sampling/sampling_logp_difference/mean": 0.010820028372108936, + "step": 65 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 4.527345299720764e-05, + "clip_ratio/low_min": 4.527345299720764e-05, + "clip_ratio/region_mean": 4.527345299720764e-05, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 745.0, + "completions/mean_length": 582.0625, + "completions/mean_terminated_length": 509.3043518066406, + "completions/min_length": 306.0, + "completions/min_terminated_length": 306.0, + "entropy": 0.3910229839384556, + "epoch": 0.035331905781584586, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.00877825915813446, + "learning_rate": 1e-05, + "loss": 0.031, + "num_tokens": 1522968.0, + "reward": 0.6875, + "reward_std": 0.2177756428718567, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.3906184434890747, + "sampling/importance_sampling_ratio/mean": 1.0001715421676636, + "sampling/importance_sampling_ratio/min": 0.7516657710075378, + "sampling/sampling_logp_difference/max": 0.32974863052368164, + "sampling/sampling_logp_difference/mean": 0.010856068693101406, + "step": 66 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 4.145937055000104e-05, + "clip_ratio/low_min": 4.145937055000104e-05, + "clip_ratio/region_mean": 4.145937055000104e-05, + "completions/clipped_ratio": 0.5, + "completions/max_length": 768.0, + "completions/max_terminated_length": 711.0, + "completions/mean_length": 656.03125, + "completions/mean_terminated_length": 544.0625, + "completions/min_length": 342.0, + "completions/min_terminated_length": 342.0, + "entropy": 0.4979759305715561, + "epoch": 0.03586723768736617, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.0032111932523548603, + "learning_rate": 1e-05, + "loss": 0.017, + "num_tokens": 1547993.0, + "reward": 0.21875, + "reward_std": 0.2041158676147461, + "rewards/accuracy_reward/mean": 0.21875, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.33022141456604, + "sampling/importance_sampling_ratio/mean": 0.99981689453125, + "sampling/importance_sampling_ratio/min": 0.5292699337005615, + "sampling/sampling_logp_difference/max": 0.6362566947937012, + "sampling/sampling_logp_difference/mean": 0.01353519968688488, + "step": 67 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 697.0, + "completions/mean_length": 469.375, + "completions/mean_terminated_length": 459.7419128417969, + "completions/min_length": 261.0, + "completions/min_terminated_length": 261.0, + "entropy": 0.43256181851029396, + "epoch": 0.03640256959314775, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.004933727905154228, + "learning_rate": 1e-05, + "loss": 0.0256, + "num_tokens": 1566565.0, + "reward": 0.8125, + "reward_std": 0.2177756428718567, + "rewards/accuracy_reward/mean": 0.8125, + "rewards/accuracy_reward/std": 0.3965577781200409, + "sampling/importance_sampling_ratio/max": 1.2607918977737427, + "sampling/importance_sampling_ratio/mean": 0.9996660351753235, + "sampling/importance_sampling_ratio/min": 0.6787375211715698, + "sampling/sampling_logp_difference/max": 0.38752079010009766, + "sampling/sampling_logp_difference/mean": 0.01155771128833294, + "step": 68 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 758.0, + "completions/mean_length": 567.375, + "completions/mean_terminated_length": 511.1999816894531, + "completions/min_length": 269.0, + "completions/min_terminated_length": 269.0, + "entropy": 0.34118273109197617, + "epoch": 0.03693790149892934, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.012756981886923313, + "learning_rate": 1e-05, + "loss": 0.0819, + "num_tokens": 1588321.0, + "reward": 0.75, + "reward_std": 0.3514062762260437, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.448341727256775, + "sampling/importance_sampling_ratio/mean": 0.9997310638427734, + "sampling/importance_sampling_ratio/min": 0.6787981390953064, + "sampling/sampling_logp_difference/max": 0.3874315023422241, + "sampling/sampling_logp_difference/mean": 0.009688274003565311, + "step": 69 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.5580258049303666e-05, + "clip_ratio/low_min": 5.5580258049303666e-05, + "clip_ratio/region_mean": 5.5580258049303666e-05, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 760.0, + "completions/mean_length": 582.4375, + "completions/mean_terminated_length": 555.9285888671875, + "completions/min_length": 208.0, + "completions/min_terminated_length": 208.0, + "entropy": 0.35030674561858177, + "epoch": 0.03747323340471092, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.006231345701962709, + "learning_rate": 1e-05, + "loss": 0.0373, + "num_tokens": 1610327.0, + "reward": 0.8125, + "reward_std": 0.2587745785713196, + "rewards/accuracy_reward/mean": 0.8125, + "rewards/accuracy_reward/std": 0.3965577781200409, + "sampling/importance_sampling_ratio/max": 1.34609055519104, + "sampling/importance_sampling_ratio/mean": 0.9999900460243225, + "sampling/importance_sampling_ratio/min": 0.585841953754425, + "sampling/sampling_logp_difference/max": 0.5347051620483398, + "sampling/sampling_logp_difference/mean": 0.010446908883750439, + "step": 70 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 6.541077891597524e-05, + "clip_ratio/low_min": 6.541077891597524e-05, + "clip_ratio/region_mean": 6.541077891597524e-05, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 605.0, + "completions/mean_length": 507.0625, + "completions/mean_terminated_length": 446.8461608886719, + "completions/min_length": 199.0, + "completions/min_terminated_length": 199.0, + "entropy": 0.530906118452549, + "epoch": 0.03800856531049251, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.02199765481054783, + "learning_rate": 1e-05, + "loss": 0.0789, + "num_tokens": 1630025.0, + "reward": 0.71875, + "reward_std": 0.4397946000099182, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.4410661458969116, + "sampling/importance_sampling_ratio/mean": 0.9997541308403015, + "sampling/importance_sampling_ratio/min": 0.5709772109985352, + "sampling/sampling_logp_difference/max": 0.560405969619751, + "sampling/sampling_logp_difference/mean": 0.01314842514693737, + "step": 71 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 695.0, + "completions/mean_length": 527.125, + "completions/mean_terminated_length": 471.5384826660156, + "completions/min_length": 334.0, + "completions/min_terminated_length": 334.0, + "entropy": 0.5136992037296295, + "epoch": 0.03854389721627409, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.012632464058697224, + "learning_rate": 1e-05, + "loss": 0.0355, + "num_tokens": 1650381.0, + "reward": 0.46875, + "reward_std": 0.24511480331420898, + "rewards/accuracy_reward/mean": 0.46875, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.2877466678619385, + "sampling/importance_sampling_ratio/mean": 0.9998239874839783, + "sampling/importance_sampling_ratio/min": 0.7674005627632141, + "sampling/sampling_logp_difference/max": 0.26474642753601074, + "sampling/sampling_logp_difference/mean": 0.013403840363025665, + "step": 72 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00011858078505611047, + "clip_ratio/low_min": 0.00011858078505611047, + "clip_ratio/region_mean": 0.00011858078505611047, + "completions/clipped_ratio": 0.3125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 683.0, + "completions/mean_length": 578.84375, + "completions/mean_terminated_length": 492.8636474609375, + "completions/min_length": 277.0, + "completions/min_terminated_length": 277.0, + "entropy": 0.47807905450463295, + "epoch": 0.03907922912205567, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.008951799012720585, + "learning_rate": 1e-05, + "loss": 0.0555, + "num_tokens": 1672192.0, + "reward": 0.40625, + "reward_std": 0.4218915104866028, + "rewards/accuracy_reward/mean": 0.40625, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.3621253967285156, + "sampling/importance_sampling_ratio/mean": 1.0000659227371216, + "sampling/importance_sampling_ratio/min": 0.4066939353942871, + "sampling/sampling_logp_difference/max": 0.8996944427490234, + "sampling/sampling_logp_difference/mean": 0.01292353868484497, + "step": 73 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 6.883259629830718e-05, + "clip_ratio/low_min": 6.883259629830718e-05, + "clip_ratio/region_mean": 6.883259629830718e-05, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 714.0, + "completions/mean_length": 511.875, + "completions/mean_terminated_length": 475.2857360839844, + "completions/min_length": 255.0, + "completions/min_terminated_length": 255.0, + "entropy": 0.330965094268322, + "epoch": 0.03961456102783726, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.005720760673284531, + "learning_rate": 1e-05, + "loss": -0.0119, + "num_tokens": 1692436.0, + "reward": 0.84375, + "reward_std": 0.22201895713806152, + "rewards/accuracy_reward/mean": 0.84375, + "rewards/accuracy_reward/std": 0.3689020276069641, + "sampling/importance_sampling_ratio/max": 1.2868467569351196, + "sampling/importance_sampling_ratio/mean": 1.000016450881958, + "sampling/importance_sampling_ratio/min": 0.5379886627197266, + "sampling/sampling_logp_difference/max": 0.6199178695678711, + "sampling/sampling_logp_difference/mean": 0.00998774729669094, + "step": 74 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 698.0, + "completions/mean_length": 538.875, + "completions/mean_terminated_length": 474.7200012207031, + "completions/min_length": 252.0, + "completions/min_terminated_length": 252.0, + "entropy": 0.43403178453445435, + "epoch": 0.04014989293361884, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.018599966540932655, + "learning_rate": 1e-05, + "loss": 0.0573, + "num_tokens": 1713048.0, + "reward": 0.8125, + "reward_std": 0.2177756428718567, + "rewards/accuracy_reward/mean": 0.8125, + "rewards/accuracy_reward/std": 0.3965577781200409, + "sampling/importance_sampling_ratio/max": 1.2929112911224365, + "sampling/importance_sampling_ratio/mean": 0.9999213218688965, + "sampling/importance_sampling_ratio/min": 0.6580591797828674, + "sampling/sampling_logp_difference/max": 0.4184603691101074, + "sampling/sampling_logp_difference/mean": 0.011468234471976757, + "step": 75 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00014653716425527819, + "clip_ratio/low_min": 0.00014653716425527819, + "clip_ratio/region_mean": 0.00014653716425527819, + "completions/clipped_ratio": 0.3125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 767.0, + "completions/mean_length": 605.78125, + "completions/mean_terminated_length": 532.0454711914062, + "completions/min_length": 337.0, + "completions/min_terminated_length": 337.0, + "entropy": 0.5242552533745766, + "epoch": 0.04068522483940043, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.019541600719094276, + "learning_rate": 1e-05, + "loss": 0.0685, + "num_tokens": 1736833.0, + "reward": 0.5625, + "reward_std": 0.5081326961517334, + "rewards/accuracy_reward/mean": 0.5625, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 1.443228840827942, + "sampling/importance_sampling_ratio/mean": 1.00018310546875, + "sampling/importance_sampling_ratio/min": 0.6059591174125671, + "sampling/sampling_logp_difference/max": 0.5009427070617676, + "sampling/sampling_logp_difference/mean": 0.01432492583990097, + "step": 76 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.489679460879415e-05, + "clip_ratio/low_min": 5.489679460879415e-05, + "clip_ratio/region_mean": 5.489679460879415e-05, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 747.0, + "completions/mean_length": 627.125, + "completions/mean_terminated_length": 572.0, + "completions/min_length": 364.0, + "completions/min_terminated_length": 364.0, + "entropy": 0.47469621524214745, + "epoch": 0.04122055674518201, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.003933256026357412, + "learning_rate": 1e-05, + "loss": 0.0347, + "num_tokens": 1760869.0, + "reward": 0.71875, + "reward_std": 0.2041158676147461, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.6860499382019043, + "sampling/importance_sampling_ratio/mean": 0.9998744130134583, + "sampling/importance_sampling_ratio/min": 0.5010150671005249, + "sampling/sampling_logp_difference/max": 0.6911191940307617, + "sampling/sampling_logp_difference/mean": 0.012595373205840588, + "step": 77 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00010221772754448466, + "clip_ratio/low_min": 0.00010221772754448466, + "clip_ratio/region_mean": 0.00010221772754448466, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 757.0, + "completions/mean_length": 509.15625, + "completions/mean_terminated_length": 461.22222900390625, + "completions/min_length": 215.0, + "completions/min_terminated_length": 215.0, + "entropy": 0.47905098646879196, + "epoch": 0.0417558886509636, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": 0.0284, + "num_tokens": 1781314.0, + "reward": 0.53125, + "reward_std": 0.0883883461356163, + "rewards/accuracy_reward/mean": 0.53125, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.3845901489257812, + "sampling/importance_sampling_ratio/mean": 0.9996688365936279, + "sampling/importance_sampling_ratio/min": 0.6213145852088928, + "sampling/sampling_logp_difference/max": 0.4759178161621094, + "sampling/sampling_logp_difference/mean": 0.013783096335828304, + "step": 78 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.40625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 759.0, + "completions/mean_length": 647.09375, + "completions/mean_terminated_length": 564.368408203125, + "completions/min_length": 364.0, + "completions/min_terminated_length": 364.0, + "entropy": 0.4179849214851856, + "epoch": 0.04229122055674518, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.007662573829293251, + "learning_rate": 1e-05, + "loss": 0.0308, + "num_tokens": 1805413.0, + "reward": 0.59375, + "reward_std": 0.3198433816432953, + "rewards/accuracy_reward/mean": 0.59375, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.4343494176864624, + "sampling/importance_sampling_ratio/mean": 1.0001981258392334, + "sampling/importance_sampling_ratio/min": 0.7690429091453552, + "sampling/sampling_logp_difference/max": 0.36071133613586426, + "sampling/sampling_logp_difference/mean": 0.011692270636558533, + "step": 79 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 9.750395838636905e-05, + "clip_ratio/low_min": 9.750395838636905e-05, + "clip_ratio/region_mean": 9.750395838636905e-05, + "completions/clipped_ratio": 0.375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 598.0, + "completions/mean_length": 588.03125, + "completions/mean_terminated_length": 480.0500183105469, + "completions/min_length": 355.0, + "completions/min_terminated_length": 355.0, + "entropy": 0.5071725882589817, + "epoch": 0.042826552462526764, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.012563863769173622, + "learning_rate": 1e-05, + "loss": 0.084, + "num_tokens": 1828502.0, + "reward": 0.5625, + "reward_std": 0.49022960662841797, + "rewards/accuracy_reward/mean": 0.5625, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 1.5360150337219238, + "sampling/importance_sampling_ratio/mean": 1.000058650970459, + "sampling/importance_sampling_ratio/min": 0.47326546907424927, + "sampling/sampling_logp_difference/max": 0.7480988502502441, + "sampling/sampling_logp_difference/mean": 0.01280238013714552, + "step": 80 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 767.0, + "completions/mean_length": 567.09375, + "completions/mean_terminated_length": 520.7307739257812, + "completions/min_length": 289.0, + "completions/min_terminated_length": 289.0, + "entropy": 0.4767908751964569, + "epoch": 0.04336188436830835, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.007349553983658552, + "learning_rate": 1e-05, + "loss": 0.0046, + "num_tokens": 1850625.0, + "reward": 0.78125, + "reward_std": 0.3061639964580536, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.4524610042572021, + "sampling/importance_sampling_ratio/mean": 1.0003424882888794, + "sampling/importance_sampling_ratio/min": 0.54655921459198, + "sampling/sampling_logp_difference/max": 0.6041126251220703, + "sampling/sampling_logp_difference/mean": 0.012623358517885208, + "step": 81 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 698.0, + "completions/mean_length": 561.9375, + "completions/mean_terminated_length": 481.3043518066406, + "completions/min_length": 278.0, + "completions/min_terminated_length": 278.0, + "entropy": 0.48736458644270897, + "epoch": 0.043897216274089934, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.007987036369740963, + "learning_rate": 1e-05, + "loss": 0.0336, + "num_tokens": 1872207.0, + "reward": 0.53125, + "reward_std": 0.3377464711666107, + "rewards/accuracy_reward/mean": 0.53125, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.511791706085205, + "sampling/importance_sampling_ratio/mean": 0.9997742772102356, + "sampling/importance_sampling_ratio/min": 0.5764255523681641, + "sampling/sampling_logp_difference/max": 0.5509090423583984, + "sampling/sampling_logp_difference/mean": 0.01264932844787836, + "step": 82 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00019177383364876732, + "clip_ratio/low_min": 0.00019177383364876732, + "clip_ratio/region_mean": 0.00019177383364876732, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 755.0, + "completions/mean_length": 644.78125, + "completions/mean_terminated_length": 596.5652465820312, + "completions/min_length": 440.0, + "completions/min_terminated_length": 440.0, + "entropy": 0.4271051101386547, + "epoch": 0.04443254817987152, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.0072464970871806145, + "learning_rate": 1e-05, + "loss": 0.0279, + "num_tokens": 1896696.0, + "reward": 0.40625, + "reward_std": 0.3198433816432953, + "rewards/accuracy_reward/mean": 0.40625, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.3683228492736816, + "sampling/importance_sampling_ratio/mean": 1.0000728368759155, + "sampling/importance_sampling_ratio/min": 0.7364410758018494, + "sampling/sampling_logp_difference/max": 0.3135857582092285, + "sampling/sampling_logp_difference/mean": 0.011938256211578846, + "step": 83 + }, + { + "clip_ratio/high_max": 0.00016138809587573633, + "clip_ratio/high_mean": 0.00016138809587573633, + "clip_ratio/low_mean": 9.469286669627763e-05, + "clip_ratio/low_min": 9.469286669627763e-05, + "clip_ratio/region_mean": 0.00025608096257201396, + "completions/clipped_ratio": 0.375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 695.0, + "completions/mean_length": 646.5625, + "completions/mean_terminated_length": 573.7000122070312, + "completions/min_length": 356.0, + "completions/min_terminated_length": 356.0, + "entropy": 0.489109069108963, + "epoch": 0.044967880085653104, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.015514599159359932, + "learning_rate": 1e-05, + "loss": 0.0385, + "num_tokens": 1921354.0, + "reward": 0.4375, + "reward_std": 0.5081326961517334, + "rewards/accuracy_reward/mean": 0.4375, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 1.5057592391967773, + "sampling/importance_sampling_ratio/mean": 1.0001661777496338, + "sampling/importance_sampling_ratio/min": 0.47354331612586975, + "sampling/sampling_logp_difference/max": 0.7475118637084961, + "sampling/sampling_logp_difference/mean": 0.013187098316848278, + "step": 84 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 739.0, + "completions/mean_length": 546.25, + "completions/mean_terminated_length": 495.0769348144531, + "completions/min_length": 318.0, + "completions/min_terminated_length": 318.0, + "entropy": 0.37117818370461464, + "epoch": 0.045503211991434686, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.003233251627534628, + "learning_rate": 1e-05, + "loss": 0.0603, + "num_tokens": 1942650.0, + "reward": 0.59375, + "reward_std": 0.22201895713806152, + "rewards/accuracy_reward/mean": 0.59375, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.415218710899353, + "sampling/importance_sampling_ratio/mean": 1.0000778436660767, + "sampling/importance_sampling_ratio/min": 0.7080608010292053, + "sampling/sampling_logp_difference/max": 0.34728407859802246, + "sampling/sampling_logp_difference/mean": 0.010211372748017311, + "step": 85 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00013728720659855753, + "clip_ratio/low_min": 0.00013728720659855753, + "clip_ratio/region_mean": 0.00013728720659855753, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 696.0, + "completions/mean_length": 522.5625, + "completions/mean_terminated_length": 465.923095703125, + "completions/min_length": 42.0, + "completions/min_terminated_length": 42.0, + "entropy": 0.44907190278172493, + "epoch": 0.046038543897216275, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.011993853375315666, + "learning_rate": 1e-05, + "loss": 0.0042, + "num_tokens": 1962612.0, + "reward": 0.5625, + "reward_std": 0.3745020925998688, + "rewards/accuracy_reward/mean": 0.5625, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 1.3522263765335083, + "sampling/importance_sampling_ratio/mean": 0.9996841549873352, + "sampling/importance_sampling_ratio/min": 0.6549509763717651, + "sampling/sampling_logp_difference/max": 0.42319488525390625, + "sampling/sampling_logp_difference/mean": 0.012766262516379356, + "step": 86 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 4.7801146138226613e-05, + "clip_ratio/low_min": 4.7801146138226613e-05, + "clip_ratio/region_mean": 4.7801146138226613e-05, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 758.0, + "completions/mean_length": 592.3125, + "completions/mean_terminated_length": 551.7692260742188, + "completions/min_length": 334.0, + "completions/min_terminated_length": 334.0, + "entropy": 0.3904706798493862, + "epoch": 0.046573875802997856, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.01267465390264988, + "learning_rate": 1e-05, + "loss": 0.0293, + "num_tokens": 1985590.0, + "reward": 0.625, + "reward_std": 0.3335031569004059, + "rewards/accuracy_reward/mean": 0.625, + "rewards/accuracy_reward/std": 0.49186936020851135, + "sampling/importance_sampling_ratio/max": 1.4399200677871704, + "sampling/importance_sampling_ratio/mean": 0.9999144673347473, + "sampling/importance_sampling_ratio/min": 0.6691920757293701, + "sampling/sampling_logp_difference/max": 0.4016841650009155, + "sampling/sampling_logp_difference/mean": 0.011088584549725056, + "step": 87 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.40625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 733.0, + "completions/mean_length": 665.09375, + "completions/mean_terminated_length": 594.6842041015625, + "completions/min_length": 291.0, + "completions/min_terminated_length": 291.0, + "entropy": 0.4168216176331043, + "epoch": 0.047109207708779445, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.003892525564879179, + "learning_rate": 1e-05, + "loss": -0.0202, + "num_tokens": 2010857.0, + "reward": 0.40625, + "reward_std": 0.3198433816432953, + "rewards/accuracy_reward/mean": 0.40625, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.4377671480178833, + "sampling/importance_sampling_ratio/mean": 1.000050663948059, + "sampling/importance_sampling_ratio/min": 0.7132178544998169, + "sampling/sampling_logp_difference/max": 0.36309123039245605, + "sampling/sampling_logp_difference/mean": 0.011454454623162746, + "step": 88 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.34375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 768.0, + "completions/mean_length": 627.46875, + "completions/mean_terminated_length": 553.857177734375, + "completions/min_length": 377.0, + "completions/min_terminated_length": 377.0, + "entropy": 0.5533029772341251, + "epoch": 0.04764453961456103, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.011965334415435791, + "learning_rate": 1e-05, + "loss": 0.0681, + "num_tokens": 2034944.0, + "reward": 0.5, + "reward_std": 0.3745020925998688, + "rewards/accuracy_reward/mean": 0.5, + "rewards/accuracy_reward/std": 0.5080004930496216, + "sampling/importance_sampling_ratio/max": 1.4319475889205933, + "sampling/importance_sampling_ratio/mean": 1.0000640153884888, + "sampling/importance_sampling_ratio/min": 0.4541526734828949, + "sampling/sampling_logp_difference/max": 0.7893218994140625, + "sampling/sampling_logp_difference/mean": 0.01460959017276764, + "step": 89 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00014825596372247674, + "clip_ratio/low_min": 0.00014825596372247674, + "clip_ratio/region_mean": 0.00014825596372247674, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 747.0, + "completions/mean_length": 615.65625, + "completions/mean_terminated_length": 556.0435180664062, + "completions/min_length": 366.0, + "completions/min_terminated_length": 366.0, + "entropy": 0.44857458397746086, + "epoch": 0.048179871520342615, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.0067571490071713924, + "learning_rate": 1e-05, + "loss": 0.0206, + "num_tokens": 2058973.0, + "reward": 0.46875, + "reward_std": 0.4355708956718445, + "rewards/accuracy_reward/mean": 0.46875, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.3036197423934937, + "sampling/importance_sampling_ratio/mean": 1.0000296831130981, + "sampling/importance_sampling_ratio/min": 0.5692623853683472, + "sampling/sampling_logp_difference/max": 0.5634138584136963, + "sampling/sampling_logp_difference/mean": 0.012198539450764656, + "step": 90 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 8.138021075865254e-05, + "clip_ratio/low_min": 8.138021075865254e-05, + "clip_ratio/region_mean": 8.138021075865254e-05, + "completions/clipped_ratio": 0.34375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 763.0, + "completions/mean_length": 599.40625, + "completions/mean_terminated_length": 511.0952453613281, + "completions/min_length": 237.0, + "completions/min_terminated_length": 237.0, + "entropy": 0.3983592949807644, + "epoch": 0.0487152034261242, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.014063858427107334, + "learning_rate": 1e-05, + "loss": 0.049, + "num_tokens": 2081882.0, + "reward": 0.5625, + "reward_std": 0.292504221200943, + "rewards/accuracy_reward/mean": 0.5625, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 1.2893673181533813, + "sampling/importance_sampling_ratio/mean": 0.9998114109039307, + "sampling/importance_sampling_ratio/min": 0.6123552322387695, + "sampling/sampling_logp_difference/max": 0.49044275283813477, + "sampling/sampling_logp_difference/mean": 0.011362754739820957, + "step": 91 + }, + { + "clip_ratio/high_max": 5.444250928121619e-05, + "clip_ratio/high_mean": 5.444250928121619e-05, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 5.444250928121619e-05, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 713.0, + "completions/mean_length": 488.09375, + "completions/mean_terminated_length": 469.433349609375, + "completions/min_length": 231.0, + "completions/min_terminated_length": 231.0, + "entropy": 0.44644933193922043, + "epoch": 0.04925053533190578, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.00947182159870863, + "learning_rate": 1e-05, + "loss": 0.0589, + "num_tokens": 2100909.0, + "reward": 0.78125, + "reward_std": 0.3471629321575165, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.4443336725234985, + "sampling/importance_sampling_ratio/mean": 0.9998829364776611, + "sampling/importance_sampling_ratio/min": 0.756104052066803, + "sampling/sampling_logp_difference/max": 0.3676481246948242, + "sampling/sampling_logp_difference/mean": 0.012396569363772869, + "step": 92 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.000111259454570245, + "clip_ratio/low_min": 0.000111259454570245, + "clip_ratio/region_mean": 0.000111259454570245, + "completions/clipped_ratio": 0.3125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 706.0, + "completions/mean_length": 620.6875, + "completions/mean_terminated_length": 553.727294921875, + "completions/min_length": 269.0, + "completions/min_terminated_length": 269.0, + "entropy": 0.47858962416648865, + "epoch": 0.04978586723768737, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.013834147714078426, + "learning_rate": 1e-05, + "loss": 0.0884, + "num_tokens": 2124595.0, + "reward": 0.53125, + "reward_std": 0.521792471408844, + "rewards/accuracy_reward/mean": 0.53125, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.6744105815887451, + "sampling/importance_sampling_ratio/mean": 1.0001462697982788, + "sampling/importance_sampling_ratio/min": 0.739568293094635, + "sampling/sampling_logp_difference/max": 0.5154612064361572, + "sampling/sampling_logp_difference/mean": 0.012730774469673634, + "step": 93 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.53125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 751.0, + "completions/mean_length": 673.3125, + "completions/mean_terminated_length": 566.0, + "completions/min_length": 300.0, + "completions/min_terminated_length": 300.0, + "entropy": 0.5602152310311794, + "epoch": 0.05032119914346895, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.007184832822531462, + "learning_rate": 1e-05, + "loss": 0.0351, + "num_tokens": 2149981.0, + "reward": 0.46875, + "reward_std": 0.38816186785697937, + "rewards/accuracy_reward/mean": 0.46875, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.4355807304382324, + "sampling/importance_sampling_ratio/mean": 0.9999159574508667, + "sampling/importance_sampling_ratio/min": 0.6217223405838013, + "sampling/sampling_logp_difference/max": 0.4752616882324219, + "sampling/sampling_logp_difference/mean": 0.012786953710019588, + "step": 94 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 714.0, + "completions/mean_length": 506.34375, + "completions/mean_terminated_length": 488.9000244140625, + "completions/min_length": 261.0, + "completions/min_terminated_length": 261.0, + "entropy": 0.3649308830499649, + "epoch": 0.05085653104925054, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.006838271860033274, + "learning_rate": 1e-05, + "loss": 0.0272, + "num_tokens": 2169376.0, + "reward": 0.875, + "reward_std": 0.2177756428718567, + "rewards/accuracy_reward/mean": 0.875, + "rewards/accuracy_reward/std": 0.33601075410842896, + "sampling/importance_sampling_ratio/max": 1.2903103828430176, + "sampling/importance_sampling_ratio/mean": 0.9998345971107483, + "sampling/importance_sampling_ratio/min": 0.4351949691772461, + "sampling/sampling_logp_difference/max": 0.8319611549377441, + "sampling/sampling_logp_difference/mean": 0.01059436984360218, + "step": 95 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 740.0, + "completions/mean_length": 564.09375, + "completions/mean_terminated_length": 517.0384521484375, + "completions/min_length": 283.0, + "completions/min_terminated_length": 283.0, + "entropy": 0.4984537586569786, + "epoch": 0.05139186295503212, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.034659527242183685, + "learning_rate": 1e-05, + "loss": -0.0084, + "num_tokens": 2190947.0, + "reward": 0.65625, + "reward_std": 0.2041158676147461, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.3317999839782715, + "sampling/importance_sampling_ratio/mean": 0.9997941851615906, + "sampling/importance_sampling_ratio/min": 0.767650306224823, + "sampling/sampling_logp_difference/max": 0.2865314483642578, + "sampling/sampling_logp_difference/mean": 0.013492072932422161, + "step": 96 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0001035625537042506, + "clip_ratio/low_min": 0.0001035625537042506, + "clip_ratio/region_mean": 0.0001035625537042506, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 757.0, + "completions/mean_length": 589.46875, + "completions/mean_terminated_length": 539.47998046875, + "completions/min_length": 284.0, + "completions/min_terminated_length": 284.0, + "entropy": 0.32192447781562805, + "epoch": 0.05192719486081371, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.005379266571253538, + "learning_rate": 1e-05, + "loss": 0.0321, + "num_tokens": 2213514.0, + "reward": 0.65625, + "reward_std": 0.3061639666557312, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.280397891998291, + "sampling/importance_sampling_ratio/mean": 0.9998688697814941, + "sampling/importance_sampling_ratio/min": 0.6917656064033508, + "sampling/sampling_logp_difference/max": 0.36850810050964355, + "sampling/sampling_logp_difference/mean": 0.009419996291399002, + "step": 97 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 731.0, + "completions/mean_length": 513.90625, + "completions/mean_terminated_length": 505.70965576171875, + "completions/min_length": 317.0, + "completions/min_terminated_length": 317.0, + "entropy": 0.30539928935468197, + "epoch": 0.05246252676659529, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": -0.0027, + "num_tokens": 2233727.0, + "reward": 0.71875, + "reward_std": 0.0883883461356163, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.3576973676681519, + "sampling/importance_sampling_ratio/mean": 1.0000336170196533, + "sampling/importance_sampling_ratio/min": 0.7603753805160522, + "sampling/sampling_logp_difference/max": 0.3057901859283447, + "sampling/sampling_logp_difference/mean": 0.00919144693762064, + "step": 98 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 711.0, + "completions/mean_length": 575.78125, + "completions/mean_terminated_length": 500.5652160644531, + "completions/min_length": 215.0, + "completions/min_terminated_length": 215.0, + "entropy": 0.42042646184563637, + "epoch": 0.05299785867237687, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.009321073070168495, + "learning_rate": 1e-05, + "loss": 0.0352, + "num_tokens": 2255712.0, + "reward": 0.71875, + "reward_std": 0.3471629321575165, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.2689825296401978, + "sampling/importance_sampling_ratio/mean": 0.9999005794525146, + "sampling/importance_sampling_ratio/min": 0.6649126410484314, + "sampling/sampling_logp_difference/max": 0.4080996513366699, + "sampling/sampling_logp_difference/mean": 0.01154857873916626, + "step": 99 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 9.067827340913936e-05, + "clip_ratio/low_min": 9.067827340913936e-05, + "clip_ratio/region_mean": 9.067827340913936e-05, + "completions/clipped_ratio": 0.5, + "completions/max_length": 768.0, + "completions/max_terminated_length": 761.0, + "completions/mean_length": 681.0, + "completions/mean_terminated_length": 594.0, + "completions/min_length": 435.0, + "completions/min_terminated_length": 435.0, + "entropy": 0.5039323531091213, + "epoch": 0.05353319057815846, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.006315401755273342, + "learning_rate": 1e-05, + "loss": 0.0556, + "num_tokens": 2281672.0, + "reward": 0.4375, + "reward_std": 0.3335031569004059, + "rewards/accuracy_reward/mean": 0.4375, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 1.447018027305603, + "sampling/importance_sampling_ratio/mean": 0.9999662637710571, + "sampling/importance_sampling_ratio/min": 0.634667694568634, + "sampling/sampling_logp_difference/max": 0.4546537399291992, + "sampling/sampling_logp_difference/mean": 0.013043881393969059, + "step": 100 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.40625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 755.0, + "completions/mean_length": 646.1875, + "completions/mean_terminated_length": 562.8421020507812, + "completions/min_length": 279.0, + "completions/min_terminated_length": 279.0, + "entropy": 0.5601640939712524, + "epoch": 0.05406852248394004, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.006256799213588238, + "learning_rate": 1e-05, + "loss": 0.0272, + "num_tokens": 2306390.0, + "reward": 0.5, + "reward_std": 0.2314550280570984, + "rewards/accuracy_reward/mean": 0.5, + "rewards/accuracy_reward/std": 0.5080004930496216, + "sampling/importance_sampling_ratio/max": 2.0, + "sampling/importance_sampling_ratio/mean": 0.9998123049736023, + "sampling/importance_sampling_ratio/min": 0.5658286213874817, + "sampling/sampling_logp_difference/max": 0.763413667678833, + "sampling/sampling_logp_difference/mean": 0.013222371228039265, + "step": 101 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 6.085686618462205e-05, + "clip_ratio/low_min": 6.085686618462205e-05, + "clip_ratio/region_mean": 6.085686618462205e-05, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 693.0, + "completions/mean_length": 509.0, + "completions/mean_terminated_length": 482.2069091796875, + "completions/min_length": 246.0, + "completions/min_terminated_length": 246.0, + "entropy": 0.4597771391272545, + "epoch": 0.05460385438972163, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.015706215053796768, + "learning_rate": 1e-05, + "loss": 0.0256, + "num_tokens": 2326190.0, + "reward": 0.6875, + "reward_std": 0.3535533845424652, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.4313578605651855, + "sampling/importance_sampling_ratio/mean": 0.9998531937599182, + "sampling/importance_sampling_ratio/min": 0.5052533149719238, + "sampling/sampling_logp_difference/max": 0.6826953887939453, + "sampling/sampling_logp_difference/mean": 0.012012065388262272, + "step": 102 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 8.138021075865254e-05, + "clip_ratio/low_min": 8.138021075865254e-05, + "clip_ratio/region_mean": 8.138021075865254e-05, + "completions/clipped_ratio": 0.59375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 765.0, + "completions/mean_length": 687.71875, + "completions/mean_terminated_length": 570.3846435546875, + "completions/min_length": 337.0, + "completions/min_terminated_length": 337.0, + "entropy": 0.3641216792166233, + "epoch": 0.05513918629550321, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.007397511508315802, + "learning_rate": 1e-05, + "loss": 0.0905, + "num_tokens": 2351853.0, + "reward": 0.4375, + "reward_std": 0.4492306709289551, + "rewards/accuracy_reward/mean": 0.4375, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 1.27895188331604, + "sampling/importance_sampling_ratio/mean": 0.9999485015869141, + "sampling/importance_sampling_ratio/min": 0.7077409625053406, + "sampling/sampling_logp_difference/max": 0.34567713737487793, + "sampling/sampling_logp_difference/mean": 0.010833405889570713, + "step": 103 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 739.0, + "completions/mean_length": 482.53125, + "completions/mean_terminated_length": 473.32257080078125, + "completions/min_length": 194.0, + "completions/min_terminated_length": 194.0, + "entropy": 0.4147394150495529, + "epoch": 0.055674518201284794, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.006192837841808796, + "learning_rate": 1e-05, + "loss": 0.0371, + "num_tokens": 2370654.0, + "reward": 0.9375, + "reward_std": 0.1157275140285492, + "rewards/accuracy_reward/mean": 0.9375, + "rewards/accuracy_reward/std": 0.24593468010425568, + "sampling/importance_sampling_ratio/max": 1.321974754333496, + "sampling/importance_sampling_ratio/mean": 1.0000211000442505, + "sampling/importance_sampling_ratio/min": 0.777760922908783, + "sampling/sampling_logp_difference/max": 0.2791266441345215, + "sampling/sampling_logp_difference/mean": 0.011425119824707508, + "step": 104 + }, + { + "clip_ratio/high_max": 9.846078319242224e-05, + "clip_ratio/high_mean": 9.846078319242224e-05, + "clip_ratio/low_mean": 9.072280590771697e-05, + "clip_ratio/low_min": 9.072280590771697e-05, + "clip_ratio/region_mean": 0.00018918358910013922, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 753.0, + "completions/mean_length": 628.0, + "completions/mean_terminated_length": 588.7999877929688, + "completions/min_length": 284.0, + "completions/min_terminated_length": 284.0, + "entropy": 0.5660983696579933, + "epoch": 0.05620985010706638, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.010016806423664093, + "learning_rate": 1e-05, + "loss": 0.08, + "num_tokens": 2394278.0, + "reward": 0.71875, + "reward_std": 0.4628904461860657, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.2932615280151367, + "sampling/importance_sampling_ratio/mean": 1.0002484321594238, + "sampling/importance_sampling_ratio/min": 0.6581019163131714, + "sampling/sampling_logp_difference/max": 0.4183955192565918, + "sampling/sampling_logp_difference/mean": 0.01387240830808878, + "step": 105 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 736.0, + "completions/mean_length": 572.59375, + "completions/mean_terminated_length": 527.5, + "completions/min_length": 173.0, + "completions/min_terminated_length": 173.0, + "entropy": 0.38742442801594734, + "epoch": 0.056745182012847964, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.012721240520477295, + "learning_rate": 1e-05, + "loss": 0.0182, + "num_tokens": 2416249.0, + "reward": 0.8125, + "reward_std": 0.2177756428718567, + "rewards/accuracy_reward/mean": 0.8125, + "rewards/accuracy_reward/std": 0.3965577781200409, + "sampling/importance_sampling_ratio/max": 1.437535047531128, + "sampling/importance_sampling_ratio/mean": 1.0001819133758545, + "sampling/importance_sampling_ratio/min": 0.655653715133667, + "sampling/sampling_logp_difference/max": 0.4221224784851074, + "sampling/sampling_logp_difference/mean": 0.01118245255202055, + "step": 106 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 9.860571663011797e-05, + "clip_ratio/low_min": 9.860571663011797e-05, + "clip_ratio/region_mean": 9.860571663011797e-05, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 732.0, + "completions/mean_length": 593.3125, + "completions/mean_terminated_length": 553.0, + "completions/min_length": 249.0, + "completions/min_terminated_length": 249.0, + "entropy": 0.3981010913848877, + "epoch": 0.05728051391862955, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.005357117857784033, + "learning_rate": 1e-05, + "loss": 0.0758, + "num_tokens": 2438947.0, + "reward": 0.6875, + "reward_std": 0.3471825420856476, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.3418827056884766, + "sampling/importance_sampling_ratio/mean": 1.000197172164917, + "sampling/importance_sampling_ratio/min": 0.5009822845458984, + "sampling/sampling_logp_difference/max": 0.6911845207214355, + "sampling/sampling_logp_difference/mean": 0.011721469461917877, + "step": 107 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00019840320237562992, + "clip_ratio/low_min": 0.00019840320237562992, + "clip_ratio/region_mean": 0.00019840320237562992, + "completions/clipped_ratio": 0.46875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 754.0, + "completions/mean_length": 664.71875, + "completions/mean_terminated_length": 573.5882568359375, + "completions/min_length": 405.0, + "completions/min_terminated_length": 405.0, + "entropy": 0.43526025488972664, + "epoch": 0.057815845824411134, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.012608899734914303, + "learning_rate": 1e-05, + "loss": 0.0713, + "num_tokens": 2464170.0, + "reward": 0.34375, + "reward_std": 0.47137710452079773, + "rewards/accuracy_reward/mean": 0.34375, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.4328020811080933, + "sampling/importance_sampling_ratio/mean": 0.9997937679290771, + "sampling/importance_sampling_ratio/min": 0.6727548837661743, + "sampling/sampling_logp_difference/max": 0.39637428522109985, + "sampling/sampling_logp_difference/mean": 0.01205401960760355, + "step": 108 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.46875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 736.0, + "completions/mean_length": 655.25, + "completions/mean_terminated_length": 555.7647094726562, + "completions/min_length": 51.0, + "completions/min_terminated_length": 51.0, + "entropy": 0.46148813143372536, + "epoch": 0.05835117773019272, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.009041559882462025, + "learning_rate": 1e-05, + "loss": 0.007, + "num_tokens": 2489610.0, + "reward": 0.5625, + "reward_std": 0.3335031569004059, + "rewards/accuracy_reward/mean": 0.5625, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 1.276477336883545, + "sampling/importance_sampling_ratio/mean": 0.9997236132621765, + "sampling/importance_sampling_ratio/min": 0.6942884922027588, + "sampling/sampling_logp_difference/max": 0.3648676872253418, + "sampling/sampling_logp_difference/mean": 0.011251892894506454, + "step": 109 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0001304120960412547, + "clip_ratio/low_min": 0.0001304120960412547, + "clip_ratio/region_mean": 0.0001304120960412547, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 732.0, + "completions/mean_length": 519.3125, + "completions/mean_terminated_length": 493.5862121582031, + "completions/min_length": 201.0, + "completions/min_terminated_length": 201.0, + "entropy": 0.43273311108350754, + "epoch": 0.058886509635974305, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.009418485686182976, + "learning_rate": 1e-05, + "loss": 0.0315, + "num_tokens": 2510188.0, + "reward": 0.5625, + "reward_std": 0.3945523500442505, + "rewards/accuracy_reward/mean": 0.5625, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 1.2939728498458862, + "sampling/importance_sampling_ratio/mean": 1.000428318977356, + "sampling/importance_sampling_ratio/min": 0.5578873753547668, + "sampling/sampling_logp_difference/max": 0.5835981369018555, + "sampling/sampling_logp_difference/mean": 0.012478448450565338, + "step": 110 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 6.443299207603559e-05, + "clip_ratio/low_min": 6.443299207603559e-05, + "clip_ratio/region_mean": 6.443299207603559e-05, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 761.0, + "completions/mean_length": 560.28125, + "completions/mean_terminated_length": 538.7930908203125, + "completions/min_length": 250.0, + "completions/min_terminated_length": 250.0, + "entropy": 0.3870910219848156, + "epoch": 0.059421841541755886, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.004824988543987274, + "learning_rate": 1e-05, + "loss": 0.0504, + "num_tokens": 2531821.0, + "reward": 0.25, + "reward_std": 0.3650856614112854, + "rewards/accuracy_reward/mean": 0.25, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.4037129878997803, + "sampling/importance_sampling_ratio/mean": 1.0002878904342651, + "sampling/importance_sampling_ratio/min": 0.6825191378593445, + "sampling/sampling_logp_difference/max": 0.38196468353271484, + "sampling/sampling_logp_difference/mean": 0.010554767213761806, + "step": 111 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.3125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 720.0, + "completions/mean_length": 587.375, + "completions/mean_terminated_length": 505.2727355957031, + "completions/min_length": 201.0, + "completions/min_terminated_length": 201.0, + "entropy": 0.6201117932796478, + "epoch": 0.059957173447537475, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.008925995789468288, + "learning_rate": 1e-05, + "loss": 0.0605, + "num_tokens": 2554177.0, + "reward": 0.5, + "reward_std": 0.3745020925998688, + "rewards/accuracy_reward/mean": 0.5, + "rewards/accuracy_reward/std": 0.5080004930496216, + "sampling/importance_sampling_ratio/max": 1.4320957660675049, + "sampling/importance_sampling_ratio/mean": 0.9996970295906067, + "sampling/importance_sampling_ratio/min": 0.69200599193573, + "sampling/sampling_logp_difference/max": 0.3681607246398926, + "sampling/sampling_logp_difference/mean": 0.014479795470833778, + "step": 112 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 4.186202204436995e-05, + "clip_ratio/low_min": 4.186202204436995e-05, + "clip_ratio/region_mean": 4.186202204436995e-05, + "completions/clipped_ratio": 0.40625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 739.0, + "completions/mean_length": 663.96875, + "completions/mean_terminated_length": 592.7894897460938, + "completions/min_length": 222.0, + "completions/min_terminated_length": 222.0, + "entropy": 0.3596794046461582, + "epoch": 0.06049250535331906, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.009059062227606773, + "learning_rate": 1e-05, + "loss": 0.0189, + "num_tokens": 2579048.0, + "reward": 0.5625, + "reward_std": 0.2177756428718567, + "rewards/accuracy_reward/mean": 0.5625, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 1.3382359743118286, + "sampling/importance_sampling_ratio/mean": 1.0003551244735718, + "sampling/importance_sampling_ratio/min": 0.6658063530921936, + "sampling/sampling_logp_difference/max": 0.4067564010620117, + "sampling/sampling_logp_difference/mean": 0.010846213437616825, + "step": 113 + }, + { + "clip_ratio/high_max": 6.130456313258037e-05, + "clip_ratio/high_mean": 6.130456313258037e-05, + "clip_ratio/low_mean": 0.00011080646800110117, + "clip_ratio/low_min": 0.00011080646800110117, + "clip_ratio/region_mean": 0.00017211103113368154, + "completions/clipped_ratio": 0.3125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 766.0, + "completions/mean_length": 626.15625, + "completions/mean_terminated_length": 561.6818237304688, + "completions/min_length": 279.0, + "completions/min_terminated_length": 279.0, + "entropy": 0.3593634217977524, + "epoch": 0.061027837259100645, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.0030713516753166914, + "learning_rate": 1e-05, + "loss": -0.0141, + "num_tokens": 2602733.0, + "reward": 0.625, + "reward_std": 0.3104073107242584, + "rewards/accuracy_reward/mean": 0.625, + "rewards/accuracy_reward/std": 0.49186936020851135, + "sampling/importance_sampling_ratio/max": 1.2897512912750244, + "sampling/importance_sampling_ratio/mean": 1.0003292560577393, + "sampling/importance_sampling_ratio/min": 0.4559897184371948, + "sampling/sampling_logp_difference/max": 0.7852849960327148, + "sampling/sampling_logp_difference/mean": 0.010105901397764683, + "step": 114 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 4.8543690354563296e-05, + "clip_ratio/low_min": 4.8543690354563296e-05, + "clip_ratio/region_mean": 4.8543690354563296e-05, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 722.0, + "completions/mean_length": 576.25, + "completions/mean_terminated_length": 548.857177734375, + "completions/min_length": 363.0, + "completions/min_terminated_length": 363.0, + "entropy": 0.35390249639749527, + "epoch": 0.06156316916488223, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.006338918581604958, + "learning_rate": 1e-05, + "loss": 0.0285, + "num_tokens": 2624925.0, + "reward": 0.5625, + "reward_std": 0.249358132481575, + "rewards/accuracy_reward/mean": 0.5625, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 1.575213074684143, + "sampling/importance_sampling_ratio/mean": 1.0000450611114502, + "sampling/importance_sampling_ratio/min": 0.6898723244667053, + "sampling/sampling_logp_difference/max": 0.4543905258178711, + "sampling/sampling_logp_difference/mean": 0.010251270607113838, + "step": 115 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 644.0, + "completions/mean_length": 552.5, + "completions/mean_terminated_length": 512.5925903320312, + "completions/min_length": 248.0, + "completions/min_terminated_length": 248.0, + "entropy": 0.3580743409693241, + "epoch": 0.06209850107066381, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.018461303785443306, + "learning_rate": 1e-05, + "loss": 0.0289, + "num_tokens": 2646693.0, + "reward": 0.75, + "reward_std": 0.3335031569004059, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.462385892868042, + "sampling/importance_sampling_ratio/mean": 1.0000616312026978, + "sampling/importance_sampling_ratio/min": 0.757928729057312, + "sampling/sampling_logp_difference/max": 0.3800692558288574, + "sampling/sampling_logp_difference/mean": 0.010540477931499481, + "step": 116 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.509034963324666e-05, + "clip_ratio/low_min": 5.509034963324666e-05, + "clip_ratio/region_mean": 5.509034963324666e-05, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 763.0, + "completions/mean_length": 523.03125, + "completions/mean_terminated_length": 497.6896667480469, + "completions/min_length": 312.0, + "completions/min_terminated_length": 312.0, + "entropy": 0.38762374222278595, + "epoch": 0.0626338329764454, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.007358966860920191, + "learning_rate": 1e-05, + "loss": 0.0195, + "num_tokens": 2667110.0, + "reward": 0.6875, + "reward_std": 0.4492306709289551, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.4650076627731323, + "sampling/importance_sampling_ratio/mean": 1.0000468492507935, + "sampling/importance_sampling_ratio/min": 0.6492045521736145, + "sampling/sampling_logp_difference/max": 0.43200743198394775, + "sampling/sampling_logp_difference/mean": 0.010364267975091934, + "step": 117 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0001240694837179035, + "clip_ratio/low_min": 0.0001240694837179035, + "clip_ratio/region_mean": 0.0001240694837179035, + "completions/clipped_ratio": 0.40625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 659.0, + "completions/mean_length": 609.875, + "completions/mean_terminated_length": 501.6842041015625, + "completions/min_length": 323.0, + "completions/min_terminated_length": 323.0, + "entropy": 0.2991199493408203, + "epoch": 0.06316916488222699, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.007873529568314552, + "learning_rate": 1e-05, + "loss": 0.0348, + "num_tokens": 2689874.0, + "reward": 0.46875, + "reward_std": 0.3061639666557312, + "rewards/accuracy_reward/mean": 0.46875, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.2994112968444824, + "sampling/importance_sampling_ratio/mean": 1.0000189542770386, + "sampling/importance_sampling_ratio/min": 0.7097275853157043, + "sampling/sampling_logp_difference/max": 0.34287405014038086, + "sampling/sampling_logp_difference/mean": 0.00929878931492567, + "step": 118 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 4.161118340562098e-05, + "clip_ratio/low_min": 4.161118340562098e-05, + "clip_ratio/region_mean": 4.161118340562098e-05, + "completions/clipped_ratio": 0.5625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 748.0, + "completions/mean_length": 721.03125, + "completions/mean_terminated_length": 660.6428833007812, + "completions/min_length": 458.0, + "completions/min_terminated_length": 458.0, + "entropy": 0.5329389944672585, + "epoch": 0.06370449678800856, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.01718723401427269, + "learning_rate": 1e-05, + "loss": 0.0414, + "num_tokens": 2717387.0, + "reward": 0.28125, + "reward_std": 0.3198433816432953, + "rewards/accuracy_reward/mean": 0.28125, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.489748239517212, + "sampling/importance_sampling_ratio/mean": 1.0000871419906616, + "sampling/importance_sampling_ratio/min": 0.7058092355728149, + "sampling/sampling_logp_difference/max": 0.39860713481903076, + "sampling/sampling_logp_difference/mean": 0.013629668392241001, + "step": 119 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 762.0, + "completions/mean_length": 585.25, + "completions/mean_terminated_length": 534.0800170898438, + "completions/min_length": 337.0, + "completions/min_terminated_length": 337.0, + "entropy": 0.43758948892354965, + "epoch": 0.06423982869379015, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.012252735905349255, + "learning_rate": 1e-05, + "loss": 0.0065, + "num_tokens": 2739611.0, + "reward": 0.59375, + "reward_std": 0.24511480331420898, + "rewards/accuracy_reward/mean": 0.59375, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.29102623462677, + "sampling/importance_sampling_ratio/mean": 1.00021231174469, + "sampling/importance_sampling_ratio/min": 0.319496750831604, + "sampling/sampling_logp_difference/max": 1.1410081386566162, + "sampling/sampling_logp_difference/mean": 0.011854034848511219, + "step": 120 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00010165071580559015, + "clip_ratio/low_min": 0.00010165071580559015, + "clip_ratio/region_mean": 0.00010165071580559015, + "completions/clipped_ratio": 0.25, + "completions/max_length": 768.0, + "completions/max_terminated_length": 765.0, + "completions/mean_length": 586.625, + "completions/mean_terminated_length": 526.1666870117188, + "completions/min_length": 343.0, + "completions/min_terminated_length": 343.0, + "entropy": 0.3946254886686802, + "epoch": 0.06477516059957174, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.002445508725941181, + "learning_rate": 1e-05, + "loss": 0.0439, + "num_tokens": 2762151.0, + "reward": 0.5, + "reward_std": 0.2925041913986206, + "rewards/accuracy_reward/mean": 0.5, + "rewards/accuracy_reward/std": 0.5080004930496216, + "sampling/importance_sampling_ratio/max": 1.2958875894546509, + "sampling/importance_sampling_ratio/mean": 0.9999931454658508, + "sampling/importance_sampling_ratio/min": 0.6989948749542236, + "sampling/sampling_logp_difference/max": 0.3581118583679199, + "sampling/sampling_logp_difference/mean": 0.011642329394817352, + "step": 121 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 767.0, + "completions/mean_length": 559.0625, + "completions/mean_terminated_length": 510.8461608886719, + "completions/min_length": 373.0, + "completions/min_terminated_length": 373.0, + "entropy": 0.41690022498369217, + "epoch": 0.06531049250535331, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.004603108856827021, + "learning_rate": 1e-05, + "loss": 0.0679, + "num_tokens": 2784537.0, + "reward": 0.6875, + "reward_std": 0.3535533845424652, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.4526787996292114, + "sampling/importance_sampling_ratio/mean": 1.000157117843628, + "sampling/importance_sampling_ratio/min": 0.7026703953742981, + "sampling/sampling_logp_difference/max": 0.3734092712402344, + "sampling/sampling_logp_difference/mean": 0.012163394130766392, + "step": 122 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 4.634779543266632e-05, + "clip_ratio/low_min": 4.634779543266632e-05, + "clip_ratio/region_mean": 4.634779543266632e-05, + "completions/clipped_ratio": 0.34375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 741.0, + "completions/mean_length": 597.96875, + "completions/mean_terminated_length": 508.90478515625, + "completions/min_length": 276.0, + "completions/min_terminated_length": 276.0, + "entropy": 0.34927118197083473, + "epoch": 0.0658458244111349, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.008883594535291195, + "learning_rate": 1e-05, + "loss": 0.0435, + "num_tokens": 2807608.0, + "reward": 0.5, + "reward_std": 0.2587745785713196, + "rewards/accuracy_reward/mean": 0.5, + "rewards/accuracy_reward/std": 0.5080004930496216, + "sampling/importance_sampling_ratio/max": 1.4907944202423096, + "sampling/importance_sampling_ratio/mean": 0.9999949932098389, + "sampling/importance_sampling_ratio/min": 0.5515795350074768, + "sampling/sampling_logp_difference/max": 0.5949692726135254, + "sampling/sampling_logp_difference/mean": 0.010144411586225033, + "step": 123 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 6.166748789837584e-05, + "clip_ratio/low_min": 6.166748789837584e-05, + "clip_ratio/region_mean": 6.166748789837584e-05, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 766.0, + "completions/mean_length": 549.375, + "completions/mean_terminated_length": 488.1600036621094, + "completions/min_length": 178.0, + "completions/min_terminated_length": 178.0, + "entropy": 0.5086312592029572, + "epoch": 0.06638115631691649, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.02062125876545906, + "learning_rate": 1e-05, + "loss": 0.129, + "num_tokens": 2828620.0, + "reward": 0.71875, + "reward_std": 0.3471629321575165, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.3955836296081543, + "sampling/importance_sampling_ratio/mean": 1.0000624656677246, + "sampling/importance_sampling_ratio/min": 0.5925219058990479, + "sampling/sampling_logp_difference/max": 0.5233674049377441, + "sampling/sampling_logp_difference/mean": 0.013531128875911236, + "step": 124 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00019864087880705483, + "clip_ratio/low_min": 0.00019864087880705483, + "clip_ratio/region_mean": 0.00019864087880705483, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 766.0, + "completions/mean_length": 629.15625, + "completions/mean_terminated_length": 574.8261108398438, + "completions/min_length": 228.0, + "completions/min_terminated_length": 228.0, + "entropy": 0.454969085752964, + "epoch": 0.06691648822269808, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.01148031372576952, + "learning_rate": 1e-05, + "loss": 0.0791, + "num_tokens": 2852721.0, + "reward": 0.4375, + "reward_std": 0.4492306709289551, + "rewards/accuracy_reward/mean": 0.4375, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 1.4142060279846191, + "sampling/importance_sampling_ratio/mean": 1.0000925064086914, + "sampling/importance_sampling_ratio/min": 0.5978776812553406, + "sampling/sampling_logp_difference/max": 0.514369010925293, + "sampling/sampling_logp_difference/mean": 0.012700111605226994, + "step": 125 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 8.633586185169406e-05, + "clip_ratio/low_min": 8.633586185169406e-05, + "clip_ratio/region_mean": 8.633586185169406e-05, + "completions/clipped_ratio": 0.3125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 739.0, + "completions/mean_length": 603.53125, + "completions/mean_terminated_length": 528.7727661132812, + "completions/min_length": 235.0, + "completions/min_terminated_length": 235.0, + "entropy": 0.4669857695698738, + "epoch": 0.06745182012847965, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.01519722305238247, + "learning_rate": 1e-05, + "loss": 0.0196, + "num_tokens": 2875546.0, + "reward": 0.25, + "reward_std": 0.4629100561141968, + "rewards/accuracy_reward/mean": 0.25, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.5019454956054688, + "sampling/importance_sampling_ratio/mean": 1.0000461339950562, + "sampling/importance_sampling_ratio/min": 0.652527928352356, + "sampling/sampling_logp_difference/max": 0.42690134048461914, + "sampling/sampling_logp_difference/mean": 0.012970094569027424, + "step": 126 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00023071422037901357, + "clip_ratio/low_min": 0.00023071422037901357, + "clip_ratio/region_mean": 0.00023071422037901357, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 700.0, + "completions/mean_length": 543.0625, + "completions/mean_terminated_length": 480.0799865722656, + "completions/min_length": 78.0, + "completions/min_terminated_length": 78.0, + "entropy": 0.709203228354454, + "epoch": 0.06798715203426124, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.012273093685507774, + "learning_rate": 1e-05, + "loss": 0.0624, + "num_tokens": 2897252.0, + "reward": 0.4375, + "reward_std": 0.47655022144317627, + "rewards/accuracy_reward/mean": 0.4375, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 1.3052661418914795, + "sampling/importance_sampling_ratio/mean": 1.0001472234725952, + "sampling/importance_sampling_ratio/min": 0.7167006731033325, + "sampling/sampling_logp_difference/max": 0.333096981048584, + "sampling/sampling_logp_difference/mean": 0.01649046689271927, + "step": 127 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.827505810884759e-05, + "clip_ratio/low_min": 5.827505810884759e-05, + "clip_ratio/region_mean": 5.827505810884759e-05, + "completions/clipped_ratio": 0.3125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 755.0, + "completions/mean_length": 566.46875, + "completions/mean_terminated_length": 474.8636474609375, + "completions/min_length": 244.0, + "completions/min_terminated_length": 244.0, + "entropy": 0.5634264834225178, + "epoch": 0.06852248394004283, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.0031958629842847586, + "learning_rate": 1e-05, + "loss": 0.0132, + "num_tokens": 2919315.0, + "reward": 0.3125, + "reward_std": 0.1157275140285492, + "rewards/accuracy_reward/mean": 0.3125, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.5357681512832642, + "sampling/importance_sampling_ratio/mean": 1.0004392862319946, + "sampling/importance_sampling_ratio/min": 0.524953305721283, + "sampling/sampling_logp_difference/max": 0.6444458961486816, + "sampling/sampling_logp_difference/mean": 0.014474543742835522, + "step": 128 + }, + { + "clip_ratio/high_max": 5.1652892580023035e-05, + "clip_ratio/high_mean": 5.1652892580023035e-05, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 5.1652892580023035e-05, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 753.0, + "completions/mean_length": 640.65625, + "completions/mean_terminated_length": 590.8261108398438, + "completions/min_length": 372.0, + "completions/min_terminated_length": 372.0, + "entropy": 0.617782287299633, + "epoch": 0.0690578158458244, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.016598865389823914, + "learning_rate": 1e-05, + "loss": 0.0449, + "num_tokens": 2944568.0, + "reward": 0.625, + "reward_std": 0.292504221200943, + "rewards/accuracy_reward/mean": 0.625, + "rewards/accuracy_reward/std": 0.49186936020851135, + "sampling/importance_sampling_ratio/max": 1.4206124544143677, + "sampling/importance_sampling_ratio/mean": 0.9997906684875488, + "sampling/importance_sampling_ratio/min": 0.6488592028617859, + "sampling/sampling_logp_difference/max": 0.43253952264785767, + "sampling/sampling_logp_difference/mean": 0.013920563273131847, + "step": 129 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00016284645244013518, + "clip_ratio/low_min": 0.00016284645244013518, + "clip_ratio/region_mean": 0.00016284645244013518, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 753.0, + "completions/mean_length": 539.59375, + "completions/mean_terminated_length": 475.6399841308594, + "completions/min_length": 304.0, + "completions/min_terminated_length": 304.0, + "entropy": 0.4281056188046932, + "epoch": 0.069593147751606, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.01196902897208929, + "learning_rate": 1e-05, + "loss": 0.0088, + "num_tokens": 2966171.0, + "reward": 0.6875, + "reward_std": 0.3514062464237213, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.2752894163131714, + "sampling/importance_sampling_ratio/mean": 0.9999509453773499, + "sampling/importance_sampling_ratio/min": 0.6416014432907104, + "sampling/sampling_logp_difference/max": 0.4437880516052246, + "sampling/sampling_logp_difference/mean": 0.01213695015758276, + "step": 130 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 682.0, + "completions/mean_length": 541.9375, + "completions/mean_terminated_length": 500.0740661621094, + "completions/min_length": 177.0, + "completions/min_terminated_length": 177.0, + "entropy": 0.3594672866165638, + "epoch": 0.07012847965738758, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.009060511365532875, + "learning_rate": 1e-05, + "loss": 0.0066, + "num_tokens": 2986961.0, + "reward": 0.625, + "reward_std": 0.4671337604522705, + "rewards/accuracy_reward/mean": 0.625, + "rewards/accuracy_reward/std": 0.49186936020851135, + "sampling/importance_sampling_ratio/max": 1.4494447708129883, + "sampling/importance_sampling_ratio/mean": 1.0001531839370728, + "sampling/importance_sampling_ratio/min": 0.6335175633430481, + "sampling/sampling_logp_difference/max": 0.4564676284790039, + "sampling/sampling_logp_difference/mean": 0.011128030717372894, + "step": 131 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 763.0, + "completions/mean_length": 624.3125, + "completions/mean_terminated_length": 584.0800170898438, + "completions/min_length": 262.0, + "completions/min_terminated_length": 262.0, + "entropy": 0.3743956610560417, + "epoch": 0.07066381156316917, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.010629815980792046, + "learning_rate": 1e-05, + "loss": 0.0168, + "num_tokens": 3010971.0, + "reward": 0.78125, + "reward_std": 0.2630178928375244, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.292447805404663, + "sampling/importance_sampling_ratio/mean": 0.9998924136161804, + "sampling/importance_sampling_ratio/min": 0.622527003288269, + "sampling/sampling_logp_difference/max": 0.4739682674407959, + "sampling/sampling_logp_difference/mean": 0.010707241483032703, + "step": 132 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 760.0, + "completions/mean_length": 542.09375, + "completions/mean_terminated_length": 500.25927734375, + "completions/min_length": 214.0, + "completions/min_terminated_length": 214.0, + "entropy": 0.3791262358427048, + "epoch": 0.07119914346895075, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.008368417620658875, + "learning_rate": 1e-05, + "loss": 0.0846, + "num_tokens": 3032086.0, + "reward": 0.8125, + "reward_std": 0.3335031569004059, + "rewards/accuracy_reward/mean": 0.8125, + "rewards/accuracy_reward/std": 0.3965577781200409, + "sampling/importance_sampling_ratio/max": 1.4262453317642212, + "sampling/importance_sampling_ratio/mean": 1.0005323886871338, + "sampling/importance_sampling_ratio/min": 0.694298267364502, + "sampling/sampling_logp_difference/max": 0.3648536205291748, + "sampling/sampling_logp_difference/mean": 0.011444810777902603, + "step": 133 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 8.72600357979536e-05, + "clip_ratio/low_min": 8.72600357979536e-05, + "clip_ratio/region_mean": 8.72600357979536e-05, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 756.0, + "completions/mean_length": 626.0, + "completions/mean_terminated_length": 570.434814453125, + "completions/min_length": 314.0, + "completions/min_terminated_length": 314.0, + "entropy": 0.40805576741695404, + "epoch": 0.07173447537473233, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.008140301331877708, + "learning_rate": 1e-05, + "loss": 0.1021, + "num_tokens": 3056126.0, + "reward": 0.75, + "reward_std": 0.4355512857437134, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.4351403713226318, + "sampling/importance_sampling_ratio/mean": 1.000121831893921, + "sampling/importance_sampling_ratio/min": 0.7144694924354553, + "sampling/sampling_logp_difference/max": 0.36126255989074707, + "sampling/sampling_logp_difference/mean": 0.012150084599852562, + "step": 134 + }, + { + "clip_ratio/high_max": 5.597850395133719e-05, + "clip_ratio/high_mean": 5.597850395133719e-05, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 5.597850395133719e-05, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 712.0, + "completions/mean_length": 462.875, + "completions/mean_terminated_length": 442.5333557128906, + "completions/min_length": 317.0, + "completions/min_terminated_length": 317.0, + "entropy": 0.4520307444036007, + "epoch": 0.07226980728051392, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.024488739669322968, + "learning_rate": 1e-05, + "loss": 0.0455, + "num_tokens": 3074610.0, + "reward": 0.84375, + "reward_std": 0.22201895713806152, + "rewards/accuracy_reward/mean": 0.84375, + "rewards/accuracy_reward/std": 0.3689020276069641, + "sampling/importance_sampling_ratio/max": 1.2992031574249268, + "sampling/importance_sampling_ratio/mean": 1.0000141859054565, + "sampling/importance_sampling_ratio/min": 0.7068355679512024, + "sampling/sampling_logp_difference/max": 0.3469572067260742, + "sampling/sampling_logp_difference/mean": 0.01334022544324398, + "step": 135 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.610413063550368e-05, + "clip_ratio/low_min": 5.610413063550368e-05, + "clip_ratio/region_mean": 5.610413063550368e-05, + "completions/clipped_ratio": 0.0, + "completions/max_length": 754.0, + "completions/max_terminated_length": 754.0, + "completions/mean_length": 527.53125, + "completions/mean_terminated_length": 527.53125, + "completions/min_length": 192.0, + "completions/min_terminated_length": 192.0, + "entropy": 0.3511754237115383, + "epoch": 0.0728051391862955, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.004012619145214558, + "learning_rate": 1e-05, + "loss": 0.0407, + "num_tokens": 3095515.0, + "reward": 0.75, + "reward_std": 0.1767766922712326, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.3161683082580566, + "sampling/importance_sampling_ratio/mean": 0.9997851848602295, + "sampling/importance_sampling_ratio/min": 0.721417248249054, + "sampling/sampling_logp_difference/max": 0.3265376091003418, + "sampling/sampling_logp_difference/mean": 0.01028917171061039, + "step": 136 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 4.83746116515249e-05, + "clip_ratio/low_min": 4.83746116515249e-05, + "clip_ratio/region_mean": 4.83746116515249e-05, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 730.0, + "completions/mean_length": 566.3125, + "completions/mean_terminated_length": 537.5, + "completions/min_length": 339.0, + "completions/min_terminated_length": 339.0, + "entropy": 0.33679792284965515, + "epoch": 0.07334047109207709, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.0023172921501100063, + "learning_rate": 1e-05, + "loss": 0.0861, + "num_tokens": 3117069.0, + "reward": 0.8125, + "reward_std": 0.2177756428718567, + "rewards/accuracy_reward/mean": 0.8125, + "rewards/accuracy_reward/std": 0.3965577781200409, + "sampling/importance_sampling_ratio/max": 1.3584282398223877, + "sampling/importance_sampling_ratio/mean": 1.0001167058944702, + "sampling/importance_sampling_ratio/min": 0.7186372876167297, + "sampling/sampling_logp_difference/max": 0.3303985595703125, + "sampling/sampling_logp_difference/mean": 0.010091305710375309, + "step": 137 + }, + { + "clip_ratio/high_max": 5.666364450007677e-05, + "clip_ratio/high_mean": 5.666364450007677e-05, + "clip_ratio/low_mean": 5.509034963324666e-05, + "clip_ratio/low_min": 5.509034963324666e-05, + "clip_ratio/region_mean": 0.00011175399413332343, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 756.0, + "completions/mean_length": 559.03125, + "completions/mean_terminated_length": 520.3333129882812, + "completions/min_length": 249.0, + "completions/min_terminated_length": 249.0, + "entropy": 0.3786024637520313, + "epoch": 0.07387580299785867, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.02267376147210598, + "learning_rate": 1e-05, + "loss": 0.0742, + "num_tokens": 3138678.0, + "reward": 0.75, + "reward_std": 0.3335031569004059, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.4510283470153809, + "sampling/importance_sampling_ratio/mean": 0.9997961521148682, + "sampling/importance_sampling_ratio/min": 0.7228491306304932, + "sampling/sampling_logp_difference/max": 0.3722724914550781, + "sampling/sampling_logp_difference/mean": 0.011123294942080975, + "step": 138 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 756.0, + "completions/mean_length": 540.09375, + "completions/mean_terminated_length": 507.5357360839844, + "completions/min_length": 241.0, + "completions/min_terminated_length": 241.0, + "entropy": 0.3804616779088974, + "epoch": 0.07441113490364026, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.011127207428216934, + "learning_rate": 1e-05, + "loss": 0.0509, + "num_tokens": 3159161.0, + "reward": 0.6875, + "reward_std": 0.3471825420856476, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.4004617929458618, + "sampling/importance_sampling_ratio/mean": 1.0003966093063354, + "sampling/importance_sampling_ratio/min": 0.7051466703414917, + "sampling/sampling_logp_difference/max": 0.3493494987487793, + "sampling/sampling_logp_difference/mean": 0.011153659783303738, + "step": 139 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 4.263301525497809e-05, + "clip_ratio/low_min": 4.263301525497809e-05, + "clip_ratio/region_mean": 4.263301525497809e-05, + "completions/clipped_ratio": 0.34375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 759.0, + "completions/mean_length": 608.53125, + "completions/mean_terminated_length": 525.0, + "completions/min_length": 338.0, + "completions/min_terminated_length": 338.0, + "entropy": 0.4498027302324772, + "epoch": 0.07494646680942184, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.003857298055663705, + "learning_rate": 1e-05, + "loss": 0.0745, + "num_tokens": 3181954.0, + "reward": 0.5625, + "reward_std": 0.3335031569004059, + "rewards/accuracy_reward/mean": 0.5625, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 1.3739570379257202, + "sampling/importance_sampling_ratio/mean": 0.9998639225959778, + "sampling/importance_sampling_ratio/min": 0.3522147238254547, + "sampling/sampling_logp_difference/max": 1.0435142517089844, + "sampling/sampling_logp_difference/mean": 0.011898073367774487, + "step": 140 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00020862984820269048, + "clip_ratio/low_min": 0.00020862984820269048, + "clip_ratio/region_mean": 0.00020862984820269048, + "completions/clipped_ratio": 0.40625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 735.0, + "completions/mean_length": 677.0, + "completions/mean_terminated_length": 614.73681640625, + "completions/min_length": 495.0, + "completions/min_terminated_length": 495.0, + "entropy": 0.4274517484009266, + "epoch": 0.07548179871520343, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.010394889861345291, + "learning_rate": 1e-05, + "loss": 0.0708, + "num_tokens": 3207506.0, + "reward": 0.5625, + "reward_std": 0.49022960662841797, + "rewards/accuracy_reward/mean": 0.5625, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 1.375177025794983, + "sampling/importance_sampling_ratio/mean": 0.999890148639679, + "sampling/importance_sampling_ratio/min": 0.594200611114502, + "sampling/sampling_logp_difference/max": 0.520538330078125, + "sampling/sampling_logp_difference/mean": 0.012031642720103264, + "step": 141 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 603.0, + "completions/mean_length": 451.5, + "completions/mean_terminated_length": 441.2903137207031, + "completions/min_length": 175.0, + "completions/min_terminated_length": 175.0, + "entropy": 0.3740456774830818, + "epoch": 0.07601713062098502, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.00951254740357399, + "learning_rate": 1e-05, + "loss": 0.0289, + "num_tokens": 3225306.0, + "reward": 0.90625, + "reward_std": 0.1293872892856598, + "rewards/accuracy_reward/mean": 0.90625, + "rewards/accuracy_reward/std": 0.2961445748806, + "sampling/importance_sampling_ratio/max": 1.455404281616211, + "sampling/importance_sampling_ratio/mean": 0.9996922016143799, + "sampling/importance_sampling_ratio/min": 0.5443582534790039, + "sampling/sampling_logp_difference/max": 0.6081476211547852, + "sampling/sampling_logp_difference/mean": 0.010931653901934624, + "step": 142 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00014522213677992113, + "clip_ratio/low_min": 0.00014522213677992113, + "clip_ratio/region_mean": 0.00014522213677992113, + "completions/clipped_ratio": 0.3125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 753.0, + "completions/mean_length": 631.84375, + "completions/mean_terminated_length": 569.95458984375, + "completions/min_length": 367.0, + "completions/min_terminated_length": 367.0, + "entropy": 0.48440078645944595, + "epoch": 0.07655246252676659, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.010871878825128078, + "learning_rate": 1e-05, + "loss": 0.0896, + "num_tokens": 3249189.0, + "reward": 0.5, + "reward_std": 0.4082317352294922, + "rewards/accuracy_reward/mean": 0.5, + "rewards/accuracy_reward/std": 0.5080004930496216, + "sampling/importance_sampling_ratio/max": 1.4944756031036377, + "sampling/importance_sampling_ratio/mean": 0.9997045397758484, + "sampling/importance_sampling_ratio/min": 0.04641878977417946, + "sampling/sampling_logp_difference/max": 3.0700509548187256, + "sampling/sampling_logp_difference/mean": 0.013384725898504257, + "step": 143 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.835667616338469e-05, + "clip_ratio/low_min": 5.835667616338469e-05, + "clip_ratio/region_mean": 5.835667616338469e-05, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 761.0, + "completions/mean_length": 521.78125, + "completions/mean_terminated_length": 476.1851806640625, + "completions/min_length": 249.0, + "completions/min_terminated_length": 249.0, + "entropy": 0.4121195822954178, + "epoch": 0.07708779443254818, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.01367057953029871, + "learning_rate": 1e-05, + "loss": -0.013, + "num_tokens": 3269502.0, + "reward": 0.6875, + "reward_std": 0.292504221200943, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.3120893239974976, + "sampling/importance_sampling_ratio/mean": 1.0002217292785645, + "sampling/importance_sampling_ratio/min": 0.6409497857093811, + "sampling/sampling_logp_difference/max": 0.44480419158935547, + "sampling/sampling_logp_difference/mean": 0.012407545000314713, + "step": 144 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 9.98272662400268e-05, + "clip_ratio/low_min": 9.98272662400268e-05, + "clip_ratio/region_mean": 9.98272662400268e-05, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 727.0, + "completions/mean_length": 566.59375, + "completions/mean_terminated_length": 520.1154174804688, + "completions/min_length": 217.0, + "completions/min_terminated_length": 217.0, + "entropy": 0.43163253366947174, + "epoch": 0.07762312633832977, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.010163567960262299, + "learning_rate": 1e-05, + "loss": 0.0858, + "num_tokens": 3291289.0, + "reward": 0.78125, + "reward_std": 0.3061639666557312, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.2963974475860596, + "sampling/importance_sampling_ratio/mean": 1.0003619194030762, + "sampling/importance_sampling_ratio/min": 0.6973382830619812, + "sampling/sampling_logp_difference/max": 0.36048460006713867, + "sampling/sampling_logp_difference/mean": 0.011822182685136795, + "step": 145 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 638.0, + "completions/mean_length": 438.0, + "completions/mean_terminated_length": 427.3548278808594, + "completions/min_length": 146.0, + "completions/min_terminated_length": 146.0, + "entropy": 0.4192674495279789, + "epoch": 0.07815845824411134, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": -0.0508, + "num_tokens": 3308793.0, + "reward": 0.84375, + "reward_std": 0.1293872892856598, + "rewards/accuracy_reward/mean": 0.84375, + "rewards/accuracy_reward/std": 0.3689020276069641, + "sampling/importance_sampling_ratio/max": 1.4280418157577515, + "sampling/importance_sampling_ratio/mean": 0.9996333122253418, + "sampling/importance_sampling_ratio/min": 0.6107430458068848, + "sampling/sampling_logp_difference/max": 0.49307894706726074, + "sampling/sampling_logp_difference/mean": 0.012461808510124683, + "step": 146 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 704.0, + "completions/mean_length": 567.40625, + "completions/mean_terminated_length": 488.9130554199219, + "completions/min_length": 289.0, + "completions/min_terminated_length": 289.0, + "entropy": 0.5919518247246742, + "epoch": 0.07869379014989293, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.009975074790418148, + "learning_rate": 1e-05, + "loss": 0.0444, + "num_tokens": 3330606.0, + "reward": 0.625, + "reward_std": 0.3335031569004059, + "rewards/accuracy_reward/mean": 0.625, + "rewards/accuracy_reward/std": 0.49186936020851135, + "sampling/importance_sampling_ratio/max": 1.3223353624343872, + "sampling/importance_sampling_ratio/mean": 0.9999397993087769, + "sampling/importance_sampling_ratio/min": 0.6502242088317871, + "sampling/sampling_logp_difference/max": 0.4304380416870117, + "sampling/sampling_logp_difference/mean": 0.014194437302649021, + "step": 147 + }, + { + "clip_ratio/high_max": 6.088650843594223e-05, + "clip_ratio/high_mean": 6.088650843594223e-05, + "clip_ratio/low_mean": 5.2764880820177495e-05, + "clip_ratio/low_min": 5.2764880820177495e-05, + "clip_ratio/region_mean": 0.00011365138925611973, + "completions/clipped_ratio": 0.25, + "completions/max_length": 768.0, + "completions/max_terminated_length": 644.0, + "completions/mean_length": 564.59375, + "completions/mean_terminated_length": 496.79168701171875, + "completions/min_length": 327.0, + "completions/min_terminated_length": 327.0, + "entropy": 0.39994125068187714, + "epoch": 0.07922912205567452, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.006876808591187, + "learning_rate": 1e-05, + "loss": -0.0087, + "num_tokens": 3352321.0, + "reward": 0.625, + "reward_std": 0.2177756428718567, + "rewards/accuracy_reward/mean": 0.625, + "rewards/accuracy_reward/std": 0.49186936020851135, + "sampling/importance_sampling_ratio/max": 1.4018239974975586, + "sampling/importance_sampling_ratio/mean": 1.0000483989715576, + "sampling/importance_sampling_ratio/min": 0.7175582051277161, + "sampling/sampling_logp_difference/max": 0.33777427673339844, + "sampling/sampling_logp_difference/mean": 0.011508024297654629, + "step": 148 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00015411724598379806, + "clip_ratio/low_min": 0.00015411724598379806, + "clip_ratio/region_mean": 0.00015411724598379806, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 746.0, + "completions/mean_length": 605.71875, + "completions/mean_terminated_length": 582.5357666015625, + "completions/min_length": 397.0, + "completions/min_terminated_length": 397.0, + "entropy": 0.33412376418709755, + "epoch": 0.07976445396145611, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.011310392990708351, + "learning_rate": 1e-05, + "loss": 0.0791, + "num_tokens": 3375168.0, + "reward": 0.75, + "reward_std": 0.3650856614112854, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.3741053342819214, + "sampling/importance_sampling_ratio/mean": 1.0000348091125488, + "sampling/importance_sampling_ratio/min": 0.6577353477478027, + "sampling/sampling_logp_difference/max": 0.4189525842666626, + "sampling/sampling_logp_difference/mean": 0.010629684664309025, + "step": 149 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00010651181219145656, + "clip_ratio/low_min": 0.00010651181219145656, + "clip_ratio/region_mean": 0.00010651181219145656, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 727.0, + "completions/mean_length": 526.53125, + "completions/mean_terminated_length": 510.433349609375, + "completions/min_length": 281.0, + "completions/min_terminated_length": 281.0, + "entropy": 0.3149258513003588, + "epoch": 0.08029978586723768, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.0036895317025482655, + "learning_rate": 1e-05, + "loss": -0.0039, + "num_tokens": 3395641.0, + "reward": 0.8125, + "reward_std": 0.2177756428718567, + "rewards/accuracy_reward/mean": 0.8125, + "rewards/accuracy_reward/std": 0.3965577781200409, + "sampling/importance_sampling_ratio/max": 1.4270235300064087, + "sampling/importance_sampling_ratio/mean": 0.9999079704284668, + "sampling/importance_sampling_ratio/min": 0.6860512495040894, + "sampling/sampling_logp_difference/max": 0.376802921295166, + "sampling/sampling_logp_difference/mean": 0.010782305151224136, + "step": 150 + }, + { + "clip_ratio/high_max": 4.77281391795259e-05, + "clip_ratio/high_mean": 4.77281391795259e-05, + "clip_ratio/low_mean": 0.0003206419605703559, + "clip_ratio/low_min": 0.0003206419605703559, + "clip_ratio/region_mean": 0.0003683700997498818, + "completions/clipped_ratio": 0.3125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 669.0, + "completions/mean_length": 615.0, + "completions/mean_terminated_length": 545.45458984375, + "completions/min_length": 262.0, + "completions/min_terminated_length": 262.0, + "entropy": 0.4121477007865906, + "epoch": 0.08083511777301927, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.00616479804739356, + "learning_rate": 1e-05, + "loss": 0.0981, + "num_tokens": 3419313.0, + "reward": 0.5, + "reward_std": 0.4671337604522705, + "rewards/accuracy_reward/mean": 0.5, + "rewards/accuracy_reward/std": 0.5080004930496216, + "sampling/importance_sampling_ratio/max": 1.6228677034378052, + "sampling/importance_sampling_ratio/mean": 1.0000944137573242, + "sampling/importance_sampling_ratio/min": 0.7266005873680115, + "sampling/sampling_logp_difference/max": 0.4841947555541992, + "sampling/sampling_logp_difference/mean": 0.012517292983829975, + "step": 151 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00013014055730309337, + "clip_ratio/low_min": 0.00013014055730309337, + "clip_ratio/region_mean": 0.00013014055730309337, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 764.0, + "completions/mean_length": 547.90625, + "completions/mean_terminated_length": 516.4642944335938, + "completions/min_length": 148.0, + "completions/min_terminated_length": 148.0, + "entropy": 0.4624267853796482, + "epoch": 0.08137044967880086, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.011894823983311653, + "learning_rate": 1e-05, + "loss": 0.0366, + "num_tokens": 3440686.0, + "reward": 0.75, + "reward_std": 0.3650856614112854, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.5203043222427368, + "sampling/importance_sampling_ratio/mean": 1.0004301071166992, + "sampling/importance_sampling_ratio/min": 0.7335498332977295, + "sampling/sampling_logp_difference/max": 0.41891050338745117, + "sampling/sampling_logp_difference/mean": 0.012454214505851269, + "step": 152 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00010090626892633736, + "clip_ratio/low_min": 0.00010090626892633736, + "clip_ratio/region_mean": 0.00010090626892633736, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 748.0, + "completions/mean_length": 571.09375, + "completions/mean_terminated_length": 542.9642944335938, + "completions/min_length": 129.0, + "completions/min_terminated_length": 129.0, + "entropy": 0.43677468225359917, + "epoch": 0.08190578158458243, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.020136717706918716, + "learning_rate": 1e-05, + "loss": 0.065, + "num_tokens": 3462585.0, + "reward": 0.71875, + "reward_std": 0.2651650309562683, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.4207556247711182, + "sampling/importance_sampling_ratio/mean": 1.0001400709152222, + "sampling/importance_sampling_ratio/min": 0.6170693635940552, + "sampling/sampling_logp_difference/max": 0.4827737808227539, + "sampling/sampling_logp_difference/mean": 0.012302085757255554, + "step": 153 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.747126488131471e-05, + "clip_ratio/low_min": 5.747126488131471e-05, + "clip_ratio/region_mean": 5.747126488131471e-05, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 757.0, + "completions/mean_length": 511.8125, + "completions/mean_terminated_length": 475.21429443359375, + "completions/min_length": 252.0, + "completions/min_terminated_length": 252.0, + "entropy": 0.4246552102267742, + "epoch": 0.08244111349036402, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.0030132178217172623, + "learning_rate": 1e-05, + "loss": 0.0813, + "num_tokens": 3483059.0, + "reward": 0.8125, + "reward_std": 0.249358132481575, + "rewards/accuracy_reward/mean": 0.8125, + "rewards/accuracy_reward/std": 0.3965577781200409, + "sampling/importance_sampling_ratio/max": 1.4191759824752808, + "sampling/importance_sampling_ratio/mean": 1.0000072717666626, + "sampling/importance_sampling_ratio/min": 0.6935054659843445, + "sampling/sampling_logp_difference/max": 0.3659961223602295, + "sampling/sampling_logp_difference/mean": 0.012803583405911922, + "step": 154 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 743.0, + "completions/mean_length": 568.84375, + "completions/mean_terminated_length": 490.9130554199219, + "completions/min_length": 260.0, + "completions/min_terminated_length": 260.0, + "entropy": 0.3456209786236286, + "epoch": 0.08297644539614561, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.008401411585509777, + "learning_rate": 1e-05, + "loss": 0.0158, + "num_tokens": 3505118.0, + "reward": 0.65625, + "reward_std": 0.1293872892856598, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.3933793306350708, + "sampling/importance_sampling_ratio/mean": 1.000069260597229, + "sampling/importance_sampling_ratio/min": 0.4956018328666687, + "sampling/sampling_logp_difference/max": 0.7019823789596558, + "sampling/sampling_logp_difference/mean": 0.010431949980556965, + "step": 155 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00012369978139759041, + "clip_ratio/low_min": 0.00012369978139759041, + "clip_ratio/region_mean": 0.00012369978139759041, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 747.0, + "completions/mean_length": 555.34375, + "completions/mean_terminated_length": 515.9629516601562, + "completions/min_length": 226.0, + "completions/min_terminated_length": 226.0, + "entropy": 0.35629772394895554, + "epoch": 0.0835117773019272, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.007527804002165794, + "learning_rate": 1e-05, + "loss": 0.0314, + "num_tokens": 3526785.0, + "reward": 0.65625, + "reward_std": 0.3471629321575165, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.6204110383987427, + "sampling/importance_sampling_ratio/mean": 1.0002415180206299, + "sampling/importance_sampling_ratio/min": 0.6880589127540588, + "sampling/sampling_logp_difference/max": 0.4826798439025879, + "sampling/sampling_logp_difference/mean": 0.011526282876729965, + "step": 156 + }, + { + "clip_ratio/high_max": 6.561679765582085e-05, + "clip_ratio/high_mean": 6.561679765582085e-05, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 6.561679765582085e-05, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 675.0, + "completions/mean_length": 469.46875, + "completions/mean_terminated_length": 459.83868408203125, + "completions/min_length": 274.0, + "completions/min_terminated_length": 274.0, + "entropy": 0.3269459344446659, + "epoch": 0.08404710920770878, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.020380010828375816, + "learning_rate": 1e-05, + "loss": 0.043, + "num_tokens": 3545336.0, + "reward": 0.96875, + "reward_std": 0.0883883461356163, + "rewards/accuracy_reward/mean": 0.96875, + "rewards/accuracy_reward/std": 0.1767766922712326, + "sampling/importance_sampling_ratio/max": 1.3741401433944702, + "sampling/importance_sampling_ratio/mean": 0.9999512434005737, + "sampling/importance_sampling_ratio/min": 0.7038865685462952, + "sampling/sampling_logp_difference/max": 0.3511381149291992, + "sampling/sampling_logp_difference/mean": 0.010684875771403313, + "step": 157 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 6.076811041566543e-05, + "clip_ratio/low_min": 6.076811041566543e-05, + "clip_ratio/region_mean": 6.076811041566543e-05, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 750.0, + "completions/mean_length": 556.21875, + "completions/mean_terminated_length": 496.91998291015625, + "completions/min_length": 331.0, + "completions/min_terminated_length": 331.0, + "entropy": 0.4623580873012543, + "epoch": 0.08458244111349036, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.009135546162724495, + "learning_rate": 1e-05, + "loss": 0.0285, + "num_tokens": 3567031.0, + "reward": 0.65625, + "reward_std": 0.2041158676147461, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.430266261100769, + "sampling/importance_sampling_ratio/mean": 0.9999651312828064, + "sampling/importance_sampling_ratio/min": 0.6599296927452087, + "sampling/sampling_logp_difference/max": 0.4156219959259033, + "sampling/sampling_logp_difference/mean": 0.013079782947897911, + "step": 158 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 7.042253855615854e-05, + "clip_ratio/low_min": 7.042253855615854e-05, + "clip_ratio/region_mean": 7.042253855615854e-05, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 711.0, + "completions/mean_length": 515.21875, + "completions/mean_terminated_length": 498.36669921875, + "completions/min_length": 222.0, + "completions/min_terminated_length": 222.0, + "entropy": 0.39342348650097847, + "epoch": 0.08511777301927195, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.01613238826394081, + "learning_rate": 1e-05, + "loss": 0.0178, + "num_tokens": 3587214.0, + "reward": 0.8125, + "reward_std": 0.2587745785713196, + "rewards/accuracy_reward/mean": 0.8125, + "rewards/accuracy_reward/std": 0.3965577781200409, + "sampling/importance_sampling_ratio/max": 1.441656470298767, + "sampling/importance_sampling_ratio/mean": 1.000588059425354, + "sampling/importance_sampling_ratio/min": 0.6850452423095703, + "sampling/sampling_logp_difference/max": 0.37827038764953613, + "sampling/sampling_logp_difference/mean": 0.01200867723673582, + "step": 159 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00010963959721266292, + "clip_ratio/low_min": 0.00010963959721266292, + "clip_ratio/region_mean": 0.00010963959721266292, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 761.0, + "completions/mean_length": 559.65625, + "completions/mean_terminated_length": 521.0740966796875, + "completions/min_length": 331.0, + "completions/min_terminated_length": 331.0, + "entropy": 0.5497498512268066, + "epoch": 0.08565310492505353, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.010425073094666004, + "learning_rate": 1e-05, + "loss": 0.1146, + "num_tokens": 3608723.0, + "reward": 0.625, + "reward_std": 0.48503684997558594, + "rewards/accuracy_reward/mean": 0.625, + "rewards/accuracy_reward/std": 0.49186936020851135, + "sampling/importance_sampling_ratio/max": 1.3026947975158691, + "sampling/importance_sampling_ratio/mean": 1.0003063678741455, + "sampling/importance_sampling_ratio/min": 0.697893500328064, + "sampling/sampling_logp_difference/max": 0.35968875885009766, + "sampling/sampling_logp_difference/mean": 0.01383618451654911, + "step": 160 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00019822225294774398, + "clip_ratio/low_min": 0.00019822225294774398, + "clip_ratio/region_mean": 0.00019822225294774398, + "completions/clipped_ratio": 0.40625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 757.0, + "completions/mean_length": 639.375, + "completions/mean_terminated_length": 551.368408203125, + "completions/min_length": 316.0, + "completions/min_terminated_length": 316.0, + "entropy": 0.6377146951854229, + "epoch": 0.08618843683083512, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.021822459995746613, + "learning_rate": 1e-05, + "loss": 0.1258, + "num_tokens": 3632943.0, + "reward": 0.375, + "reward_std": 0.4492306709289551, + "rewards/accuracy_reward/mean": 0.375, + "rewards/accuracy_reward/std": 0.49186936020851135, + "sampling/importance_sampling_ratio/max": 1.2977290153503418, + "sampling/importance_sampling_ratio/mean": 0.9999541640281677, + "sampling/importance_sampling_ratio/min": 0.7110767960548401, + "sampling/sampling_logp_difference/max": 0.3409748077392578, + "sampling/sampling_logp_difference/mean": 0.01600494794547558, + "step": 161 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00011524299407028593, + "clip_ratio/low_min": 0.00011524299407028593, + "clip_ratio/region_mean": 0.00011524299407028593, + "completions/clipped_ratio": 0.375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 746.0, + "completions/mean_length": 595.1875, + "completions/mean_terminated_length": 491.5, + "completions/min_length": 214.0, + "completions/min_terminated_length": 214.0, + "entropy": 0.688156247138977, + "epoch": 0.0867237687366167, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.024438703432679176, + "learning_rate": 1e-05, + "loss": 0.0774, + "num_tokens": 3655837.0, + "reward": 0.5625, + "reward_std": 0.3335031569004059, + "rewards/accuracy_reward/mean": 0.5625, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 1.3606370687484741, + "sampling/importance_sampling_ratio/mean": 1.0005593299865723, + "sampling/importance_sampling_ratio/min": 0.5256821513175964, + "sampling/sampling_logp_difference/max": 0.6430585384368896, + "sampling/sampling_logp_difference/mean": 0.015769150108098984, + "step": 162 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.3146257414482534e-05, + "clip_ratio/low_min": 5.3146257414482534e-05, + "clip_ratio/region_mean": 5.3146257414482534e-05, + "completions/clipped_ratio": 0.40625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 739.0, + "completions/mean_length": 620.71875, + "completions/mean_terminated_length": 519.9473876953125, + "completions/min_length": 334.0, + "completions/min_terminated_length": 334.0, + "entropy": 0.6735220775008202, + "epoch": 0.0872591006423983, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.0015876619145274162, + "learning_rate": 1e-05, + "loss": -0.011, + "num_tokens": 3680036.0, + "reward": 0.21875, + "reward_std": 0.0883883461356163, + "rewards/accuracy_reward/mean": 0.21875, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.445966124534607, + "sampling/importance_sampling_ratio/mean": 0.9998120665550232, + "sampling/importance_sampling_ratio/min": 0.7061793208122253, + "sampling/sampling_logp_difference/max": 0.36877763271331787, + "sampling/sampling_logp_difference/mean": 0.018158145248889923, + "step": 163 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 6.313131598290056e-05, + "clip_ratio/low_min": 6.313131598290056e-05, + "clip_ratio/region_mean": 6.313131598290056e-05, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 729.0, + "completions/mean_length": 552.53125, + "completions/mean_terminated_length": 512.629638671875, + "completions/min_length": 291.0, + "completions/min_terminated_length": 291.0, + "entropy": 0.4686658903956413, + "epoch": 0.08779443254817987, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.014513184316456318, + "learning_rate": 1e-05, + "loss": 0.0026, + "num_tokens": 3701349.0, + "reward": 0.5625, + "reward_std": 0.2925041913986206, + "rewards/accuracy_reward/mean": 0.5625, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 1.3749197721481323, + "sampling/importance_sampling_ratio/mean": 0.9998638033866882, + "sampling/importance_sampling_ratio/min": 0.743603527545929, + "sampling/sampling_logp_difference/max": 0.31839537620544434, + "sampling/sampling_logp_difference/mean": 0.012418043799698353, + "step": 164 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 4.291108780307695e-05, + "clip_ratio/low_min": 4.291108780307695e-05, + "clip_ratio/region_mean": 4.291108780307695e-05, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 702.0, + "completions/mean_length": 572.625, + "completions/mean_terminated_length": 527.5384521484375, + "completions/min_length": 325.0, + "completions/min_terminated_length": 325.0, + "entropy": 0.38484838977456093, + "epoch": 0.08832976445396146, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.008719516918063164, + "learning_rate": 1e-05, + "loss": 0.0423, + "num_tokens": 3722873.0, + "reward": 0.53125, + "reward_std": 0.24511480331420898, + "rewards/accuracy_reward/mean": 0.53125, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.4222228527069092, + "sampling/importance_sampling_ratio/mean": 0.9997617602348328, + "sampling/importance_sampling_ratio/min": 0.5860301852226257, + "sampling/sampling_logp_difference/max": 0.53438401222229, + "sampling/sampling_logp_difference/mean": 0.011152560822665691, + "step": 165 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00011565401655388996, + "clip_ratio/low_min": 0.00011565401655388996, + "clip_ratio/region_mean": 0.00011565401655388996, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 756.0, + "completions/mean_length": 555.40625, + "completions/mean_terminated_length": 525.0357666015625, + "completions/min_length": 307.0, + "completions/min_terminated_length": 307.0, + "entropy": 0.3937302567064762, + "epoch": 0.08886509635974305, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.01666826754808426, + "learning_rate": 1e-05, + "loss": 0.0691, + "num_tokens": 3744302.0, + "reward": 0.8125, + "reward_std": 0.3335031569004059, + "rewards/accuracy_reward/mean": 0.8125, + "rewards/accuracy_reward/std": 0.3965577781200409, + "sampling/importance_sampling_ratio/max": 1.4650695323944092, + "sampling/importance_sampling_ratio/mean": 0.9999182820320129, + "sampling/importance_sampling_ratio/min": 0.6995806097984314, + "sampling/sampling_logp_difference/max": 0.38190269470214844, + "sampling/sampling_logp_difference/mean": 0.011928286403417587, + "step": 166 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 4.9465768825029954e-05, + "clip_ratio/low_min": 4.9465768825029954e-05, + "clip_ratio/region_mean": 4.9465768825029954e-05, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 748.0, + "completions/mean_length": 618.34375, + "completions/mean_terminated_length": 590.629638671875, + "completions/min_length": 308.0, + "completions/min_terminated_length": 308.0, + "entropy": 0.42345913872122765, + "epoch": 0.08940042826552462, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.020113669335842133, + "learning_rate": 1e-05, + "loss": 0.0251, + "num_tokens": 3767561.0, + "reward": 0.75, + "reward_std": 0.4261348247528076, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.5204405784606934, + "sampling/importance_sampling_ratio/mean": 0.9999188184738159, + "sampling/importance_sampling_ratio/min": 0.6613926887512207, + "sampling/sampling_logp_difference/max": 0.41900014877319336, + "sampling/sampling_logp_difference/mean": 0.01242354791611433, + "step": 167 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.518763646250591e-05, + "clip_ratio/low_min": 5.518763646250591e-05, + "clip_ratio/region_mean": 5.518763646250591e-05, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 763.0, + "completions/mean_length": 587.15625, + "completions/mean_terminated_length": 545.423095703125, + "completions/min_length": 295.0, + "completions/min_terminated_length": 295.0, + "entropy": 0.5502735823392868, + "epoch": 0.08993576017130621, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.009137660264968872, + "learning_rate": 1e-05, + "loss": 0.0746, + "num_tokens": 3789854.0, + "reward": 0.71875, + "reward_std": 0.3608423173427582, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.3804175853729248, + "sampling/importance_sampling_ratio/mean": 0.9998465776443481, + "sampling/importance_sampling_ratio/min": 0.41815128922462463, + "sampling/sampling_logp_difference/max": 0.8719120025634766, + "sampling/sampling_logp_difference/mean": 0.013950812630355358, + "step": 168 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 768.0, + "completions/max_terminated_length": 727.0, + "completions/mean_length": 538.34375, + "completions/mean_terminated_length": 461.79168701171875, + "completions/min_length": 192.0, + "completions/min_terminated_length": 192.0, + "entropy": 0.5538909025490284, + "epoch": 0.0904710920770878, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.002491187071427703, + "learning_rate": 1e-05, + "loss": -0.006, + "num_tokens": 3811297.0, + "reward": 0.6875, + "reward_std": 0.1767766922712326, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.396300196647644, + "sampling/importance_sampling_ratio/mean": 1.0003321170806885, + "sampling/importance_sampling_ratio/min": 0.7034684419631958, + "sampling/sampling_logp_difference/max": 0.3517322540283203, + "sampling/sampling_logp_difference/mean": 0.01397632621228695, + "step": 169 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 648.0, + "completions/mean_length": 493.21875, + "completions/mean_terminated_length": 453.96429443359375, + "completions/min_length": 277.0, + "completions/min_terminated_length": 277.0, + "entropy": 0.49675028398633003, + "epoch": 0.09100642398286937, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.0036047326866537333, + "learning_rate": 1e-05, + "loss": 0.0405, + "num_tokens": 3830944.0, + "reward": 0.71875, + "reward_std": 0.3198433816432953, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.3590514659881592, + "sampling/importance_sampling_ratio/mean": 1.0000966787338257, + "sampling/importance_sampling_ratio/min": 0.6605715155601501, + "sampling/sampling_logp_difference/max": 0.41464996337890625, + "sampling/sampling_logp_difference/mean": 0.014946396462619305, + "step": 170 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.941064591752365e-05, + "clip_ratio/low_min": 5.941064591752365e-05, + "clip_ratio/region_mean": 5.941064591752365e-05, + "completions/clipped_ratio": 0.34375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 733.0, + "completions/mean_length": 617.0, + "completions/mean_terminated_length": 537.90478515625, + "completions/min_length": 246.0, + "completions/min_terminated_length": 246.0, + "entropy": 0.33820777013897896, + "epoch": 0.09154175588865096, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.007598424796015024, + "learning_rate": 1e-05, + "loss": 0.0358, + "num_tokens": 3854616.0, + "reward": 0.4375, + "reward_std": 0.2177756428718567, + "rewards/accuracy_reward/mean": 0.4375, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 1.322767734527588, + "sampling/importance_sampling_ratio/mean": 1.0002580881118774, + "sampling/importance_sampling_ratio/min": 0.7275824546813965, + "sampling/sampling_logp_difference/max": 0.31802797317504883, + "sampling/sampling_logp_difference/mean": 0.010352818295359612, + "step": 171 + }, + { + "clip_ratio/high_max": 7.931471918709576e-05, + "clip_ratio/high_mean": 7.931471918709576e-05, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 7.931471918709576e-05, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 757.0, + "completions/mean_length": 482.625, + "completions/mean_terminated_length": 473.4193420410156, + "completions/min_length": 266.0, + "completions/min_terminated_length": 266.0, + "entropy": 0.37833021581172943, + "epoch": 0.09207708779443255, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.002545375842601061, + "learning_rate": 1e-05, + "loss": -0.009, + "num_tokens": 3873516.0, + "reward": 0.9375, + "reward_std": 0.1767766922712326, + "rewards/accuracy_reward/mean": 0.9375, + "rewards/accuracy_reward/std": 0.24593468010425568, + "sampling/importance_sampling_ratio/max": 1.3294203281402588, + "sampling/importance_sampling_ratio/mean": 1.0000876188278198, + "sampling/importance_sampling_ratio/min": 0.7000898718833923, + "sampling/sampling_logp_difference/max": 0.3565465211868286, + "sampling/sampling_logp_difference/mean": 0.011979430913925171, + "step": 172 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 9.448358468944207e-05, + "clip_ratio/low_min": 9.448358468944207e-05, + "clip_ratio/region_mean": 9.448358468944207e-05, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 727.0, + "completions/mean_length": 543.09375, + "completions/mean_terminated_length": 510.96429443359375, + "completions/min_length": 320.0, + "completions/min_terminated_length": 320.0, + "entropy": 0.4378381334245205, + "epoch": 0.09261241970021414, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.0038268915377557278, + "learning_rate": 1e-05, + "loss": 0.0027, + "num_tokens": 3894975.0, + "reward": 0.53125, + "reward_std": 0.0883883461356163, + "rewards/accuracy_reward/mean": 0.53125, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.4210575819015503, + "sampling/importance_sampling_ratio/mean": 1.0002058744430542, + "sampling/importance_sampling_ratio/min": 0.6890934705734253, + "sampling/sampling_logp_difference/max": 0.3723783493041992, + "sampling/sampling_logp_difference/mean": 0.013296155259013176, + "step": 173 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 732.0, + "completions/mean_length": 470.59375, + "completions/mean_terminated_length": 450.7666931152344, + "completions/min_length": 198.0, + "completions/min_terminated_length": 198.0, + "entropy": 0.37185758352279663, + "epoch": 0.09314775160599571, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.023561233654618263, + "learning_rate": 1e-05, + "loss": 0.04, + "num_tokens": 3913578.0, + "reward": 0.78125, + "reward_std": 0.3608423173427582, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.4191864728927612, + "sampling/importance_sampling_ratio/mean": 1.000483751296997, + "sampling/importance_sampling_ratio/min": 0.6901246905326843, + "sampling/sampling_logp_difference/max": 0.3708829879760742, + "sampling/sampling_logp_difference/mean": 0.011572744697332382, + "step": 174 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 728.0, + "completions/mean_length": 529.375, + "completions/mean_terminated_length": 521.6774291992188, + "completions/min_length": 306.0, + "completions/min_terminated_length": 306.0, + "entropy": 0.3523942269384861, + "epoch": 0.0936830835117773, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.0074403719045221806, + "learning_rate": 1e-05, + "loss": 0.0161, + "num_tokens": 3934438.0, + "reward": 0.65625, + "reward_std": 0.22201895713806152, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.410582423210144, + "sampling/importance_sampling_ratio/mean": 0.9998390674591064, + "sampling/importance_sampling_ratio/min": 0.6631200313568115, + "sampling/sampling_logp_difference/max": 0.4107992649078369, + "sampling/sampling_logp_difference/mean": 0.011772280558943748, + "step": 175 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 768.0, + "completions/max_terminated_length": 683.0, + "completions/mean_length": 536.65625, + "completions/mean_terminated_length": 459.54168701171875, + "completions/min_length": 342.0, + "completions/min_terminated_length": 342.0, + "entropy": 0.5548071675002575, + "epoch": 0.09421841541755889, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.00894884392619133, + "learning_rate": 1e-05, + "loss": 0.0421, + "num_tokens": 3955827.0, + "reward": 0.3125, + "reward_std": 0.249358132481575, + "rewards/accuracy_reward/mean": 0.3125, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.5687905550003052, + "sampling/importance_sampling_ratio/mean": 1.0000354051589966, + "sampling/importance_sampling_ratio/min": 0.697902500629425, + "sampling/sampling_logp_difference/max": 0.4503049850463867, + "sampling/sampling_logp_difference/mean": 0.014390457421541214, + "step": 176 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 8.932847413234413e-05, + "clip_ratio/low_min": 8.932847413234413e-05, + "clip_ratio/region_mean": 8.932847413234413e-05, + "completions/clipped_ratio": 0.4375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 738.0, + "completions/mean_length": 637.34375, + "completions/mean_terminated_length": 535.7222290039062, + "completions/min_length": 301.0, + "completions/min_terminated_length": 301.0, + "entropy": 0.42246196419000626, + "epoch": 0.09475374732334046, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.008636223152279854, + "learning_rate": 1e-05, + "loss": -0.0237, + "num_tokens": 3980702.0, + "reward": 0.3125, + "reward_std": 0.3745020925998688, + "rewards/accuracy_reward/mean": 0.3125, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.3391796350479126, + "sampling/importance_sampling_ratio/mean": 0.9999925494194031, + "sampling/importance_sampling_ratio/min": 0.5327543020248413, + "sampling/sampling_logp_difference/max": 0.629694938659668, + "sampling/sampling_logp_difference/mean": 0.011958497576415539, + "step": 177 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 750.0, + "completions/mean_length": 466.6875, + "completions/mean_terminated_length": 382.3199768066406, + "completions/min_length": 219.0, + "completions/min_terminated_length": 219.0, + "entropy": 0.6823995895683765, + "epoch": 0.09528907922912205, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.005268024280667305, + "learning_rate": 1e-05, + "loss": 0.0028, + "num_tokens": 3999092.0, + "reward": 0.5625, + "reward_std": 0.1157275140285492, + "rewards/accuracy_reward/mean": 0.5625, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 1.514682650566101, + "sampling/importance_sampling_ratio/mean": 1.0001227855682373, + "sampling/importance_sampling_ratio/min": 0.7262601852416992, + "sampling/sampling_logp_difference/max": 0.4152059555053711, + "sampling/sampling_logp_difference/mean": 0.016032084822654724, + "step": 178 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 6.463288445957005e-05, + "clip_ratio/low_min": 6.463288445957005e-05, + "clip_ratio/region_mean": 6.463288445957005e-05, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 692.0, + "completions/mean_length": 490.625, + "completions/mean_terminated_length": 461.9310302734375, + "completions/min_length": 274.0, + "completions/min_terminated_length": 274.0, + "entropy": 0.5883337892591953, + "epoch": 0.09582441113490364, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.018840458244085312, + "learning_rate": 1e-05, + "loss": -0.0069, + "num_tokens": 4018368.0, + "reward": 0.53125, + "reward_std": 0.3471629321575165, + "rewards/accuracy_reward/mean": 0.53125, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.6545922756195068, + "sampling/importance_sampling_ratio/mean": 0.9998740553855896, + "sampling/importance_sampling_ratio/min": 0.5516250133514404, + "sampling/sampling_logp_difference/max": 0.5948867797851562, + "sampling/sampling_logp_difference/mean": 0.014858265407383442, + "step": 179 + }, + { + "clip_ratio/high_max": 5.3236795793054625e-05, + "clip_ratio/high_mean": 5.3236795793054625e-05, + "clip_ratio/low_mean": 5.484861685545184e-05, + "clip_ratio/low_min": 5.484861685545184e-05, + "clip_ratio/region_mean": 0.00010808541264850646, + "completions/clipped_ratio": 0.25, + "completions/max_length": 768.0, + "completions/max_terminated_length": 752.0, + "completions/mean_length": 601.78125, + "completions/mean_terminated_length": 546.375, + "completions/min_length": 306.0, + "completions/min_terminated_length": 306.0, + "entropy": 0.3723117969930172, + "epoch": 0.09635974304068523, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.008200142532587051, + "learning_rate": 1e-05, + "loss": 0.0146, + "num_tokens": 4041609.0, + "reward": 0.71875, + "reward_std": 0.3471629321575165, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.2989987134933472, + "sampling/importance_sampling_ratio/mean": 1.000217080116272, + "sampling/importance_sampling_ratio/min": 0.7527726292610168, + "sampling/sampling_logp_difference/max": 0.28399205207824707, + "sampling/sampling_logp_difference/mean": 0.011099657043814659, + "step": 180 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 6.228201527846977e-05, + "clip_ratio/low_min": 6.228201527846977e-05, + "clip_ratio/region_mean": 6.228201527846977e-05, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 714.0, + "completions/mean_length": 418.21875, + "completions/mean_terminated_length": 406.93548583984375, + "completions/min_length": 155.0, + "completions/min_terminated_length": 155.0, + "entropy": 0.4289981424808502, + "epoch": 0.0968950749464668, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.005340190604329109, + "learning_rate": 1e-05, + "loss": -0.0135, + "num_tokens": 4058584.0, + "reward": 0.78125, + "reward_std": 0.24511480331420898, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.5157135725021362, + "sampling/importance_sampling_ratio/mean": 0.9994528889656067, + "sampling/importance_sampling_ratio/min": 0.7148792743682861, + "sampling/sampling_logp_difference/max": 0.41588640213012695, + "sampling/sampling_logp_difference/mean": 0.013451691716909409, + "step": 181 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 768.0, + "completions/max_terminated_length": 719.0, + "completions/mean_length": 585.3125, + "completions/mean_terminated_length": 524.4166870117188, + "completions/min_length": 229.0, + "completions/min_terminated_length": 229.0, + "entropy": 0.3565916270017624, + "epoch": 0.0974304068522484, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.008056404069066048, + "learning_rate": 1e-05, + "loss": 0.0141, + "num_tokens": 4080922.0, + "reward": 0.65625, + "reward_std": 0.38816186785697937, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.3575481176376343, + "sampling/importance_sampling_ratio/mean": 1.0001626014709473, + "sampling/importance_sampling_ratio/min": 0.6891089081764221, + "sampling/sampling_logp_difference/max": 0.37235593795776367, + "sampling/sampling_logp_difference/mean": 0.010848519392311573, + "step": 182 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00014377634579432197, + "clip_ratio/low_min": 0.00014377634579432197, + "clip_ratio/region_mean": 0.00014377634579432197, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 757.0, + "completions/mean_length": 571.25, + "completions/mean_terminated_length": 550.8965454101562, + "completions/min_length": 320.0, + "completions/min_terminated_length": 320.0, + "entropy": 0.33851267769932747, + "epoch": 0.09796573875802998, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.018591763451695442, + "learning_rate": 1e-05, + "loss": 0.0479, + "num_tokens": 4103602.0, + "reward": 0.75, + "reward_std": 0.1767766922712326, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.4362455606460571, + "sampling/importance_sampling_ratio/mean": 1.0002412796020508, + "sampling/importance_sampling_ratio/min": 0.7441653609275818, + "sampling/sampling_logp_difference/max": 0.362032413482666, + "sampling/sampling_logp_difference/mean": 0.010752060450613499, + "step": 183 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 8.833922038320452e-05, + "clip_ratio/low_min": 8.833922038320452e-05, + "clip_ratio/region_mean": 8.833922038320452e-05, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 758.0, + "completions/mean_length": 572.6875, + "completions/mean_terminated_length": 518.0, + "completions/min_length": 329.0, + "completions/min_terminated_length": 329.0, + "entropy": 0.4437847100198269, + "epoch": 0.09850107066381156, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.032799091190099716, + "learning_rate": 1e-05, + "loss": 0.0756, + "num_tokens": 4126248.0, + "reward": 0.71875, + "reward_std": 0.35564959049224854, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 2.0, + "sampling/importance_sampling_ratio/mean": 0.9999086856842041, + "sampling/importance_sampling_ratio/min": 0.6396790146827698, + "sampling/sampling_logp_difference/max": 0.7101085186004639, + "sampling/sampling_logp_difference/mean": 0.012340809218585491, + "step": 184 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.278716344037093e-05, + "clip_ratio/low_min": 5.278716344037093e-05, + "clip_ratio/region_mean": 5.278716344037093e-05, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 752.0, + "completions/mean_length": 584.28125, + "completions/mean_terminated_length": 550.25927734375, + "completions/min_length": 323.0, + "completions/min_terminated_length": 323.0, + "entropy": 0.39026234298944473, + "epoch": 0.09903640256959315, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.006969504989683628, + "learning_rate": 1e-05, + "loss": 0.0439, + "num_tokens": 4148705.0, + "reward": 0.8125, + "reward_std": 0.3335031569004059, + "rewards/accuracy_reward/mean": 0.8125, + "rewards/accuracy_reward/std": 0.3965577781200409, + "sampling/importance_sampling_ratio/max": 1.47065007686615, + "sampling/importance_sampling_ratio/mean": 1.0001626014709473, + "sampling/importance_sampling_ratio/min": 0.6491507291793823, + "sampling/sampling_logp_difference/max": 0.43209028244018555, + "sampling/sampling_logp_difference/mean": 0.012676937505602837, + "step": 185 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00020644002506742254, + "clip_ratio/low_min": 0.00020644002506742254, + "clip_ratio/region_mean": 0.00020644002506742254, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 761.0, + "completions/mean_length": 624.90625, + "completions/mean_terminated_length": 584.8399658203125, + "completions/min_length": 288.0, + "completions/min_terminated_length": 288.0, + "entropy": 0.4165581353008747, + "epoch": 0.09957173447537473, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.010572649538516998, + "learning_rate": 1e-05, + "loss": 0.0231, + "num_tokens": 4172662.0, + "reward": 0.59375, + "reward_std": 0.4218915104866028, + "rewards/accuracy_reward/mean": 0.59375, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.907720923423767, + "sampling/importance_sampling_ratio/mean": 0.9999308586120605, + "sampling/importance_sampling_ratio/min": 0.6693372130393982, + "sampling/sampling_logp_difference/max": 0.645909309387207, + "sampling/sampling_logp_difference/mean": 0.011922947131097317, + "step": 186 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00020344323638710193, + "clip_ratio/low_min": 0.00020344323638710193, + "clip_ratio/region_mean": 0.00020344323638710193, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 671.0, + "completions/mean_length": 431.28125, + "completions/mean_terminated_length": 408.8333435058594, + "completions/min_length": 198.0, + "completions/min_terminated_length": 198.0, + "entropy": 0.4276481121778488, + "epoch": 0.10010706638115632, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.007953577674925327, + "learning_rate": 1e-05, + "loss": 0.0361, + "num_tokens": 4189735.0, + "reward": 0.25, + "reward_std": 0.3335031569004059, + "rewards/accuracy_reward/mean": 0.25, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.3328940868377686, + "sampling/importance_sampling_ratio/mean": 1.0002760887145996, + "sampling/importance_sampling_ratio/min": 0.6931687593460083, + "sampling/sampling_logp_difference/max": 0.3664817810058594, + "sampling/sampling_logp_difference/mean": 0.013134675100445747, + "step": 187 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00010754277172964066, + "clip_ratio/low_min": 0.00010754277172964066, + "clip_ratio/region_mean": 0.00010754277172964066, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 690.0, + "completions/mean_length": 556.96875, + "completions/mean_terminated_length": 517.888916015625, + "completions/min_length": 335.0, + "completions/min_terminated_length": 335.0, + "entropy": 0.4502966143190861, + "epoch": 0.1006423982869379, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.015682239085435867, + "learning_rate": 1e-05, + "loss": 0.0952, + "num_tokens": 4211462.0, + "reward": 0.65625, + "reward_std": 0.3608423173427582, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.3125203847885132, + "sampling/importance_sampling_ratio/mean": 0.9996594190597534, + "sampling/importance_sampling_ratio/min": 0.5370357036590576, + "sampling/sampling_logp_difference/max": 0.6216907501220703, + "sampling/sampling_logp_difference/mean": 0.012791034765541553, + "step": 188 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 4.763719334732741e-05, + "clip_ratio/low_min": 4.763719334732741e-05, + "clip_ratio/region_mean": 4.763719334732741e-05, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 758.0, + "completions/mean_length": 570.28125, + "completions/mean_terminated_length": 514.9199829101562, + "completions/min_length": 275.0, + "completions/min_terminated_length": 275.0, + "entropy": 0.46106133610010147, + "epoch": 0.10117773019271949, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.007562241051346064, + "learning_rate": 1e-05, + "loss": -0.0074, + "num_tokens": 4233487.0, + "reward": 0.59375, + "reward_std": 0.3377465009689331, + "rewards/accuracy_reward/mean": 0.59375, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.3308215141296387, + "sampling/importance_sampling_ratio/mean": 0.9997634291648865, + "sampling/importance_sampling_ratio/min": 0.5934041142463684, + "sampling/sampling_logp_difference/max": 0.5218796730041504, + "sampling/sampling_logp_difference/mean": 0.013349337503314018, + "step": 189 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0001560395503474865, + "clip_ratio/low_min": 0.0001560395503474865, + "clip_ratio/region_mean": 0.0001560395503474865, + "completions/clipped_ratio": 0.5, + "completions/max_length": 768.0, + "completions/max_terminated_length": 713.0, + "completions/mean_length": 650.53125, + "completions/mean_terminated_length": 533.0625, + "completions/min_length": 280.0, + "completions/min_terminated_length": 280.0, + "entropy": 0.46249398961663246, + "epoch": 0.10171306209850108, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.0019809517543762922, + "learning_rate": 1e-05, + "loss": 0.0118, + "num_tokens": 4258840.0, + "reward": 0.0625, + "reward_std": 0.1767766922712326, + "rewards/accuracy_reward/mean": 0.0625, + "rewards/accuracy_reward/std": 0.24593468010425568, + "sampling/importance_sampling_ratio/max": 1.507697343826294, + "sampling/importance_sampling_ratio/mean": 0.9998849630355835, + "sampling/importance_sampling_ratio/min": 0.6196276545524597, + "sampling/sampling_logp_difference/max": 0.4786365032196045, + "sampling/sampling_logp_difference/mean": 0.013673453591763973, + "step": 190 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0001461528445361182, + "clip_ratio/low_min": 0.0001461528445361182, + "clip_ratio/region_mean": 0.0001461528445361182, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 766.0, + "completions/mean_length": 653.9375, + "completions/mean_terminated_length": 609.3043823242188, + "completions/min_length": 418.0, + "completions/min_terminated_length": 418.0, + "entropy": 0.33082620427012444, + "epoch": 0.10224839400428265, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.007062896620482206, + "learning_rate": 1e-05, + "loss": 0.0543, + "num_tokens": 4284094.0, + "reward": 0.53125, + "reward_std": 0.4397946000099182, + "rewards/accuracy_reward/mean": 0.53125, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.336458683013916, + "sampling/importance_sampling_ratio/mean": 0.999947726726532, + "sampling/importance_sampling_ratio/min": 0.644375205039978, + "sampling/sampling_logp_difference/max": 0.43947410583496094, + "sampling/sampling_logp_difference/mean": 0.01081669982522726, + "step": 191 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0001768947513482999, + "clip_ratio/low_min": 0.0001768947513482999, + "clip_ratio/region_mean": 0.0001768947513482999, + "completions/clipped_ratio": 0.375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 713.0, + "completions/mean_length": 640.125, + "completions/mean_terminated_length": 563.4000244140625, + "completions/min_length": 347.0, + "completions/min_terminated_length": 347.0, + "entropy": 0.42493443191051483, + "epoch": 0.10278372591006424, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.012724976055324078, + "learning_rate": 1e-05, + "loss": 0.0633, + "num_tokens": 4308482.0, + "reward": 0.3125, + "reward_std": 0.3514062464237213, + "rewards/accuracy_reward/mean": 0.3125, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.426673173904419, + "sampling/importance_sampling_ratio/mean": 0.9997580051422119, + "sampling/importance_sampling_ratio/min": 0.571729302406311, + "sampling/sampling_logp_difference/max": 0.5590896606445312, + "sampling/sampling_logp_difference/mean": 0.013163600116968155, + "step": 192 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.3125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 675.0, + "completions/mean_length": 585.34375, + "completions/mean_terminated_length": 502.3182067871094, + "completions/min_length": 279.0, + "completions/min_terminated_length": 279.0, + "entropy": 0.5026186592876911, + "epoch": 0.10331905781584583, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.005640897434204817, + "learning_rate": 1e-05, + "loss": 0.0759, + "num_tokens": 4331141.0, + "reward": 0.625, + "reward_std": 0.2177756428718567, + "rewards/accuracy_reward/mean": 0.625, + "rewards/accuracy_reward/std": 0.49186936020851135, + "sampling/importance_sampling_ratio/max": 1.4223015308380127, + "sampling/importance_sampling_ratio/mean": 0.9998229742050171, + "sampling/importance_sampling_ratio/min": 0.6328842639923096, + "sampling/sampling_logp_difference/max": 0.45746779441833496, + "sampling/sampling_logp_difference/mean": 0.015014126896858215, + "step": 193 + }, + { + "clip_ratio/high_max": 4.4642856664722785e-05, + "clip_ratio/high_mean": 4.4642856664722785e-05, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 4.4642856664722785e-05, + "completions/clipped_ratio": 0.46875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 725.0, + "completions/mean_length": 624.28125, + "completions/mean_terminated_length": 497.4705810546875, + "completions/min_length": 311.0, + "completions/min_terminated_length": 311.0, + "entropy": 0.5654965825378895, + "epoch": 0.10385438972162742, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.010605625808238983, + "learning_rate": 1e-05, + "loss": 0.0616, + "num_tokens": 4355278.0, + "reward": 0.4375, + "reward_std": 0.3335031569004059, + "rewards/accuracy_reward/mean": 0.4375, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 1.486506700515747, + "sampling/importance_sampling_ratio/mean": 0.999529242515564, + "sampling/importance_sampling_ratio/min": 0.6654889583587646, + "sampling/sampling_logp_difference/max": 0.40723323822021484, + "sampling/sampling_logp_difference/mean": 0.015287080779671669, + "step": 194 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 711.0, + "completions/mean_length": 507.34375, + "completions/mean_terminated_length": 470.107177734375, + "completions/min_length": 291.0, + "completions/min_terminated_length": 291.0, + "entropy": 0.4125858396291733, + "epoch": 0.10438972162740899, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.007816245779395103, + "learning_rate": 1e-05, + "loss": 0.0552, + "num_tokens": 4375073.0, + "reward": 0.8125, + "reward_std": 0.249358132481575, + "rewards/accuracy_reward/mean": 0.8125, + "rewards/accuracy_reward/std": 0.3965577781200409, + "sampling/importance_sampling_ratio/max": 1.4488462209701538, + "sampling/importance_sampling_ratio/mean": 0.9996964335441589, + "sampling/importance_sampling_ratio/min": 0.6287140846252441, + "sampling/sampling_logp_difference/max": 0.4640786647796631, + "sampling/sampling_logp_difference/mean": 0.012324759736657143, + "step": 195 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 651.0, + "completions/mean_length": 497.375, + "completions/mean_terminated_length": 469.3793029785156, + "completions/min_length": 262.0, + "completions/min_terminated_length": 262.0, + "entropy": 0.3259465880692005, + "epoch": 0.10492505353319058, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.019756974652409554, + "learning_rate": 1e-05, + "loss": 0.0587, + "num_tokens": 4394677.0, + "reward": 0.78125, + "reward_std": 0.4218915104866028, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.435684084892273, + "sampling/importance_sampling_ratio/mean": 1.0003255605697632, + "sampling/importance_sampling_ratio/min": 0.6057069301605225, + "sampling/sampling_logp_difference/max": 0.5013589859008789, + "sampling/sampling_logp_difference/mean": 0.011160098947584629, + "step": 196 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.484861685545184e-05, + "clip_ratio/low_min": 5.484861685545184e-05, + "clip_ratio/region_mean": 5.484861685545184e-05, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 716.0, + "completions/mean_length": 523.46875, + "completions/mean_terminated_length": 478.1851806640625, + "completions/min_length": 283.0, + "completions/min_terminated_length": 283.0, + "entropy": 0.41297532618045807, + "epoch": 0.10546038543897217, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.003697456093505025, + "learning_rate": 1e-05, + "loss": 0.0223, + "num_tokens": 4415124.0, + "reward": 0.75, + "reward_std": 0.1767766922712326, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.3766028881072998, + "sampling/importance_sampling_ratio/mean": 1.0000152587890625, + "sampling/importance_sampling_ratio/min": 0.6974722743034363, + "sampling/sampling_logp_difference/max": 0.3602924346923828, + "sampling/sampling_logp_difference/mean": 0.012368491850793362, + "step": 197 + }, + { + "clip_ratio/high_max": 5.451373726828024e-05, + "clip_ratio/high_mean": 5.451373726828024e-05, + "clip_ratio/low_mean": 0.00015712289678049274, + "clip_ratio/low_min": 0.00015712289678049274, + "clip_ratio/region_mean": 0.0002116366413247306, + "completions/clipped_ratio": 0.3125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 704.0, + "completions/mean_length": 593.09375, + "completions/mean_terminated_length": 513.5909423828125, + "completions/min_length": 351.0, + "completions/min_terminated_length": 351.0, + "entropy": 0.3787360694259405, + "epoch": 0.10599571734475374, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.0069842226803302765, + "learning_rate": 1e-05, + "loss": 0.116, + "num_tokens": 4438199.0, + "reward": 0.46875, + "reward_std": 0.4397946000099182, + "rewards/accuracy_reward/mean": 0.46875, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.346086025238037, + "sampling/importance_sampling_ratio/mean": 0.9994926452636719, + "sampling/importance_sampling_ratio/min": 0.6092809438705444, + "sampling/sampling_logp_difference/max": 0.49547576904296875, + "sampling/sampling_logp_difference/mean": 0.012435591779649258, + "step": 198 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 733.0, + "completions/mean_length": 552.0625, + "completions/mean_terminated_length": 502.23077392578125, + "completions/min_length": 232.0, + "completions/min_terminated_length": 232.0, + "entropy": 0.3516416922211647, + "epoch": 0.10653104925053533, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": 0.0351, + "num_tokens": 4459329.0, + "reward": 0.65625, + "reward_std": 0.1293872892856598, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.3627244234085083, + "sampling/importance_sampling_ratio/mean": 1.0005079507827759, + "sampling/importance_sampling_ratio/min": 0.6949403285980225, + "sampling/sampling_logp_difference/max": 0.36392927169799805, + "sampling/sampling_logp_difference/mean": 0.011472605168819427, + "step": 199 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 6.0444875998655334e-05, + "clip_ratio/low_min": 6.0444875998655334e-05, + "clip_ratio/region_mean": 6.0444875998655334e-05, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 742.0, + "completions/mean_length": 489.0625, + "completions/mean_terminated_length": 470.4667053222656, + "completions/min_length": 201.0, + "completions/min_terminated_length": 201.0, + "entropy": 0.43827785551548004, + "epoch": 0.10706638115631692, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.006218034774065018, + "learning_rate": 1e-05, + "loss": 0.1175, + "num_tokens": 4478507.0, + "reward": 0.75, + "reward_std": 0.3514062464237213, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.473978877067566, + "sampling/importance_sampling_ratio/mean": 0.9999464750289917, + "sampling/importance_sampling_ratio/min": 0.6813673973083496, + "sampling/sampling_logp_difference/max": 0.38796544075012207, + "sampling/sampling_logp_difference/mean": 0.013890287838876247, + "step": 200 + }, + { + "clip_ratio/high_max": 6.459948053816333e-05, + "clip_ratio/high_mean": 6.459948053816333e-05, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 6.459948053816333e-05, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 742.0, + "completions/mean_length": 472.90625, + "completions/mean_terminated_length": 463.3870849609375, + "completions/min_length": 192.0, + "completions/min_terminated_length": 192.0, + "entropy": 0.36613602563738823, + "epoch": 0.1076017130620985, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.002796619199216366, + "learning_rate": 1e-05, + "loss": 0.0344, + "num_tokens": 4496936.0, + "reward": 0.96875, + "reward_std": 0.0883883461356163, + "rewards/accuracy_reward/mean": 0.96875, + "rewards/accuracy_reward/std": 0.1767766922712326, + "sampling/importance_sampling_ratio/max": 1.561207890510559, + "sampling/importance_sampling_ratio/mean": 1.0001063346862793, + "sampling/importance_sampling_ratio/min": 0.6822316646575928, + "sampling/sampling_logp_difference/max": 0.44545984268188477, + "sampling/sampling_logp_difference/mean": 0.011922389268875122, + "step": 201 + }, + { + "clip_ratio/high_max": 4.932912270305678e-05, + "clip_ratio/high_mean": 4.932912270305678e-05, + "clip_ratio/low_mean": 5.267593587632291e-05, + "clip_ratio/low_min": 5.267593587632291e-05, + "clip_ratio/region_mean": 0.00010200505857937969, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 768.0, + "completions/mean_length": 574.40625, + "completions/mean_terminated_length": 538.5555419921875, + "completions/min_length": 273.0, + "completions/min_terminated_length": 273.0, + "entropy": 0.323601009324193, + "epoch": 0.10813704496788008, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.011047563515603542, + "learning_rate": 1e-05, + "loss": 0.0461, + "num_tokens": 4518797.0, + "reward": 0.875, + "reward_std": 0.2314550280570984, + "rewards/accuracy_reward/mean": 0.875, + "rewards/accuracy_reward/std": 0.33601075410842896, + "sampling/importance_sampling_ratio/max": 1.422509789466858, + "sampling/importance_sampling_ratio/mean": 0.9997566938400269, + "sampling/importance_sampling_ratio/min": 0.5621358752250671, + "sampling/sampling_logp_difference/max": 0.5760116577148438, + "sampling/sampling_logp_difference/mean": 0.010617411695420742, + "step": 202 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 7.418397581204772e-05, + "clip_ratio/low_min": 7.418397581204772e-05, + "clip_ratio/region_mean": 7.418397581204772e-05, + "completions/clipped_ratio": 0.0, + "completions/max_length": 708.0, + "completions/max_terminated_length": 708.0, + "completions/mean_length": 488.59375, + "completions/mean_terminated_length": 488.59375, + "completions/min_length": 92.0, + "completions/min_terminated_length": 92.0, + "entropy": 0.3624422922730446, + "epoch": 0.10867237687366167, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.012993577867746353, + "learning_rate": 1e-05, + "loss": 0.0867, + "num_tokens": 4537864.0, + "reward": 0.75, + "reward_std": 0.3745020925998688, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.447041630744934, + "sampling/importance_sampling_ratio/mean": 0.9999805688858032, + "sampling/importance_sampling_ratio/min": 0.7270780205726624, + "sampling/sampling_logp_difference/max": 0.3695211410522461, + "sampling/sampling_logp_difference/mean": 0.01146488357335329, + "step": 203 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 9.194557060254738e-05, + "clip_ratio/low_min": 9.194557060254738e-05, + "clip_ratio/region_mean": 9.194557060254738e-05, + "completions/clipped_ratio": 0.34375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 757.0, + "completions/mean_length": 642.96875, + "completions/mean_terminated_length": 577.4761962890625, + "completions/min_length": 360.0, + "completions/min_terminated_length": 360.0, + "entropy": 0.4151980020105839, + "epoch": 0.10920770877944326, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.005442460533231497, + "learning_rate": 1e-05, + "loss": 0.0616, + "num_tokens": 4562223.0, + "reward": 0.40625, + "reward_std": 0.4534739851951599, + "rewards/accuracy_reward/mean": 0.40625, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.459665298461914, + "sampling/importance_sampling_ratio/mean": 1.0001134872436523, + "sampling/importance_sampling_ratio/min": 0.7412106394767761, + "sampling/sampling_logp_difference/max": 0.3782072067260742, + "sampling/sampling_logp_difference/mean": 0.012796086259186268, + "step": 204 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 8.480325777782127e-05, + "clip_ratio/low_min": 8.480325777782127e-05, + "clip_ratio/region_mean": 8.480325777782127e-05, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 738.0, + "completions/mean_length": 459.90625, + "completions/mean_terminated_length": 428.03448486328125, + "completions/min_length": 259.0, + "completions/min_terminated_length": 259.0, + "entropy": 0.5418744124472141, + "epoch": 0.10974304068522484, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.009512354619801044, + "learning_rate": 1e-05, + "loss": -0.0017, + "num_tokens": 4580988.0, + "reward": 0.6875, + "reward_std": 0.3924051821231842, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.3982853889465332, + "sampling/importance_sampling_ratio/mean": 1.0001652240753174, + "sampling/importance_sampling_ratio/min": 0.6685997247695923, + "sampling/sampling_logp_difference/max": 0.4025697708129883, + "sampling/sampling_logp_difference/mean": 0.01568397879600525, + "step": 205 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 726.0, + "completions/mean_length": 508.5, + "completions/mean_terminated_length": 500.1290283203125, + "completions/min_length": 246.0, + "completions/min_terminated_length": 246.0, + "entropy": 0.35495569556951523, + "epoch": 0.11027837259100642, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.005454814527183771, + "learning_rate": 1e-05, + "loss": 0.0625, + "num_tokens": 4601236.0, + "reward": 0.84375, + "reward_std": 0.24511480331420898, + "rewards/accuracy_reward/mean": 0.84375, + "rewards/accuracy_reward/std": 0.3689020276069641, + "sampling/importance_sampling_ratio/max": 1.3583024740219116, + "sampling/importance_sampling_ratio/mean": 1.00021493434906, + "sampling/importance_sampling_ratio/min": 0.41868987679481506, + "sampling/sampling_logp_difference/max": 0.8706247806549072, + "sampling/sampling_logp_difference/mean": 0.01119942031800747, + "step": 206 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00011782906949520111, + "clip_ratio/low_min": 0.00011782906949520111, + "clip_ratio/region_mean": 0.00011782906949520111, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 752.0, + "completions/mean_length": 536.0, + "completions/mean_terminated_length": 471.03997802734375, + "completions/min_length": 288.0, + "completions/min_terminated_length": 288.0, + "entropy": 0.46669817715883255, + "epoch": 0.11081370449678801, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.01467866264283657, + "learning_rate": 1e-05, + "loss": 0.1617, + "num_tokens": 4621980.0, + "reward": 0.53125, + "reward_std": 0.4218915104866028, + "rewards/accuracy_reward/mean": 0.53125, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.2943097352981567, + "sampling/importance_sampling_ratio/mean": 0.9996358156204224, + "sampling/importance_sampling_ratio/min": 0.6145366430282593, + "sampling/sampling_logp_difference/max": 0.48688673973083496, + "sampling/sampling_logp_difference/mean": 0.013230583630502224, + "step": 207 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 4.5110067731002346e-05, + "clip_ratio/low_min": 4.5110067731002346e-05, + "clip_ratio/region_mean": 4.5110067731002346e-05, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 756.0, + "completions/mean_length": 606.8125, + "completions/mean_terminated_length": 576.9629516601562, + "completions/min_length": 340.0, + "completions/min_terminated_length": 340.0, + "entropy": 0.4399042911827564, + "epoch": 0.11134903640256959, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.015372531488537788, + "learning_rate": 1e-05, + "loss": 0.0271, + "num_tokens": 4645102.0, + "reward": 0.75, + "reward_std": 0.2587745785713196, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 2.0, + "sampling/importance_sampling_ratio/mean": 1.0002062320709229, + "sampling/importance_sampling_ratio/min": 0.6881018280982971, + "sampling/sampling_logp_difference/max": 0.806495189666748, + "sampling/sampling_logp_difference/mean": 0.01302797719836235, + "step": 208 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 673.0, + "completions/mean_length": 489.0, + "completions/mean_terminated_length": 460.137939453125, + "completions/min_length": 269.0, + "completions/min_terminated_length": 269.0, + "entropy": 0.44632333144545555, + "epoch": 0.11188436830835118, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.005208560265600681, + "learning_rate": 1e-05, + "loss": 0.0688, + "num_tokens": 4664198.0, + "reward": 0.53125, + "reward_std": 0.3471629321575165, + "rewards/accuracy_reward/mean": 0.53125, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.4133529663085938, + "sampling/importance_sampling_ratio/mean": 1.0002318620681763, + "sampling/importance_sampling_ratio/min": 0.6086702942848206, + "sampling/sampling_logp_difference/max": 0.4964785575866699, + "sampling/sampling_logp_difference/mean": 0.013368175365030766, + "step": 209 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0001271186483791098, + "clip_ratio/low_min": 0.0001271186483791098, + "clip_ratio/region_mean": 0.0001271186483791098, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 748.0, + "completions/mean_length": 480.25, + "completions/mean_terminated_length": 413.8461608886719, + "completions/min_length": 138.0, + "completions/min_terminated_length": 138.0, + "entropy": 0.4682172201573849, + "epoch": 0.11241970021413276, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.015574317425489426, + "learning_rate": 1e-05, + "loss": 0.058, + "num_tokens": 4683006.0, + "reward": 0.625, + "reward_std": 0.3104073107242584, + "rewards/accuracy_reward/mean": 0.625, + "rewards/accuracy_reward/std": 0.49186936020851135, + "sampling/importance_sampling_ratio/max": 1.48087477684021, + "sampling/importance_sampling_ratio/mean": 1.000162124633789, + "sampling/importance_sampling_ratio/min": 0.6997669339179993, + "sampling/sampling_logp_difference/max": 0.39263296127319336, + "sampling/sampling_logp_difference/mean": 0.01364071387797594, + "step": 210 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 735.0, + "completions/mean_length": 544.9375, + "completions/mean_terminated_length": 493.4615478515625, + "completions/min_length": 354.0, + "completions/min_terminated_length": 354.0, + "entropy": 0.3690958619117737, + "epoch": 0.11295503211991435, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.010793657973408699, + "learning_rate": 1e-05, + "loss": 0.0356, + "num_tokens": 4703948.0, + "reward": 0.65625, + "reward_std": 0.1293872892856598, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.4460468292236328, + "sampling/importance_sampling_ratio/mean": 1.000065565109253, + "sampling/importance_sampling_ratio/min": 0.7057710289955139, + "sampling/sampling_logp_difference/max": 0.3688335418701172, + "sampling/sampling_logp_difference/mean": 0.011855688877403736, + "step": 211 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 708.0, + "completions/mean_length": 487.625, + "completions/mean_terminated_length": 468.933349609375, + "completions/min_length": 315.0, + "completions/min_terminated_length": 315.0, + "entropy": 0.39685725048184395, + "epoch": 0.11349036402569593, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.01544248778373003, + "learning_rate": 1e-05, + "loss": 0.0475, + "num_tokens": 4723304.0, + "reward": 0.65625, + "reward_std": 0.3608423173427582, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.290382742881775, + "sampling/importance_sampling_ratio/mean": 0.9997295141220093, + "sampling/importance_sampling_ratio/min": 0.6577038168907166, + "sampling/sampling_logp_difference/max": 0.41900062561035156, + "sampling/sampling_logp_difference/mean": 0.012623356655240059, + "step": 212 + }, + { + "clip_ratio/high_max": 4.5306271204026416e-05, + "clip_ratio/high_mean": 4.5306271204026416e-05, + "clip_ratio/low_mean": 0.00014086080045672134, + "clip_ratio/low_min": 0.00014086080045672134, + "clip_ratio/region_mean": 0.00018616707166074775, + "completions/clipped_ratio": 0.40625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 752.0, + "completions/mean_length": 641.3125, + "completions/mean_terminated_length": 554.631591796875, + "completions/min_length": 394.0, + "completions/min_terminated_length": 394.0, + "entropy": 0.528442993760109, + "epoch": 0.11402569593147752, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.005189127754420042, + "learning_rate": 1e-05, + "loss": 0.0175, + "num_tokens": 4747714.0, + "reward": 0.53125, + "reward_std": 0.2651650309562683, + "rewards/accuracy_reward/mean": 0.53125, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.4936127662658691, + "sampling/importance_sampling_ratio/mean": 0.9998847246170044, + "sampling/importance_sampling_ratio/min": 0.6634731888771057, + "sampling/sampling_logp_difference/max": 0.4102668762207031, + "sampling/sampling_logp_difference/mean": 0.014573306776583195, + "step": 213 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00010960105282720178, + "clip_ratio/low_min": 0.00010960105282720178, + "clip_ratio/region_mean": 0.00010960105282720178, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 679.0, + "completions/mean_length": 561.40625, + "completions/mean_terminated_length": 513.7307739257812, + "completions/min_length": 316.0, + "completions/min_terminated_length": 316.0, + "entropy": 0.4899198245257139, + "epoch": 0.1145610278372591, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.017145702615380287, + "learning_rate": 1e-05, + "loss": 0.0085, + "num_tokens": 4769695.0, + "reward": 0.625, + "reward_std": 0.3335031569004059, + "rewards/accuracy_reward/mean": 0.625, + "rewards/accuracy_reward/std": 0.49186936020851135, + "sampling/importance_sampling_ratio/max": 1.40604829788208, + "sampling/importance_sampling_ratio/mean": 0.9998758435249329, + "sampling/importance_sampling_ratio/min": 0.6533907055854797, + "sampling/sampling_logp_difference/max": 0.4255800247192383, + "sampling/sampling_logp_difference/mean": 0.014461631886661053, + "step": 214 + }, + { + "clip_ratio/high_max": 5.921364208916202e-05, + "clip_ratio/high_mean": 5.921364208916202e-05, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 5.921364208916202e-05, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 761.0, + "completions/mean_length": 494.28125, + "completions/mean_terminated_length": 476.0333557128906, + "completions/min_length": 233.0, + "completions/min_terminated_length": 233.0, + "entropy": 0.33969808742403984, + "epoch": 0.11509635974304068, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.007109393365681171, + "learning_rate": 1e-05, + "loss": -0.0182, + "num_tokens": 4789488.0, + "reward": 0.90625, + "reward_std": 0.2041158676147461, + "rewards/accuracy_reward/mean": 0.90625, + "rewards/accuracy_reward/std": 0.2961445748806, + "sampling/importance_sampling_ratio/max": 1.5345942974090576, + "sampling/importance_sampling_ratio/mean": 1.0005249977111816, + "sampling/importance_sampling_ratio/min": 0.5697072744369507, + "sampling/sampling_logp_difference/max": 0.5626325607299805, + "sampling/sampling_logp_difference/mean": 0.0109400674700737, + "step": 215 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 765.0, + "completions/mean_length": 496.65625, + "completions/mean_terminated_length": 468.5862121582031, + "completions/min_length": 228.0, + "completions/min_terminated_length": 228.0, + "entropy": 0.4533826895058155, + "epoch": 0.11563169164882227, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.0041758473962545395, + "learning_rate": 1e-05, + "loss": 0.0326, + "num_tokens": 4809077.0, + "reward": 0.75, + "reward_std": 0.1767766922712326, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.7522910833358765, + "sampling/importance_sampling_ratio/mean": 1.0000641345977783, + "sampling/importance_sampling_ratio/min": 0.49681904911994934, + "sampling/sampling_logp_difference/max": 0.6995294094085693, + "sampling/sampling_logp_difference/mean": 0.013441476039588451, + "step": 216 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00011785111200879328, + "clip_ratio/low_min": 0.00011785111200879328, + "clip_ratio/region_mean": 0.00011785111200879328, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 705.0, + "completions/mean_length": 537.71875, + "completions/mean_terminated_length": 504.8214416503906, + "completions/min_length": 284.0, + "completions/min_terminated_length": 284.0, + "entropy": 0.43806789815425873, + "epoch": 0.11616702355460386, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.008020127192139626, + "learning_rate": 1e-05, + "loss": 0.0196, + "num_tokens": 4829884.0, + "reward": 0.71875, + "reward_std": 0.2630178928375244, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.5796546936035156, + "sampling/importance_sampling_ratio/mean": 0.9999369978904724, + "sampling/importance_sampling_ratio/min": 0.6530105471611023, + "sampling/sampling_logp_difference/max": 0.45720624923706055, + "sampling/sampling_logp_difference/mean": 0.012817000970244408, + "step": 217 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0002680727484403178, + "clip_ratio/low_min": 0.0002680727484403178, + "clip_ratio/region_mean": 0.0002680727484403178, + "completions/clipped_ratio": 0.5, + "completions/max_length": 768.0, + "completions/max_terminated_length": 697.0, + "completions/mean_length": 665.96875, + "completions/mean_terminated_length": 563.9375, + "completions/min_length": 452.0, + "completions/min_terminated_length": 452.0, + "entropy": 0.5910607874393463, + "epoch": 0.11670235546038545, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.009125817567110062, + "learning_rate": 1e-05, + "loss": 0.0537, + "num_tokens": 4855643.0, + "reward": 0.3125, + "reward_std": 0.3514062464237213, + "rewards/accuracy_reward/mean": 0.3125, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.4317469596862793, + "sampling/importance_sampling_ratio/mean": 0.9997588992118835, + "sampling/importance_sampling_ratio/min": 0.4819125831127167, + "sampling/sampling_logp_difference/max": 0.7299926280975342, + "sampling/sampling_logp_difference/mean": 0.016217265278100967, + "step": 218 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00032712527900002897, + "clip_ratio/low_min": 0.00032712527900002897, + "clip_ratio/region_mean": 0.00032712527900002897, + "completions/clipped_ratio": 0.25, + "completions/max_length": 768.0, + "completions/max_terminated_length": 702.0, + "completions/mean_length": 572.15625, + "completions/mean_terminated_length": 506.875, + "completions/min_length": 327.0, + "completions/min_terminated_length": 327.0, + "entropy": 0.48503001406788826, + "epoch": 0.11723768736616702, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.011963547207415104, + "learning_rate": 1e-05, + "loss": 0.0439, + "num_tokens": 4878128.0, + "reward": 0.46875, + "reward_std": 0.3471629321575165, + "rewards/accuracy_reward/mean": 0.46875, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.5091300010681152, + "sampling/importance_sampling_ratio/mean": 0.999998927116394, + "sampling/importance_sampling_ratio/min": 0.6804026365280151, + "sampling/sampling_logp_difference/max": 0.4115333557128906, + "sampling/sampling_logp_difference/mean": 0.014249210245907307, + "step": 219 + }, + { + "clip_ratio/high_max": 6.670223956461996e-05, + "clip_ratio/high_mean": 6.670223956461996e-05, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 6.670223956461996e-05, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 717.0, + "completions/mean_length": 504.15625, + "completions/mean_terminated_length": 466.46429443359375, + "completions/min_length": 232.0, + "completions/min_terminated_length": 232.0, + "entropy": 0.5912829264998436, + "epoch": 0.11777301927194861, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.01798270083963871, + "learning_rate": 1e-05, + "loss": 0.042, + "num_tokens": 4898125.0, + "reward": 0.78125, + "reward_std": 0.24511480331420898, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.4862390756607056, + "sampling/importance_sampling_ratio/mean": 1.0002179145812988, + "sampling/importance_sampling_ratio/min": 0.6412653923034668, + "sampling/sampling_logp_difference/max": 0.44431185722351074, + "sampling/sampling_logp_difference/mean": 0.013367424719035625, + "step": 220 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.689576573786326e-05, + "clip_ratio/low_min": 5.689576573786326e-05, + "clip_ratio/region_mean": 5.689576573786326e-05, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 757.0, + "completions/mean_length": 538.5, + "completions/mean_terminated_length": 523.2000122070312, + "completions/min_length": 309.0, + "completions/min_terminated_length": 309.0, + "entropy": 0.39681804180145264, + "epoch": 0.1183083511777302, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.004230021964758635, + "learning_rate": 1e-05, + "loss": 0.0221, + "num_tokens": 4918917.0, + "reward": 0.875, + "reward_std": 0.292504221200943, + "rewards/accuracy_reward/mean": 0.875, + "rewards/accuracy_reward/std": 0.33601075410842896, + "sampling/importance_sampling_ratio/max": 1.4435296058654785, + "sampling/importance_sampling_ratio/mean": 0.9999265074729919, + "sampling/importance_sampling_ratio/min": 0.6420924663543701, + "sampling/sampling_logp_difference/max": 0.4430229663848877, + "sampling/sampling_logp_difference/mean": 0.012738961726427078, + "step": 221 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00011799021012848243, + "clip_ratio/low_min": 0.00011799021012848243, + "clip_ratio/region_mean": 0.00011799021012848243, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 742.0, + "completions/mean_length": 506.8125, + "completions/mean_terminated_length": 469.5000305175781, + "completions/min_length": 303.0, + "completions/min_terminated_length": 303.0, + "entropy": 0.391423586755991, + "epoch": 0.11884368308351177, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.0145324831828475, + "learning_rate": 1e-05, + "loss": 0.0036, + "num_tokens": 4938575.0, + "reward": 0.59375, + "reward_std": 0.3787454068660736, + "rewards/accuracy_reward/mean": 0.59375, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.4798321723937988, + "sampling/importance_sampling_ratio/mean": 1.0004429817199707, + "sampling/importance_sampling_ratio/min": 0.6881264448165894, + "sampling/sampling_logp_difference/max": 0.39192867279052734, + "sampling/sampling_logp_difference/mean": 0.01311441045254469, + "step": 222 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00010432070484966971, + "clip_ratio/low_min": 0.00010432070484966971, + "clip_ratio/region_mean": 0.00010432070484966971, + "completions/clipped_ratio": 0.34375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 693.0, + "completions/mean_length": 548.84375, + "completions/mean_terminated_length": 434.0476379394531, + "completions/min_length": 251.0, + "completions/min_terminated_length": 251.0, + "entropy": 0.3972848244011402, + "epoch": 0.11937901498929336, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": 0.0416, + "num_tokens": 4959962.0, + "reward": 0.46875, + "reward_std": 0.24511480331420898, + "rewards/accuracy_reward/mean": 0.46875, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.345275640487671, + "sampling/importance_sampling_ratio/mean": 1.0000486373901367, + "sampling/importance_sampling_ratio/min": 0.6178843975067139, + "sampling/sampling_logp_difference/max": 0.48145389556884766, + "sampling/sampling_logp_difference/mean": 0.012422376312315464, + "step": 223 + }, + { + "clip_ratio/high_max": 4.792944673681632e-05, + "clip_ratio/high_mean": 4.792944673681632e-05, + "clip_ratio/low_mean": 5.4537522373721004e-05, + "clip_ratio/low_min": 5.4537522373721004e-05, + "clip_ratio/region_mean": 0.00010246696911053732, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 738.0, + "completions/mean_length": 565.96875, + "completions/mean_terminated_length": 519.34619140625, + "completions/min_length": 308.0, + "completions/min_terminated_length": 308.0, + "entropy": 0.42224637046456337, + "epoch": 0.11991434689507495, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.007893293164670467, + "learning_rate": 1e-05, + "loss": 0.097, + "num_tokens": 4981217.0, + "reward": 0.75, + "reward_std": 0.3945523500442505, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.8203226327896118, + "sampling/importance_sampling_ratio/mean": 1.0004703998565674, + "sampling/importance_sampling_ratio/min": 0.5529597997665405, + "sampling/sampling_logp_difference/max": 0.5990138053894043, + "sampling/sampling_logp_difference/mean": 0.012788786552846432, + "step": 224 + }, + { + "clip_ratio/high_max": 6.648935959674418e-05, + "clip_ratio/high_mean": 6.648935959674418e-05, + "clip_ratio/low_mean": 0.0001456488753319718, + "clip_ratio/low_min": 0.0001456488753319718, + "clip_ratio/region_mean": 0.00021213823492871597, + "completions/clipped_ratio": 0.375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 680.0, + "completions/mean_length": 610.59375, + "completions/mean_terminated_length": 516.1500244140625, + "completions/min_length": 210.0, + "completions/min_terminated_length": 210.0, + "entropy": 0.5052645802497864, + "epoch": 0.12044967880085652, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.00404767319560051, + "learning_rate": 1e-05, + "loss": 0.0522, + "num_tokens": 5004828.0, + "reward": 0.4375, + "reward_std": 0.292504221200943, + "rewards/accuracy_reward/mean": 0.4375, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 1.5891460180282593, + "sampling/importance_sampling_ratio/mean": 0.9998494982719421, + "sampling/importance_sampling_ratio/min": 0.6003414392471313, + "sampling/sampling_logp_difference/max": 0.5102567672729492, + "sampling/sampling_logp_difference/mean": 0.014668923802673817, + "step": 225 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.8112505939789116e-05, + "clip_ratio/low_min": 5.8112505939789116e-05, + "clip_ratio/region_mean": 5.8112505939789116e-05, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 752.0, + "completions/mean_length": 471.5, + "completions/mean_terminated_length": 451.7333679199219, + "completions/min_length": 127.0, + "completions/min_terminated_length": 127.0, + "entropy": 0.502563439309597, + "epoch": 0.12098501070663811, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.02190265990793705, + "learning_rate": 1e-05, + "loss": -0.002, + "num_tokens": 5023972.0, + "reward": 0.625, + "reward_std": 0.3335031569004059, + "rewards/accuracy_reward/mean": 0.625, + "rewards/accuracy_reward/std": 0.49186936020851135, + "sampling/importance_sampling_ratio/max": 1.3722007274627686, + "sampling/importance_sampling_ratio/mean": 0.9995844960212708, + "sampling/importance_sampling_ratio/min": 0.5808719992637634, + "sampling/sampling_logp_difference/max": 0.5432248115539551, + "sampling/sampling_logp_difference/mean": 0.013688228093087673, + "step": 226 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00019691576017066836, + "clip_ratio/low_min": 0.00019691576017066836, + "clip_ratio/region_mean": 0.00019691576017066836, + "completions/clipped_ratio": 0.4375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 766.0, + "completions/mean_length": 644.46875, + "completions/mean_terminated_length": 548.388916015625, + "completions/min_length": 257.0, + "completions/min_terminated_length": 257.0, + "entropy": 0.7140659429132938, + "epoch": 0.1215203426124197, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.01852392591536045, + "learning_rate": 1e-05, + "loss": 0.0198, + "num_tokens": 5049363.0, + "reward": 0.34375, + "reward_std": 0.2041158676147461, + "rewards/accuracy_reward/mean": 0.34375, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.6167372465133667, + "sampling/importance_sampling_ratio/mean": 1.0000755786895752, + "sampling/importance_sampling_ratio/min": 0.6656412482261658, + "sampling/sampling_logp_difference/max": 0.480410099029541, + "sampling/sampling_logp_difference/mean": 0.017896713688969612, + "step": 227 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 713.0, + "completions/mean_length": 562.59375, + "completions/mean_terminated_length": 524.5555419921875, + "completions/min_length": 368.0, + "completions/min_terminated_length": 368.0, + "entropy": 0.3955683335661888, + "epoch": 0.12205567451820129, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.004563800059258938, + "learning_rate": 1e-05, + "loss": 0.0843, + "num_tokens": 5071078.0, + "reward": 0.65625, + "reward_std": 0.2041158676147461, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.3121706247329712, + "sampling/importance_sampling_ratio/mean": 1.0000855922698975, + "sampling/importance_sampling_ratio/min": 0.3818032443523407, + "sampling/sampling_logp_difference/max": 0.9628498554229736, + "sampling/sampling_logp_difference/mean": 0.01162648480385542, + "step": 228 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 759.0, + "completions/mean_length": 584.78125, + "completions/mean_terminated_length": 550.8518676757812, + "completions/min_length": 327.0, + "completions/min_terminated_length": 327.0, + "entropy": 0.3032823149114847, + "epoch": 0.12259100642398287, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.004989946726709604, + "learning_rate": 1e-05, + "loss": -0.0156, + "num_tokens": 5093527.0, + "reward": 0.65625, + "reward_std": 0.3198433816432953, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.8463772535324097, + "sampling/importance_sampling_ratio/mean": 1.0001384019851685, + "sampling/importance_sampling_ratio/min": 0.5978705883026123, + "sampling/sampling_logp_difference/max": 0.6132254600524902, + "sampling/sampling_logp_difference/mean": 0.01033024676144123, + "step": 229 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 6.303580448729917e-05, + "clip_ratio/low_min": 6.303580448729917e-05, + "clip_ratio/region_mean": 6.303580448729917e-05, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 711.0, + "completions/mean_length": 464.75, + "completions/mean_terminated_length": 444.5333557128906, + "completions/min_length": 280.0, + "completions/min_terminated_length": 280.0, + "entropy": 0.3863627575337887, + "epoch": 0.12312633832976445, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.005575043149292469, + "learning_rate": 1e-05, + "loss": 0.0657, + "num_tokens": 5112143.0, + "reward": 0.78125, + "reward_std": 0.3377465009689331, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.5059571266174316, + "sampling/importance_sampling_ratio/mean": 0.9998601078987122, + "sampling/importance_sampling_ratio/min": 0.7042276263237, + "sampling/sampling_logp_difference/max": 0.40942859649658203, + "sampling/sampling_logp_difference/mean": 0.013208088465034962, + "step": 230 + }, + { + "clip_ratio/high_max": 4.976114723831415e-05, + "clip_ratio/high_mean": 4.976114723831415e-05, + "clip_ratio/low_mean": 4.9407113692723215e-05, + "clip_ratio/low_min": 4.9407113692723215e-05, + "clip_ratio/region_mean": 9.916826093103737e-05, + "completions/clipped_ratio": 0.3125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 711.0, + "completions/mean_length": 540.0625, + "completions/mean_terminated_length": 436.4545593261719, + "completions/min_length": 203.0, + "completions/min_terminated_length": 203.0, + "entropy": 0.5061399415135384, + "epoch": 0.12366167023554604, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": -0.0552, + "num_tokens": 5133089.0, + "reward": 0.625, + "reward_std": 0.2177756428718567, + "rewards/accuracy_reward/mean": 0.625, + "rewards/accuracy_reward/std": 0.49186936020851135, + "sampling/importance_sampling_ratio/max": 1.3599090576171875, + "sampling/importance_sampling_ratio/mean": 1.0001215934753418, + "sampling/importance_sampling_ratio/min": 0.29600051045417786, + "sampling/sampling_logp_difference/max": 1.2173941135406494, + "sampling/sampling_logp_difference/mean": 0.014033789746463299, + "step": 231 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00010252309220959432, + "clip_ratio/low_min": 0.00010252309220959432, + "clip_ratio/region_mean": 0.00010252309220959432, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 761.0, + "completions/mean_length": 533.46875, + "completions/mean_terminated_length": 509.2069091796875, + "completions/min_length": 193.0, + "completions/min_terminated_length": 193.0, + "entropy": 0.391541488468647, + "epoch": 0.12419700214132762, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.021787628531455994, + "learning_rate": 1e-05, + "loss": 0.0325, + "num_tokens": 5153784.0, + "reward": 0.78125, + "reward_std": 0.3377464711666107, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.3615431785583496, + "sampling/importance_sampling_ratio/mean": 0.9998968243598938, + "sampling/importance_sampling_ratio/min": 0.5969482064247131, + "sampling/sampling_logp_difference/max": 0.5159249305725098, + "sampling/sampling_logp_difference/mean": 0.012499446049332619, + "step": 232 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00020247400607331656, + "clip_ratio/low_min": 0.00020247400607331656, + "clip_ratio/region_mean": 0.00020247400607331656, + "completions/clipped_ratio": 0.5, + "completions/max_length": 768.0, + "completions/max_terminated_length": 658.0, + "completions/mean_length": 625.28125, + "completions/mean_terminated_length": 482.5625, + "completions/min_length": 213.0, + "completions/min_terminated_length": 213.0, + "entropy": 0.4108714498579502, + "epoch": 0.1247323340471092, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.01006588339805603, + "learning_rate": 1e-05, + "loss": 0.0793, + "num_tokens": 5177801.0, + "reward": 0.5, + "reward_std": 0.47655022144317627, + "rewards/accuracy_reward/mean": 0.5, + "rewards/accuracy_reward/std": 0.5080004930496216, + "sampling/importance_sampling_ratio/max": 1.3702772855758667, + "sampling/importance_sampling_ratio/mean": 0.9998976588249207, + "sampling/importance_sampling_ratio/min": 0.48900964856147766, + "sampling/sampling_logp_difference/max": 0.7153730392456055, + "sampling/sampling_logp_difference/mean": 0.011515702120959759, + "step": 233 + }, + { + "clip_ratio/high_max": 9.656446491135284e-05, + "clip_ratio/high_mean": 9.656446491135284e-05, + "clip_ratio/low_mean": 9.113035412156023e-05, + "clip_ratio/low_min": 9.113035412156023e-05, + "clip_ratio/region_mean": 0.00018769481903291307, + "completions/clipped_ratio": 0.34375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 703.0, + "completions/mean_length": 600.125, + "completions/mean_terminated_length": 512.1904907226562, + "completions/min_length": 280.0, + "completions/min_terminated_length": 280.0, + "entropy": 0.46367712691426277, + "epoch": 0.1252676659528908, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.006998525932431221, + "learning_rate": 1e-05, + "loss": 0.0301, + "num_tokens": 5200581.0, + "reward": 0.5625, + "reward_std": 0.4082317352294922, + "rewards/accuracy_reward/mean": 0.5625, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 1.6151835918426514, + "sampling/importance_sampling_ratio/mean": 0.999909520149231, + "sampling/importance_sampling_ratio/min": 0.7212191224098206, + "sampling/sampling_logp_difference/max": 0.4794485569000244, + "sampling/sampling_logp_difference/mean": 0.01389430183917284, + "step": 234 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 732.0, + "completions/mean_length": 460.5, + "completions/mean_terminated_length": 440.0000305175781, + "completions/min_length": 208.0, + "completions/min_terminated_length": 208.0, + "entropy": 0.45329560339450836, + "epoch": 0.12580299785867238, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.003511700313538313, + "learning_rate": 1e-05, + "loss": 0.0792, + "num_tokens": 5219517.0, + "reward": 0.6875, + "reward_std": 0.1767766922712326, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.8255608081817627, + "sampling/importance_sampling_ratio/mean": 0.999989926815033, + "sampling/importance_sampling_ratio/min": 0.7056424021720886, + "sampling/sampling_logp_difference/max": 0.6018872261047363, + "sampling/sampling_logp_difference/mean": 0.01322560291737318, + "step": 235 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 747.0, + "completions/mean_length": 538.09375, + "completions/mean_terminated_length": 485.0384826660156, + "completions/min_length": 158.0, + "completions/min_terminated_length": 158.0, + "entropy": 0.4739189185202122, + "epoch": 0.12633832976445397, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": -0.0018, + "num_tokens": 5241744.0, + "reward": 0.6875, + "reward_std": 0.1157275140285492, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.3244550228118896, + "sampling/importance_sampling_ratio/mean": 0.9999309182167053, + "sampling/importance_sampling_ratio/min": 0.44490817189216614, + "sampling/sampling_logp_difference/max": 0.8098874092102051, + "sampling/sampling_logp_difference/mean": 0.013555406592786312, + "step": 236 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 4.805843855137937e-05, + "clip_ratio/low_min": 4.805843855137937e-05, + "clip_ratio/region_mean": 4.805843855137937e-05, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 655.0, + "completions/mean_length": 522.96875, + "completions/mean_terminated_length": 427.08697509765625, + "completions/min_length": 223.0, + "completions/min_terminated_length": 223.0, + "entropy": 0.4349859729409218, + "epoch": 0.12687366167023553, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.014106686227023602, + "learning_rate": 1e-05, + "loss": 0.1133, + "num_tokens": 5262583.0, + "reward": 0.6875, + "reward_std": 0.292504221200943, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.8777523040771484, + "sampling/importance_sampling_ratio/mean": 1.000289797782898, + "sampling/importance_sampling_ratio/min": 0.663354218006134, + "sampling/sampling_logp_difference/max": 0.6300754547119141, + "sampling/sampling_logp_difference/mean": 0.013021737337112427, + "step": 237 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 761.0, + "completions/mean_length": 566.34375, + "completions/mean_terminated_length": 537.5357666015625, + "completions/min_length": 81.0, + "completions/min_terminated_length": 81.0, + "entropy": 0.3597859516739845, + "epoch": 0.12740899357601712, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.010164322331547737, + "learning_rate": 1e-05, + "loss": 0.0136, + "num_tokens": 5284354.0, + "reward": 0.8125, + "reward_std": 0.2177756428718567, + "rewards/accuracy_reward/mean": 0.8125, + "rewards/accuracy_reward/std": 0.3965577781200409, + "sampling/importance_sampling_ratio/max": 1.2968369722366333, + "sampling/importance_sampling_ratio/mean": 0.9998154640197754, + "sampling/importance_sampling_ratio/min": 0.6164509057998657, + "sampling/sampling_logp_difference/max": 0.4837765693664551, + "sampling/sampling_logp_difference/mean": 0.010971475392580032, + "step": 238 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.3125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 683.0, + "completions/mean_length": 570.6875, + "completions/mean_terminated_length": 481.0, + "completions/min_length": 281.0, + "completions/min_terminated_length": 281.0, + "entropy": 0.42755215615034103, + "epoch": 0.1279443254817987, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.00346189271658659, + "learning_rate": 1e-05, + "loss": -0.0083, + "num_tokens": 5306896.0, + "reward": 0.65625, + "reward_std": 0.2041158676147461, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.620949625968933, + "sampling/importance_sampling_ratio/mean": 1.0003163814544678, + "sampling/importance_sampling_ratio/min": 0.5884921550750732, + "sampling/sampling_logp_difference/max": 0.5301916599273682, + "sampling/sampling_logp_difference/mean": 0.013137415051460266, + "step": 239 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 6.300403038039804e-05, + "clip_ratio/low_min": 6.300403038039804e-05, + "clip_ratio/region_mean": 6.300403038039804e-05, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 705.0, + "completions/mean_length": 486.90625, + "completions/mean_terminated_length": 434.85186767578125, + "completions/min_length": 164.0, + "completions/min_terminated_length": 164.0, + "entropy": 0.4919372908771038, + "epoch": 0.1284796573875803, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.013900874182581902, + "learning_rate": 1e-05, + "loss": 0.0791, + "num_tokens": 5326453.0, + "reward": 0.625, + "reward_std": 0.2177756428718567, + "rewards/accuracy_reward/mean": 0.625, + "rewards/accuracy_reward/std": 0.49186936020851135, + "sampling/importance_sampling_ratio/max": 1.276257872581482, + "sampling/importance_sampling_ratio/mean": 1.0000147819519043, + "sampling/importance_sampling_ratio/min": 0.6976582407951355, + "sampling/sampling_logp_difference/max": 0.36002588272094727, + "sampling/sampling_logp_difference/mean": 0.01409463956952095, + "step": 240 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 9.580544792697765e-05, + "clip_ratio/low_min": 9.580544792697765e-05, + "clip_ratio/region_mean": 9.580544792697765e-05, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 767.0, + "completions/mean_length": 543.0, + "completions/mean_terminated_length": 510.857177734375, + "completions/min_length": 292.0, + "completions/min_terminated_length": 292.0, + "entropy": 0.442779716104269, + "epoch": 0.1290149892933619, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.01613631099462509, + "learning_rate": 1e-05, + "loss": 0.068, + "num_tokens": 5347565.0, + "reward": 0.6875, + "reward_std": 0.4355512857437134, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.4268854856491089, + "sampling/importance_sampling_ratio/mean": 1.0003812313079834, + "sampling/importance_sampling_ratio/min": 0.6826814413070679, + "sampling/sampling_logp_difference/max": 0.3817269802093506, + "sampling/sampling_logp_difference/mean": 0.013200968503952026, + "step": 241 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 697.0, + "completions/mean_length": 568.21875, + "completions/mean_terminated_length": 512.2799682617188, + "completions/min_length": 325.0, + "completions/min_terminated_length": 325.0, + "entropy": 0.36066998913884163, + "epoch": 0.12955032119914348, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.02156771533191204, + "learning_rate": 1e-05, + "loss": 0.0185, + "num_tokens": 5369700.0, + "reward": 0.71875, + "reward_std": 0.0883883461356163, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.430492639541626, + "sampling/importance_sampling_ratio/mean": 0.9997615218162537, + "sampling/importance_sampling_ratio/min": 0.641115128993988, + "sampling/sampling_logp_difference/max": 0.4445462226867676, + "sampling/sampling_logp_difference/mean": 0.011267188936471939, + "step": 242 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 750.0, + "completions/mean_length": 489.125, + "completions/mean_terminated_length": 470.5333557128906, + "completions/min_length": 269.0, + "completions/min_terminated_length": 269.0, + "entropy": 0.35818955674767494, + "epoch": 0.13008565310492506, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.01729537546634674, + "learning_rate": 1e-05, + "loss": 0.0497, + "num_tokens": 5388928.0, + "reward": 0.71875, + "reward_std": 0.3787454068660736, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.608560562133789, + "sampling/importance_sampling_ratio/mean": 1.000006079673767, + "sampling/importance_sampling_ratio/min": 0.6876170635223389, + "sampling/sampling_logp_difference/max": 0.4753396511077881, + "sampling/sampling_logp_difference/mean": 0.011617233045399189, + "step": 243 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00011165935939061455, + "clip_ratio/low_min": 0.00011165935939061455, + "clip_ratio/region_mean": 0.00011165935939061455, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 761.0, + "completions/mean_length": 503.4375, + "completions/mean_terminated_length": 476.0689697265625, + "completions/min_length": 277.0, + "completions/min_terminated_length": 277.0, + "entropy": 0.4210197776556015, + "epoch": 0.13062098501070663, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.010469350032508373, + "learning_rate": 1e-05, + "loss": 0.0743, + "num_tokens": 5408782.0, + "reward": 0.71875, + "reward_std": 0.24511480331420898, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.4662446975708008, + "sampling/importance_sampling_ratio/mean": 1.0000981092453003, + "sampling/importance_sampling_ratio/min": 0.7117865085601807, + "sampling/sampling_logp_difference/max": 0.382704496383667, + "sampling/sampling_logp_difference/mean": 0.01223764382302761, + "step": 244 + }, + { + "clip_ratio/high_max": 6.517205474665388e-05, + "clip_ratio/high_mean": 6.517205474665388e-05, + "clip_ratio/low_mean": 4.8543690354563296e-05, + "clip_ratio/low_min": 4.8543690354563296e-05, + "clip_ratio/region_mean": 0.00011371574510121718, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 723.0, + "completions/mean_length": 613.71875, + "completions/mean_terminated_length": 553.3478393554688, + "completions/min_length": 408.0, + "completions/min_terminated_length": 408.0, + "entropy": 0.47519996762275696, + "epoch": 0.1311563169164882, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.005689447280019522, + "learning_rate": 1e-05, + "loss": 0.0899, + "num_tokens": 5432653.0, + "reward": 0.59375, + "reward_std": 0.3471629321575165, + "rewards/accuracy_reward/mean": 0.59375, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.4368473291397095, + "sampling/importance_sampling_ratio/mean": 1.0000447034835815, + "sampling/importance_sampling_ratio/min": 0.6039672493934631, + "sampling/sampling_logp_difference/max": 0.5042352676391602, + "sampling/sampling_logp_difference/mean": 0.013447093777358532, + "step": 245 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 732.0, + "completions/mean_length": 510.125, + "completions/mean_terminated_length": 492.933349609375, + "completions/min_length": 238.0, + "completions/min_terminated_length": 238.0, + "entropy": 0.379620973020792, + "epoch": 0.1316916488222698, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.015214172191917896, + "learning_rate": 1e-05, + "loss": 0.0442, + "num_tokens": 5452585.0, + "reward": 0.78125, + "reward_std": 0.3377464711666107, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.4035483598709106, + "sampling/importance_sampling_ratio/mean": 0.9997231364250183, + "sampling/importance_sampling_ratio/min": 0.701497495174408, + "sampling/sampling_logp_difference/max": 0.3545379638671875, + "sampling/sampling_logp_difference/mean": 0.011838559992611408, + "step": 246 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 756.0, + "completions/mean_length": 493.15625, + "completions/mean_terminated_length": 464.72412109375, + "completions/min_length": 282.0, + "completions/min_terminated_length": 282.0, + "entropy": 0.47054021805524826, + "epoch": 0.1322269807280514, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.013052414171397686, + "learning_rate": 1e-05, + "loss": 0.0219, + "num_tokens": 5472206.0, + "reward": 0.78125, + "reward_std": 0.3377464711666107, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.5128657817840576, + "sampling/importance_sampling_ratio/mean": 0.9999228119850159, + "sampling/importance_sampling_ratio/min": 0.6195247173309326, + "sampling/sampling_logp_difference/max": 0.4788026809692383, + "sampling/sampling_logp_difference/mean": 0.013454243540763855, + "step": 247 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 620.0, + "completions/mean_length": 466.65625, + "completions/mean_terminated_length": 446.5666809082031, + "completions/min_length": 228.0, + "completions/min_terminated_length": 228.0, + "entropy": 0.3530896082520485, + "epoch": 0.13276231263383298, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.004537623841315508, + "learning_rate": 1e-05, + "loss": 0.094, + "num_tokens": 5490395.0, + "reward": 0.875, + "reward_std": 0.292504221200943, + "rewards/accuracy_reward/mean": 0.875, + "rewards/accuracy_reward/std": 0.33601075410842896, + "sampling/importance_sampling_ratio/max": 1.4455835819244385, + "sampling/importance_sampling_ratio/mean": 1.000109076499939, + "sampling/importance_sampling_ratio/min": 0.6275488138198853, + "sampling/sampling_logp_difference/max": 0.46593379974365234, + "sampling/sampling_logp_difference/mean": 0.011388519778847694, + "step": 248 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00011779661872424185, + "clip_ratio/low_min": 0.00011779661872424185, + "clip_ratio/region_mean": 0.00011779661872424185, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 715.0, + "completions/mean_length": 511.34375, + "completions/mean_terminated_length": 484.7930908203125, + "completions/min_length": 265.0, + "completions/min_terminated_length": 265.0, + "entropy": 0.4604758098721504, + "epoch": 0.13329764453961457, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.018156999722123146, + "learning_rate": 1e-05, + "loss": 0.0354, + "num_tokens": 5510750.0, + "reward": 0.46875, + "reward_std": 0.35564959049224854, + "rewards/accuracy_reward/mean": 0.46875, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.4280118942260742, + "sampling/importance_sampling_ratio/mean": 1.0001708269119263, + "sampling/importance_sampling_ratio/min": 0.6212390065193176, + "sampling/sampling_logp_difference/max": 0.47603940963745117, + "sampling/sampling_logp_difference/mean": 0.011983473785221577, + "step": 249 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 622.0, + "completions/mean_length": 464.625, + "completions/mean_terminated_length": 394.6153869628906, + "completions/min_length": 191.0, + "completions/min_terminated_length": 191.0, + "entropy": 0.577805370092392, + "epoch": 0.13383297644539616, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.009487269446253777, + "learning_rate": 1e-05, + "loss": 0.0476, + "num_tokens": 5529762.0, + "reward": 0.59375, + "reward_std": 0.24511480331420898, + "rewards/accuracy_reward/mean": 0.59375, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.3565547466278076, + "sampling/importance_sampling_ratio/mean": 0.9998091459274292, + "sampling/importance_sampling_ratio/min": 0.549450695514679, + "sampling/sampling_logp_difference/max": 0.5988361835479736, + "sampling/sampling_logp_difference/mean": 0.01527494564652443, + "step": 250 + }, + { + "clip_ratio/high_max": 7.936507608974352e-05, + "clip_ratio/high_mean": 7.936507608974352e-05, + "clip_ratio/low_mean": 5.755064557888545e-05, + "clip_ratio/low_min": 5.755064557888545e-05, + "clip_ratio/region_mean": 0.00013691572166862898, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 760.0, + "completions/mean_length": 466.75, + "completions/mean_terminated_length": 435.5862121582031, + "completions/min_length": 257.0, + "completions/min_terminated_length": 257.0, + "entropy": 0.36938830465078354, + "epoch": 0.13436830835117772, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.015231940895318985, + "learning_rate": 1e-05, + "loss": 0.0357, + "num_tokens": 5548282.0, + "reward": 0.53125, + "reward_std": 0.3808925747871399, + "rewards/accuracy_reward/mean": 0.53125, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.4448702335357666, + "sampling/importance_sampling_ratio/mean": 0.9998409152030945, + "sampling/importance_sampling_ratio/min": 0.5664653182029724, + "sampling/sampling_logp_difference/max": 0.5683393478393555, + "sampling/sampling_logp_difference/mean": 0.012200850062072277, + "step": 251 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.40625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 766.0, + "completions/mean_length": 632.78125, + "completions/mean_terminated_length": 540.26318359375, + "completions/min_length": 274.0, + "completions/min_terminated_length": 274.0, + "entropy": 0.5005229525268078, + "epoch": 0.1349036402569593, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.008063266053795815, + "learning_rate": 1e-05, + "loss": -0.0078, + "num_tokens": 5573043.0, + "reward": 0.34375, + "reward_std": 0.3608423173427582, + "rewards/accuracy_reward/mean": 0.34375, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.525425910949707, + "sampling/importance_sampling_ratio/mean": 1.0000256299972534, + "sampling/importance_sampling_ratio/min": 0.6791707277297974, + "sampling/sampling_logp_difference/max": 0.4222736358642578, + "sampling/sampling_logp_difference/mean": 0.014093023724853992, + "step": 252 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 600.0, + "completions/max_terminated_length": 600.0, + "completions/mean_length": 410.25, + "completions/mean_terminated_length": 410.25, + "completions/min_length": 196.0, + "completions/min_terminated_length": 196.0, + "entropy": 0.2839566823095083, + "epoch": 0.1354389721627409, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.0196959488093853, + "learning_rate": 1e-05, + "loss": -0.0341, + "num_tokens": 5589523.0, + "reward": 0.90625, + "reward_std": 0.2041158676147461, + "rewards/accuracy_reward/mean": 0.90625, + "rewards/accuracy_reward/std": 0.2961445748806, + "sampling/importance_sampling_ratio/max": 1.3948116302490234, + "sampling/importance_sampling_ratio/mean": 0.9999096393585205, + "sampling/importance_sampling_ratio/min": 0.5720772743225098, + "sampling/sampling_logp_difference/max": 0.5584812164306641, + "sampling/sampling_logp_difference/mean": 0.009745274670422077, + "step": 253 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 7.212925265775993e-05, + "clip_ratio/low_min": 7.212925265775993e-05, + "clip_ratio/region_mean": 7.212925265775993e-05, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 731.0, + "completions/mean_length": 534.6875, + "completions/mean_terminated_length": 491.4814758300781, + "completions/min_length": 226.0, + "completions/min_terminated_length": 226.0, + "entropy": 0.4521285966038704, + "epoch": 0.13597430406852248, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.01752840168774128, + "learning_rate": 1e-05, + "loss": 0.016, + "num_tokens": 5610433.0, + "reward": 0.75, + "reward_std": 0.26726123690605164, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.389081597328186, + "sampling/importance_sampling_ratio/mean": 0.9998871684074402, + "sampling/importance_sampling_ratio/min": 0.7004028558731079, + "sampling/sampling_logp_difference/max": 0.35609960556030273, + "sampling/sampling_logp_difference/mean": 0.013969801366329193, + "step": 254 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 766.0, + "completions/mean_length": 460.15625, + "completions/mean_terminated_length": 439.63336181640625, + "completions/min_length": 224.0, + "completions/min_terminated_length": 224.0, + "entropy": 0.3674822635948658, + "epoch": 0.13650963597430407, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.011356225237250328, + "learning_rate": 1e-05, + "loss": 0.0501, + "num_tokens": 5628414.0, + "reward": 0.65625, + "reward_std": 0.3471629321575165, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.422326922416687, + "sampling/importance_sampling_ratio/mean": 1.0003477334976196, + "sampling/importance_sampling_ratio/min": 0.6975710988044739, + "sampling/sampling_logp_difference/max": 0.3601508140563965, + "sampling/sampling_logp_difference/mean": 0.011541708372533321, + "step": 255 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 6.041565939085558e-05, + "clip_ratio/low_min": 6.041565939085558e-05, + "clip_ratio/region_mean": 6.041565939085558e-05, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 748.0, + "completions/mean_length": 487.15625, + "completions/mean_terminated_length": 435.1481628417969, + "completions/min_length": 250.0, + "completions/min_terminated_length": 250.0, + "entropy": 0.4013260640203953, + "epoch": 0.13704496788008566, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.013696649111807346, + "learning_rate": 1e-05, + "loss": 0.0458, + "num_tokens": 5647571.0, + "reward": 0.46875, + "reward_std": 0.4218915104866028, + "rewards/accuracy_reward/mean": 0.46875, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.4437862634658813, + "sampling/importance_sampling_ratio/mean": 1.000393271446228, + "sampling/importance_sampling_ratio/min": 0.7036470770835876, + "sampling/sampling_logp_difference/max": 0.36726903915405273, + "sampling/sampling_logp_difference/mean": 0.012749886140227318, + "step": 256 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 647.0, + "completions/max_terminated_length": 647.0, + "completions/mean_length": 445.03125, + "completions/mean_terminated_length": 445.03125, + "completions/min_length": 277.0, + "completions/min_terminated_length": 277.0, + "entropy": 0.34894824028015137, + "epoch": 0.13758029978586725, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.002513580024242401, + "learning_rate": 1e-05, + "loss": 0.0504, + "num_tokens": 5665012.0, + "reward": 0.8125, + "reward_std": 0.2177756428718567, + "rewards/accuracy_reward/mean": 0.8125, + "rewards/accuracy_reward/std": 0.3965577781200409, + "sampling/importance_sampling_ratio/max": 2.0, + "sampling/importance_sampling_ratio/mean": 0.9998238682746887, + "sampling/importance_sampling_ratio/min": 0.635778546333313, + "sampling/sampling_logp_difference/max": 0.6938328742980957, + "sampling/sampling_logp_difference/mean": 0.012405021116137505, + "step": 257 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 7.01065655448474e-05, + "clip_ratio/low_min": 7.01065655448474e-05, + "clip_ratio/region_mean": 7.01065655448474e-05, + "completions/clipped_ratio": 0.0, + "completions/max_length": 672.0, + "completions/max_terminated_length": 672.0, + "completions/mean_length": 451.0, + "completions/mean_terminated_length": 451.0, + "completions/min_length": 285.0, + "completions/min_terminated_length": 285.0, + "entropy": 0.3489055410027504, + "epoch": 0.1381156316916488, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.006209916900843382, + "learning_rate": 1e-05, + "loss": 0.0214, + "num_tokens": 5682900.0, + "reward": 0.75, + "reward_std": 0.2314550280570984, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.4003722667694092, + "sampling/importance_sampling_ratio/mean": 0.99953693151474, + "sampling/importance_sampling_ratio/min": 0.6066949367523193, + "sampling/sampling_logp_difference/max": 0.4997291564941406, + "sampling/sampling_logp_difference/mean": 0.012071177363395691, + "step": 258 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.002000762033276e-05, + "clip_ratio/low_min": 5.002000762033276e-05, + "clip_ratio/region_mean": 5.002000762033276e-05, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 764.0, + "completions/mean_length": 618.65625, + "completions/mean_terminated_length": 560.2174072265625, + "completions/min_length": 239.0, + "completions/min_terminated_length": 239.0, + "entropy": 0.5505796335637569, + "epoch": 0.1386509635974304, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.004257765598595142, + "learning_rate": 1e-05, + "loss": 0.0181, + "num_tokens": 5706433.0, + "reward": 0.25, + "reward_std": 0.2587745785713196, + "rewards/accuracy_reward/mean": 0.25, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.7159582376480103, + "sampling/importance_sampling_ratio/mean": 1.000075340270996, + "sampling/importance_sampling_ratio/min": 0.5809783935546875, + "sampling/sampling_logp_difference/max": 0.5430417060852051, + "sampling/sampling_logp_difference/mean": 0.013373611494898796, + "step": 259 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 7.467144314432517e-05, + "clip_ratio/low_min": 7.467144314432517e-05, + "clip_ratio/region_mean": 7.467144314432517e-05, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 734.0, + "completions/mean_length": 479.5, + "completions/mean_terminated_length": 460.2666931152344, + "completions/min_length": 182.0, + "completions/min_terminated_length": 182.0, + "entropy": 0.508996956050396, + "epoch": 0.139186295503212, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.013239891268312931, + "learning_rate": 1e-05, + "loss": 0.0312, + "num_tokens": 5725161.0, + "reward": 0.65625, + "reward_std": 0.3471629321575165, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.2882016897201538, + "sampling/importance_sampling_ratio/mean": 0.9997792840003967, + "sampling/importance_sampling_ratio/min": 0.5010967254638672, + "sampling/sampling_logp_difference/max": 0.6909561157226562, + "sampling/sampling_logp_difference/mean": 0.01288861408829689, + "step": 260 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 9.622786456020549e-05, + "clip_ratio/low_min": 9.622786456020549e-05, + "clip_ratio/region_mean": 9.622786456020549e-05, + "completions/clipped_ratio": 0.375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 754.0, + "completions/mean_length": 636.1875, + "completions/mean_terminated_length": 557.1000366210938, + "completions/min_length": 382.0, + "completions/min_terminated_length": 382.0, + "entropy": 0.5438135378062725, + "epoch": 0.13972162740899358, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.007354440167546272, + "learning_rate": 1e-05, + "loss": 0.0466, + "num_tokens": 5749543.0, + "reward": 0.375, + "reward_std": 0.2177756428718567, + "rewards/accuracy_reward/mean": 0.375, + "rewards/accuracy_reward/std": 0.49186936020851135, + "sampling/importance_sampling_ratio/max": 1.8053772449493408, + "sampling/importance_sampling_ratio/mean": 1.0000406503677368, + "sampling/importance_sampling_ratio/min": 0.6841415166854858, + "sampling/sampling_logp_difference/max": 0.5907695293426514, + "sampling/sampling_logp_difference/mean": 0.01508672907948494, + "step": 261 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00017080266115954146, + "clip_ratio/low_min": 0.00017080266115954146, + "clip_ratio/region_mean": 0.00017080266115954146, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 693.0, + "completions/mean_length": 423.375, + "completions/mean_terminated_length": 374.14288330078125, + "completions/min_length": 171.0, + "completions/min_terminated_length": 171.0, + "entropy": 0.4612344726920128, + "epoch": 0.14025695931477516, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.011318651027977467, + "learning_rate": 1e-05, + "loss": 0.041, + "num_tokens": 5766467.0, + "reward": 0.625, + "reward_std": 0.49871626496315, + "rewards/accuracy_reward/mean": 0.625, + "rewards/accuracy_reward/std": 0.49186936020851135, + "sampling/importance_sampling_ratio/max": 1.363800287246704, + "sampling/importance_sampling_ratio/mean": 1.0003305673599243, + "sampling/importance_sampling_ratio/min": 0.5737609267234802, + "sampling/sampling_logp_difference/max": 0.5555424690246582, + "sampling/sampling_logp_difference/mean": 0.012841030955314636, + "step": 262 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.000243418642639881, + "clip_ratio/low_min": 0.000243418642639881, + "clip_ratio/region_mean": 0.000243418642639881, + "completions/clipped_ratio": 0.25, + "completions/max_length": 768.0, + "completions/max_terminated_length": 760.0, + "completions/mean_length": 629.1875, + "completions/mean_terminated_length": 582.9166870117188, + "completions/min_length": 365.0, + "completions/min_terminated_length": 365.0, + "entropy": 0.5080892257392406, + "epoch": 0.14079229122055675, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.011443656869232655, + "learning_rate": 1e-05, + "loss": 0.0449, + "num_tokens": 5790153.0, + "reward": 0.46875, + "reward_std": 0.3471629321575165, + "rewards/accuracy_reward/mean": 0.46875, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.5307055711746216, + "sampling/importance_sampling_ratio/mean": 1.000052809715271, + "sampling/importance_sampling_ratio/min": 0.3311429023742676, + "sampling/sampling_logp_difference/max": 1.1052052974700928, + "sampling/sampling_logp_difference/mean": 0.014086173847317696, + "step": 263 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.771006544819102e-05, + "clip_ratio/low_min": 5.771006544819102e-05, + "clip_ratio/region_mean": 5.771006544819102e-05, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 702.0, + "completions/mean_length": 508.25, + "completions/mean_terminated_length": 448.3077087402344, + "completions/min_length": 246.0, + "completions/min_terminated_length": 246.0, + "entropy": 0.3992535434663296, + "epoch": 0.14132762312633834, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.025639759376645088, + "learning_rate": 1e-05, + "loss": 0.01, + "num_tokens": 5810369.0, + "reward": 0.28125, + "reward_std": 0.0883883461356163, + "rewards/accuracy_reward/mean": 0.28125, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.3812230825424194, + "sampling/importance_sampling_ratio/mean": 1.0002771615982056, + "sampling/importance_sampling_ratio/min": 0.6831875443458557, + "sampling/sampling_logp_difference/max": 0.3809858560562134, + "sampling/sampling_logp_difference/mean": 0.012271767482161522, + "step": 264 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 6.275100167840719e-05, + "clip_ratio/low_min": 6.275100167840719e-05, + "clip_ratio/region_mean": 6.275100167840719e-05, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 739.0, + "completions/mean_length": 567.59375, + "completions/mean_terminated_length": 538.9642944335938, + "completions/min_length": 336.0, + "completions/min_terminated_length": 336.0, + "entropy": 0.4798845537006855, + "epoch": 0.1418629550321199, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.006482654716819525, + "learning_rate": 1e-05, + "loss": 0.0399, + "num_tokens": 5832204.0, + "reward": 0.71875, + "reward_std": 0.2041158676147461, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.3956878185272217, + "sampling/importance_sampling_ratio/mean": 1.0003331899642944, + "sampling/importance_sampling_ratio/min": 0.6024718880653381, + "sampling/sampling_logp_difference/max": 0.5067142248153687, + "sampling/sampling_logp_difference/mean": 0.013263930566608906, + "step": 265 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0001232134090969339, + "clip_ratio/low_min": 0.0001232134090969339, + "clip_ratio/region_mean": 0.0001232134090969339, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 712.0, + "completions/mean_length": 577.28125, + "completions/mean_terminated_length": 550.0357666015625, + "completions/min_length": 266.0, + "completions/min_terminated_length": 266.0, + "entropy": 0.4998796731233597, + "epoch": 0.1423982869379015, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": 0.0423, + "num_tokens": 5854829.0, + "reward": 0.78125, + "reward_std": 0.2630178928375244, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.3215910196304321, + "sampling/importance_sampling_ratio/mean": 1.000644564628601, + "sampling/importance_sampling_ratio/min": 0.6367124915122986, + "sampling/sampling_logp_difference/max": 0.45143699645996094, + "sampling/sampling_logp_difference/mean": 0.01436928752809763, + "step": 266 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 709.0, + "completions/mean_length": 445.59375, + "completions/mean_terminated_length": 435.19354248046875, + "completions/min_length": 212.0, + "completions/min_terminated_length": 212.0, + "entropy": 0.3613501340150833, + "epoch": 0.14293361884368308, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": 0.0861, + "num_tokens": 5872408.0, + "reward": 0.9375, + "reward_std": 0.1767766922712326, + "rewards/accuracy_reward/mean": 0.9375, + "rewards/accuracy_reward/std": 0.24593468010425568, + "sampling/importance_sampling_ratio/max": 1.4321389198303223, + "sampling/importance_sampling_ratio/mean": 0.9999266266822815, + "sampling/importance_sampling_ratio/min": 0.7264004945755005, + "sampling/sampling_logp_difference/max": 0.35916900634765625, + "sampling/sampling_logp_difference/mean": 0.011632355861365795, + "step": 267 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 673.0, + "completions/mean_length": 519.40625, + "completions/mean_terminated_length": 502.8333740234375, + "completions/min_length": 348.0, + "completions/min_terminated_length": 348.0, + "entropy": 0.32485054805874825, + "epoch": 0.14346895074946467, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.006728060077875853, + "learning_rate": 1e-05, + "loss": 0.0159, + "num_tokens": 5892653.0, + "reward": 0.84375, + "reward_std": 0.3061639964580536, + "rewards/accuracy_reward/mean": 0.84375, + "rewards/accuracy_reward/std": 0.3689020276069641, + "sampling/importance_sampling_ratio/max": 1.497738242149353, + "sampling/importance_sampling_ratio/mean": 1.0001980066299438, + "sampling/importance_sampling_ratio/min": 0.684363067150116, + "sampling/sampling_logp_difference/max": 0.40395617485046387, + "sampling/sampling_logp_difference/mean": 0.010887635871767998, + "step": 268 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00013224303984316066, + "clip_ratio/low_min": 0.00013224303984316066, + "clip_ratio/region_mean": 0.00013224303984316066, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 729.0, + "completions/mean_length": 468.09375, + "completions/mean_terminated_length": 448.10003662109375, + "completions/min_length": 279.0, + "completions/min_terminated_length": 279.0, + "entropy": 0.40673037990927696, + "epoch": 0.14400428265524626, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.010233355686068535, + "learning_rate": 1e-05, + "loss": -0.0387, + "num_tokens": 5910984.0, + "reward": 0.5625, + "reward_std": 0.3471825420856476, + "rewards/accuracy_reward/mean": 0.5625, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 1.5287798643112183, + "sampling/importance_sampling_ratio/mean": 0.999657928943634, + "sampling/importance_sampling_ratio/min": 0.7221192121505737, + "sampling/sampling_logp_difference/max": 0.4244699478149414, + "sampling/sampling_logp_difference/mean": 0.012970454059541225, + "step": 269 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00020064205455128103, + "clip_ratio/low_min": 0.00020064205455128103, + "clip_ratio/region_mean": 0.00020064205455128103, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 522.0, + "completions/mean_length": 445.59375, + "completions/mean_terminated_length": 385.8888854980469, + "completions/min_length": 201.0, + "completions/min_terminated_length": 201.0, + "entropy": 0.46112507954239845, + "epoch": 0.14453961456102785, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.019001614302396774, + "learning_rate": 1e-05, + "loss": 0.0536, + "num_tokens": 5928531.0, + "reward": 0.5625, + "reward_std": 0.3514062762260437, + "rewards/accuracy_reward/mean": 0.5625, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 1.360998272895813, + "sampling/importance_sampling_ratio/mean": 0.9998810887336731, + "sampling/importance_sampling_ratio/min": 0.5097512602806091, + "sampling/sampling_logp_difference/max": 0.6738324165344238, + "sampling/sampling_logp_difference/mean": 0.01386426854878664, + "step": 270 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 4.9860391300171614e-05, + "clip_ratio/low_min": 4.9860391300171614e-05, + "clip_ratio/region_mean": 4.9860391300171614e-05, + "completions/clipped_ratio": 0.3125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 706.0, + "completions/mean_length": 608.34375, + "completions/mean_terminated_length": 535.7727661132812, + "completions/min_length": 277.0, + "completions/min_terminated_length": 277.0, + "entropy": 0.44793014228343964, + "epoch": 0.14507494646680943, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.0058585829101502895, + "learning_rate": 1e-05, + "loss": 0.0425, + "num_tokens": 5951374.0, + "reward": 0.625, + "reward_std": 0.3335031569004059, + "rewards/accuracy_reward/mean": 0.625, + "rewards/accuracy_reward/std": 0.49186936020851135, + "sampling/importance_sampling_ratio/max": 1.7821135520935059, + "sampling/importance_sampling_ratio/mean": 1.0003591775894165, + "sampling/importance_sampling_ratio/min": 0.7277841567993164, + "sampling/sampling_logp_difference/max": 0.5778000354766846, + "sampling/sampling_logp_difference/mean": 0.012861361727118492, + "step": 271 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 6.380806735251099e-05, + "clip_ratio/low_min": 6.380806735251099e-05, + "clip_ratio/region_mean": 6.380806735251099e-05, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 659.0, + "completions/mean_length": 478.78125, + "completions/mean_terminated_length": 459.5000305175781, + "completions/min_length": 298.0, + "completions/min_terminated_length": 298.0, + "entropy": 0.39119698852300644, + "epoch": 0.145610278372591, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.02509799413383007, + "learning_rate": 1e-05, + "loss": 0.0942, + "num_tokens": 5970415.0, + "reward": 0.71875, + "reward_std": 0.3808925747871399, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.4179117679595947, + "sampling/importance_sampling_ratio/mean": 0.9994146823883057, + "sampling/importance_sampling_ratio/min": 0.6299112439155579, + "sampling/sampling_logp_difference/max": 0.4621763229370117, + "sampling/sampling_logp_difference/mean": 0.012396533973515034, + "step": 272 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 760.0, + "completions/mean_length": 484.125, + "completions/mean_terminated_length": 404.6399841308594, + "completions/min_length": 188.0, + "completions/min_terminated_length": 188.0, + "entropy": 0.48611050844192505, + "epoch": 0.14614561027837258, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.020668551325798035, + "learning_rate": 1e-05, + "loss": 0.0815, + "num_tokens": 5989203.0, + "reward": 0.40625, + "reward_std": 0.3471629321575165, + "rewards/accuracy_reward/mean": 0.40625, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.5681416988372803, + "sampling/importance_sampling_ratio/mean": 1.0000240802764893, + "sampling/importance_sampling_ratio/min": 0.7110249400138855, + "sampling/sampling_logp_difference/max": 0.4498913288116455, + "sampling/sampling_logp_difference/mean": 0.014063272625207901, + "step": 273 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 733.0, + "completions/mean_length": 518.78125, + "completions/mean_terminated_length": 493.0, + "completions/min_length": 267.0, + "completions/min_terminated_length": 267.0, + "entropy": 0.33634426072239876, + "epoch": 0.14668094218415417, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.01658659800887108, + "learning_rate": 1e-05, + "loss": -0.0045, + "num_tokens": 6009292.0, + "reward": 0.53125, + "reward_std": 0.3198433816432953, + "rewards/accuracy_reward/mean": 0.53125, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.4125926494598389, + "sampling/importance_sampling_ratio/mean": 1.0000663995742798, + "sampling/importance_sampling_ratio/min": 0.5959081649780273, + "sampling/sampling_logp_difference/max": 0.5176687240600586, + "sampling/sampling_logp_difference/mean": 0.011553918942809105, + "step": 274 + }, + { + "clip_ratio/high_max": 6.281406967900693e-05, + "clip_ratio/high_mean": 6.281406967900693e-05, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 6.281406967900693e-05, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 684.0, + "completions/mean_length": 444.28125, + "completions/mean_terminated_length": 410.7930908203125, + "completions/min_length": 201.0, + "completions/min_terminated_length": 201.0, + "entropy": 0.4818809889256954, + "epoch": 0.14721627408993576, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.023966826498508453, + "learning_rate": 1e-05, + "loss": -0.0106, + "num_tokens": 6027237.0, + "reward": 0.71875, + "reward_std": 0.3471629321575165, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.4246875047683716, + "sampling/importance_sampling_ratio/mean": 1.0001494884490967, + "sampling/importance_sampling_ratio/min": 0.6532939672470093, + "sampling/sampling_logp_difference/max": 0.42572808265686035, + "sampling/sampling_logp_difference/mean": 0.015202755108475685, + "step": 275 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 723.0, + "completions/mean_length": 482.40625, + "completions/mean_terminated_length": 463.36669921875, + "completions/min_length": 272.0, + "completions/min_terminated_length": 272.0, + "entropy": 0.4784157834947109, + "epoch": 0.14775160599571735, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": -0.0169, + "num_tokens": 6046514.0, + "reward": 0.875, + "reward_std": 0.13363061845302582, + "rewards/accuracy_reward/mean": 0.875, + "rewards/accuracy_reward/std": 0.33601075410842896, + "sampling/importance_sampling_ratio/max": 1.4301837682724, + "sampling/importance_sampling_ratio/mean": 1.0002397298812866, + "sampling/importance_sampling_ratio/min": 0.6200202107429504, + "sampling/sampling_logp_difference/max": 0.47800326347351074, + "sampling/sampling_logp_difference/mean": 0.013694281689822674, + "step": 276 + }, + { + "clip_ratio/high_max": 5.287647945806384e-05, + "clip_ratio/high_mean": 5.287647945806384e-05, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 5.287647945806384e-05, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 708.0, + "completions/mean_length": 504.875, + "completions/mean_terminated_length": 477.6551818847656, + "completions/min_length": 270.0, + "completions/min_terminated_length": 270.0, + "entropy": 0.3827531524002552, + "epoch": 0.14828693790149894, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.012952609919011593, + "learning_rate": 1e-05, + "loss": -0.0061, + "num_tokens": 6066726.0, + "reward": 0.6875, + "reward_std": 0.3924052119255066, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.362608551979065, + "sampling/importance_sampling_ratio/mean": 1.0001440048217773, + "sampling/importance_sampling_ratio/min": 0.6704936027526855, + "sampling/sampling_logp_difference/max": 0.39974117279052734, + "sampling/sampling_logp_difference/mean": 0.011616443283855915, + "step": 277 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 718.0, + "completions/mean_length": 463.6875, + "completions/mean_terminated_length": 420.21429443359375, + "completions/min_length": 255.0, + "completions/min_terminated_length": 255.0, + "entropy": 0.37863264977931976, + "epoch": 0.14882226980728053, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.01699407957494259, + "learning_rate": 1e-05, + "loss": 0.0916, + "num_tokens": 6085332.0, + "reward": 0.59375, + "reward_std": 0.3608423173427582, + "rewards/accuracy_reward/mean": 0.59375, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.336287498474121, + "sampling/importance_sampling_ratio/mean": 0.9995310306549072, + "sampling/importance_sampling_ratio/min": 0.6001459360122681, + "sampling/sampling_logp_difference/max": 0.510582447052002, + "sampling/sampling_logp_difference/mean": 0.012289334088563919, + "step": 278 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00038535775820491835, + "clip_ratio/low_min": 0.00038535775820491835, + "clip_ratio/region_mean": 0.00038535775820491835, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 752.0, + "completions/mean_length": 561.4375, + "completions/mean_terminated_length": 503.5999755859375, + "completions/min_length": 319.0, + "completions/min_terminated_length": 319.0, + "entropy": 0.5481689944863319, + "epoch": 0.1493576017130621, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.019931795075535774, + "learning_rate": 1e-05, + "loss": 0.102, + "num_tokens": 6107858.0, + "reward": 0.5625, + "reward_std": 0.3514062762260437, + "rewards/accuracy_reward/mean": 0.5625, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 1.4254674911499023, + "sampling/importance_sampling_ratio/mean": 1.000180721282959, + "sampling/importance_sampling_ratio/min": 0.6351988911628723, + "sampling/sampling_logp_difference/max": 0.45381712913513184, + "sampling/sampling_logp_difference/mean": 0.01599624752998352, + "step": 279 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 8.821453957352787e-05, + "clip_ratio/low_min": 8.821453957352787e-05, + "clip_ratio/region_mean": 8.821453957352787e-05, + "completions/clipped_ratio": 0.0, + "completions/max_length": 598.0, + "completions/max_terminated_length": 598.0, + "completions/mean_length": 414.5, + "completions/mean_terminated_length": 414.5, + "completions/min_length": 200.0, + "completions/min_terminated_length": 200.0, + "entropy": 0.31437014415860176, + "epoch": 0.14989293361884368, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.014123039320111275, + "learning_rate": 1e-05, + "loss": -0.059, + "num_tokens": 6124210.0, + "reward": 0.90625, + "reward_std": 0.2041158676147461, + "rewards/accuracy_reward/mean": 0.90625, + "rewards/accuracy_reward/std": 0.2961445748806, + "sampling/importance_sampling_ratio/max": 1.435381531715393, + "sampling/importance_sampling_ratio/mean": 1.0001773834228516, + "sampling/importance_sampling_ratio/min": 0.6299055218696594, + "sampling/sampling_logp_difference/max": 0.4621853828430176, + "sampling/sampling_logp_difference/mean": 0.01068188063800335, + "step": 280 + }, + { + "clip_ratio/high_max": 7.110352453310043e-05, + "clip_ratio/high_mean": 7.110352453310043e-05, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 7.110352453310043e-05, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 746.0, + "completions/mean_length": 525.15625, + "completions/mean_terminated_length": 508.9667053222656, + "completions/min_length": 353.0, + "completions/min_terminated_length": 353.0, + "entropy": 0.4922599531710148, + "epoch": 0.15042826552462527, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.025635600090026855, + "learning_rate": 1e-05, + "loss": 0.0608, + "num_tokens": 6144887.0, + "reward": 0.8125, + "reward_std": 0.3104073107242584, + "rewards/accuracy_reward/mean": 0.8125, + "rewards/accuracy_reward/std": 0.3965577781200409, + "sampling/importance_sampling_ratio/max": 1.5917540788650513, + "sampling/importance_sampling_ratio/mean": 1.0004243850708008, + "sampling/importance_sampling_ratio/min": 0.609991729259491, + "sampling/sampling_logp_difference/max": 0.4943099021911621, + "sampling/sampling_logp_difference/mean": 0.013856773264706135, + "step": 281 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.2410901844268665e-05, + "clip_ratio/low_min": 5.2410901844268665e-05, + "clip_ratio/region_mean": 5.2410901844268665e-05, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 716.0, + "completions/mean_length": 526.4375, + "completions/mean_terminated_length": 481.7037048339844, + "completions/min_length": 169.0, + "completions/min_terminated_length": 169.0, + "entropy": 0.3575453422963619, + "epoch": 0.15096359743040685, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.008215086534619331, + "learning_rate": 1e-05, + "loss": 0.0063, + "num_tokens": 6165165.0, + "reward": 0.71875, + "reward_std": 0.2630178928375244, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.4350372552871704, + "sampling/importance_sampling_ratio/mean": 1.0001972913742065, + "sampling/importance_sampling_ratio/min": 0.626156210899353, + "sampling/sampling_logp_difference/max": 0.4681553840637207, + "sampling/sampling_logp_difference/mean": 0.011345704086124897, + "step": 282 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00010989011207129806, + "clip_ratio/low_min": 0.00010989011207129806, + "clip_ratio/region_mean": 0.00010989011207129806, + "completions/clipped_ratio": 0.40625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 698.0, + "completions/mean_length": 613.9375, + "completions/mean_terminated_length": 508.52630615234375, + "completions/min_length": 330.0, + "completions/min_terminated_length": 330.0, + "entropy": 0.5087274610996246, + "epoch": 0.15149892933618844, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.017519496381282806, + "learning_rate": 1e-05, + "loss": 0.0681, + "num_tokens": 6188891.0, + "reward": 0.46875, + "reward_std": 0.3198433816432953, + "rewards/accuracy_reward/mean": 0.46875, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.4468004703521729, + "sampling/importance_sampling_ratio/mean": 0.9999304413795471, + "sampling/importance_sampling_ratio/min": 0.6442550420761108, + "sampling/sampling_logp_difference/max": 0.4396606683731079, + "sampling/sampling_logp_difference/mean": 0.013716845773160458, + "step": 283 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 686.0, + "completions/mean_length": 489.78125, + "completions/mean_terminated_length": 480.8064270019531, + "completions/min_length": 251.0, + "completions/min_terminated_length": 251.0, + "entropy": 0.3560822978615761, + "epoch": 0.15203426124197003, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.023355163633823395, + "learning_rate": 1e-05, + "loss": -0.0782, + "num_tokens": 6208372.0, + "reward": 0.84375, + "reward_std": 0.22201895713806152, + "rewards/accuracy_reward/mean": 0.84375, + "rewards/accuracy_reward/std": 0.3689020276069641, + "sampling/importance_sampling_ratio/max": 1.2881642580032349, + "sampling/importance_sampling_ratio/mean": 0.9998629093170166, + "sampling/importance_sampling_ratio/min": 0.6603497862815857, + "sampling/sampling_logp_difference/max": 0.41498565673828125, + "sampling/sampling_logp_difference/mean": 0.01101180911064148, + "step": 284 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 736.0, + "completions/mean_length": 517.46875, + "completions/mean_terminated_length": 481.6785888671875, + "completions/min_length": 231.0, + "completions/min_terminated_length": 231.0, + "entropy": 0.43186208605766296, + "epoch": 0.15256959314775162, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.01367235742509365, + "learning_rate": 1e-05, + "loss": -0.0141, + "num_tokens": 6228891.0, + "reward": 0.5625, + "reward_std": 0.249358132481575, + "rewards/accuracy_reward/mean": 0.5625, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 1.514527440071106, + "sampling/importance_sampling_ratio/mean": 1.0001575946807861, + "sampling/importance_sampling_ratio/min": 0.6505802273750305, + "sampling/sampling_logp_difference/max": 0.42989063262939453, + "sampling/sampling_logp_difference/mean": 0.01313034538179636, + "step": 285 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 702.0, + "completions/mean_length": 443.3125, + "completions/mean_terminated_length": 432.83868408203125, + "completions/min_length": 162.0, + "completions/min_terminated_length": 162.0, + "entropy": 0.479502160102129, + "epoch": 0.15310492505353318, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.007370191626250744, + "learning_rate": 1e-05, + "loss": 0.0603, + "num_tokens": 6246645.0, + "reward": 0.875, + "reward_std": 0.292504221200943, + "rewards/accuracy_reward/mean": 0.875, + "rewards/accuracy_reward/std": 0.33601075410842896, + "sampling/importance_sampling_ratio/max": 1.4618957042694092, + "sampling/importance_sampling_ratio/mean": 0.9996649622917175, + "sampling/importance_sampling_ratio/min": 0.44690823554992676, + "sampling/sampling_logp_difference/max": 0.8054020404815674, + "sampling/sampling_logp_difference/mean": 0.014546182006597519, + "step": 286 + }, + { + "clip_ratio/high_max": 6.361323175951838e-05, + "clip_ratio/high_mean": 6.361323175951838e-05, + "clip_ratio/low_mean": 0.00018266693950863555, + "clip_ratio/low_min": 0.00018266693950863555, + "clip_ratio/region_mean": 0.00024628017126815394, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 709.0, + "completions/mean_length": 548.0625, + "completions/mean_terminated_length": 507.3333435058594, + "completions/min_length": 331.0, + "completions/min_terminated_length": 331.0, + "entropy": 0.5106589309871197, + "epoch": 0.15364025695931477, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.009625596925616264, + "learning_rate": 1e-05, + "loss": 0.017, + "num_tokens": 6267807.0, + "reward": 0.40625, + "reward_std": 0.3061639666557312, + "rewards/accuracy_reward/mean": 0.40625, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.4405272006988525, + "sampling/importance_sampling_ratio/mean": 0.9999322891235352, + "sampling/importance_sampling_ratio/min": 0.6387335062026978, + "sampling/sampling_logp_difference/max": 0.44826793670654297, + "sampling/sampling_logp_difference/mean": 0.014671443961560726, + "step": 287 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00018225424355478026, + "clip_ratio/low_min": 0.00018225424355478026, + "clip_ratio/region_mean": 0.00018225424355478026, + "completions/clipped_ratio": 0.34375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 765.0, + "completions/mean_length": 643.71875, + "completions/mean_terminated_length": 578.6190795898438, + "completions/min_length": 384.0, + "completions/min_terminated_length": 384.0, + "entropy": 0.4791031889617443, + "epoch": 0.15417558886509636, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.005762732587754726, + "learning_rate": 1e-05, + "loss": 0.0713, + "num_tokens": 6292846.0, + "reward": 0.5625, + "reward_std": 0.292504221200943, + "rewards/accuracy_reward/mean": 0.5625, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 1.5718718767166138, + "sampling/importance_sampling_ratio/mean": 1.0001534223556519, + "sampling/importance_sampling_ratio/min": 0.7144895792007446, + "sampling/sampling_logp_difference/max": 0.4522671699523926, + "sampling/sampling_logp_difference/mean": 0.013097360730171204, + "step": 288 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 764.0, + "completions/mean_length": 547.25, + "completions/mean_terminated_length": 496.3077087402344, + "completions/min_length": 314.0, + "completions/min_terminated_length": 314.0, + "entropy": 0.3792828842997551, + "epoch": 0.15471092077087795, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.02202201448380947, + "learning_rate": 1e-05, + "loss": 0.009, + "num_tokens": 6314030.0, + "reward": 0.625, + "reward_std": 0.4355512857437134, + "rewards/accuracy_reward/mean": 0.625, + "rewards/accuracy_reward/std": 0.49186936020851135, + "sampling/importance_sampling_ratio/max": 1.3409948348999023, + "sampling/importance_sampling_ratio/mean": 0.999615490436554, + "sampling/importance_sampling_ratio/min": 0.6302874088287354, + "sampling/sampling_logp_difference/max": 0.4615793228149414, + "sampling/sampling_logp_difference/mean": 0.011571809649467468, + "step": 289 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00012770119064953178, + "clip_ratio/low_min": 0.00012770119064953178, + "clip_ratio/region_mean": 0.00012770119064953178, + "completions/clipped_ratio": 0.25, + "completions/max_length": 768.0, + "completions/max_terminated_length": 743.0, + "completions/mean_length": 557.40625, + "completions/mean_terminated_length": 487.2083435058594, + "completions/min_length": 222.0, + "completions/min_terminated_length": 222.0, + "entropy": 0.43026646226644516, + "epoch": 0.15524625267665954, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.008927677758038044, + "learning_rate": 1e-05, + "loss": 0.0973, + "num_tokens": 6335739.0, + "reward": 0.46875, + "reward_std": 0.3198433816432953, + "rewards/accuracy_reward/mean": 0.46875, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.3829481601715088, + "sampling/importance_sampling_ratio/mean": 0.9999781250953674, + "sampling/importance_sampling_ratio/min": 0.6179466843605042, + "sampling/sampling_logp_difference/max": 0.4813530445098877, + "sampling/sampling_logp_difference/mean": 0.012422534637153149, + "step": 290 + }, + { + "clip_ratio/high_max": 5.6407941883662716e-05, + "clip_ratio/high_mean": 5.6407941883662716e-05, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 5.6407941883662716e-05, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 748.0, + "completions/mean_length": 527.53125, + "completions/mean_terminated_length": 511.5000305175781, + "completions/min_length": 228.0, + "completions/min_terminated_length": 228.0, + "entropy": 0.27904924005270004, + "epoch": 0.15578158458244112, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.009527213871479034, + "learning_rate": 1e-05, + "loss": 0.0204, + "num_tokens": 6356084.0, + "reward": 0.90625, + "reward_std": 0.2041158676147461, + "rewards/accuracy_reward/mean": 0.90625, + "rewards/accuracy_reward/std": 0.2961445748806, + "sampling/importance_sampling_ratio/max": 1.3129595518112183, + "sampling/importance_sampling_ratio/mean": 0.999772310256958, + "sampling/importance_sampling_ratio/min": 0.6971725821495056, + "sampling/sampling_logp_difference/max": 0.36072230339050293, + "sampling/sampling_logp_difference/mean": 0.01001154538244009, + "step": 291 + }, + { + "clip_ratio/high_max": 6.297229265328497e-05, + "clip_ratio/high_mean": 6.297229265328497e-05, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 6.297229265328497e-05, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 759.0, + "completions/mean_length": 512.34375, + "completions/mean_terminated_length": 504.0967712402344, + "completions/min_length": 281.0, + "completions/min_terminated_length": 281.0, + "entropy": 0.42052092403173447, + "epoch": 0.15631691648822268, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.019503682851791382, + "learning_rate": 1e-05, + "loss": 0.0766, + "num_tokens": 6376359.0, + "reward": 0.84375, + "reward_std": 0.3198433816432953, + "rewards/accuracy_reward/mean": 0.84375, + "rewards/accuracy_reward/std": 0.3689020276069641, + "sampling/importance_sampling_ratio/max": 1.3344718217849731, + "sampling/importance_sampling_ratio/mean": 1.0001314878463745, + "sampling/importance_sampling_ratio/min": 0.7134794592857361, + "sampling/sampling_logp_difference/max": 0.3376016616821289, + "sampling/sampling_logp_difference/mean": 0.012851390056312084, + "step": 292 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 726.0, + "completions/mean_length": 546.3125, + "completions/mean_terminated_length": 531.5333862304688, + "completions/min_length": 280.0, + "completions/min_terminated_length": 280.0, + "entropy": 0.6059800870716572, + "epoch": 0.15685224839400427, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.008632062003016472, + "learning_rate": 1e-05, + "loss": -0.0295, + "num_tokens": 6398113.0, + "reward": 0.40625, + "reward_std": 0.3377465009689331, + "rewards/accuracy_reward/mean": 0.40625, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 2.0, + "sampling/importance_sampling_ratio/mean": 0.9996324777603149, + "sampling/importance_sampling_ratio/min": 0.6368066668510437, + "sampling/sampling_logp_difference/max": 0.7614378929138184, + "sampling/sampling_logp_difference/mean": 0.013707511126995087, + "step": 293 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 686.0, + "completions/mean_length": 492.90625, + "completions/mean_terminated_length": 441.96295166015625, + "completions/min_length": 267.0, + "completions/min_terminated_length": 267.0, + "entropy": 0.47353770211338997, + "epoch": 0.15738758029978586, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.016395388171076775, + "learning_rate": 1e-05, + "loss": 0.0199, + "num_tokens": 6417526.0, + "reward": 0.59375, + "reward_std": 0.1293872892856598, + "rewards/accuracy_reward/mean": 0.59375, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.4660577774047852, + "sampling/importance_sampling_ratio/mean": 0.9998359084129333, + "sampling/importance_sampling_ratio/min": 0.37846291065216064, + "sampling/sampling_logp_difference/max": 0.9716372489929199, + "sampling/sampling_logp_difference/mean": 0.014490845613181591, + "step": 294 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00014403185195988044, + "clip_ratio/low_min": 0.00014403185195988044, + "clip_ratio/region_mean": 0.00014403185195988044, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 728.0, + "completions/mean_length": 432.6875, + "completions/mean_terminated_length": 410.3333435058594, + "completions/min_length": 245.0, + "completions/min_terminated_length": 245.0, + "entropy": 0.4177595153450966, + "epoch": 0.15792291220556745, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.02344527281820774, + "learning_rate": 1e-05, + "loss": 0.021, + "num_tokens": 6435132.0, + "reward": 0.6875, + "reward_std": 0.3335031569004059, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.3946726322174072, + "sampling/importance_sampling_ratio/mean": 0.999739944934845, + "sampling/importance_sampling_ratio/min": 0.6931111216545105, + "sampling/sampling_logp_difference/max": 0.3665649890899658, + "sampling/sampling_logp_difference/mean": 0.013593433424830437, + "step": 295 + }, + { + "clip_ratio/high_max": 5.482456253957935e-05, + "clip_ratio/high_mean": 5.482456253957935e-05, + "clip_ratio/low_mean": 0.0001080386973626446, + "clip_ratio/low_min": 0.0001080386973626446, + "clip_ratio/region_mean": 0.00016286325990222394, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 713.0, + "completions/mean_length": 482.71875, + "completions/mean_terminated_length": 473.51611328125, + "completions/min_length": 170.0, + "completions/min_terminated_length": 170.0, + "entropy": 0.4640315771102905, + "epoch": 0.15845824411134904, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.01397132221609354, + "learning_rate": 1e-05, + "loss": -0.0418, + "num_tokens": 6453883.0, + "reward": 0.5, + "reward_std": 0.4355512857437134, + "rewards/accuracy_reward/mean": 0.5, + "rewards/accuracy_reward/std": 0.5080004930496216, + "sampling/importance_sampling_ratio/max": 1.367895483970642, + "sampling/importance_sampling_ratio/mean": 0.9997069835662842, + "sampling/importance_sampling_ratio/min": 0.7424412369728088, + "sampling/sampling_logp_difference/max": 0.31327342987060547, + "sampling/sampling_logp_difference/mean": 0.012333623133599758, + "step": 296 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 763.0, + "completions/mean_length": 526.40625, + "completions/mean_terminated_length": 470.65386962890625, + "completions/min_length": 186.0, + "completions/min_terminated_length": 186.0, + "entropy": 0.5003989450633526, + "epoch": 0.15899357601713063, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.008706404827535152, + "learning_rate": 1e-05, + "loss": 0.0654, + "num_tokens": 6474472.0, + "reward": 0.5, + "reward_std": 0.3514062464237213, + "rewards/accuracy_reward/mean": 0.5, + "rewards/accuracy_reward/std": 0.5080004930496216, + "sampling/importance_sampling_ratio/max": 1.3327733278274536, + "sampling/importance_sampling_ratio/mean": 1.0000064373016357, + "sampling/importance_sampling_ratio/min": 0.7041017413139343, + "sampling/sampling_logp_difference/max": 0.350832462310791, + "sampling/sampling_logp_difference/mean": 0.014013871550559998, + "step": 297 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 766.0, + "completions/max_terminated_length": 766.0, + "completions/mean_length": 422.6875, + "completions/mean_terminated_length": 422.6875, + "completions/min_length": 251.0, + "completions/min_terminated_length": 251.0, + "entropy": 0.34741342440247536, + "epoch": 0.15952890792291222, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.01617433875799179, + "learning_rate": 1e-05, + "loss": -0.0165, + "num_tokens": 6491294.0, + "reward": 0.96875, + "reward_std": 0.0883883461356163, + "rewards/accuracy_reward/mean": 0.96875, + "rewards/accuracy_reward/std": 0.1767766922712326, + "sampling/importance_sampling_ratio/max": 1.7172808647155762, + "sampling/importance_sampling_ratio/mean": 1.0002919435501099, + "sampling/importance_sampling_ratio/min": 0.6275865435600281, + "sampling/sampling_logp_difference/max": 0.5407421588897705, + "sampling/sampling_logp_difference/mean": 0.011559183709323406, + "step": 298 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 768.0, + "completions/max_terminated_length": 723.0, + "completions/mean_length": 553.90625, + "completions/mean_terminated_length": 482.54168701171875, + "completions/min_length": 313.0, + "completions/min_terminated_length": 313.0, + "entropy": 0.5882219597697258, + "epoch": 0.16006423982869378, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": 0.0, + "num_tokens": 6512995.0, + "reward": 0.5, + "reward_std": 0.0, + "rewards/accuracy_reward/mean": 0.5, + "rewards/accuracy_reward/std": 0.5080004930496216, + "sampling/importance_sampling_ratio/max": 1.7449169158935547, + "sampling/importance_sampling_ratio/mean": 0.9997621774673462, + "sampling/importance_sampling_ratio/min": 0.6054978370666504, + "sampling/sampling_logp_difference/max": 0.5567069053649902, + "sampling/sampling_logp_difference/mean": 0.013056526891887188, + "step": 299 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 755.0, + "completions/mean_length": 587.875, + "completions/mean_terminated_length": 537.4400024414062, + "completions/min_length": 328.0, + "completions/min_terminated_length": 328.0, + "entropy": 0.3660525269806385, + "epoch": 0.16059957173447537, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.008583523333072662, + "learning_rate": 1e-05, + "loss": 0.1076, + "num_tokens": 6535807.0, + "reward": 0.71875, + "reward_std": 0.4534739851951599, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.3342469930648804, + "sampling/importance_sampling_ratio/mean": 1.0001211166381836, + "sampling/importance_sampling_ratio/min": 0.619476318359375, + "sampling/sampling_logp_difference/max": 0.4788808822631836, + "sampling/sampling_logp_difference/mean": 0.011281672865152359, + "step": 300 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 7.720815483480692e-05, + "clip_ratio/low_min": 7.720815483480692e-05, + "clip_ratio/region_mean": 7.720815483480692e-05, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 748.0, + "completions/mean_length": 476.46875, + "completions/mean_terminated_length": 434.8214416503906, + "completions/min_length": 199.0, + "completions/min_terminated_length": 199.0, + "entropy": 0.37877320498228073, + "epoch": 0.16113490364025695, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.01476945262402296, + "learning_rate": 1e-05, + "loss": 0.0181, + "num_tokens": 6554766.0, + "reward": 0.71875, + "reward_std": 0.35564959049224854, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.467990517616272, + "sampling/importance_sampling_ratio/mean": 1.000206708908081, + "sampling/importance_sampling_ratio/min": 0.5913010239601135, + "sampling/sampling_logp_difference/max": 0.5254299640655518, + "sampling/sampling_logp_difference/mean": 0.012068652547895908, + "step": 301 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 730.0, + "completions/mean_length": 539.59375, + "completions/mean_terminated_length": 524.36669921875, + "completions/min_length": 361.0, + "completions/min_terminated_length": 361.0, + "entropy": 0.4354598969221115, + "epoch": 0.16167023554603854, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.011717413552105427, + "learning_rate": 1e-05, + "loss": 0.0041, + "num_tokens": 6575857.0, + "reward": 0.78125, + "reward_std": 0.3608423173427582, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.3261480331420898, + "sampling/importance_sampling_ratio/mean": 0.9998117089271545, + "sampling/importance_sampling_ratio/min": 0.6434427499771118, + "sampling/sampling_logp_difference/max": 0.44092226028442383, + "sampling/sampling_logp_difference/mean": 0.01168701983988285, + "step": 302 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00011061946861445904, + "clip_ratio/low_min": 0.00011061946861445904, + "clip_ratio/region_mean": 0.00011061946861445904, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 733.0, + "completions/mean_length": 577.0, + "completions/mean_terminated_length": 502.2608642578125, + "completions/min_length": 280.0, + "completions/min_terminated_length": 280.0, + "entropy": 0.4856537878513336, + "epoch": 0.16220556745182013, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.012045544572174549, + "learning_rate": 1e-05, + "loss": 0.0295, + "num_tokens": 6597857.0, + "reward": 0.5625, + "reward_std": 0.3514062464237213, + "rewards/accuracy_reward/mean": 0.5625, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 1.597886085510254, + "sampling/importance_sampling_ratio/mean": 1.0002676248550415, + "sampling/importance_sampling_ratio/min": 0.6344450116157532, + "sampling/sampling_logp_difference/max": 0.46868157386779785, + "sampling/sampling_logp_difference/mean": 0.014197330921888351, + "step": 303 + }, + { + "clip_ratio/high_max": 4.763719334732741e-05, + "clip_ratio/high_mean": 4.763719334732741e-05, + "clip_ratio/low_mean": 5.755064557888545e-05, + "clip_ratio/low_min": 5.755064557888545e-05, + "clip_ratio/region_mean": 0.00010518783892621286, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 666.0, + "completions/mean_length": 502.75, + "completions/mean_terminated_length": 428.47998046875, + "completions/min_length": 145.0, + "completions/min_terminated_length": 145.0, + "entropy": 0.5767744444310665, + "epoch": 0.16274089935760172, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.011516324244439602, + "learning_rate": 1e-05, + "loss": 0.0064, + "num_tokens": 6617729.0, + "reward": 0.6875, + "reward_std": 0.2177756428718567, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.509402871131897, + "sampling/importance_sampling_ratio/mean": 0.999971330165863, + "sampling/importance_sampling_ratio/min": 0.7389633059501648, + "sampling/sampling_logp_difference/max": 0.4117140769958496, + "sampling/sampling_logp_difference/mean": 0.014930041506886482, + "step": 304 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 4.978096330887638e-05, + "clip_ratio/low_min": 4.978096330887638e-05, + "clip_ratio/region_mean": 4.978096330887638e-05, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 740.0, + "completions/mean_length": 478.28125, + "completions/mean_terminated_length": 424.629638671875, + "completions/min_length": 163.0, + "completions/min_terminated_length": 163.0, + "entropy": 0.5792366713285446, + "epoch": 0.1632762312633833, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.008899957872927189, + "learning_rate": 1e-05, + "loss": -0.0167, + "num_tokens": 6636690.0, + "reward": 0.40625, + "reward_std": 0.378745436668396, + "rewards/accuracy_reward/mean": 0.40625, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.5502060651779175, + "sampling/importance_sampling_ratio/mean": 1.000200867652893, + "sampling/importance_sampling_ratio/min": 0.6895433664321899, + "sampling/sampling_logp_difference/max": 0.4383878707885742, + "sampling/sampling_logp_difference/mean": 0.016736779361963272, + "step": 305 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0002110216701112222, + "clip_ratio/low_min": 0.0002110216701112222, + "clip_ratio/region_mean": 0.0002110216701112222, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 718.0, + "completions/mean_length": 565.375, + "completions/mean_terminated_length": 486.08697509765625, + "completions/min_length": 282.0, + "completions/min_terminated_length": 282.0, + "entropy": 0.48659036308526993, + "epoch": 0.16381156316916487, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.02768571302294731, + "learning_rate": 1e-05, + "loss": 0.0458, + "num_tokens": 6658542.0, + "reward": 0.59375, + "reward_std": 0.3198433816432953, + "rewards/accuracy_reward/mean": 0.59375, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.319368839263916, + "sampling/importance_sampling_ratio/mean": 1.000253438949585, + "sampling/importance_sampling_ratio/min": 0.6450673937797546, + "sampling/sampling_logp_difference/max": 0.4384005069732666, + "sampling/sampling_logp_difference/mean": 0.013724671676754951, + "step": 306 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 4.8962006985675544e-05, + "clip_ratio/low_min": 4.8962006985675544e-05, + "clip_ratio/region_mean": 4.8962006985675544e-05, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 738.0, + "completions/mean_length": 533.90625, + "completions/mean_terminated_length": 509.6896667480469, + "completions/min_length": 302.0, + "completions/min_terminated_length": 302.0, + "entropy": 0.27842517755925655, + "epoch": 0.16434689507494646, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.012529374100267887, + "learning_rate": 1e-05, + "loss": -0.0076, + "num_tokens": 6680067.0, + "reward": 0.78125, + "reward_std": 0.3377464711666107, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.3033475875854492, + "sampling/importance_sampling_ratio/mean": 0.9997432231903076, + "sampling/importance_sampling_ratio/min": 0.6831303238868713, + "sampling/sampling_logp_difference/max": 0.3810696601867676, + "sampling/sampling_logp_difference/mean": 0.009572218172252178, + "step": 307 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.53125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 674.0, + "completions/mean_length": 648.8125, + "completions/mean_terminated_length": 513.7333374023438, + "completions/min_length": 253.0, + "completions/min_terminated_length": 253.0, + "entropy": 0.48289478570222855, + "epoch": 0.16488222698072805, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.00882739294320345, + "learning_rate": 1e-05, + "loss": 0.0201, + "num_tokens": 6705421.0, + "reward": 0.28125, + "reward_std": 0.2630178928375244, + "rewards/accuracy_reward/mean": 0.28125, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.4294395446777344, + "sampling/importance_sampling_ratio/mean": 0.9995339512825012, + "sampling/importance_sampling_ratio/min": 0.5330876111984253, + "sampling/sampling_logp_difference/max": 0.6290695667266846, + "sampling/sampling_logp_difference/mean": 0.012598089873790741, + "step": 308 + }, + { + "clip_ratio/high_max": 8.549931953893974e-05, + "clip_ratio/high_mean": 8.549931953893974e-05, + "clip_ratio/low_mean": 0.0001661965434323065, + "clip_ratio/low_min": 0.0001661965434323065, + "clip_ratio/region_mean": 0.00025169586297124624, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 747.0, + "completions/mean_length": 486.59375, + "completions/mean_terminated_length": 467.8333435058594, + "completions/min_length": 197.0, + "completions/min_terminated_length": 197.0, + "entropy": 0.38671203702688217, + "epoch": 0.16541755888650964, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.013369970954954624, + "learning_rate": 1e-05, + "loss": 0.074, + "num_tokens": 6724352.0, + "reward": 0.84375, + "reward_std": 0.3061639666557312, + "rewards/accuracy_reward/mean": 0.84375, + "rewards/accuracy_reward/std": 0.3689020276069641, + "sampling/importance_sampling_ratio/max": 1.4490798711776733, + "sampling/importance_sampling_ratio/mean": 1.0000407695770264, + "sampling/importance_sampling_ratio/min": 0.6715885400772095, + "sampling/sampling_logp_difference/max": 0.39810943603515625, + "sampling/sampling_logp_difference/mean": 0.011065604165196419, + "step": 309 + }, + { + "clip_ratio/high_max": 4.9270791350863874e-05, + "clip_ratio/high_mean": 4.9270791350863874e-05, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 4.9270791350863874e-05, + "completions/clipped_ratio": 0.34375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 749.0, + "completions/mean_length": 592.40625, + "completions/mean_terminated_length": 500.4285888671875, + "completions/min_length": 241.0, + "completions/min_terminated_length": 241.0, + "entropy": 0.5058122612535954, + "epoch": 0.16595289079229122, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.021446535363793373, + "learning_rate": 1e-05, + "loss": 0.0343, + "num_tokens": 6747189.0, + "reward": 0.625, + "reward_std": 0.2925041913986206, + "rewards/accuracy_reward/mean": 0.625, + "rewards/accuracy_reward/std": 0.49186936020851135, + "sampling/importance_sampling_ratio/max": 1.4081288576126099, + "sampling/importance_sampling_ratio/mean": 0.9999823570251465, + "sampling/importance_sampling_ratio/min": 0.5873938798904419, + "sampling/sampling_logp_difference/max": 0.5320596694946289, + "sampling/sampling_logp_difference/mean": 0.012491296976804733, + "step": 310 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 764.0, + "completions/mean_length": 525.34375, + "completions/mean_terminated_length": 469.3461608886719, + "completions/min_length": 244.0, + "completions/min_terminated_length": 244.0, + "entropy": 0.38173941895365715, + "epoch": 0.1664882226980728, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.008090285584330559, + "learning_rate": 1e-05, + "loss": 0.0175, + "num_tokens": 6768536.0, + "reward": 0.5625, + "reward_std": 0.3471825420856476, + "rewards/accuracy_reward/mean": 0.5625, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 1.4165804386138916, + "sampling/importance_sampling_ratio/mean": 1.0000367164611816, + "sampling/importance_sampling_ratio/min": 0.5375379323959351, + "sampling/sampling_logp_difference/max": 0.6207559108734131, + "sampling/sampling_logp_difference/mean": 0.011909333989024162, + "step": 311 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 8.865247946232557e-05, + "clip_ratio/low_min": 8.865247946232557e-05, + "clip_ratio/region_mean": 8.865247946232557e-05, + "completions/clipped_ratio": 0.0, + "completions/max_length": 761.0, + "completions/max_terminated_length": 761.0, + "completions/mean_length": 407.5, + "completions/mean_terminated_length": 407.5, + "completions/min_length": 218.0, + "completions/min_terminated_length": 218.0, + "entropy": 0.4366965591907501, + "epoch": 0.1670235546038544, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.017805999144911766, + "learning_rate": 1e-05, + "loss": -0.0471, + "num_tokens": 6784560.0, + "reward": 0.8125, + "reward_std": 0.3335031569004059, + "rewards/accuracy_reward/mean": 0.8125, + "rewards/accuracy_reward/std": 0.3965577781200409, + "sampling/importance_sampling_ratio/max": 1.3484771251678467, + "sampling/importance_sampling_ratio/mean": 1.0002574920654297, + "sampling/importance_sampling_ratio/min": 0.7097929120063782, + "sampling/sampling_logp_difference/max": 0.34278202056884766, + "sampling/sampling_logp_difference/mean": 0.012549689039587975, + "step": 312 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 758.0, + "completions/mean_length": 474.25, + "completions/mean_terminated_length": 454.66668701171875, + "completions/min_length": 272.0, + "completions/min_terminated_length": 272.0, + "entropy": 0.3349683992564678, + "epoch": 0.16755888650963596, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.010974748060107231, + "learning_rate": 1e-05, + "loss": 0.0115, + "num_tokens": 6803768.0, + "reward": 0.84375, + "reward_std": 0.22201895713806152, + "rewards/accuracy_reward/mean": 0.84375, + "rewards/accuracy_reward/std": 0.3689020276069641, + "sampling/importance_sampling_ratio/max": 1.2873549461364746, + "sampling/importance_sampling_ratio/mean": 0.9998072385787964, + "sampling/importance_sampling_ratio/min": 0.7049721479415894, + "sampling/sampling_logp_difference/max": 0.3495969772338867, + "sampling/sampling_logp_difference/mean": 0.010414966382086277, + "step": 313 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 736.0, + "completions/max_terminated_length": 736.0, + "completions/mean_length": 406.3125, + "completions/mean_terminated_length": 406.3125, + "completions/min_length": 227.0, + "completions/min_terminated_length": 227.0, + "entropy": 0.36525509133934975, + "epoch": 0.16809421841541755, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": 0.0, + "num_tokens": 6820194.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/accuracy_reward/mean": 1.0, + "rewards/accuracy_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.4298120737075806, + "sampling/importance_sampling_ratio/mean": 1.0004063844680786, + "sampling/importance_sampling_ratio/min": 0.7078427076339722, + "sampling/sampling_logp_difference/max": 0.3575429916381836, + "sampling/sampling_logp_difference/mean": 0.011604657396674156, + "step": 314 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 758.0, + "completions/mean_length": 463.625, + "completions/mean_terminated_length": 453.8064270019531, + "completions/min_length": 289.0, + "completions/min_terminated_length": 289.0, + "entropy": 0.3767676129937172, + "epoch": 0.16862955032119914, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.012036073952913284, + "learning_rate": 1e-05, + "loss": 0.0586, + "num_tokens": 6838502.0, + "reward": 0.84375, + "reward_std": 0.24511480331420898, + "rewards/accuracy_reward/mean": 0.84375, + "rewards/accuracy_reward/std": 0.3689020276069641, + "sampling/importance_sampling_ratio/max": 2.0, + "sampling/importance_sampling_ratio/mean": 1.000014305114746, + "sampling/importance_sampling_ratio/min": 0.6360156536102295, + "sampling/sampling_logp_difference/max": 0.7063698768615723, + "sampling/sampling_logp_difference/mean": 0.011565844528377056, + "step": 315 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 768.0, + "completions/max_terminated_length": 666.0, + "completions/mean_length": 564.125, + "completions/mean_terminated_length": 496.16668701171875, + "completions/min_length": 314.0, + "completions/min_terminated_length": 314.0, + "entropy": 0.6370006389915943, + "epoch": 0.16916488222698073, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.002451046369969845, + "learning_rate": 1e-05, + "loss": -0.0041, + "num_tokens": 6860202.0, + "reward": 0.46875, + "reward_std": 0.0883883461356163, + "rewards/accuracy_reward/mean": 0.46875, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.4470359086990356, + "sampling/importance_sampling_ratio/mean": 1.000141978263855, + "sampling/importance_sampling_ratio/min": 0.43400317430496216, + "sampling/sampling_logp_difference/max": 0.8347034454345703, + "sampling/sampling_logp_difference/mean": 0.016150325536727905, + "step": 316 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 682.0, + "completions/mean_length": 508.40625, + "completions/mean_terminated_length": 491.10003662109375, + "completions/min_length": 266.0, + "completions/min_terminated_length": 266.0, + "entropy": 0.43147315084934235, + "epoch": 0.16970021413276232, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.026230446994304657, + "learning_rate": 1e-05, + "loss": 0.0165, + "num_tokens": 6880095.0, + "reward": 0.6875, + "reward_std": 0.44403791427612305, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.3381121158599854, + "sampling/importance_sampling_ratio/mean": 1.0000815391540527, + "sampling/importance_sampling_ratio/min": 0.6977789998054504, + "sampling/sampling_logp_difference/max": 0.35985279083251953, + "sampling/sampling_logp_difference/mean": 0.01168067567050457, + "step": 317 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 8.394895849050954e-05, + "clip_ratio/low_min": 8.394895849050954e-05, + "clip_ratio/region_mean": 8.394895849050954e-05, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 655.0, + "completions/mean_length": 453.0625, + "completions/mean_terminated_length": 442.9031982421875, + "completions/min_length": 155.0, + "completions/min_terminated_length": 155.0, + "entropy": 0.493406031280756, + "epoch": 0.1702355460385439, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.026222607120871544, + "learning_rate": 1e-05, + "loss": -0.0027, + "num_tokens": 6898313.0, + "reward": 0.6875, + "reward_std": 0.2587745785713196, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.5384159088134766, + "sampling/importance_sampling_ratio/mean": 1.0001434087753296, + "sampling/importance_sampling_ratio/min": 0.7131491899490356, + "sampling/sampling_logp_difference/max": 0.430753231048584, + "sampling/sampling_logp_difference/mean": 0.012421787716448307, + "step": 318 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 717.0, + "completions/mean_length": 699.15625, + "completions/mean_terminated_length": 584.4166870117188, + "completions/min_length": 429.0, + "completions/min_terminated_length": 429.0, + "entropy": 0.5640444606542587, + "epoch": 0.1707708779443255, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.0030002526473253965, + "learning_rate": 1e-05, + "loss": 0.0239, + "num_tokens": 6925246.0, + "reward": 0.1875, + "reward_std": 0.1157275140285492, + "rewards/accuracy_reward/mean": 0.1875, + "rewards/accuracy_reward/std": 0.3965577781200409, + "sampling/importance_sampling_ratio/max": 1.5129494667053223, + "sampling/importance_sampling_ratio/mean": 0.9998442530632019, + "sampling/importance_sampling_ratio/min": 0.6620175242424011, + "sampling/sampling_logp_difference/max": 0.4140610694885254, + "sampling/sampling_logp_difference/mean": 0.01600034348666668, + "step": 319 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 718.0, + "completions/mean_length": 561.375, + "completions/mean_terminated_length": 531.857177734375, + "completions/min_length": 338.0, + "completions/min_terminated_length": 338.0, + "entropy": 0.300706734880805, + "epoch": 0.17130620985010706, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.004973437171429396, + "learning_rate": 1e-05, + "loss": 0.0244, + "num_tokens": 6947458.0, + "reward": 0.90625, + "reward_std": 0.1293872892856598, + "rewards/accuracy_reward/mean": 0.90625, + "rewards/accuracy_reward/std": 0.2961445748806, + "sampling/importance_sampling_ratio/max": 1.4563747644424438, + "sampling/importance_sampling_ratio/mean": 0.9997153282165527, + "sampling/importance_sampling_ratio/min": 0.6179333329200745, + "sampling/sampling_logp_difference/max": 0.48137474060058594, + "sampling/sampling_logp_difference/mean": 0.010129036381840706, + "step": 320 + }, + { + "clip_ratio/high_max": 6.436663534259424e-05, + "clip_ratio/high_mean": 6.436663534259424e-05, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 6.436663534259424e-05, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 751.0, + "completions/mean_length": 539.75, + "completions/mean_terminated_length": 507.14288330078125, + "completions/min_length": 318.0, + "completions/min_terminated_length": 318.0, + "entropy": 0.3287808820605278, + "epoch": 0.17184154175588864, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.008796916343271732, + "learning_rate": 1e-05, + "loss": 0.0392, + "num_tokens": 6968218.0, + "reward": 0.5625, + "reward_std": 0.4671337604522705, + "rewards/accuracy_reward/mean": 0.5625, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 1.3601312637329102, + "sampling/importance_sampling_ratio/mean": 1.0003015995025635, + "sampling/importance_sampling_ratio/min": 0.5666290521621704, + "sampling/sampling_logp_difference/max": 0.5680503845214844, + "sampling/sampling_logp_difference/mean": 0.01035685557872057, + "step": 321 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 768.0, + "completions/max_terminated_length": 762.0, + "completions/mean_length": 590.96875, + "completions/mean_terminated_length": 531.9583740234375, + "completions/min_length": 332.0, + "completions/min_terminated_length": 332.0, + "entropy": 0.5468134395778179, + "epoch": 0.17237687366167023, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.0027159431483596563, + "learning_rate": 1e-05, + "loss": 0.0476, + "num_tokens": 6991185.0, + "reward": 0.53125, + "reward_std": 0.24511480331420898, + "rewards/accuracy_reward/mean": 0.53125, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.4517290592193604, + "sampling/importance_sampling_ratio/mean": 0.9999667406082153, + "sampling/importance_sampling_ratio/min": 0.5871145725250244, + "sampling/sampling_logp_difference/max": 0.5325352549552917, + "sampling/sampling_logp_difference/mean": 0.014989539049565792, + "step": 322 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 612.0, + "completions/mean_length": 479.625, + "completions/mean_terminated_length": 438.4285888671875, + "completions/min_length": 207.0, + "completions/min_terminated_length": 207.0, + "entropy": 0.42262522876262665, + "epoch": 0.17291220556745182, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.001966841984540224, + "learning_rate": 1e-05, + "loss": 0.0262, + "num_tokens": 7010477.0, + "reward": 0.6875, + "reward_std": 0.1767766922712326, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.287473440170288, + "sampling/importance_sampling_ratio/mean": 0.9999221563339233, + "sampling/importance_sampling_ratio/min": 0.6411926746368408, + "sampling/sampling_logp_difference/max": 0.4444253444671631, + "sampling/sampling_logp_difference/mean": 0.013051128946244717, + "step": 323 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 725.0, + "completions/mean_length": 490.125, + "completions/mean_terminated_length": 461.3793029785156, + "completions/min_length": 82.0, + "completions/min_terminated_length": 82.0, + "entropy": 0.5104870870709419, + "epoch": 0.1734475374732334, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.003290761960670352, + "learning_rate": 1e-05, + "loss": 0.0344, + "num_tokens": 7029929.0, + "reward": 0.71875, + "reward_std": 0.0883883461356163, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.9737179279327393, + "sampling/importance_sampling_ratio/mean": 0.9998589158058167, + "sampling/importance_sampling_ratio/min": 0.6362985372543335, + "sampling/sampling_logp_difference/max": 0.6799190044403076, + "sampling/sampling_logp_difference/mean": 0.01370849646627903, + "step": 324 + }, + { + "clip_ratio/high_max": 5.776340185548179e-05, + "clip_ratio/high_mean": 5.776340185548179e-05, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 5.776340185548179e-05, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 738.0, + "completions/mean_length": 502.21875, + "completions/mean_terminated_length": 484.5000305175781, + "completions/min_length": 104.0, + "completions/min_terminated_length": 104.0, + "entropy": 0.36331427469849586, + "epoch": 0.173982869379015, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.0124629195779562, + "learning_rate": 1e-05, + "loss": -0.0282, + "num_tokens": 7049960.0, + "reward": 0.78125, + "reward_std": 0.3471629321575165, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.3294010162353516, + "sampling/importance_sampling_ratio/mean": 0.9997987151145935, + "sampling/importance_sampling_ratio/min": 0.5725442171096802, + "sampling/sampling_logp_difference/max": 0.5576653480529785, + "sampling/sampling_logp_difference/mean": 0.011005447246134281, + "step": 325 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 717.0, + "completions/mean_length": 506.6875, + "completions/mean_terminated_length": 469.357177734375, + "completions/min_length": 279.0, + "completions/min_terminated_length": 279.0, + "entropy": 0.3884149491786957, + "epoch": 0.1745182012847966, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": -0.0168, + "num_tokens": 7070006.0, + "reward": 0.71875, + "reward_std": 0.0883883461356163, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.3052456378936768, + "sampling/importance_sampling_ratio/mean": 0.9998556971549988, + "sampling/importance_sampling_ratio/min": 0.6920611262321472, + "sampling/sampling_logp_difference/max": 0.36808109283447266, + "sampling/sampling_logp_difference/mean": 0.011671111918985844, + "step": 326 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 747.0, + "completions/mean_length": 524.65625, + "completions/mean_terminated_length": 499.4827575683594, + "completions/min_length": 260.0, + "completions/min_terminated_length": 260.0, + "entropy": 0.5351394787430763, + "epoch": 0.17505353319057815, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.015136069618165493, + "learning_rate": 1e-05, + "loss": 0.0329, + "num_tokens": 7090403.0, + "reward": 0.75, + "reward_std": 0.2587745785713196, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.3184289932250977, + "sampling/importance_sampling_ratio/mean": 0.999699592590332, + "sampling/importance_sampling_ratio/min": 0.6287856101989746, + "sampling/sampling_logp_difference/max": 0.46396493911743164, + "sampling/sampling_logp_difference/mean": 0.013748998753726482, + "step": 327 + }, + { + "clip_ratio/high_max": 7.557436765637249e-05, + "clip_ratio/high_mean": 7.557436765637249e-05, + "clip_ratio/low_mean": 5.258729652268812e-05, + "clip_ratio/low_min": 5.258729652268812e-05, + "clip_ratio/region_mean": 0.0001281616641790606, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 768.0, + "completions/mean_length": 526.375, + "completions/mean_terminated_length": 510.2666931152344, + "completions/min_length": 198.0, + "completions/min_terminated_length": 198.0, + "entropy": 0.4058164283633232, + "epoch": 0.17558886509635974, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.004701528698205948, + "learning_rate": 1e-05, + "loss": 0.0075, + "num_tokens": 7111367.0, + "reward": 0.8125, + "reward_std": 0.2177756428718567, + "rewards/accuracy_reward/mean": 0.8125, + "rewards/accuracy_reward/std": 0.3965577781200409, + "sampling/importance_sampling_ratio/max": 1.4546986818313599, + "sampling/importance_sampling_ratio/mean": 0.9999401569366455, + "sampling/importance_sampling_ratio/min": 0.6548423171043396, + "sampling/sampling_logp_difference/max": 0.42336082458496094, + "sampling/sampling_logp_difference/mean": 0.01178866159170866, + "step": 328 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.623032120638527e-05, + "clip_ratio/low_min": 5.623032120638527e-05, + "clip_ratio/region_mean": 5.623032120638527e-05, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 668.0, + "completions/mean_length": 496.09375, + "completions/mean_terminated_length": 457.2500305175781, + "completions/min_length": 283.0, + "completions/min_terminated_length": 283.0, + "entropy": 0.43937700241804123, + "epoch": 0.17612419700214133, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.010809614323079586, + "learning_rate": 1e-05, + "loss": 0.0993, + "num_tokens": 7131090.0, + "reward": 0.59375, + "reward_std": 0.4944729208946228, + "rewards/accuracy_reward/mean": 0.59375, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.4216543436050415, + "sampling/importance_sampling_ratio/mean": 1.0000747442245483, + "sampling/importance_sampling_ratio/min": 0.753114640712738, + "sampling/sampling_logp_difference/max": 0.3518211841583252, + "sampling/sampling_logp_difference/mean": 0.012104019522666931, + "step": 329 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 4.774637272930704e-05, + "clip_ratio/low_min": 4.774637272930704e-05, + "clip_ratio/region_mean": 4.774637272930704e-05, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 765.0, + "completions/mean_length": 547.1875, + "completions/mean_terminated_length": 524.3448486328125, + "completions/min_length": 198.0, + "completions/min_terminated_length": 198.0, + "entropy": 0.5076432041823864, + "epoch": 0.1766595289079229, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.014773234724998474, + "learning_rate": 1e-05, + "loss": -0.0199, + "num_tokens": 7152880.0, + "reward": 0.78125, + "reward_std": 0.3377465009689331, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.3977811336517334, + "sampling/importance_sampling_ratio/mean": 0.9996352195739746, + "sampling/importance_sampling_ratio/min": 0.5488889813423157, + "sampling/sampling_logp_difference/max": 0.5998589992523193, + "sampling/sampling_logp_difference/mean": 0.014740390703082085, + "step": 330 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 6.582411879207939e-05, + "clip_ratio/low_min": 6.582411879207939e-05, + "clip_ratio/region_mean": 6.582411879207939e-05, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 709.0, + "completions/mean_length": 448.6875, + "completions/mean_terminated_length": 438.3870849609375, + "completions/min_length": 213.0, + "completions/min_terminated_length": 213.0, + "entropy": 0.49838024750351906, + "epoch": 0.1771948608137045, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.005636438727378845, + "learning_rate": 1e-05, + "loss": -0.0361, + "num_tokens": 7170654.0, + "reward": 0.71875, + "reward_std": 0.2651650309562683, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.6480814218521118, + "sampling/importance_sampling_ratio/mean": 1.0003423690795898, + "sampling/importance_sampling_ratio/min": 0.3878595232963562, + "sampling/sampling_logp_difference/max": 0.9471120834350586, + "sampling/sampling_logp_difference/mean": 0.014354714192450047, + "step": 331 + }, + { + "clip_ratio/high_max": 0.00011916110815946013, + "clip_ratio/high_mean": 0.00011916110815946013, + "clip_ratio/low_mean": 7.090187136782333e-05, + "clip_ratio/low_min": 7.090187136782333e-05, + "clip_ratio/region_mean": 0.00019006297952728346, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 717.0, + "completions/mean_length": 508.125, + "completions/mean_terminated_length": 499.7419128417969, + "completions/min_length": 283.0, + "completions/min_terminated_length": 283.0, + "entropy": 0.5146004147827625, + "epoch": 0.1777301927194861, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.015472235158085823, + "learning_rate": 1e-05, + "loss": 0.0445, + "num_tokens": 7190322.0, + "reward": 0.6875, + "reward_std": 0.4671337604522705, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.4304981231689453, + "sampling/importance_sampling_ratio/mean": 1.0004292726516724, + "sampling/importance_sampling_ratio/min": 0.5301676392555237, + "sampling/sampling_logp_difference/max": 0.6345620155334473, + "sampling/sampling_logp_difference/mean": 0.013377378694713116, + "step": 332 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 765.0, + "completions/mean_length": 593.53125, + "completions/mean_terminated_length": 525.2608642578125, + "completions/min_length": 320.0, + "completions/min_terminated_length": 320.0, + "entropy": 0.4686245284974575, + "epoch": 0.17826552462526768, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.012375380843877792, + "learning_rate": 1e-05, + "loss": -0.0692, + "num_tokens": 7212899.0, + "reward": 0.4375, + "reward_std": 0.4492306709289551, + "rewards/accuracy_reward/mean": 0.4375, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 1.428321123123169, + "sampling/importance_sampling_ratio/mean": 1.0000466108322144, + "sampling/importance_sampling_ratio/min": 0.6934236288070679, + "sampling/sampling_logp_difference/max": 0.36611413955688477, + "sampling/sampling_logp_difference/mean": 0.012941376306116581, + "step": 333 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00011114602239103988, + "clip_ratio/low_min": 0.00011114602239103988, + "clip_ratio/region_mean": 0.00011114602239103988, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 709.0, + "completions/mean_length": 528.28125, + "completions/mean_terminated_length": 494.0357360839844, + "completions/min_length": 326.0, + "completions/min_terminated_length": 326.0, + "entropy": 0.8232426717877388, + "epoch": 0.17880085653104924, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.020998461171984673, + "learning_rate": 1e-05, + "loss": 0.0593, + "num_tokens": 7233724.0, + "reward": 0.6875, + "reward_std": 0.4082317352294922, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.5058468580245972, + "sampling/importance_sampling_ratio/mean": 1.0002719163894653, + "sampling/importance_sampling_ratio/min": 0.7372462749481201, + "sampling/sampling_logp_difference/max": 0.40935540199279785, + "sampling/sampling_logp_difference/mean": 0.01719389110803604, + "step": 334 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00016946071264101192, + "clip_ratio/low_min": 0.00016946071264101192, + "clip_ratio/region_mean": 0.00016946071264101192, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 756.0, + "completions/mean_length": 575.71875, + "completions/mean_terminated_length": 521.8800048828125, + "completions/min_length": 348.0, + "completions/min_terminated_length": 348.0, + "entropy": 0.4106746129691601, + "epoch": 0.17933618843683083, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.01226749550551176, + "learning_rate": 1e-05, + "loss": 0.0232, + "num_tokens": 7256003.0, + "reward": 0.71875, + "reward_std": 0.3471629321575165, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.431685447692871, + "sampling/importance_sampling_ratio/mean": 0.999666690826416, + "sampling/importance_sampling_ratio/min": 0.6186524033546448, + "sampling/sampling_logp_difference/max": 0.4802117347717285, + "sampling/sampling_logp_difference/mean": 0.012043134309351444, + "step": 335 + }, + { + "clip_ratio/high_max": 0.00012898632849100977, + "clip_ratio/high_mean": 0.00012898632849100977, + "clip_ratio/low_mean": 5.43714668310713e-05, + "clip_ratio/low_min": 5.43714668310713e-05, + "clip_ratio/region_mean": 0.00018335779532208107, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 739.0, + "completions/mean_length": 537.71875, + "completions/mean_terminated_length": 522.36669921875, + "completions/min_length": 198.0, + "completions/min_terminated_length": 198.0, + "entropy": 0.31302709877491, + "epoch": 0.17987152034261242, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.008577987551689148, + "learning_rate": 1e-05, + "loss": 0.0453, + "num_tokens": 7277146.0, + "reward": 0.84375, + "reward_std": 0.22201895713806152, + "rewards/accuracy_reward/mean": 0.84375, + "rewards/accuracy_reward/std": 0.3689020276069641, + "sampling/importance_sampling_ratio/max": 1.3239972591400146, + "sampling/importance_sampling_ratio/mean": 1.000171184539795, + "sampling/importance_sampling_ratio/min": 0.7074315547943115, + "sampling/sampling_logp_difference/max": 0.3461143970489502, + "sampling/sampling_logp_difference/mean": 0.009894965216517448, + "step": 336 + }, + { + "clip_ratio/high_max": 9.130658509093337e-05, + "clip_ratio/high_mean": 9.130658509093337e-05, + "clip_ratio/low_mean": 0.00020746150767081417, + "clip_ratio/low_min": 0.00020746150767081417, + "clip_ratio/region_mean": 0.00029876809276174754, + "completions/clipped_ratio": 0.375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 753.0, + "completions/mean_length": 606.71875, + "completions/mean_terminated_length": 509.95001220703125, + "completions/min_length": 316.0, + "completions/min_terminated_length": 316.0, + "entropy": 0.39334094151854515, + "epoch": 0.180406852248394, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.00736922025680542, + "learning_rate": 1e-05, + "loss": 0.0068, + "num_tokens": 7301329.0, + "reward": 0.28125, + "reward_std": 0.3471629321575165, + "rewards/accuracy_reward/mean": 0.28125, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.4514373540878296, + "sampling/importance_sampling_ratio/mean": 0.9997961521148682, + "sampling/importance_sampling_ratio/min": 0.5209622979164124, + "sampling/sampling_logp_difference/max": 0.6520776748657227, + "sampling/sampling_logp_difference/mean": 0.011272238567471504, + "step": 337 + }, + { + "clip_ratio/high_max": 0.000170951709151268, + "clip_ratio/high_mean": 0.000170951709151268, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.000170951709151268, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 758.0, + "completions/mean_length": 416.8125, + "completions/mean_terminated_length": 405.4838562011719, + "completions/min_length": 208.0, + "completions/min_terminated_length": 208.0, + "entropy": 0.38587110862135887, + "epoch": 0.1809421841541756, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.014478067867457867, + "learning_rate": 1e-05, + "loss": -0.0152, + "num_tokens": 7318211.0, + "reward": 0.90625, + "reward_std": 0.2041158676147461, + "rewards/accuracy_reward/mean": 0.90625, + "rewards/accuracy_reward/std": 0.2961445748806, + "sampling/importance_sampling_ratio/max": 1.4607062339782715, + "sampling/importance_sampling_ratio/mean": 0.9998483061790466, + "sampling/importance_sampling_ratio/min": 0.3506035804748535, + "sampling/sampling_logp_difference/max": 1.0480990409851074, + "sampling/sampling_logp_difference/mean": 0.011547443456947803, + "step": 338 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 677.0, + "completions/mean_length": 445.46875, + "completions/mean_terminated_length": 435.06451416015625, + "completions/min_length": 243.0, + "completions/min_terminated_length": 243.0, + "entropy": 0.35046200081706047, + "epoch": 0.18147751605995718, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.004316780716180801, + "learning_rate": 1e-05, + "loss": 0.0528, + "num_tokens": 7336042.0, + "reward": 0.9375, + "reward_std": 0.1157275140285492, + "rewards/accuracy_reward/mean": 0.9375, + "rewards/accuracy_reward/std": 0.24593468010425568, + "sampling/importance_sampling_ratio/max": 1.3374011516571045, + "sampling/importance_sampling_ratio/mean": 0.9998047947883606, + "sampling/importance_sampling_ratio/min": 0.6119147539138794, + "sampling/sampling_logp_difference/max": 0.4911623001098633, + "sampling/sampling_logp_difference/mean": 0.010776880197227001, + "step": 339 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00014489953173324466, + "clip_ratio/low_min": 0.00014489953173324466, + "clip_ratio/region_mean": 0.00014489953173324466, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 752.0, + "completions/mean_length": 534.125, + "completions/mean_terminated_length": 490.8148193359375, + "completions/min_length": 294.0, + "completions/min_terminated_length": 294.0, + "entropy": 0.3672318812459707, + "epoch": 0.18201284796573874, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.012201718054711819, + "learning_rate": 1e-05, + "loss": 0.0498, + "num_tokens": 7356454.0, + "reward": 0.875, + "reward_std": 0.13363061845302582, + "rewards/accuracy_reward/mean": 0.875, + "rewards/accuracy_reward/std": 0.33601075410842896, + "sampling/importance_sampling_ratio/max": 1.293308138847351, + "sampling/importance_sampling_ratio/mean": 0.9997571706771851, + "sampling/importance_sampling_ratio/min": 0.695732057094574, + "sampling/sampling_logp_difference/max": 0.362790584564209, + "sampling/sampling_logp_difference/mean": 0.010314418002963066, + "step": 340 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0001675299572525546, + "clip_ratio/low_min": 0.0001675299572525546, + "clip_ratio/region_mean": 0.0001675299572525546, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 710.0, + "completions/mean_length": 529.1875, + "completions/mean_terminated_length": 474.0769348144531, + "completions/min_length": 246.0, + "completions/min_terminated_length": 246.0, + "entropy": 0.34470563009381294, + "epoch": 0.18254817987152033, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.01242291834205389, + "learning_rate": 1e-05, + "loss": 0.0236, + "num_tokens": 7377572.0, + "reward": 0.6875, + "reward_std": 0.2587745785713196, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.7681902647018433, + "sampling/importance_sampling_ratio/mean": 1.0000383853912354, + "sampling/importance_sampling_ratio/min": 0.7239490151405334, + "sampling/sampling_logp_difference/max": 0.5699565410614014, + "sampling/sampling_logp_difference/mean": 0.011041436344385147, + "step": 341 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 738.0, + "completions/max_terminated_length": 738.0, + "completions/mean_length": 475.09375, + "completions/mean_terminated_length": 475.09375, + "completions/min_length": 299.0, + "completions/min_terminated_length": 299.0, + "entropy": 0.32042665779590607, + "epoch": 0.18308351177730192, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.023323800414800644, + "learning_rate": 1e-05, + "loss": -0.0088, + "num_tokens": 7396639.0, + "reward": 0.9375, + "reward_std": 0.1767766922712326, + "rewards/accuracy_reward/mean": 0.9375, + "rewards/accuracy_reward/std": 0.24593468010425568, + "sampling/importance_sampling_ratio/max": 1.3497538566589355, + "sampling/importance_sampling_ratio/mean": 1.0001760721206665, + "sampling/importance_sampling_ratio/min": 0.7312887907028198, + "sampling/sampling_logp_difference/max": 0.31294679641723633, + "sampling/sampling_logp_difference/mean": 0.011018609628081322, + "step": 342 + }, + { + "clip_ratio/high_max": 6.076811041566543e-05, + "clip_ratio/high_mean": 6.076811041566543e-05, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 6.076811041566543e-05, + "completions/clipped_ratio": 0.0, + "completions/max_length": 673.0, + "completions/max_terminated_length": 673.0, + "completions/mean_length": 433.0625, + "completions/mean_terminated_length": 433.0625, + "completions/min_length": 208.0, + "completions/min_terminated_length": 208.0, + "entropy": 0.44570066407322884, + "epoch": 0.1836188436830835, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.026565412059426308, + "learning_rate": 1e-05, + "loss": 0.0513, + "num_tokens": 7413953.0, + "reward": 0.875, + "reward_std": 0.2177756428718567, + "rewards/accuracy_reward/mean": 0.875, + "rewards/accuracy_reward/std": 0.33601075410842896, + "sampling/importance_sampling_ratio/max": 1.3588019609451294, + "sampling/importance_sampling_ratio/mean": 1.0004217624664307, + "sampling/importance_sampling_ratio/min": 0.33628877997398376, + "sampling/sampling_logp_difference/max": 1.089785099029541, + "sampling/sampling_logp_difference/mean": 0.01277589425444603, + "step": 343 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 768.0, + "completions/max_terminated_length": 686.0, + "completions/mean_length": 562.03125, + "completions/mean_terminated_length": 493.375, + "completions/min_length": 339.0, + "completions/min_terminated_length": 339.0, + "entropy": 0.5396241508424282, + "epoch": 0.1841541755888651, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": 0.0097, + "num_tokens": 7436346.0, + "reward": 0.4375, + "reward_std": 0.1157275140285492, + "rewards/accuracy_reward/mean": 0.4375, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 1.8712308406829834, + "sampling/importance_sampling_ratio/mean": 1.000185251235962, + "sampling/importance_sampling_ratio/min": 0.5464312434196472, + "sampling/sampling_logp_difference/max": 0.6265964508056641, + "sampling/sampling_logp_difference/mean": 0.01332900021225214, + "step": 344 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 758.0, + "completions/mean_length": 602.46875, + "completions/mean_terminated_length": 537.6956787109375, + "completions/min_length": 317.0, + "completions/min_terminated_length": 317.0, + "entropy": 0.4268195964396, + "epoch": 0.1846895074946467, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.011327230371534824, + "learning_rate": 1e-05, + "loss": 0.055, + "num_tokens": 7459385.0, + "reward": 0.6875, + "reward_std": 0.38298875093460083, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.4862500429153442, + "sampling/importance_sampling_ratio/mean": 1.0000451803207397, + "sampling/importance_sampling_ratio/min": 0.7432152628898621, + "sampling/sampling_logp_difference/max": 0.3962562084197998, + "sampling/sampling_logp_difference/mean": 0.01288197934627533, + "step": 345 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 682.0, + "completions/mean_length": 511.03125, + "completions/mean_terminated_length": 451.73077392578125, + "completions/min_length": 238.0, + "completions/min_terminated_length": 238.0, + "entropy": 0.41892142966389656, + "epoch": 0.18522483940042828, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.008036869578063488, + "learning_rate": 1e-05, + "loss": 0.0459, + "num_tokens": 7479498.0, + "reward": 0.65625, + "reward_std": 0.22201895713806152, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.2727625370025635, + "sampling/importance_sampling_ratio/mean": 1.0000818967819214, + "sampling/importance_sampling_ratio/min": 0.6897485256195068, + "sampling/sampling_logp_difference/max": 0.3714282512664795, + "sampling/sampling_logp_difference/mean": 0.012019198387861252, + "step": 346 + }, + { + "clip_ratio/high_max": 4.4770775275537744e-05, + "clip_ratio/high_mean": 4.4770775275537744e-05, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 4.4770775275537744e-05, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 746.0, + "completions/mean_length": 587.21875, + "completions/mean_terminated_length": 536.5999755859375, + "completions/min_length": 143.0, + "completions/min_terminated_length": 143.0, + "entropy": 0.5027334354817867, + "epoch": 0.18576017130620984, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.009729636833071709, + "learning_rate": 1e-05, + "loss": -0.0369, + "num_tokens": 7502217.0, + "reward": 0.5, + "reward_std": 0.3335031569004059, + "rewards/accuracy_reward/mean": 0.5, + "rewards/accuracy_reward/std": 0.5080004930496216, + "sampling/importance_sampling_ratio/max": 1.3486765623092651, + "sampling/importance_sampling_ratio/mean": 0.9997532367706299, + "sampling/importance_sampling_ratio/min": 0.6723280549049377, + "sampling/sampling_logp_difference/max": 0.39700889587402344, + "sampling/sampling_logp_difference/mean": 0.013065842911601067, + "step": 347 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 700.0, + "completions/mean_length": 423.5, + "completions/mean_terminated_length": 412.3870849609375, + "completions/min_length": 215.0, + "completions/min_terminated_length": 215.0, + "entropy": 0.37799080833792686, + "epoch": 0.18629550321199143, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.004395653028041124, + "learning_rate": 1e-05, + "loss": 0.0827, + "num_tokens": 7519377.0, + "reward": 0.90625, + "reward_std": 0.2651650309562683, + "rewards/accuracy_reward/mean": 0.90625, + "rewards/accuracy_reward/std": 0.2961445748806, + "sampling/importance_sampling_ratio/max": 1.3875583410263062, + "sampling/importance_sampling_ratio/mean": 1.0003392696380615, + "sampling/importance_sampling_ratio/min": 0.6281449198722839, + "sampling/sampling_logp_difference/max": 0.4649844169616699, + "sampling/sampling_logp_difference/mean": 0.01179863978177309, + "step": 348 + }, + { + "clip_ratio/high_max": 0.0001152073746197857, + "clip_ratio/high_mean": 0.0001152073746197857, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0001152073746197857, + "completions/clipped_ratio": 0.3125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 526.0, + "completions/mean_length": 476.84375, + "completions/mean_terminated_length": 344.5, + "completions/min_length": 209.0, + "completions/min_terminated_length": 209.0, + "entropy": 0.4453147351741791, + "epoch": 0.18683083511777301, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.006316605024039745, + "learning_rate": 1e-05, + "loss": 0.0121, + "num_tokens": 7538028.0, + "reward": 0.65625, + "reward_std": 0.2041158676147461, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.3541252613067627, + "sampling/importance_sampling_ratio/mean": 0.9995315670967102, + "sampling/importance_sampling_ratio/min": 0.30046942830085754, + "sampling/sampling_logp_difference/max": 1.202409267425537, + "sampling/sampling_logp_difference/mean": 0.013666864484548569, + "step": 349 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 598.0, + "completions/mean_length": 512.59375, + "completions/mean_terminated_length": 476.107177734375, + "completions/min_length": 336.0, + "completions/min_terminated_length": 336.0, + "entropy": 0.4042432829737663, + "epoch": 0.1873661670235546, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": 0.0, + "num_tokens": 7558263.0, + "reward": 0.75, + "reward_std": 0.0, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.3103748559951782, + "sampling/importance_sampling_ratio/mean": 0.999916672706604, + "sampling/importance_sampling_ratio/min": 0.6854010820388794, + "sampling/sampling_logp_difference/max": 0.37775111198425293, + "sampling/sampling_logp_difference/mean": 0.011970157735049725, + "step": 350 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 762.0, + "completions/mean_length": 464.0625, + "completions/mean_terminated_length": 407.77777099609375, + "completions/min_length": 279.0, + "completions/min_terminated_length": 279.0, + "entropy": 0.40735117346048355, + "epoch": 0.1879014989293362, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.00286686304025352, + "learning_rate": 1e-05, + "loss": 0.0318, + "num_tokens": 7576881.0, + "reward": 0.5, + "reward_std": 0.1767766922712326, + "rewards/accuracy_reward/mean": 0.5, + "rewards/accuracy_reward/std": 0.5080004930496216, + "sampling/importance_sampling_ratio/max": 1.4542512893676758, + "sampling/importance_sampling_ratio/mean": 0.9998769760131836, + "sampling/importance_sampling_ratio/min": 0.6443073153495789, + "sampling/sampling_logp_difference/max": 0.43957948684692383, + "sampling/sampling_logp_difference/mean": 0.012052088975906372, + "step": 351 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.278716344037093e-05, + "clip_ratio/low_min": 5.278716344037093e-05, + "clip_ratio/region_mean": 5.278716344037093e-05, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 623.0, + "completions/mean_length": 515.625, + "completions/mean_terminated_length": 479.5714416503906, + "completions/min_length": 341.0, + "completions/min_terminated_length": 341.0, + "entropy": 0.3545593172311783, + "epoch": 0.18843683083511778, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.011616687290370464, + "learning_rate": 1e-05, + "loss": 0.0368, + "num_tokens": 7597133.0, + "reward": 0.625, + "reward_std": 0.5081326961517334, + "rewards/accuracy_reward/mean": 0.625, + "rewards/accuracy_reward/std": 0.49186936020851135, + "sampling/importance_sampling_ratio/max": 1.3291075229644775, + "sampling/importance_sampling_ratio/mean": 0.9997459650039673, + "sampling/importance_sampling_ratio/min": 0.10021040588617325, + "sampling/sampling_logp_difference/max": 2.300483226776123, + "sampling/sampling_logp_difference/mean": 0.011193163692951202, + "step": 352 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 4.301445369492285e-05, + "clip_ratio/low_min": 4.301445369492285e-05, + "clip_ratio/region_mean": 4.301445369492285e-05, + "completions/clipped_ratio": 0.34375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 638.0, + "completions/mean_length": 554.25, + "completions/mean_terminated_length": 442.2857360839844, + "completions/min_length": 166.0, + "completions/min_terminated_length": 166.0, + "entropy": 0.6081601269543171, + "epoch": 0.18897216274089937, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": 0.0228, + "num_tokens": 7618645.0, + "reward": 0.5625, + "reward_std": 0.1157275140285492, + "rewards/accuracy_reward/mean": 0.5625, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 1.8886375427246094, + "sampling/importance_sampling_ratio/mean": 0.9999396204948425, + "sampling/importance_sampling_ratio/min": 0.6797575950622559, + "sampling/sampling_logp_difference/max": 0.6358556747436523, + "sampling/sampling_logp_difference/mean": 0.017190592363476753, + "step": 353 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 695.0, + "completions/mean_length": 529.9375, + "completions/mean_terminated_length": 522.258056640625, + "completions/min_length": 376.0, + "completions/min_terminated_length": 376.0, + "entropy": 0.41264915466308594, + "epoch": 0.18950749464668093, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.005830404348671436, + "learning_rate": 1e-05, + "loss": 0.0366, + "num_tokens": 7639059.0, + "reward": 0.84375, + "reward_std": 0.3808925747871399, + "rewards/accuracy_reward/mean": 0.84375, + "rewards/accuracy_reward/std": 0.3689020276069641, + "sampling/importance_sampling_ratio/max": 1.6129010915756226, + "sampling/importance_sampling_ratio/mean": 0.9999990463256836, + "sampling/importance_sampling_ratio/min": 0.704312801361084, + "sampling/sampling_logp_difference/max": 0.47803449630737305, + "sampling/sampling_logp_difference/mean": 0.011893495917320251, + "step": 354 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00010156382631976157, + "clip_ratio/low_min": 0.00010156382631976157, + "clip_ratio/region_mean": 0.00010156382631976157, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 763.0, + "completions/mean_length": 622.84375, + "completions/mean_terminated_length": 589.34619140625, + "completions/min_length": 249.0, + "completions/min_terminated_length": 249.0, + "entropy": 0.33784014731645584, + "epoch": 0.19004282655246252, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.0122241685166955, + "learning_rate": 1e-05, + "loss": -0.025, + "num_tokens": 7662854.0, + "reward": 0.375, + "reward_std": 0.2314550280570984, + "rewards/accuracy_reward/mean": 0.375, + "rewards/accuracy_reward/std": 0.49186936020851135, + "sampling/importance_sampling_ratio/max": 1.5569756031036377, + "sampling/importance_sampling_ratio/mean": 1.000101923942566, + "sampling/importance_sampling_ratio/min": 0.6211375594139099, + "sampling/sampling_logp_difference/max": 0.47620272636413574, + "sampling/sampling_logp_difference/mean": 0.010734748095273972, + "step": 355 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 718.0, + "completions/mean_length": 611.75, + "completions/mean_terminated_length": 550.6087036132812, + "completions/min_length": 275.0, + "completions/min_terminated_length": 275.0, + "entropy": 0.4817916601896286, + "epoch": 0.1905781584582441, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.005454639904201031, + "learning_rate": 1e-05, + "loss": 0.0828, + "num_tokens": 7687350.0, + "reward": 0.65625, + "reward_std": 0.2041158676147461, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 2.0, + "sampling/importance_sampling_ratio/mean": 1.0002686977386475, + "sampling/importance_sampling_ratio/min": 0.6121079325675964, + "sampling/sampling_logp_difference/max": 0.8192362785339355, + "sampling/sampling_logp_difference/mean": 0.013691947795450687, + "step": 356 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 711.0, + "completions/mean_length": 481.21875, + "completions/mean_terminated_length": 451.5517272949219, + "completions/min_length": 191.0, + "completions/min_terminated_length": 191.0, + "entropy": 0.32394311763346195, + "epoch": 0.1911134903640257, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.015326720662415028, + "learning_rate": 1e-05, + "loss": 0.0694, + "num_tokens": 7705861.0, + "reward": 0.78125, + "reward_std": 0.24511480331420898, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.3399609327316284, + "sampling/importance_sampling_ratio/mean": 0.9996441006660461, + "sampling/importance_sampling_ratio/min": 0.41950923204421997, + "sampling/sampling_logp_difference/max": 0.8686697483062744, + "sampling/sampling_logp_difference/mean": 0.010584529489278793, + "step": 357 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 4.978096330887638e-05, + "clip_ratio/low_min": 4.978096330887638e-05, + "clip_ratio/region_mean": 4.978096330887638e-05, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 767.0, + "completions/mean_length": 570.4375, + "completions/mean_terminated_length": 533.8518676757812, + "completions/min_length": 271.0, + "completions/min_terminated_length": 271.0, + "entropy": 0.348613727837801, + "epoch": 0.19164882226980728, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.00653418805450201, + "learning_rate": 1e-05, + "loss": -0.0025, + "num_tokens": 7727643.0, + "reward": 0.59375, + "reward_std": 0.1293872892856598, + "rewards/accuracy_reward/mean": 0.59375, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.364234447479248, + "sampling/importance_sampling_ratio/mean": 0.999725878238678, + "sampling/importance_sampling_ratio/min": 0.6682811379432678, + "sampling/sampling_logp_difference/max": 0.4030463695526123, + "sampling/sampling_logp_difference/mean": 0.010410048067569733, + "step": 358 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 625.0, + "completions/mean_length": 413.59375, + "completions/mean_terminated_length": 402.1612854003906, + "completions/min_length": 297.0, + "completions/min_terminated_length": 297.0, + "entropy": 0.4846441000699997, + "epoch": 0.19218415417558887, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.014523014426231384, + "learning_rate": 1e-05, + "loss": 0.0435, + "num_tokens": 7744902.0, + "reward": 0.84375, + "reward_std": 0.1293872892856598, + "rewards/accuracy_reward/mean": 0.84375, + "rewards/accuracy_reward/std": 0.3689020276069641, + "sampling/importance_sampling_ratio/max": 1.3490887880325317, + "sampling/importance_sampling_ratio/mean": 0.9997272491455078, + "sampling/importance_sampling_ratio/min": 0.6908149719238281, + "sampling/sampling_logp_difference/max": 0.36988329887390137, + "sampling/sampling_logp_difference/mean": 0.014691982418298721, + "step": 359 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 698.0, + "completions/mean_length": 512.09375, + "completions/mean_terminated_length": 485.6206970214844, + "completions/min_length": 324.0, + "completions/min_terminated_length": 324.0, + "entropy": 0.37720469012856483, + "epoch": 0.19271948608137046, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.008529460057616234, + "learning_rate": 1e-05, + "loss": 0.0046, + "num_tokens": 7764865.0, + "reward": 0.8125, + "reward_std": 0.249358132481575, + "rewards/accuracy_reward/mean": 0.8125, + "rewards/accuracy_reward/std": 0.3965577781200409, + "sampling/importance_sampling_ratio/max": 1.376272201538086, + "sampling/importance_sampling_ratio/mean": 0.9998212456703186, + "sampling/importance_sampling_ratio/min": 0.7713286876678467, + "sampling/sampling_logp_difference/max": 0.3193786144256592, + "sampling/sampling_logp_difference/mean": 0.011417688801884651, + "step": 360 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 4.948535206494853e-05, + "clip_ratio/low_min": 4.948535206494853e-05, + "clip_ratio/region_mean": 4.948535206494853e-05, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 720.0, + "completions/mean_length": 542.0625, + "completions/mean_terminated_length": 489.923095703125, + "completions/min_length": 181.0, + "completions/min_terminated_length": 181.0, + "entropy": 0.4329718202352524, + "epoch": 0.19325481798715202, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.023381182923913002, + "learning_rate": 1e-05, + "loss": 0.1034, + "num_tokens": 7786211.0, + "reward": 0.53125, + "reward_std": 0.2651650309562683, + "rewards/accuracy_reward/mean": 0.53125, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.4117883443832397, + "sampling/importance_sampling_ratio/mean": 0.9998693466186523, + "sampling/importance_sampling_ratio/min": 0.6523434519767761, + "sampling/sampling_logp_difference/max": 0.4271841049194336, + "sampling/sampling_logp_difference/mean": 0.013179978355765343, + "step": 361 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 6.446622137445956e-05, + "clip_ratio/low_min": 6.446622137445956e-05, + "clip_ratio/region_mean": 6.446622137445956e-05, + "completions/clipped_ratio": 0.25, + "completions/max_length": 768.0, + "completions/max_terminated_length": 702.0, + "completions/mean_length": 537.5, + "completions/mean_terminated_length": 460.66668701171875, + "completions/min_length": 223.0, + "completions/min_terminated_length": 223.0, + "entropy": 0.592044323682785, + "epoch": 0.1937901498929336, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.007650850340723991, + "learning_rate": 1e-05, + "loss": 0.1212, + "num_tokens": 7807651.0, + "reward": 0.6875, + "reward_std": 0.3335031569004059, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 2.0, + "sampling/importance_sampling_ratio/mean": 1.0002923011779785, + "sampling/importance_sampling_ratio/min": 0.682411789894104, + "sampling/sampling_logp_difference/max": 0.8413052558898926, + "sampling/sampling_logp_difference/mean": 0.014110127463936806, + "step": 362 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 9.919505100697279e-05, + "clip_ratio/low_min": 9.919505100697279e-05, + "clip_ratio/region_mean": 9.919505100697279e-05, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 765.0, + "completions/mean_length": 586.40625, + "completions/mean_terminated_length": 535.5599975585938, + "completions/min_length": 303.0, + "completions/min_terminated_length": 303.0, + "entropy": 0.47988008335232735, + "epoch": 0.1943254817987152, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.018716705963015556, + "learning_rate": 1e-05, + "loss": 0.0611, + "num_tokens": 7830504.0, + "reward": 0.46875, + "reward_std": 0.4628904461860657, + "rewards/accuracy_reward/mean": 0.46875, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.5832464694976807, + "sampling/importance_sampling_ratio/mean": 0.9997760057449341, + "sampling/importance_sampling_ratio/min": 0.595163881778717, + "sampling/sampling_logp_difference/max": 0.518918514251709, + "sampling/sampling_logp_difference/mean": 0.013171184808015823, + "step": 363 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.58534411538858e-05, + "clip_ratio/low_min": 5.58534411538858e-05, + "clip_ratio/region_mean": 5.58534411538858e-05, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 759.0, + "completions/mean_length": 532.53125, + "completions/mean_terminated_length": 498.89288330078125, + "completions/min_length": 183.0, + "completions/min_terminated_length": 183.0, + "entropy": 0.3653341978788376, + "epoch": 0.1948608137044968, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.011009089648723602, + "learning_rate": 1e-05, + "loss": 0.0511, + "num_tokens": 7851737.0, + "reward": 0.5625, + "reward_std": 0.4261348247528076, + "rewards/accuracy_reward/mean": 0.5625, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 1.3536845445632935, + "sampling/importance_sampling_ratio/mean": 0.9997643828392029, + "sampling/importance_sampling_ratio/min": 0.7109105587005615, + "sampling/sampling_logp_difference/max": 0.34120869636535645, + "sampling/sampling_logp_difference/mean": 0.011702951975166798, + "step": 364 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 7.014590664766729e-05, + "clip_ratio/low_min": 7.014590664766729e-05, + "clip_ratio/region_mean": 7.014590664766729e-05, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 767.0, + "completions/mean_length": 531.4375, + "completions/mean_terminated_length": 523.8064575195312, + "completions/min_length": 290.0, + "completions/min_terminated_length": 290.0, + "entropy": 0.3485301695764065, + "epoch": 0.19539614561027838, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.009981510229408741, + "learning_rate": 1e-05, + "loss": -0.0505, + "num_tokens": 7872471.0, + "reward": 0.5625, + "reward_std": 0.2177756428718567, + "rewards/accuracy_reward/mean": 0.5625, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 1.4363633394241333, + "sampling/importance_sampling_ratio/mean": 1.00010347366333, + "sampling/importance_sampling_ratio/min": 0.7413499355316162, + "sampling/sampling_logp_difference/max": 0.36211442947387695, + "sampling/sampling_logp_difference/mean": 0.010392917320132256, + "step": 365 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 725.0, + "completions/mean_length": 444.5625, + "completions/mean_terminated_length": 434.1290283203125, + "completions/min_length": 203.0, + "completions/min_terminated_length": 203.0, + "entropy": 0.48997560515999794, + "epoch": 0.19593147751605997, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.008400565013289452, + "learning_rate": 1e-05, + "loss": 0.0457, + "num_tokens": 7890321.0, + "reward": 0.78125, + "reward_std": 0.24511480331420898, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.5290173292160034, + "sampling/importance_sampling_ratio/mean": 1.00031578540802, + "sampling/importance_sampling_ratio/min": 0.7160792946815491, + "sampling/sampling_logp_difference/max": 0.4246252775192261, + "sampling/sampling_logp_difference/mean": 0.013884284533560276, + "step": 366 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0001172342490463052, + "clip_ratio/low_min": 0.0001172342490463052, + "clip_ratio/region_mean": 0.0001172342490463052, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 653.0, + "completions/mean_length": 497.875, + "completions/mean_terminated_length": 435.5384826660156, + "completions/min_length": 221.0, + "completions/min_terminated_length": 221.0, + "entropy": 0.5482137557119131, + "epoch": 0.19646680942184155, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.005809069145470858, + "learning_rate": 1e-05, + "loss": 0.0779, + "num_tokens": 7910421.0, + "reward": 0.59375, + "reward_std": 0.3987956643104553, + "rewards/accuracy_reward/mean": 0.59375, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.635168194770813, + "sampling/importance_sampling_ratio/mean": 1.000160813331604, + "sampling/importance_sampling_ratio/min": 0.588154137134552, + "sampling/sampling_logp_difference/max": 0.5307661890983582, + "sampling/sampling_logp_difference/mean": 0.014795972965657711, + "step": 367 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 7.716049731243402e-05, + "clip_ratio/low_min": 7.716049731243402e-05, + "clip_ratio/region_mean": 7.716049731243402e-05, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 712.0, + "completions/mean_length": 452.625, + "completions/mean_terminated_length": 442.45159912109375, + "completions/min_length": 307.0, + "completions/min_terminated_length": 307.0, + "entropy": 0.42601926252245903, + "epoch": 0.19700214132762311, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.02773178741335869, + "learning_rate": 1e-05, + "loss": -0.0051, + "num_tokens": 7928681.0, + "reward": 0.75, + "reward_std": 0.1767766922712326, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.3831844329833984, + "sampling/importance_sampling_ratio/mean": 0.9999629259109497, + "sampling/importance_sampling_ratio/min": 0.46010899543762207, + "sampling/sampling_logp_difference/max": 0.7762918472290039, + "sampling/sampling_logp_difference/mean": 0.01247339602559805, + "step": 368 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 652.0, + "completions/mean_length": 434.0625, + "completions/mean_terminated_length": 423.2903137207031, + "completions/min_length": 201.0, + "completions/min_terminated_length": 201.0, + "entropy": 0.39646005257964134, + "epoch": 0.1975374732334047, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.004643204156309366, + "learning_rate": 1e-05, + "loss": -0.0065, + "num_tokens": 7945955.0, + "reward": 0.9375, + "reward_std": 0.1767766922712326, + "rewards/accuracy_reward/mean": 0.9375, + "rewards/accuracy_reward/std": 0.24593468010425568, + "sampling/importance_sampling_ratio/max": 1.3710025548934937, + "sampling/importance_sampling_ratio/mean": 1.000269889831543, + "sampling/importance_sampling_ratio/min": 0.7469781041145325, + "sampling/sampling_logp_difference/max": 0.31554222106933594, + "sampling/sampling_logp_difference/mean": 0.012254216708242893, + "step": 369 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00010352126264479011, + "clip_ratio/low_min": 0.00010352126264479011, + "clip_ratio/region_mean": 0.00010352126264479011, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 756.0, + "completions/mean_length": 484.40625, + "completions/mean_terminated_length": 475.258056640625, + "completions/min_length": 184.0, + "completions/min_terminated_length": 184.0, + "entropy": 0.4069571755826473, + "epoch": 0.1980728051391863, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.010718288831412792, + "learning_rate": 1e-05, + "loss": 0.0475, + "num_tokens": 7965232.0, + "reward": 0.5625, + "reward_std": 0.49022960662841797, + "rewards/accuracy_reward/mean": 0.5625, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 1.3774203062057495, + "sampling/importance_sampling_ratio/mean": 0.9999377131462097, + "sampling/importance_sampling_ratio/min": 0.3378658592700958, + "sampling/sampling_logp_difference/max": 1.085106372833252, + "sampling/sampling_logp_difference/mean": 0.013025488704442978, + "step": 370 + }, + { + "clip_ratio/high_max": 5.8247904235031456e-05, + "clip_ratio/high_mean": 5.8247904235031456e-05, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 5.8247904235031456e-05, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 743.0, + "completions/mean_length": 507.46875, + "completions/mean_terminated_length": 480.5172424316406, + "completions/min_length": 150.0, + "completions/min_terminated_length": 150.0, + "entropy": 0.48590512201189995, + "epoch": 0.19860813704496788, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.006943477783352137, + "learning_rate": 1e-05, + "loss": -0.0175, + "num_tokens": 7985663.0, + "reward": 0.625, + "reward_std": 0.48503684997558594, + "rewards/accuracy_reward/mean": 0.625, + "rewards/accuracy_reward/std": 0.49186936020851135, + "sampling/importance_sampling_ratio/max": 1.3872870206832886, + "sampling/importance_sampling_ratio/mean": 1.0000364780426025, + "sampling/importance_sampling_ratio/min": 0.6696125268936157, + "sampling/sampling_logp_difference/max": 0.40105605125427246, + "sampling/sampling_logp_difference/mean": 0.01292111724615097, + "step": 371 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0001097573185688816, + "clip_ratio/low_min": 0.0001097573185688816, + "clip_ratio/region_mean": 0.0001097573185688816, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 742.0, + "completions/mean_length": 521.46875, + "completions/mean_terminated_length": 513.51611328125, + "completions/min_length": 314.0, + "completions/min_terminated_length": 314.0, + "entropy": 0.3042990453541279, + "epoch": 0.19914346895074947, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.009631731547415257, + "learning_rate": 1e-05, + "loss": 0.049, + "num_tokens": 8005766.0, + "reward": 0.65625, + "reward_std": 0.4944729208946228, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.4142330884933472, + "sampling/importance_sampling_ratio/mean": 1.0002249479293823, + "sampling/importance_sampling_ratio/min": 0.6511433720588684, + "sampling/sampling_logp_difference/max": 0.42902541160583496, + "sampling/sampling_logp_difference/mean": 0.010262439027428627, + "step": 372 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0002464971912559122, + "clip_ratio/low_min": 0.0002464971912559122, + "clip_ratio/region_mean": 0.0002464971912559122, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 710.0, + "completions/mean_length": 543.90625, + "completions/mean_terminated_length": 502.40740966796875, + "completions/min_length": 266.0, + "completions/min_terminated_length": 266.0, + "entropy": 0.38599660992622375, + "epoch": 0.19967880085653106, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.014723707921802998, + "learning_rate": 1e-05, + "loss": -0.0421, + "num_tokens": 8027483.0, + "reward": 0.6875, + "reward_std": 0.3335031569004059, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.6832029819488525, + "sampling/importance_sampling_ratio/mean": 1.0002552270889282, + "sampling/importance_sampling_ratio/min": 0.6958727836608887, + "sampling/sampling_logp_difference/max": 0.5206985473632812, + "sampling/sampling_logp_difference/mean": 0.01208306010812521, + "step": 373 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.43714668310713e-05, + "clip_ratio/low_min": 5.43714668310713e-05, + "clip_ratio/region_mean": 5.43714668310713e-05, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 717.0, + "completions/mean_length": 483.96875, + "completions/mean_terminated_length": 465.0333557128906, + "completions/min_length": 270.0, + "completions/min_terminated_length": 270.0, + "entropy": 0.3956117704510689, + "epoch": 0.20021413276231265, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.004858255386352539, + "learning_rate": 1e-05, + "loss": -0.0041, + "num_tokens": 8046394.0, + "reward": 0.78125, + "reward_std": 0.0883883461356163, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.5563654899597168, + "sampling/importance_sampling_ratio/mean": 1.0002444982528687, + "sampling/importance_sampling_ratio/min": 0.11141053587198257, + "sampling/sampling_logp_difference/max": 2.194533348083496, + "sampling/sampling_logp_difference/mean": 0.01166938804090023, + "step": 374 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 672.0, + "completions/mean_length": 526.875, + "completions/mean_terminated_length": 459.3599853515625, + "completions/min_length": 274.0, + "completions/min_terminated_length": 274.0, + "entropy": 0.5160121358931065, + "epoch": 0.2007494646680942, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.0033759973011910915, + "learning_rate": 1e-05, + "loss": -0.0044, + "num_tokens": 8067190.0, + "reward": 0.6875, + "reward_std": 0.1767766922712326, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.4209799766540527, + "sampling/importance_sampling_ratio/mean": 1.0001325607299805, + "sampling/importance_sampling_ratio/min": 0.6531583070755005, + "sampling/sampling_logp_difference/max": 0.4259357452392578, + "sampling/sampling_logp_difference/mean": 0.015096231363713741, + "step": 375 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.477651211549528e-05, + "clip_ratio/low_min": 5.477651211549528e-05, + "clip_ratio/region_mean": 5.477651211549528e-05, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 734.0, + "completions/mean_length": 530.53125, + "completions/mean_terminated_length": 522.8709716796875, + "completions/min_length": 306.0, + "completions/min_terminated_length": 306.0, + "entropy": 0.40019306913018227, + "epoch": 0.2012847965738758, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.02143034338951111, + "learning_rate": 1e-05, + "loss": -0.0122, + "num_tokens": 8088055.0, + "reward": 0.875, + "reward_std": 0.292504221200943, + "rewards/accuracy_reward/mean": 0.875, + "rewards/accuracy_reward/std": 0.33601075410842896, + "sampling/importance_sampling_ratio/max": 1.4617897272109985, + "sampling/importance_sampling_ratio/mean": 1.0000956058502197, + "sampling/importance_sampling_ratio/min": 0.7446685433387756, + "sampling/sampling_logp_difference/max": 0.37966156005859375, + "sampling/sampling_logp_difference/mean": 0.01237609051167965, + "step": 376 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 692.0, + "completions/mean_length": 570.65625, + "completions/mean_terminated_length": 534.1111450195312, + "completions/min_length": 310.0, + "completions/min_terminated_length": 310.0, + "entropy": 0.39847778528928757, + "epoch": 0.20182012847965738, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.00591615354642272, + "learning_rate": 1e-05, + "loss": 0.0247, + "num_tokens": 8110692.0, + "reward": 0.65625, + "reward_std": 0.3061639964580536, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.358012318611145, + "sampling/importance_sampling_ratio/mean": 0.9998806118965149, + "sampling/importance_sampling_ratio/min": 0.7165917158126831, + "sampling/sampling_logp_difference/max": 0.3332490921020508, + "sampling/sampling_logp_difference/mean": 0.012140526436269283, + "step": 377 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 757.0, + "completions/mean_length": 535.65625, + "completions/mean_terminated_length": 511.6206970214844, + "completions/min_length": 267.0, + "completions/min_terminated_length": 267.0, + "entropy": 0.3456450626254082, + "epoch": 0.20235546038543897, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.008054766803979874, + "learning_rate": 1e-05, + "loss": 0.0065, + "num_tokens": 8131313.0, + "reward": 0.75, + "reward_std": 0.3514062464237213, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.320526361465454, + "sampling/importance_sampling_ratio/mean": 0.9997600317001343, + "sampling/importance_sampling_ratio/min": 0.5832955241203308, + "sampling/sampling_logp_difference/max": 0.5390613079071045, + "sampling/sampling_logp_difference/mean": 0.011443529278039932, + "step": 378 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 709.0, + "completions/mean_length": 542.6875, + "completions/mean_terminated_length": 490.69232177734375, + "completions/min_length": 171.0, + "completions/min_terminated_length": 171.0, + "entropy": 0.4418714940547943, + "epoch": 0.20289079229122056, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.006573774851858616, + "learning_rate": 1e-05, + "loss": 0.0737, + "num_tokens": 8152487.0, + "reward": 0.6875, + "reward_std": 0.292504221200943, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.425748348236084, + "sampling/importance_sampling_ratio/mean": 0.9998875260353088, + "sampling/importance_sampling_ratio/min": 0.6758825182914734, + "sampling/sampling_logp_difference/max": 0.3917360305786133, + "sampling/sampling_logp_difference/mean": 0.013822292909026146, + "step": 379 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.9073725424241275e-05, + "clip_ratio/low_min": 5.9073725424241275e-05, + "clip_ratio/region_mean": 5.9073725424241275e-05, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 703.0, + "completions/mean_length": 507.6875, + "completions/mean_terminated_length": 447.6153869628906, + "completions/min_length": 222.0, + "completions/min_terminated_length": 222.0, + "entropy": 0.45735814049839973, + "epoch": 0.20342612419700215, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.027993470430374146, + "learning_rate": 1e-05, + "loss": 0.0774, + "num_tokens": 8172277.0, + "reward": 0.6875, + "reward_std": 0.47655022144317627, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.36721670627594, + "sampling/importance_sampling_ratio/mean": 0.9996927976608276, + "sampling/importance_sampling_ratio/min": 0.7227435111999512, + "sampling/sampling_logp_difference/max": 0.3247009515762329, + "sampling/sampling_logp_difference/mean": 0.012482824735343456, + "step": 380 + }, + { + "clip_ratio/high_max": 6.853070226497948e-05, + "clip_ratio/high_mean": 6.853070226497948e-05, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 6.853070226497948e-05, + "completions/clipped_ratio": 0.0, + "completions/max_length": 758.0, + "completions/max_terminated_length": 758.0, + "completions/mean_length": 444.875, + "completions/mean_terminated_length": 444.875, + "completions/min_length": 219.0, + "completions/min_terminated_length": 219.0, + "entropy": 0.3879472576081753, + "epoch": 0.20396145610278374, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.004540951922535896, + "learning_rate": 1e-05, + "loss": -0.0301, + "num_tokens": 8190201.0, + "reward": 0.90625, + "reward_std": 0.2651650309562683, + "rewards/accuracy_reward/mean": 0.90625, + "rewards/accuracy_reward/std": 0.2961445748806, + "sampling/importance_sampling_ratio/max": 2.0, + "sampling/importance_sampling_ratio/mean": 1.0005786418914795, + "sampling/importance_sampling_ratio/min": 0.6149527430534363, + "sampling/sampling_logp_difference/max": 0.7482748031616211, + "sampling/sampling_logp_difference/mean": 0.011853198520839214, + "step": 381 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 697.0, + "completions/mean_length": 475.09375, + "completions/mean_terminated_length": 455.5666809082031, + "completions/min_length": 253.0, + "completions/min_terminated_length": 253.0, + "entropy": 0.3254448063671589, + "epoch": 0.2044967880085653, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.008202667348086834, + "learning_rate": 1e-05, + "loss": 0.035, + "num_tokens": 8208916.0, + "reward": 0.90625, + "reward_std": 0.2041158676147461, + "rewards/accuracy_reward/mean": 0.90625, + "rewards/accuracy_reward/std": 0.2961445748806, + "sampling/importance_sampling_ratio/max": 1.3376712799072266, + "sampling/importance_sampling_ratio/mean": 0.9995779991149902, + "sampling/importance_sampling_ratio/min": 0.744231641292572, + "sampling/sampling_logp_difference/max": 0.29540300369262695, + "sampling/sampling_logp_difference/mean": 0.0104160001501441, + "step": 382 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 678.0, + "completions/mean_length": 515.0625, + "completions/mean_terminated_length": 456.69232177734375, + "completions/min_length": 226.0, + "completions/min_terminated_length": 226.0, + "entropy": 0.3882271386682987, + "epoch": 0.2050321199143469, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.026752524077892303, + "learning_rate": 1e-05, + "loss": 0.0897, + "num_tokens": 8228910.0, + "reward": 0.78125, + "reward_std": 0.3377465009689331, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.364371657371521, + "sampling/importance_sampling_ratio/mean": 0.9997440576553345, + "sampling/importance_sampling_ratio/min": 0.6323453187942505, + "sampling/sampling_logp_difference/max": 0.45831966400146484, + "sampling/sampling_logp_difference/mean": 0.012213868089020252, + "step": 383 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00015898891433607787, + "clip_ratio/low_min": 0.00015898891433607787, + "clip_ratio/region_mean": 0.00015898891433607787, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 720.0, + "completions/mean_length": 549.4375, + "completions/mean_terminated_length": 488.239990234375, + "completions/min_length": 334.0, + "completions/min_terminated_length": 334.0, + "entropy": 0.46953893080353737, + "epoch": 0.20556745182012848, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.006220671813935041, + "learning_rate": 1e-05, + "loss": -0.0059, + "num_tokens": 8250524.0, + "reward": 0.59375, + "reward_std": 0.2041158676147461, + "rewards/accuracy_reward/mean": 0.59375, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.3521066904067993, + "sampling/importance_sampling_ratio/mean": 0.9997193813323975, + "sampling/importance_sampling_ratio/min": 0.6970633268356323, + "sampling/sampling_logp_difference/max": 0.36087894439697266, + "sampling/sampling_logp_difference/mean": 0.014271720312535763, + "step": 384 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 9.784736175788566e-05, + "clip_ratio/low_min": 9.784736175788566e-05, + "clip_ratio/region_mean": 9.784736175788566e-05, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 764.0, + "completions/mean_length": 594.21875, + "completions/mean_terminated_length": 554.1154174804688, + "completions/min_length": 336.0, + "completions/min_terminated_length": 336.0, + "entropy": 0.3626043200492859, + "epoch": 0.20610278372591007, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.005539539735764265, + "learning_rate": 1e-05, + "loss": 0.0681, + "num_tokens": 8273387.0, + "reward": 0.5625, + "reward_std": 0.3335031569004059, + "rewards/accuracy_reward/mean": 0.5625, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 1.3743711709976196, + "sampling/importance_sampling_ratio/mean": 1.0000483989715576, + "sampling/importance_sampling_ratio/min": 0.713853120803833, + "sampling/sampling_logp_difference/max": 0.3370780944824219, + "sampling/sampling_logp_difference/mean": 0.01183868758380413, + "step": 385 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00013117517664795741, + "clip_ratio/low_min": 0.00013117517664795741, + "clip_ratio/region_mean": 0.00013117517664795741, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 723.0, + "completions/mean_length": 493.78125, + "completions/mean_terminated_length": 484.9354553222656, + "completions/min_length": 248.0, + "completions/min_terminated_length": 248.0, + "entropy": 0.367743544280529, + "epoch": 0.20663811563169165, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.015349820256233215, + "learning_rate": 1e-05, + "loss": -0.0061, + "num_tokens": 8292796.0, + "reward": 0.59375, + "reward_std": 0.3787454068660736, + "rewards/accuracy_reward/mean": 0.59375, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.6753965616226196, + "sampling/importance_sampling_ratio/mean": 0.9997833967208862, + "sampling/importance_sampling_ratio/min": 0.7073305249214172, + "sampling/sampling_logp_difference/max": 0.516049861907959, + "sampling/sampling_logp_difference/mean": 0.011631221510469913, + "step": 386 + }, + { + "clip_ratio/high_max": 5.792400406789966e-05, + "clip_ratio/high_mean": 5.792400406789966e-05, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 5.792400406789966e-05, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 762.0, + "completions/mean_length": 559.3125, + "completions/mean_terminated_length": 500.8799743652344, + "completions/min_length": 179.0, + "completions/min_terminated_length": 179.0, + "entropy": 0.3384836260229349, + "epoch": 0.20717344753747324, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.016733651980757713, + "learning_rate": 1e-05, + "loss": 0.0209, + "num_tokens": 8314382.0, + "reward": 0.40625, + "reward_std": 0.24511480331420898, + "rewards/accuracy_reward/mean": 0.40625, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.7089895009994507, + "sampling/importance_sampling_ratio/mean": 1.0001564025878906, + "sampling/importance_sampling_ratio/min": 0.2909180521965027, + "sampling/sampling_logp_difference/max": 1.2347136735916138, + "sampling/sampling_logp_difference/mean": 0.010874203406274319, + "step": 387 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 761.0, + "completions/mean_length": 553.59375, + "completions/mean_terminated_length": 504.11541748046875, + "completions/min_length": 222.0, + "completions/min_terminated_length": 222.0, + "entropy": 0.3786693848669529, + "epoch": 0.20770877944325483, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.004186462145298719, + "learning_rate": 1e-05, + "loss": 0.0287, + "num_tokens": 8335961.0, + "reward": 0.65625, + "reward_std": 0.2041158676147461, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.6064395904541016, + "sampling/importance_sampling_ratio/mean": 1.0000622272491455, + "sampling/importance_sampling_ratio/min": 0.6084471344947815, + "sampling/sampling_logp_difference/max": 0.4968452453613281, + "sampling/sampling_logp_difference/mean": 0.011787356808781624, + "step": 388 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 4.708097912953235e-05, + "clip_ratio/low_min": 4.708097912953235e-05, + "clip_ratio/region_mean": 4.708097912953235e-05, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 732.0, + "completions/mean_length": 517.6875, + "completions/mean_terminated_length": 481.9285888671875, + "completions/min_length": 229.0, + "completions/min_terminated_length": 229.0, + "entropy": 0.3944498412311077, + "epoch": 0.2082441113490364, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.007791217882186174, + "learning_rate": 1e-05, + "loss": 0.0828, + "num_tokens": 8355847.0, + "reward": 0.59375, + "reward_std": 0.4218915104866028, + "rewards/accuracy_reward/mean": 0.59375, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.2994695901870728, + "sampling/importance_sampling_ratio/mean": 0.9997397065162659, + "sampling/importance_sampling_ratio/min": 0.7719744443893433, + "sampling/sampling_logp_difference/max": 0.26195621490478516, + "sampling/sampling_logp_difference/mean": 0.011835463345050812, + "step": 389 + }, + { + "clip_ratio/high_max": 7.318500865949318e-05, + "clip_ratio/high_mean": 7.318500865949318e-05, + "clip_ratio/low_mean": 5.441880784928799e-05, + "clip_ratio/low_min": 5.441880784928799e-05, + "clip_ratio/region_mean": 0.00012760381650878116, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 720.0, + "completions/mean_length": 499.84375, + "completions/mean_terminated_length": 424.7599792480469, + "completions/min_length": 146.0, + "completions/min_terminated_length": 146.0, + "entropy": 0.35400139540433884, + "epoch": 0.20877944325481798, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.005581251811236143, + "learning_rate": 1e-05, + "loss": 0.0019, + "num_tokens": 8375642.0, + "reward": 0.5625, + "reward_std": 0.292504221200943, + "rewards/accuracy_reward/mean": 0.5625, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 1.4042198657989502, + "sampling/importance_sampling_ratio/mean": 1.0000568628311157, + "sampling/importance_sampling_ratio/min": 0.6956577897071838, + "sampling/sampling_logp_difference/max": 0.3628973960876465, + "sampling/sampling_logp_difference/mean": 0.012154696509242058, + "step": 390 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 651.0, + "completions/mean_length": 431.71875, + "completions/mean_terminated_length": 420.8709411621094, + "completions/min_length": 191.0, + "completions/min_terminated_length": 191.0, + "entropy": 0.3674179017543793, + "epoch": 0.20931477516059957, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.007779018487781286, + "learning_rate": 1e-05, + "loss": 0.0409, + "num_tokens": 8392905.0, + "reward": 0.5, + "reward_std": 0.2314550280570984, + "rewards/accuracy_reward/mean": 0.5, + "rewards/accuracy_reward/std": 0.5080004930496216, + "sampling/importance_sampling_ratio/max": 1.2777687311172485, + "sampling/importance_sampling_ratio/mean": 0.9992403984069824, + "sampling/importance_sampling_ratio/min": 0.5686952471733093, + "sampling/sampling_logp_difference/max": 0.5644105672836304, + "sampling/sampling_logp_difference/mean": 0.01222455594688654, + "step": 391 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 729.0, + "completions/mean_length": 440.09375, + "completions/mean_terminated_length": 429.51611328125, + "completions/min_length": 219.0, + "completions/min_terminated_length": 219.0, + "entropy": 0.4279603026807308, + "epoch": 0.20985010706638116, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.012629266828298569, + "learning_rate": 1e-05, + "loss": 0.0188, + "num_tokens": 8410556.0, + "reward": 0.875, + "reward_std": 0.13363061845302582, + "rewards/accuracy_reward/mean": 0.875, + "rewards/accuracy_reward/std": 0.33601075410842896, + "sampling/importance_sampling_ratio/max": 1.569334864616394, + "sampling/importance_sampling_ratio/mean": 0.9999269843101501, + "sampling/importance_sampling_ratio/min": 0.5857587456703186, + "sampling/sampling_logp_difference/max": 0.5348472595214844, + "sampling/sampling_logp_difference/mean": 0.013155401684343815, + "step": 392 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 768.0, + "completions/max_terminated_length": 753.0, + "completions/mean_length": 566.34375, + "completions/mean_terminated_length": 499.125, + "completions/min_length": 266.0, + "completions/min_terminated_length": 266.0, + "entropy": 0.5298543311655521, + "epoch": 0.21038543897216275, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.005383187904953957, + "learning_rate": 1e-05, + "loss": 0.0662, + "num_tokens": 8432511.0, + "reward": 0.625, + "reward_std": 0.2177756428718567, + "rewards/accuracy_reward/mean": 0.625, + "rewards/accuracy_reward/std": 0.49186936020851135, + "sampling/importance_sampling_ratio/max": 1.8232184648513794, + "sampling/importance_sampling_ratio/mean": 0.9999746680259705, + "sampling/importance_sampling_ratio/min": 0.47589820623397827, + "sampling/sampling_logp_difference/max": 0.742551326751709, + "sampling/sampling_logp_difference/mean": 0.015069807879626751, + "step": 393 + }, + { + "clip_ratio/high_max": 7.163323607528582e-05, + "clip_ratio/high_mean": 7.163323607528582e-05, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 7.163323607528582e-05, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 740.0, + "completions/mean_length": 510.53125, + "completions/mean_terminated_length": 483.89654541015625, + "completions/min_length": 293.0, + "completions/min_terminated_length": 293.0, + "entropy": 0.5083822384476662, + "epoch": 0.21092077087794434, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.011491530574858189, + "learning_rate": 1e-05, + "loss": 0.0172, + "num_tokens": 8452752.0, + "reward": 0.65625, + "reward_std": 0.4628904461860657, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.39710533618927, + "sampling/importance_sampling_ratio/mean": 1.00003182888031, + "sampling/importance_sampling_ratio/min": 0.6281613707542419, + "sampling/sampling_logp_difference/max": 0.46495819091796875, + "sampling/sampling_logp_difference/mean": 0.01529776118695736, + "step": 394 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 762.0, + "completions/mean_length": 532.5625, + "completions/mean_terminated_length": 498.9285888671875, + "completions/min_length": 286.0, + "completions/min_terminated_length": 286.0, + "entropy": 0.3021841011941433, + "epoch": 0.2114561027837259, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.0022210676688700914, + "learning_rate": 1e-05, + "loss": 0.0027, + "num_tokens": 8474066.0, + "reward": 0.90625, + "reward_std": 0.2041158676147461, + "rewards/accuracy_reward/mean": 0.90625, + "rewards/accuracy_reward/std": 0.2961445748806, + "sampling/importance_sampling_ratio/max": 1.4248532056808472, + "sampling/importance_sampling_ratio/mean": 1.000141978263855, + "sampling/importance_sampling_ratio/min": 0.6691626310348511, + "sampling/sampling_logp_difference/max": 0.40172815322875977, + "sampling/sampling_logp_difference/mean": 0.01092034112662077, + "step": 395 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 6.994963769102469e-05, + "clip_ratio/low_min": 6.994963769102469e-05, + "clip_ratio/region_mean": 6.994963769102469e-05, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 722.0, + "completions/mean_length": 537.15625, + "completions/mean_terminated_length": 513.27587890625, + "completions/min_length": 341.0, + "completions/min_terminated_length": 341.0, + "entropy": 0.28629919327795506, + "epoch": 0.21199143468950749, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.008062201552093029, + "learning_rate": 1e-05, + "loss": 0.0084, + "num_tokens": 8495055.0, + "reward": 0.65625, + "reward_std": 0.4218915104866028, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.4054651260375977, + "sampling/importance_sampling_ratio/mean": 0.9998922944068909, + "sampling/importance_sampling_ratio/min": 0.697835385799408, + "sampling/sampling_logp_difference/max": 0.3597719669342041, + "sampling/sampling_logp_difference/mean": 0.01047180313616991, + "step": 396 + }, + { + "clip_ratio/high_max": 6.33874224149622e-05, + "clip_ratio/high_mean": 6.33874224149622e-05, + "clip_ratio/low_mean": 6.091617979109287e-05, + "clip_ratio/low_min": 6.091617979109287e-05, + "clip_ratio/region_mean": 0.00012430360220605507, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 701.0, + "completions/mean_length": 437.625, + "completions/mean_terminated_length": 426.9677429199219, + "completions/min_length": 168.0, + "completions/min_terminated_length": 168.0, + "entropy": 0.4095449633896351, + "epoch": 0.21252676659528907, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": 0.0227, + "num_tokens": 8513019.0, + "reward": 0.65625, + "reward_std": 0.1293872892856598, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.421972632408142, + "sampling/importance_sampling_ratio/mean": 0.9997876286506653, + "sampling/importance_sampling_ratio/min": 0.6967024803161621, + "sampling/sampling_logp_difference/max": 0.36139678955078125, + "sampling/sampling_logp_difference/mean": 0.012393293902277946, + "step": 397 + }, + { + "clip_ratio/high_max": 5.1292572607053444e-05, + "clip_ratio/high_mean": 5.1292572607053444e-05, + "clip_ratio/low_mean": 0.00012902263915748335, + "clip_ratio/low_min": 0.00012902263915748335, + "clip_ratio/region_mean": 0.0001803152117645368, + "completions/clipped_ratio": 0.25, + "completions/max_length": 768.0, + "completions/max_terminated_length": 762.0, + "completions/mean_length": 544.46875, + "completions/mean_terminated_length": 469.9583435058594, + "completions/min_length": 192.0, + "completions/min_terminated_length": 192.0, + "entropy": 0.4786128140985966, + "epoch": 0.21306209850107066, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.005949270911514759, + "learning_rate": 1e-05, + "loss": 0.1088, + "num_tokens": 8535058.0, + "reward": 0.59375, + "reward_std": 0.4218915104866028, + "rewards/accuracy_reward/mean": 0.59375, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.7323617935180664, + "sampling/importance_sampling_ratio/mean": 1.0003728866577148, + "sampling/importance_sampling_ratio/min": 0.7132600545883179, + "sampling/sampling_logp_difference/max": 0.5494856834411621, + "sampling/sampling_logp_difference/mean": 0.014721707440912724, + "step": 398 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 729.0, + "completions/mean_length": 531.15625, + "completions/mean_terminated_length": 497.3214416503906, + "completions/min_length": 254.0, + "completions/min_terminated_length": 254.0, + "entropy": 0.4673791192471981, + "epoch": 0.21359743040685225, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.011197878047823906, + "learning_rate": 1e-05, + "loss": 0.0846, + "num_tokens": 8555911.0, + "reward": 0.59375, + "reward_std": 0.22201895713806152, + "rewards/accuracy_reward/mean": 0.59375, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.3088711500167847, + "sampling/importance_sampling_ratio/mean": 0.9998037815093994, + "sampling/importance_sampling_ratio/min": 0.7026304006576538, + "sampling/sampling_logp_difference/max": 0.3529243469238281, + "sampling/sampling_logp_difference/mean": 0.013329234905540943, + "step": 399 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 722.0, + "completions/max_terminated_length": 722.0, + "completions/mean_length": 493.53125, + "completions/mean_terminated_length": 493.53125, + "completions/min_length": 281.0, + "completions/min_terminated_length": 281.0, + "entropy": 0.31775000505149364, + "epoch": 0.21413276231263384, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.011138384230434895, + "learning_rate": 1e-05, + "loss": 0.0623, + "num_tokens": 8575296.0, + "reward": 0.59375, + "reward_std": 0.3608423173427582, + "rewards/accuracy_reward/mean": 0.59375, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.3432323932647705, + "sampling/importance_sampling_ratio/mean": 0.9998267889022827, + "sampling/importance_sampling_ratio/min": 0.6971604228019714, + "sampling/sampling_logp_difference/max": 0.36073970794677734, + "sampling/sampling_logp_difference/mean": 0.010663994587957859, + "step": 400 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.390254591475241e-05, + "clip_ratio/low_min": 5.390254591475241e-05, + "clip_ratio/region_mean": 5.390254591475241e-05, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 734.0, + "completions/mean_length": 539.09375, + "completions/mean_terminated_length": 531.7096557617188, + "completions/min_length": 298.0, + "completions/min_terminated_length": 298.0, + "entropy": 0.2961991000920534, + "epoch": 0.21466809421841543, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.00423061940819025, + "learning_rate": 1e-05, + "loss": 0.0515, + "num_tokens": 8596011.0, + "reward": 0.875, + "reward_std": 0.2925041913986206, + "rewards/accuracy_reward/mean": 0.875, + "rewards/accuracy_reward/std": 0.33601075410842896, + "sampling/importance_sampling_ratio/max": 1.3731248378753662, + "sampling/importance_sampling_ratio/mean": 0.9998635649681091, + "sampling/importance_sampling_ratio/min": 0.6961231827735901, + "sampling/sampling_logp_difference/max": 0.3622286319732666, + "sampling/sampling_logp_difference/mean": 0.010276572778820992, + "step": 401 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 656.0, + "completions/mean_length": 422.84375, + "completions/mean_terminated_length": 411.70965576171875, + "completions/min_length": 231.0, + "completions/min_terminated_length": 231.0, + "entropy": 0.40191569551825523, + "epoch": 0.215203426124197, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.01045410055667162, + "learning_rate": 1e-05, + "loss": 0.004, + "num_tokens": 8612758.0, + "reward": 0.5625, + "reward_std": 0.2177756428718567, + "rewards/accuracy_reward/mean": 0.5625, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 1.458146095275879, + "sampling/importance_sampling_ratio/mean": 1.000295877456665, + "sampling/importance_sampling_ratio/min": 0.7797122597694397, + "sampling/sampling_logp_difference/max": 0.3771657943725586, + "sampling/sampling_logp_difference/mean": 0.01333489827811718, + "step": 402 + }, + { + "clip_ratio/high_max": 8.344459638465196e-05, + "clip_ratio/high_mean": 8.344459638465196e-05, + "clip_ratio/low_mean": 5.926979793002829e-05, + "clip_ratio/low_min": 5.926979793002829e-05, + "clip_ratio/region_mean": 0.00014271439431468025, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 719.0, + "completions/mean_length": 538.03125, + "completions/mean_terminated_length": 484.9615478515625, + "completions/min_length": 232.0, + "completions/min_terminated_length": 232.0, + "entropy": 0.34591904282569885, + "epoch": 0.21573875802997858, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.011563985608518124, + "learning_rate": 1e-05, + "loss": -0.0105, + "num_tokens": 8633623.0, + "reward": 0.6875, + "reward_std": 0.3335031569004059, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.3086165189743042, + "sampling/importance_sampling_ratio/mean": 0.9997710585594177, + "sampling/importance_sampling_ratio/min": 0.6319659352302551, + "sampling/sampling_logp_difference/max": 0.4589197635650635, + "sampling/sampling_logp_difference/mean": 0.011281891725957394, + "step": 403 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00012117157530155964, + "clip_ratio/low_min": 0.00012117157530155964, + "clip_ratio/region_mean": 0.00012117157530155964, + "completions/clipped_ratio": 0.0, + "completions/max_length": 724.0, + "completions/max_terminated_length": 724.0, + "completions/mean_length": 470.75, + "completions/mean_terminated_length": 470.75, + "completions/min_length": 179.0, + "completions/min_terminated_length": 179.0, + "entropy": 0.4018666222691536, + "epoch": 0.21627408993576017, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.010249024257063866, + "learning_rate": 1e-05, + "loss": 0.0037, + "num_tokens": 8652103.0, + "reward": 0.8125, + "reward_std": 0.249358132481575, + "rewards/accuracy_reward/mean": 0.8125, + "rewards/accuracy_reward/std": 0.3965577781200409, + "sampling/importance_sampling_ratio/max": 1.3936488628387451, + "sampling/importance_sampling_ratio/mean": 0.9999995827674866, + "sampling/importance_sampling_ratio/min": 0.6596998572349548, + "sampling/sampling_logp_difference/max": 0.4159703254699707, + "sampling/sampling_logp_difference/mean": 0.011900828219950199, + "step": 404 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 596.0, + "completions/mean_length": 474.90625, + "completions/mean_terminated_length": 465.45159912109375, + "completions/min_length": 312.0, + "completions/min_terminated_length": 312.0, + "entropy": 0.3899528533220291, + "epoch": 0.21680942184154176, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.006876398343592882, + "learning_rate": 1e-05, + "loss": 0.0244, + "num_tokens": 8671068.0, + "reward": 0.90625, + "reward_std": 0.2041158676147461, + "rewards/accuracy_reward/mean": 0.90625, + "rewards/accuracy_reward/std": 0.2961445748806, + "sampling/importance_sampling_ratio/max": 1.4994722604751587, + "sampling/importance_sampling_ratio/mean": 1.0000793933868408, + "sampling/importance_sampling_ratio/min": 0.5886525511741638, + "sampling/sampling_logp_difference/max": 0.5299191474914551, + "sampling/sampling_logp_difference/mean": 0.012460133992135525, + "step": 405 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 6.0386471886886284e-05, + "clip_ratio/low_min": 6.0386471886886284e-05, + "clip_ratio/region_mean": 6.0386471886886284e-05, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 694.0, + "completions/mean_length": 434.53125, + "completions/mean_terminated_length": 423.774169921875, + "completions/min_length": 200.0, + "completions/min_terminated_length": 200.0, + "entropy": 0.43231403082609177, + "epoch": 0.21734475374732334, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.012683926150202751, + "learning_rate": 1e-05, + "loss": -0.0279, + "num_tokens": 8688589.0, + "reward": 0.53125, + "reward_std": 0.378745436668396, + "rewards/accuracy_reward/mean": 0.53125, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.3972766399383545, + "sampling/importance_sampling_ratio/mean": 0.9998230338096619, + "sampling/importance_sampling_ratio/min": 0.7335515022277832, + "sampling/sampling_logp_difference/max": 0.33452510833740234, + "sampling/sampling_logp_difference/mean": 0.012979553081095219, + "step": 406 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 765.0, + "completions/mean_length": 536.46875, + "completions/mean_terminated_length": 521.0333862304688, + "completions/min_length": 297.0, + "completions/min_terminated_length": 297.0, + "entropy": 0.36300141736865044, + "epoch": 0.21788008565310493, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.011247067712247372, + "learning_rate": 1e-05, + "loss": 0.0251, + "num_tokens": 8709852.0, + "reward": 0.59375, + "reward_std": 0.3987956643104553, + "rewards/accuracy_reward/mean": 0.59375, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.4313021898269653, + "sampling/importance_sampling_ratio/mean": 1.0001040697097778, + "sampling/importance_sampling_ratio/min": 0.6243442296981812, + "sampling/sampling_logp_difference/max": 0.4710533618927002, + "sampling/sampling_logp_difference/mean": 0.010918519459664822, + "step": 407 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0002676450021681376, + "clip_ratio/low_min": 0.0002676450021681376, + "clip_ratio/region_mean": 0.0002676450021681376, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 764.0, + "completions/mean_length": 558.375, + "completions/mean_terminated_length": 510.0000305175781, + "completions/min_length": 265.0, + "completions/min_terminated_length": 265.0, + "entropy": 0.34893396869301796, + "epoch": 0.21841541755888652, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.01758604310452938, + "learning_rate": 1e-05, + "loss": 0.0475, + "num_tokens": 8731064.0, + "reward": 0.71875, + "reward_std": 0.2041158676147461, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.296985387802124, + "sampling/importance_sampling_ratio/mean": 0.9999855160713196, + "sampling/importance_sampling_ratio/min": 0.6867310404777527, + "sampling/sampling_logp_difference/max": 0.3758125305175781, + "sampling/sampling_logp_difference/mean": 0.010774402879178524, + "step": 408 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00021035455574747175, + "clip_ratio/low_min": 0.00021035455574747175, + "clip_ratio/region_mean": 0.00021035455574747175, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 758.0, + "completions/mean_length": 492.59375, + "completions/mean_terminated_length": 415.47998046875, + "completions/min_length": 215.0, + "completions/min_terminated_length": 215.0, + "entropy": 0.6372664421796799, + "epoch": 0.21895074946466808, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.025644909590482712, + "learning_rate": 1e-05, + "loss": 0.0392, + "num_tokens": 8750483.0, + "reward": 0.375, + "reward_std": 0.3745020925998688, + "rewards/accuracy_reward/mean": 0.375, + "rewards/accuracy_reward/std": 0.49186936020851135, + "sampling/importance_sampling_ratio/max": 1.5307456254959106, + "sampling/importance_sampling_ratio/mean": 0.9998341798782349, + "sampling/importance_sampling_ratio/min": 0.6213971972465515, + "sampling/sampling_logp_difference/max": 0.4757847785949707, + "sampling/sampling_logp_difference/mean": 0.017566129565238953, + "step": 409 + }, + { + "clip_ratio/high_max": 5.830223744851537e-05, + "clip_ratio/high_mean": 5.830223744851537e-05, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 5.830223744851537e-05, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 767.0, + "completions/mean_length": 527.75, + "completions/mean_terminated_length": 511.7333679199219, + "completions/min_length": 294.0, + "completions/min_terminated_length": 294.0, + "entropy": 0.4586259126663208, + "epoch": 0.21948608137044967, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.015534854494035244, + "learning_rate": 1e-05, + "loss": -0.017, + "num_tokens": 8771459.0, + "reward": 0.625, + "reward_std": 0.4629100561141968, + "rewards/accuracy_reward/mean": 0.625, + "rewards/accuracy_reward/std": 0.49186936020851135, + "sampling/importance_sampling_ratio/max": 1.3773834705352783, + "sampling/importance_sampling_ratio/mean": 0.9997857809066772, + "sampling/importance_sampling_ratio/min": 0.6078636646270752, + "sampling/sampling_logp_difference/max": 0.4978046417236328, + "sampling/sampling_logp_difference/mean": 0.01344833429902792, + "step": 410 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 4.5938992116134614e-05, + "clip_ratio/low_min": 4.5938992116134614e-05, + "clip_ratio/region_mean": 4.5938992116134614e-05, + "completions/clipped_ratio": 0.375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 711.0, + "completions/mean_length": 607.65625, + "completions/mean_terminated_length": 511.45001220703125, + "completions/min_length": 138.0, + "completions/min_terminated_length": 138.0, + "entropy": 0.7120710089802742, + "epoch": 0.22002141327623126, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.014514628797769547, + "learning_rate": 1e-05, + "loss": 0.0509, + "num_tokens": 8795328.0, + "reward": 0.375, + "reward_std": 0.2314550280570984, + "rewards/accuracy_reward/mean": 0.375, + "rewards/accuracy_reward/std": 0.49186936020851135, + "sampling/importance_sampling_ratio/max": 2.0, + "sampling/importance_sampling_ratio/mean": 1.0008165836334229, + "sampling/importance_sampling_ratio/min": 0.7169780731201172, + "sampling/sampling_logp_difference/max": 0.7082998752593994, + "sampling/sampling_logp_difference/mean": 0.017513783648610115, + "step": 411 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 756.0, + "completions/mean_length": 597.875, + "completions/mean_terminated_length": 566.370361328125, + "completions/min_length": 348.0, + "completions/min_terminated_length": 348.0, + "entropy": 0.47999877482652664, + "epoch": 0.22055674518201285, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.013548356480896473, + "learning_rate": 1e-05, + "loss": 0.0417, + "num_tokens": 8818004.0, + "reward": 0.59375, + "reward_std": 0.1293872892856598, + "rewards/accuracy_reward/mean": 0.59375, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.6668263673782349, + "sampling/importance_sampling_ratio/mean": 0.9998438954353333, + "sampling/importance_sampling_ratio/min": 0.6898931264877319, + "sampling/sampling_logp_difference/max": 0.5109214782714844, + "sampling/sampling_logp_difference/mean": 0.014204300940036774, + "step": 412 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0001049721431627404, + "clip_ratio/low_min": 0.0001049721431627404, + "clip_ratio/region_mean": 0.0001049721431627404, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 720.0, + "completions/mean_length": 552.375, + "completions/mean_terminated_length": 502.61541748046875, + "completions/min_length": 246.0, + "completions/min_terminated_length": 246.0, + "entropy": 0.545205969363451, + "epoch": 0.22109207708779444, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.01512906327843666, + "learning_rate": 1e-05, + "loss": 0.0318, + "num_tokens": 8840008.0, + "reward": 0.625, + "reward_std": 0.292504221200943, + "rewards/accuracy_reward/mean": 0.625, + "rewards/accuracy_reward/std": 0.49186936020851135, + "sampling/importance_sampling_ratio/max": 1.3915132284164429, + "sampling/importance_sampling_ratio/mean": 0.999944269657135, + "sampling/importance_sampling_ratio/min": 0.777584433555603, + "sampling/sampling_logp_difference/max": 0.33039188385009766, + "sampling/sampling_logp_difference/mean": 0.012664245441555977, + "step": 413 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 768.0, + "completions/max_terminated_length": 608.0, + "completions/mean_length": 515.875, + "completions/mean_terminated_length": 431.8333435058594, + "completions/min_length": 291.0, + "completions/min_terminated_length": 291.0, + "entropy": 0.3869321160018444, + "epoch": 0.22162740899357602, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.0113857202231884, + "learning_rate": 1e-05, + "loss": 0.0105, + "num_tokens": 8860116.0, + "reward": 0.46875, + "reward_std": 0.3198433816432953, + "rewards/accuracy_reward/mean": 0.46875, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.4158447980880737, + "sampling/importance_sampling_ratio/mean": 1.0004041194915771, + "sampling/importance_sampling_ratio/min": 0.6664666533470154, + "sampling/sampling_logp_difference/max": 0.405765175819397, + "sampling/sampling_logp_difference/mean": 0.012336570769548416, + "step": 414 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00019005159265361726, + "clip_ratio/low_min": 0.00019005159265361726, + "clip_ratio/region_mean": 0.00019005159265361726, + "completions/clipped_ratio": 0.34375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 705.0, + "completions/mean_length": 643.03125, + "completions/mean_terminated_length": 577.5714111328125, + "completions/min_length": 427.0, + "completions/min_terminated_length": 427.0, + "entropy": 0.4384213499724865, + "epoch": 0.2221627408993576, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.006695670075714588, + "learning_rate": 1e-05, + "loss": 0.0061, + "num_tokens": 8884813.0, + "reward": 0.3125, + "reward_std": 0.4082317352294922, + "rewards/accuracy_reward/mean": 0.3125, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.500514030456543, + "sampling/importance_sampling_ratio/mean": 0.9998024106025696, + "sampling/importance_sampling_ratio/min": 0.6033892631530762, + "sampling/sampling_logp_difference/max": 0.505192756652832, + "sampling/sampling_logp_difference/mean": 0.012900110334157944, + "step": 415 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00010464841761859134, + "clip_ratio/low_min": 0.00010464841761859134, + "clip_ratio/region_mean": 0.00010464841761859134, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 693.0, + "completions/mean_length": 557.1875, + "completions/mean_terminated_length": 508.5384826660156, + "completions/min_length": 238.0, + "completions/min_terminated_length": 238.0, + "entropy": 0.5661557614803314, + "epoch": 0.22269807280513917, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.008390724658966064, + "learning_rate": 1e-05, + "loss": -0.0036, + "num_tokens": 8906075.0, + "reward": 0.71875, + "reward_std": 0.2041158676147461, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.381516933441162, + "sampling/importance_sampling_ratio/mean": 0.9999334216117859, + "sampling/importance_sampling_ratio/min": 0.6935417056083679, + "sampling/sampling_logp_difference/max": 0.36594390869140625, + "sampling/sampling_logp_difference/mean": 0.014079151675105095, + "step": 416 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 6.592826684936881e-05, + "clip_ratio/low_min": 6.592826684936881e-05, + "clip_ratio/region_mean": 6.592826684936881e-05, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 751.0, + "completions/mean_length": 525.78125, + "completions/mean_terminated_length": 469.8846435546875, + "completions/min_length": 135.0, + "completions/min_terminated_length": 135.0, + "entropy": 0.5091826841235161, + "epoch": 0.22323340471092076, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.006459562107920647, + "learning_rate": 1e-05, + "loss": 0.0079, + "num_tokens": 8926860.0, + "reward": 0.65625, + "reward_std": 0.2041158676147461, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.4588712453842163, + "sampling/importance_sampling_ratio/mean": 1.0000801086425781, + "sampling/importance_sampling_ratio/min": 0.5136346817016602, + "sampling/sampling_logp_difference/max": 0.6662429571151733, + "sampling/sampling_logp_difference/mean": 0.01519847009330988, + "step": 417 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00013404825585894287, + "clip_ratio/low_min": 0.00013404825585894287, + "clip_ratio/region_mean": 0.00013404825585894287, + "completions/clipped_ratio": 0.34375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 622.0, + "completions/mean_length": 562.625, + "completions/mean_terminated_length": 455.0476379394531, + "completions/min_length": 345.0, + "completions/min_terminated_length": 345.0, + "entropy": 0.577263955026865, + "epoch": 0.22376873661670235, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.006603385787457228, + "learning_rate": 1e-05, + "loss": 0.0551, + "num_tokens": 8949544.0, + "reward": 0.46875, + "reward_std": 0.2041158676147461, + "rewards/accuracy_reward/mean": 0.46875, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.3305896520614624, + "sampling/importance_sampling_ratio/mean": 0.9996999502182007, + "sampling/importance_sampling_ratio/min": 0.6139088869094849, + "sampling/sampling_logp_difference/max": 0.4879087209701538, + "sampling/sampling_logp_difference/mean": 0.016703732311725616, + "step": 418 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00021168820967432112, + "clip_ratio/low_min": 0.00021168820967432112, + "clip_ratio/region_mean": 0.00021168820967432112, + "completions/clipped_ratio": 0.34375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 694.0, + "completions/mean_length": 593.21875, + "completions/mean_terminated_length": 501.66668701171875, + "completions/min_length": 288.0, + "completions/min_terminated_length": 288.0, + "entropy": 0.45473482087254524, + "epoch": 0.22430406852248394, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.021779675036668777, + "learning_rate": 1e-05, + "loss": 0.0528, + "num_tokens": 8972319.0, + "reward": 0.59375, + "reward_std": 0.3471629321575165, + "rewards/accuracy_reward/mean": 0.59375, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.4410138130187988, + "sampling/importance_sampling_ratio/mean": 0.9995977878570557, + "sampling/importance_sampling_ratio/min": 0.5860862135887146, + "sampling/sampling_logp_difference/max": 0.5342884063720703, + "sampling/sampling_logp_difference/mean": 0.013226356357336044, + "step": 419 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 763.0, + "completions/mean_length": 577.1875, + "completions/mean_terminated_length": 533.1538696289062, + "completions/min_length": 317.0, + "completions/min_terminated_length": 317.0, + "entropy": 0.35899868980050087, + "epoch": 0.22483940042826553, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.006980070844292641, + "learning_rate": 1e-05, + "loss": 0.102, + "num_tokens": 8994421.0, + "reward": 0.625, + "reward_std": 0.3335031569004059, + "rewards/accuracy_reward/mean": 0.625, + "rewards/accuracy_reward/std": 0.49186936020851135, + "sampling/importance_sampling_ratio/max": 1.3777568340301514, + "sampling/importance_sampling_ratio/mean": 0.9996013641357422, + "sampling/importance_sampling_ratio/min": 0.6605517864227295, + "sampling/sampling_logp_difference/max": 0.41467976570129395, + "sampling/sampling_logp_difference/mean": 0.011273551732301712, + "step": 420 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0001349163503618911, + "clip_ratio/low_min": 0.0001349163503618911, + "clip_ratio/region_mean": 0.0001349163503618911, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 742.0, + "completions/mean_length": 579.34375, + "completions/mean_terminated_length": 544.4074096679688, + "completions/min_length": 317.0, + "completions/min_terminated_length": 317.0, + "entropy": 0.4214400425553322, + "epoch": 0.22537473233404712, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.01015987154096365, + "learning_rate": 1e-05, + "loss": 0.0362, + "num_tokens": 9016784.0, + "reward": 0.53125, + "reward_std": 0.4765698313713074, + "rewards/accuracy_reward/mean": 0.53125, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.442196249961853, + "sampling/importance_sampling_ratio/mean": 1.0000865459442139, + "sampling/importance_sampling_ratio/min": 0.6789780259132385, + "sampling/sampling_logp_difference/max": 0.38716650009155273, + "sampling/sampling_logp_difference/mean": 0.012763852253556252, + "step": 421 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0001649361729505472, + "clip_ratio/low_min": 0.0001649361729505472, + "clip_ratio/region_mean": 0.0001649361729505472, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 765.0, + "completions/mean_length": 540.9375, + "completions/mean_terminated_length": 517.4483032226562, + "completions/min_length": 306.0, + "completions/min_terminated_length": 306.0, + "entropy": 0.46569474786520004, + "epoch": 0.2259100642398287, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.013653812929987907, + "learning_rate": 1e-05, + "loss": -0.0051, + "num_tokens": 9038358.0, + "reward": 0.59375, + "reward_std": 0.3061639964580536, + "rewards/accuracy_reward/mean": 0.59375, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.4148699045181274, + "sampling/importance_sampling_ratio/mean": 0.999466061592102, + "sampling/importance_sampling_ratio/min": 0.6482210755348206, + "sampling/sampling_logp_difference/max": 0.43352341651916504, + "sampling/sampling_logp_difference/mean": 0.013205149210989475, + "step": 422 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 766.0, + "completions/mean_length": 446.875, + "completions/mean_terminated_length": 425.4666748046875, + "completions/min_length": 163.0, + "completions/min_terminated_length": 163.0, + "entropy": 0.4123445525765419, + "epoch": 0.22644539614561027, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.0027087004855275154, + "learning_rate": 1e-05, + "loss": -0.0216, + "num_tokens": 9056074.0, + "reward": 0.71875, + "reward_std": 0.0883883461356163, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.374659538269043, + "sampling/importance_sampling_ratio/mean": 0.9997585415840149, + "sampling/importance_sampling_ratio/min": 0.7262780666351318, + "sampling/sampling_logp_difference/max": 0.3198223114013672, + "sampling/sampling_logp_difference/mean": 0.012693490833044052, + "step": 423 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 6.067961294320412e-05, + "clip_ratio/low_min": 6.067961294320412e-05, + "clip_ratio/region_mean": 6.067961294320412e-05, + "completions/clipped_ratio": 0.3125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 757.0, + "completions/mean_length": 578.9375, + "completions/mean_terminated_length": 493.0, + "completions/min_length": 193.0, + "completions/min_terminated_length": 193.0, + "entropy": 0.44280479848384857, + "epoch": 0.22698072805139186, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.010412955656647682, + "learning_rate": 1e-05, + "loss": 0.0155, + "num_tokens": 9078776.0, + "reward": 0.65625, + "reward_std": 0.22201895713806152, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.4163012504577637, + "sampling/importance_sampling_ratio/mean": 1.0002821683883667, + "sampling/importance_sampling_ratio/min": 0.3965229392051697, + "sampling/sampling_logp_difference/max": 0.9250214099884033, + "sampling/sampling_logp_difference/mean": 0.012683121487498283, + "step": 424 + }, + { + "clip_ratio/high_max": 6.403688166756183e-05, + "clip_ratio/high_mean": 6.403688166756183e-05, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 6.403688166756183e-05, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 744.0, + "completions/mean_length": 550.84375, + "completions/mean_terminated_length": 519.8214721679688, + "completions/min_length": 269.0, + "completions/min_terminated_length": 269.0, + "entropy": 0.3804459273815155, + "epoch": 0.22751605995717344, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.0024838228709995747, + "learning_rate": 1e-05, + "loss": 0.0207, + "num_tokens": 9099835.0, + "reward": 0.71875, + "reward_std": 0.0883883461356163, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.4718800783157349, + "sampling/importance_sampling_ratio/mean": 1.0002789497375488, + "sampling/importance_sampling_ratio/min": 0.6511872410774231, + "sampling/sampling_logp_difference/max": 0.42895805835723877, + "sampling/sampling_logp_difference/mean": 0.012384253554046154, + "step": 425 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.924170545767993e-05, + "clip_ratio/low_min": 5.924170545767993e-05, + "clip_ratio/region_mean": 5.924170545767993e-05, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 726.0, + "completions/mean_length": 535.75, + "completions/mean_terminated_length": 502.5714416503906, + "completions/min_length": 252.0, + "completions/min_terminated_length": 252.0, + "entropy": 0.29265643656253815, + "epoch": 0.22805139186295503, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.012128359638154507, + "learning_rate": 1e-05, + "loss": -0.0059, + "num_tokens": 9120731.0, + "reward": 0.75, + "reward_std": 0.26726123690605164, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.426264762878418, + "sampling/importance_sampling_ratio/mean": 0.9998127222061157, + "sampling/importance_sampling_ratio/min": 0.695228636264801, + "sampling/sampling_logp_difference/max": 0.36351442337036133, + "sampling/sampling_logp_difference/mean": 0.010166767984628677, + "step": 426 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 6.175889575388283e-05, + "clip_ratio/low_min": 6.175889575388283e-05, + "clip_ratio/region_mean": 6.175889575388283e-05, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 759.0, + "completions/mean_length": 533.40625, + "completions/mean_terminated_length": 479.2692565917969, + "completions/min_length": 269.0, + "completions/min_terminated_length": 269.0, + "entropy": 0.41359881311655045, + "epoch": 0.22858672376873662, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": 0.0279, + "num_tokens": 9141696.0, + "reward": 0.65625, + "reward_std": 0.1293872892856598, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.388490915298462, + "sampling/importance_sampling_ratio/mean": 0.9996965527534485, + "sampling/importance_sampling_ratio/min": 0.6807646155357361, + "sampling/sampling_logp_difference/max": 0.3845386505126953, + "sampling/sampling_logp_difference/mean": 0.012603675946593285, + "step": 427 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.418292130343616e-05, + "clip_ratio/low_min": 5.418292130343616e-05, + "clip_ratio/region_mean": 5.418292130343616e-05, + "completions/clipped_ratio": 0.0, + "completions/max_length": 744.0, + "completions/max_terminated_length": 744.0, + "completions/mean_length": 461.34375, + "completions/mean_terminated_length": 461.34375, + "completions/min_length": 221.0, + "completions/min_terminated_length": 221.0, + "entropy": 0.3670461028814316, + "epoch": 0.2291220556745182, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.008405312895774841, + "learning_rate": 1e-05, + "loss": -0.0129, + "num_tokens": 9160003.0, + "reward": 0.6875, + "reward_std": 0.3335031569004059, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.3555933237075806, + "sampling/importance_sampling_ratio/mean": 1.0002845525741577, + "sampling/importance_sampling_ratio/min": 0.6224300861358643, + "sampling/sampling_logp_difference/max": 0.4741239547729492, + "sampling/sampling_logp_difference/mean": 0.012463947758078575, + "step": 428 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 689.0, + "completions/mean_length": 556.90625, + "completions/mean_terminated_length": 508.19232177734375, + "completions/min_length": 302.0, + "completions/min_terminated_length": 302.0, + "entropy": 0.35653635300695896, + "epoch": 0.2296573875802998, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.031224975362420082, + "learning_rate": 1e-05, + "loss": 0.0507, + "num_tokens": 9181712.0, + "reward": 0.5625, + "reward_std": 0.3335031569004059, + "rewards/accuracy_reward/mean": 0.5625, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 1.421380877494812, + "sampling/importance_sampling_ratio/mean": 1.000276803970337, + "sampling/importance_sampling_ratio/min": 0.6906217932701111, + "sampling/sampling_logp_difference/max": 0.3701629638671875, + "sampling/sampling_logp_difference/mean": 0.011170031502842903, + "step": 429 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 635.0, + "completions/mean_length": 421.78125, + "completions/mean_terminated_length": 410.6128845214844, + "completions/min_length": 191.0, + "completions/min_terminated_length": 191.0, + "entropy": 0.38883281126618385, + "epoch": 0.23019271948608136, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.0073440903797745705, + "learning_rate": 1e-05, + "loss": 0.0618, + "num_tokens": 9198737.0, + "reward": 0.8125, + "reward_std": 0.2587745785713196, + "rewards/accuracy_reward/mean": 0.8125, + "rewards/accuracy_reward/std": 0.3965577781200409, + "sampling/importance_sampling_ratio/max": 1.3637741804122925, + "sampling/importance_sampling_ratio/mean": 1.000226616859436, + "sampling/importance_sampling_ratio/min": 0.659216582775116, + "sampling/sampling_logp_difference/max": 0.4167032241821289, + "sampling/sampling_logp_difference/mean": 0.01179863978177309, + "step": 430 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 694.0, + "completions/mean_length": 475.125, + "completions/mean_terminated_length": 433.2857360839844, + "completions/min_length": 189.0, + "completions/min_terminated_length": 189.0, + "entropy": 0.44823867455124855, + "epoch": 0.23072805139186295, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.009986692108213902, + "learning_rate": 1e-05, + "loss": 0.0481, + "num_tokens": 9217573.0, + "reward": 0.6875, + "reward_std": 0.4492306709289551, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.271128535270691, + "sampling/importance_sampling_ratio/mean": 1.0000205039978027, + "sampling/importance_sampling_ratio/min": 0.6460689306259155, + "sampling/sampling_logp_difference/max": 0.43684911727905273, + "sampling/sampling_logp_difference/mean": 0.01257572416216135, + "step": 431 + }, + { + "clip_ratio/high_max": 0.00011637931675068103, + "clip_ratio/high_mean": 0.00011637931675068103, + "clip_ratio/low_mean": 6.053268589312211e-05, + "clip_ratio/low_min": 6.053268589312211e-05, + "clip_ratio/region_mean": 0.00017691200264380313, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 759.0, + "completions/mean_length": 529.46875, + "completions/mean_terminated_length": 513.5667114257812, + "completions/min_length": 316.0, + "completions/min_terminated_length": 316.0, + "entropy": 0.42199476435780525, + "epoch": 0.23126338329764454, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.004911072552204132, + "learning_rate": 1e-05, + "loss": -0.0107, + "num_tokens": 9238444.0, + "reward": 0.8125, + "reward_std": 0.2177756428718567, + "rewards/accuracy_reward/mean": 0.8125, + "rewards/accuracy_reward/std": 0.3965577781200409, + "sampling/importance_sampling_ratio/max": 1.4605900049209595, + "sampling/importance_sampling_ratio/mean": 1.0000569820404053, + "sampling/importance_sampling_ratio/min": 0.6634466052055359, + "sampling/sampling_logp_difference/max": 0.4103069305419922, + "sampling/sampling_logp_difference/mean": 0.012371168471872807, + "step": 432 + }, + { + "clip_ratio/high_max": 5.587840860243887e-05, + "clip_ratio/high_mean": 5.587840860243887e-05, + "clip_ratio/low_mean": 4.570384044200182e-05, + "clip_ratio/low_min": 4.570384044200182e-05, + "clip_ratio/region_mean": 0.00010158224904444069, + "completions/clipped_ratio": 0.3125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 737.0, + "completions/mean_length": 625.03125, + "completions/mean_terminated_length": 560.0454711914062, + "completions/min_length": 323.0, + "completions/min_terminated_length": 323.0, + "entropy": 0.29868466779589653, + "epoch": 0.23179871520342613, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.008090803399682045, + "learning_rate": 1e-05, + "loss": 0.0503, + "num_tokens": 9262277.0, + "reward": 0.5625, + "reward_std": 0.3514062464237213, + "rewards/accuracy_reward/mean": 0.5625, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 1.2700191736221313, + "sampling/importance_sampling_ratio/mean": 0.9997373223304749, + "sampling/importance_sampling_ratio/min": 0.7011478543281555, + "sampling/sampling_logp_difference/max": 0.35503649711608887, + "sampling/sampling_logp_difference/mean": 0.009685932658612728, + "step": 433 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.34375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 763.0, + "completions/mean_length": 612.3125, + "completions/mean_terminated_length": 530.7619018554688, + "completions/min_length": 205.0, + "completions/min_terminated_length": 205.0, + "entropy": 0.5361443646252155, + "epoch": 0.23233404710920771, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.006447171326726675, + "learning_rate": 1e-05, + "loss": 0.0005, + "num_tokens": 9285511.0, + "reward": 0.40625, + "reward_std": 0.1293872892856598, + "rewards/accuracy_reward/mean": 0.40625, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.6125704050064087, + "sampling/importance_sampling_ratio/mean": 0.9995403289794922, + "sampling/importance_sampling_ratio/min": 0.45170289278030396, + "sampling/sampling_logp_difference/max": 0.7947306632995605, + "sampling/sampling_logp_difference/mean": 0.014863980002701283, + "step": 434 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 659.0, + "completions/max_terminated_length": 659.0, + "completions/mean_length": 437.78125, + "completions/mean_terminated_length": 437.78125, + "completions/min_length": 272.0, + "completions/min_terminated_length": 272.0, + "entropy": 0.4513319730758667, + "epoch": 0.2328693790149893, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.011153290048241615, + "learning_rate": 1e-05, + "loss": -0.0011, + "num_tokens": 9303280.0, + "reward": 0.90625, + "reward_std": 0.1293872892856598, + "rewards/accuracy_reward/mean": 0.90625, + "rewards/accuracy_reward/std": 0.2961445748806, + "sampling/importance_sampling_ratio/max": 1.4422086477279663, + "sampling/importance_sampling_ratio/mean": 1.0000663995742798, + "sampling/importance_sampling_ratio/min": 0.6965842247009277, + "sampling/sampling_logp_difference/max": 0.36617565155029297, + "sampling/sampling_logp_difference/mean": 0.012972470372915268, + "step": 435 + }, + { + "clip_ratio/high_max": 5.9580554079730064e-05, + "clip_ratio/high_mean": 5.9580554079730064e-05, + "clip_ratio/low_mean": 0.0001631690247450024, + "clip_ratio/low_min": 0.0001631690247450024, + "clip_ratio/region_mean": 0.00022274957882473245, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 714.0, + "completions/mean_length": 554.59375, + "completions/mean_terminated_length": 515.0740966796875, + "completions/min_length": 322.0, + "completions/min_terminated_length": 322.0, + "entropy": 0.4475087486207485, + "epoch": 0.2334047109207709, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.008825229480862617, + "learning_rate": 1e-05, + "loss": 0.1122, + "num_tokens": 9324755.0, + "reward": 0.53125, + "reward_std": 0.4218915104866028, + "rewards/accuracy_reward/mean": 0.53125, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.4316606521606445, + "sampling/importance_sampling_ratio/mean": 0.9998408555984497, + "sampling/importance_sampling_ratio/min": 0.6648994088172913, + "sampling/sampling_logp_difference/max": 0.40811944007873535, + "sampling/sampling_logp_difference/mean": 0.012117262929677963, + "step": 436 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 756.0, + "completions/mean_length": 527.34375, + "completions/mean_terminated_length": 482.77777099609375, + "completions/min_length": 202.0, + "completions/min_terminated_length": 202.0, + "entropy": 0.2966017406433821, + "epoch": 0.23394004282655245, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.010183276608586311, + "learning_rate": 1e-05, + "loss": 0.0377, + "num_tokens": 9344862.0, + "reward": 0.6875, + "reward_std": 0.47655022144317627, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.282387375831604, + "sampling/importance_sampling_ratio/mean": 1.000028371810913, + "sampling/importance_sampling_ratio/min": 0.6990740299224854, + "sampling/sampling_logp_difference/max": 0.3579986095428467, + "sampling/sampling_logp_difference/mean": 0.009799963794648647, + "step": 437 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 768.0, + "completions/max_terminated_length": 602.0, + "completions/mean_length": 522.25, + "completions/mean_terminated_length": 440.3333435058594, + "completions/min_length": 298.0, + "completions/min_terminated_length": 298.0, + "entropy": 0.3842923603951931, + "epoch": 0.23447537473233404, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.0069622802548110485, + "learning_rate": 1e-05, + "loss": 0.0322, + "num_tokens": 9365838.0, + "reward": 0.53125, + "reward_std": 0.2041158676147461, + "rewards/accuracy_reward/mean": 0.53125, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.5250685214996338, + "sampling/importance_sampling_ratio/mean": 1.0001485347747803, + "sampling/importance_sampling_ratio/min": 0.6450940370559692, + "sampling/sampling_logp_difference/max": 0.43835926055908203, + "sampling/sampling_logp_difference/mean": 0.011143209412693977, + "step": 438 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 6.325911090243608e-05, + "clip_ratio/low_min": 6.325911090243608e-05, + "clip_ratio/region_mean": 6.325911090243608e-05, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 629.0, + "completions/mean_length": 459.875, + "completions/mean_terminated_length": 449.9354553222656, + "completions/min_length": 206.0, + "completions/min_terminated_length": 206.0, + "entropy": 0.35073431953787804, + "epoch": 0.23501070663811563, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.012068981304764748, + "learning_rate": 1e-05, + "loss": 0.0094, + "num_tokens": 9384074.0, + "reward": 0.78125, + "reward_std": 0.2630178928375244, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.4477179050445557, + "sampling/importance_sampling_ratio/mean": 0.9997803568840027, + "sampling/importance_sampling_ratio/min": 0.5760422945022583, + "sampling/sampling_logp_difference/max": 0.5515742301940918, + "sampling/sampling_logp_difference/mean": 0.011955702677369118, + "step": 439 + }, + { + "clip_ratio/high_max": 5.910165418754332e-05, + "clip_ratio/high_mean": 5.910165418754332e-05, + "clip_ratio/low_mean": 0.00018559396994533017, + "clip_ratio/low_min": 0.00018559396994533017, + "clip_ratio/region_mean": 0.0002446956241328735, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 716.0, + "completions/mean_length": 519.8125, + "completions/mean_terminated_length": 473.85186767578125, + "completions/min_length": 264.0, + "completions/min_terminated_length": 264.0, + "entropy": 0.403911329805851, + "epoch": 0.23554603854389722, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.012297307141125202, + "learning_rate": 1e-05, + "loss": 0.0837, + "num_tokens": 9404580.0, + "reward": 0.59375, + "reward_std": 0.3966485261917114, + "rewards/accuracy_reward/mean": 0.59375, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.2885267734527588, + "sampling/importance_sampling_ratio/mean": 0.9999293684959412, + "sampling/importance_sampling_ratio/min": 0.7447843551635742, + "sampling/sampling_logp_difference/max": 0.2946605682373047, + "sampling/sampling_logp_difference/mean": 0.012314294464886189, + "step": 440 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 745.0, + "completions/mean_length": 441.90625, + "completions/mean_terminated_length": 420.16668701171875, + "completions/min_length": 219.0, + "completions/min_terminated_length": 219.0, + "entropy": 0.43974559381604195, + "epoch": 0.2360813704496788, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.0028224484995007515, + "learning_rate": 1e-05, + "loss": 0.0734, + "num_tokens": 9422121.0, + "reward": 0.65625, + "reward_std": 0.2041158676147461, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.3143641948699951, + "sampling/importance_sampling_ratio/mean": 0.9996448755264282, + "sampling/importance_sampling_ratio/min": 0.6420266628265381, + "sampling/sampling_logp_difference/max": 0.44312548637390137, + "sampling/sampling_logp_difference/mean": 0.014007256366312504, + "step": 441 + }, + { + "clip_ratio/high_max": 6.027000927133486e-05, + "clip_ratio/high_mean": 6.027000927133486e-05, + "clip_ratio/low_mean": 6.027000927133486e-05, + "clip_ratio/low_min": 6.027000927133486e-05, + "clip_ratio/region_mean": 0.00012054001854266971, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 762.0, + "completions/mean_length": 535.21875, + "completions/mean_terminated_length": 527.7096557617188, + "completions/min_length": 273.0, + "completions/min_terminated_length": 273.0, + "entropy": 0.3419300578534603, + "epoch": 0.2366167023554604, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.006149868480861187, + "learning_rate": 1e-05, + "loss": 0.0248, + "num_tokens": 9443168.0, + "reward": 0.78125, + "reward_std": 0.3061639964580536, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.4472057819366455, + "sampling/importance_sampling_ratio/mean": 0.9997043013572693, + "sampling/importance_sampling_ratio/min": 0.6846362352371216, + "sampling/sampling_logp_difference/max": 0.37886762619018555, + "sampling/sampling_logp_difference/mean": 0.011754102073609829, + "step": 442 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.718206739402376e-05, + "clip_ratio/low_min": 5.718206739402376e-05, + "clip_ratio/region_mean": 5.718206739402376e-05, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 697.0, + "completions/mean_length": 532.875, + "completions/mean_terminated_length": 499.2857360839844, + "completions/min_length": 277.0, + "completions/min_terminated_length": 277.0, + "entropy": 0.4245343003422022, + "epoch": 0.23715203426124196, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.029914360493421555, + "learning_rate": 1e-05, + "loss": 0.0002, + "num_tokens": 9463580.0, + "reward": 0.8125, + "reward_std": 0.2177756428718567, + "rewards/accuracy_reward/mean": 0.8125, + "rewards/accuracy_reward/std": 0.3965577781200409, + "sampling/importance_sampling_ratio/max": 1.3767448663711548, + "sampling/importance_sampling_ratio/mean": 1.0006240606307983, + "sampling/importance_sampling_ratio/min": 0.6944502592086792, + "sampling/sampling_logp_difference/max": 0.36463475227355957, + "sampling/sampling_logp_difference/mean": 0.01291370764374733, + "step": 443 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 648.0, + "completions/mean_length": 436.125, + "completions/mean_terminated_length": 425.4193420410156, + "completions/min_length": 226.0, + "completions/min_terminated_length": 226.0, + "entropy": 0.3653532285243273, + "epoch": 0.23768736616702354, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": 0.013, + "num_tokens": 9481056.0, + "reward": 0.875, + "reward_std": 0.13363061845302582, + "rewards/accuracy_reward/mean": 0.875, + "rewards/accuracy_reward/std": 0.33601075410842896, + "sampling/importance_sampling_ratio/max": 1.454010009765625, + "sampling/importance_sampling_ratio/mean": 0.9999028444290161, + "sampling/importance_sampling_ratio/min": 0.5122338533401489, + "sampling/sampling_logp_difference/max": 0.6689740419387817, + "sampling/sampling_logp_difference/mean": 0.011901103891432285, + "step": 444 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.8548008382786065e-05, + "clip_ratio/low_min": 5.8548008382786065e-05, + "clip_ratio/region_mean": 5.8548008382786065e-05, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 756.0, + "completions/mean_length": 572.0625, + "completions/mean_terminated_length": 551.7930908203125, + "completions/min_length": 296.0, + "completions/min_terminated_length": 296.0, + "entropy": 0.30460595712065697, + "epoch": 0.23822269807280513, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.011782202869653702, + "learning_rate": 1e-05, + "loss": 0.0696, + "num_tokens": 9502930.0, + "reward": 0.84375, + "reward_std": 0.3198433816432953, + "rewards/accuracy_reward/mean": 0.84375, + "rewards/accuracy_reward/std": 0.3689020276069641, + "sampling/importance_sampling_ratio/max": 1.3174147605895996, + "sampling/importance_sampling_ratio/mean": 0.9998862147331238, + "sampling/importance_sampling_ratio/min": 0.4322245717048645, + "sampling/sampling_logp_difference/max": 0.8388099670410156, + "sampling/sampling_logp_difference/mean": 0.01028999499976635, + "step": 445 + }, + { + "clip_ratio/high_max": 5.460900138132274e-05, + "clip_ratio/high_mean": 5.460900138132274e-05, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 5.460900138132274e-05, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 718.0, + "completions/mean_length": 520.9375, + "completions/mean_terminated_length": 504.4667053222656, + "completions/min_length": 294.0, + "completions/min_terminated_length": 294.0, + "entropy": 0.31242641247808933, + "epoch": 0.23875802997858672, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.007522552274167538, + "learning_rate": 1e-05, + "loss": 0.0914, + "num_tokens": 9523456.0, + "reward": 0.875, + "reward_std": 0.292504221200943, + "rewards/accuracy_reward/mean": 0.875, + "rewards/accuracy_reward/std": 0.33601075410842896, + "sampling/importance_sampling_ratio/max": 1.4370394945144653, + "sampling/importance_sampling_ratio/mean": 0.9999423623085022, + "sampling/importance_sampling_ratio/min": 0.6511749029159546, + "sampling/sampling_logp_difference/max": 0.42897701263427734, + "sampling/sampling_logp_difference/mean": 0.010918579064309597, + "step": 446 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00011043973427149467, + "clip_ratio/low_min": 0.00011043973427149467, + "clip_ratio/region_mean": 0.00011043973427149467, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 738.0, + "completions/mean_length": 569.28125, + "completions/mean_terminated_length": 562.8709716796875, + "completions/min_length": 357.0, + "completions/min_terminated_length": 357.0, + "entropy": 0.33172520995140076, + "epoch": 0.2392933618843683, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.0036930718924850225, + "learning_rate": 1e-05, + "loss": 0.0097, + "num_tokens": 9545521.0, + "reward": 0.6875, + "reward_std": 0.292504221200943, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.4489433765411377, + "sampling/importance_sampling_ratio/mean": 1.0001510381698608, + "sampling/importance_sampling_ratio/min": 0.6575350761413574, + "sampling/sampling_logp_difference/max": 0.41925716400146484, + "sampling/sampling_logp_difference/mean": 0.011404524557292461, + "step": 447 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 724.0, + "completions/mean_length": 486.84375, + "completions/mean_terminated_length": 477.774169921875, + "completions/min_length": 293.0, + "completions/min_terminated_length": 293.0, + "entropy": 0.33927514031529427, + "epoch": 0.2398286937901499, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.005942976102232933, + "learning_rate": 1e-05, + "loss": 0.0153, + "num_tokens": 9565124.0, + "reward": 0.8125, + "reward_std": 0.249358132481575, + "rewards/accuracy_reward/mean": 0.8125, + "rewards/accuracy_reward/std": 0.3965577781200409, + "sampling/importance_sampling_ratio/max": 1.4997282028198242, + "sampling/importance_sampling_ratio/mean": 0.9999534487724304, + "sampling/importance_sampling_ratio/min": 0.6569938063621521, + "sampling/sampling_logp_difference/max": 0.42008066177368164, + "sampling/sampling_logp_difference/mean": 0.011414112523198128, + "step": 448 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.256518124951981e-05, + "clip_ratio/low_min": 5.256518124951981e-05, + "clip_ratio/region_mean": 5.256518124951981e-05, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 735.0, + "completions/mean_length": 550.125, + "completions/mean_terminated_length": 509.77777099609375, + "completions/min_length": 241.0, + "completions/min_terminated_length": 241.0, + "entropy": 0.4012523554265499, + "epoch": 0.2403640256959315, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.020486872643232346, + "learning_rate": 1e-05, + "loss": 0.0501, + "num_tokens": 9586368.0, + "reward": 0.71875, + "reward_std": 0.3608423173427582, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.4344967603683472, + "sampling/importance_sampling_ratio/mean": 1.00015127658844, + "sampling/importance_sampling_ratio/min": 0.5887377858161926, + "sampling/sampling_logp_difference/max": 0.5297744274139404, + "sampling/sampling_logp_difference/mean": 0.012876774184405804, + "step": 449 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 748.0, + "completions/mean_length": 509.75, + "completions/mean_terminated_length": 472.857177734375, + "completions/min_length": 302.0, + "completions/min_terminated_length": 302.0, + "entropy": 0.3603517934679985, + "epoch": 0.24089935760171305, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.016826961189508438, + "learning_rate": 1e-05, + "loss": 0.0223, + "num_tokens": 9606056.0, + "reward": 0.9375, + "reward_std": 0.1157275140285492, + "rewards/accuracy_reward/mean": 0.9375, + "rewards/accuracy_reward/std": 0.24593468010425568, + "sampling/importance_sampling_ratio/max": 1.350253701210022, + "sampling/importance_sampling_ratio/mean": 0.9999425411224365, + "sampling/importance_sampling_ratio/min": 0.6953261494636536, + "sampling/sampling_logp_difference/max": 0.3633742332458496, + "sampling/sampling_logp_difference/mean": 0.011786971241235733, + "step": 450 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 7.094211468938738e-05, + "clip_ratio/low_min": 7.094211468938738e-05, + "clip_ratio/region_mean": 7.094211468938738e-05, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 677.0, + "completions/mean_length": 507.84375, + "completions/mean_terminated_length": 499.45159912109375, + "completions/min_length": 353.0, + "completions/min_terminated_length": 353.0, + "entropy": 0.3076478410512209, + "epoch": 0.24143468950749464, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.0046203541569411755, + "learning_rate": 1e-05, + "loss": -0.0137, + "num_tokens": 9625915.0, + "reward": 0.84375, + "reward_std": 0.22201895713806152, + "rewards/accuracy_reward/mean": 0.84375, + "rewards/accuracy_reward/std": 0.3689020276069641, + "sampling/importance_sampling_ratio/max": 1.342712163925171, + "sampling/importance_sampling_ratio/mean": 0.9998106956481934, + "sampling/importance_sampling_ratio/min": 0.687568187713623, + "sampling/sampling_logp_difference/max": 0.37459421157836914, + "sampling/sampling_logp_difference/mean": 0.010666599497199059, + "step": 451 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 6.188119004946202e-05, + "clip_ratio/low_min": 6.188119004946202e-05, + "clip_ratio/region_mean": 6.188119004946202e-05, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 762.0, + "completions/mean_length": 421.4375, + "completions/mean_terminated_length": 398.3333435058594, + "completions/min_length": 223.0, + "completions/min_terminated_length": 223.0, + "entropy": 0.3537568338215351, + "epoch": 0.24197002141327623, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.011190691962838173, + "learning_rate": 1e-05, + "loss": -0.0793, + "num_tokens": 9642721.0, + "reward": 0.6875, + "reward_std": 0.2177756428718567, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.3687958717346191, + "sampling/importance_sampling_ratio/mean": 0.9997307062149048, + "sampling/importance_sampling_ratio/min": 0.7300587296485901, + "sampling/sampling_logp_difference/max": 0.31463027000427246, + "sampling/sampling_logp_difference/mean": 0.011748920194804668, + "step": 452 + }, + { + "clip_ratio/high_max": 5.193186370888725e-05, + "clip_ratio/high_mean": 5.193186370888725e-05, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 5.193186370888725e-05, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 711.0, + "completions/mean_length": 530.03125, + "completions/mean_terminated_length": 485.96295166015625, + "completions/min_length": 261.0, + "completions/min_terminated_length": 261.0, + "entropy": 0.4937206022441387, + "epoch": 0.24250535331905781, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.04192846640944481, + "learning_rate": 1e-05, + "loss": 0.0909, + "num_tokens": 9663274.0, + "reward": 0.71875, + "reward_std": 0.4218915104866028, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.475556492805481, + "sampling/importance_sampling_ratio/mean": 0.9997404217720032, + "sampling/importance_sampling_ratio/min": 0.6958999037742615, + "sampling/sampling_logp_difference/max": 0.3890352249145508, + "sampling/sampling_logp_difference/mean": 0.0142365712672472, + "step": 453 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 6.160670454846695e-05, + "clip_ratio/low_min": 6.160670454846695e-05, + "clip_ratio/region_mean": 6.160670454846695e-05, + "completions/clipped_ratio": 0.3125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 749.0, + "completions/mean_length": 636.46875, + "completions/mean_terminated_length": 576.6818237304688, + "completions/min_length": 330.0, + "completions/min_terminated_length": 330.0, + "entropy": 0.3610347770154476, + "epoch": 0.2430406852248394, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.014986919239163399, + "learning_rate": 1e-05, + "loss": 0.0566, + "num_tokens": 9687753.0, + "reward": 0.46875, + "reward_std": 0.378745436668396, + "rewards/accuracy_reward/mean": 0.46875, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.6268322467803955, + "sampling/importance_sampling_ratio/mean": 1.000038504600525, + "sampling/importance_sampling_ratio/min": 0.7378782033920288, + "sampling/sampling_logp_difference/max": 0.4866347312927246, + "sampling/sampling_logp_difference/mean": 0.011606104671955109, + "step": 454 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 8.884150884114206e-05, + "clip_ratio/low_min": 8.884150884114206e-05, + "clip_ratio/region_mean": 8.884150884114206e-05, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 743.0, + "completions/mean_length": 564.46875, + "completions/mean_terminated_length": 526.7777709960938, + "completions/min_length": 267.0, + "completions/min_terminated_length": 267.0, + "entropy": 0.44936030358076096, + "epoch": 0.243576017130621, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.009507955051958561, + "learning_rate": 1e-05, + "loss": 0.0201, + "num_tokens": 9710056.0, + "reward": 0.6875, + "reward_std": 0.3335031569004059, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.4472140073776245, + "sampling/importance_sampling_ratio/mean": 0.9998016953468323, + "sampling/importance_sampling_ratio/min": 0.6794170141220093, + "sampling/sampling_logp_difference/max": 0.3865201473236084, + "sampling/sampling_logp_difference/mean": 0.014226510189473629, + "step": 455 + }, + { + "clip_ratio/high_max": 6.157635652925819e-05, + "clip_ratio/high_mean": 6.157635652925819e-05, + "clip_ratio/low_mean": 0.0001727002309053205, + "clip_ratio/low_min": 0.0001727002309053205, + "clip_ratio/region_mean": 0.0002342765874345787, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 763.0, + "completions/mean_length": 587.75, + "completions/mean_terminated_length": 517.2174072265625, + "completions/min_length": 289.0, + "completions/min_terminated_length": 289.0, + "entropy": 0.4764968156814575, + "epoch": 0.24411134903640258, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.013710142113268375, + "learning_rate": 1e-05, + "loss": 0.0979, + "num_tokens": 9732720.0, + "reward": 0.5625, + "reward_std": 0.3945523500442505, + "rewards/accuracy_reward/mean": 0.5625, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 2.0, + "sampling/importance_sampling_ratio/mean": 0.9998855590820312, + "sampling/importance_sampling_ratio/min": 0.6379536986351013, + "sampling/sampling_logp_difference/max": 1.1404428482055664, + "sampling/sampling_logp_difference/mean": 0.014291264116764069, + "step": 456 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 657.0, + "completions/mean_length": 465.8125, + "completions/mean_terminated_length": 456.06451416015625, + "completions/min_length": 325.0, + "completions/min_terminated_length": 325.0, + "entropy": 0.33709372393786907, + "epoch": 0.24464668094218414, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.006640094332396984, + "learning_rate": 1e-05, + "loss": 0.0199, + "num_tokens": 9751570.0, + "reward": 0.71875, + "reward_std": 0.3198433816432953, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.4198437929153442, + "sampling/importance_sampling_ratio/mean": 0.9998038411140442, + "sampling/importance_sampling_ratio/min": 0.7205827832221985, + "sampling/sampling_logp_difference/max": 0.35054683685302734, + "sampling/sampling_logp_difference/mean": 0.011600518599152565, + "step": 457 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.747126488131471e-05, + "clip_ratio/low_min": 5.747126488131471e-05, + "clip_ratio/region_mean": 5.747126488131471e-05, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 718.0, + "completions/mean_length": 470.03125, + "completions/mean_terminated_length": 401.2692565917969, + "completions/min_length": 214.0, + "completions/min_terminated_length": 214.0, + "entropy": 0.3748924769461155, + "epoch": 0.24518201284796573, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.019087394699454308, + "learning_rate": 1e-05, + "loss": -0.0011, + "num_tokens": 9770083.0, + "reward": 0.8125, + "reward_std": 0.1157275140285492, + "rewards/accuracy_reward/mean": 0.8125, + "rewards/accuracy_reward/std": 0.3965577781200409, + "sampling/importance_sampling_ratio/max": 1.4516117572784424, + "sampling/importance_sampling_ratio/mean": 1.000480055809021, + "sampling/importance_sampling_ratio/min": 0.7660260200500488, + "sampling/sampling_logp_difference/max": 0.37267446517944336, + "sampling/sampling_logp_difference/mean": 0.012263812124729156, + "step": 458 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 8.229097875300795e-05, + "clip_ratio/low_min": 8.229097875300795e-05, + "clip_ratio/region_mean": 8.229097875300795e-05, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 628.0, + "completions/mean_length": 453.46875, + "completions/mean_terminated_length": 443.32257080078125, + "completions/min_length": 204.0, + "completions/min_terminated_length": 204.0, + "entropy": 0.3883685953915119, + "epoch": 0.24571734475374732, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.02708948403596878, + "learning_rate": 1e-05, + "loss": 0.0567, + "num_tokens": 9788242.0, + "reward": 0.84375, + "reward_std": 0.24511480331420898, + "rewards/accuracy_reward/mean": 0.84375, + "rewards/accuracy_reward/std": 0.3689020276069641, + "sampling/importance_sampling_ratio/max": 1.4311306476593018, + "sampling/importance_sampling_ratio/mean": 0.9997379183769226, + "sampling/importance_sampling_ratio/min": 0.6925398707389832, + "sampling/sampling_logp_difference/max": 0.367389440536499, + "sampling/sampling_logp_difference/mean": 0.011722617782652378, + "step": 459 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00011693237320287153, + "clip_ratio/low_min": 0.00011693237320287153, + "clip_ratio/region_mean": 0.00011693237320287153, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 702.0, + "completions/mean_length": 484.53125, + "completions/mean_terminated_length": 465.63336181640625, + "completions/min_length": 313.0, + "completions/min_terminated_length": 313.0, + "entropy": 0.3766840063035488, + "epoch": 0.2462526766595289, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.0047309487126767635, + "learning_rate": 1e-05, + "loss": 0.08, + "num_tokens": 9806971.0, + "reward": 0.875, + "reward_std": 0.2314550280570984, + "rewards/accuracy_reward/mean": 0.875, + "rewards/accuracy_reward/std": 0.33601075410842896, + "sampling/importance_sampling_ratio/max": 1.5795408487319946, + "sampling/importance_sampling_ratio/mean": 1.0000979900360107, + "sampling/importance_sampling_ratio/min": 0.716194748878479, + "sampling/sampling_logp_difference/max": 0.4571342468261719, + "sampling/sampling_logp_difference/mean": 0.011670341715216637, + "step": 460 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.55308761249762e-05, + "clip_ratio/low_min": 5.55308761249762e-05, + "clip_ratio/region_mean": 5.55308761249762e-05, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 672.0, + "completions/mean_length": 498.59375, + "completions/mean_terminated_length": 480.63336181640625, + "completions/min_length": 276.0, + "completions/min_terminated_length": 276.0, + "entropy": 0.34036383405327797, + "epoch": 0.2467880085653105, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.013544617220759392, + "learning_rate": 1e-05, + "loss": 0.0187, + "num_tokens": 9826606.0, + "reward": 0.75, + "reward_std": 0.2587745785713196, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.3156120777130127, + "sampling/importance_sampling_ratio/mean": 0.99959397315979, + "sampling/importance_sampling_ratio/min": 0.7834674715995789, + "sampling/sampling_logp_difference/max": 0.27430200576782227, + "sampling/sampling_logp_difference/mean": 0.011032242327928543, + "step": 461 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 745.0, + "completions/mean_length": 510.09375, + "completions/mean_terminated_length": 492.9000244140625, + "completions/min_length": 307.0, + "completions/min_terminated_length": 307.0, + "entropy": 0.3001006357371807, + "epoch": 0.24732334047109208, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.007335676345974207, + "learning_rate": 1e-05, + "loss": 0.049, + "num_tokens": 9846777.0, + "reward": 0.875, + "reward_std": 0.2177756428718567, + "rewards/accuracy_reward/mean": 0.875, + "rewards/accuracy_reward/std": 0.33601075410842896, + "sampling/importance_sampling_ratio/max": 1.365645170211792, + "sampling/importance_sampling_ratio/mean": 0.9996696710586548, + "sampling/importance_sampling_ratio/min": 0.6994103193283081, + "sampling/sampling_logp_difference/max": 0.35751771926879883, + "sampling/sampling_logp_difference/mean": 0.01061292365193367, + "step": 462 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.7843590184347704e-05, + "clip_ratio/low_min": 5.7843590184347704e-05, + "clip_ratio/region_mean": 5.7843590184347704e-05, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 765.0, + "completions/mean_length": 574.71875, + "completions/mean_terminated_length": 538.9259033203125, + "completions/min_length": 330.0, + "completions/min_terminated_length": 330.0, + "entropy": 0.4043949991464615, + "epoch": 0.24785867237687367, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.007336011156439781, + "learning_rate": 1e-05, + "loss": 0.0149, + "num_tokens": 9869168.0, + "reward": 0.75, + "reward_std": 0.3335031569004059, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.3946592807769775, + "sampling/importance_sampling_ratio/mean": 0.9998060464859009, + "sampling/importance_sampling_ratio/min": 0.7594910860061646, + "sampling/sampling_logp_difference/max": 0.33265018463134766, + "sampling/sampling_logp_difference/mean": 0.0124510508030653, + "step": 463 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 9.449973003938794e-05, + "clip_ratio/low_min": 9.449973003938794e-05, + "clip_ratio/region_mean": 9.449973003938794e-05, + "completions/clipped_ratio": 0.5, + "completions/max_length": 768.0, + "completions/max_terminated_length": 761.0, + "completions/mean_length": 640.71875, + "completions/mean_terminated_length": 513.4375, + "completions/min_length": 267.0, + "completions/min_terminated_length": 267.0, + "entropy": 0.3673878461122513, + "epoch": 0.24839400428265523, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.002930504735559225, + "learning_rate": 1e-05, + "loss": 0.0505, + "num_tokens": 9894095.0, + "reward": 0.4375, + "reward_std": 0.2177756428718567, + "rewards/accuracy_reward/mean": 0.4375, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 1.408188819885254, + "sampling/importance_sampling_ratio/mean": 1.0000945329666138, + "sampling/importance_sampling_ratio/min": 0.6061339974403381, + "sampling/sampling_logp_difference/max": 0.5006542205810547, + "sampling/sampling_logp_difference/mean": 0.011721408925950527, + "step": 464 + }, + { + "clip_ratio/high_max": 5.408913784776814e-05, + "clip_ratio/high_mean": 5.408913784776814e-05, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 5.408913784776814e-05, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 686.0, + "completions/mean_length": 473.59375, + "completions/mean_terminated_length": 464.0967712402344, + "completions/min_length": 214.0, + "completions/min_terminated_length": 214.0, + "entropy": 0.305716123431921, + "epoch": 0.24892933618843682, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.01655631512403488, + "learning_rate": 1e-05, + "loss": -0.0151, + "num_tokens": 9912890.0, + "reward": 0.71875, + "reward_std": 0.35564959049224854, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.4440534114837646, + "sampling/importance_sampling_ratio/mean": 0.9998975992202759, + "sampling/importance_sampling_ratio/min": 0.45198091864585876, + "sampling/sampling_logp_difference/max": 0.7941153049468994, + "sampling/sampling_logp_difference/mean": 0.010904813185334206, + "step": 465 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0001181104889838025, + "clip_ratio/low_min": 0.0001181104889838025, + "clip_ratio/region_mean": 0.0001181104889838025, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 728.0, + "completions/mean_length": 562.28125, + "completions/mean_terminated_length": 481.7826232910156, + "completions/min_length": 277.0, + "completions/min_terminated_length": 277.0, + "entropy": 0.5426614657044411, + "epoch": 0.2494646680942184, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.0032838040497153997, + "learning_rate": 1e-05, + "loss": 0.0348, + "num_tokens": 9934723.0, + "reward": 0.15625, + "reward_std": 0.3198433816432953, + "rewards/accuracy_reward/mean": 0.15625, + "rewards/accuracy_reward/std": 0.3689020276069641, + "sampling/importance_sampling_ratio/max": 1.4727178812026978, + "sampling/importance_sampling_ratio/mean": 1.00029456615448, + "sampling/importance_sampling_ratio/min": 0.677629292011261, + "sampling/sampling_logp_difference/max": 0.38915491104125977, + "sampling/sampling_logp_difference/mean": 0.015486430376768112, + "step": 466 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 732.0, + "completions/mean_length": 563.625, + "completions/mean_terminated_length": 534.4285888671875, + "completions/min_length": 62.0, + "completions/min_terminated_length": 62.0, + "entropy": 0.3231046833097935, + "epoch": 0.25, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.017085576429963112, + "learning_rate": 1e-05, + "loss": 0.0034, + "num_tokens": 9956231.0, + "reward": 0.6875, + "reward_std": 0.4808131456375122, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.3188414573669434, + "sampling/importance_sampling_ratio/mean": 1.0000824928283691, + "sampling/importance_sampling_ratio/min": 0.7050997614860535, + "sampling/sampling_logp_difference/max": 0.34941601753234863, + "sampling/sampling_logp_difference/mean": 0.011022663675248623, + "step": 467 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0001145213027484715, + "clip_ratio/low_min": 0.0001145213027484715, + "clip_ratio/region_mean": 0.0001145213027484715, + "completions/clipped_ratio": 0.34375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 683.0, + "completions/mean_length": 584.28125, + "completions/mean_terminated_length": 488.0476379394531, + "completions/min_length": 252.0, + "completions/min_terminated_length": 252.0, + "entropy": 0.44339311495423317, + "epoch": 0.2505353319057816, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.017508333548903465, + "learning_rate": 1e-05, + "loss": 0.0851, + "num_tokens": 9978784.0, + "reward": 0.28125, + "reward_std": 0.24511480331420898, + "rewards/accuracy_reward/mean": 0.28125, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.358830213546753, + "sampling/importance_sampling_ratio/mean": 1.0002084970474243, + "sampling/importance_sampling_ratio/min": 0.7079991698265076, + "sampling/sampling_logp_difference/max": 0.34531235694885254, + "sampling/sampling_logp_difference/mean": 0.012901067733764648, + "step": 468 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00015160703333094716, + "clip_ratio/low_min": 0.00015160703333094716, + "clip_ratio/region_mean": 0.00015160703333094716, + "completions/clipped_ratio": 0.0, + "completions/max_length": 715.0, + "completions/max_terminated_length": 715.0, + "completions/mean_length": 433.3125, + "completions/mean_terminated_length": 433.3125, + "completions/min_length": 259.0, + "completions/min_terminated_length": 259.0, + "entropy": 0.45748865604400635, + "epoch": 0.2510706638115632, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.02379726991057396, + "learning_rate": 1e-05, + "loss": 0.0276, + "num_tokens": 9995986.0, + "reward": 0.6875, + "reward_std": 0.1157275140285492, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.419661045074463, + "sampling/importance_sampling_ratio/mean": 1.0002655982971191, + "sampling/importance_sampling_ratio/min": 0.6812078952789307, + "sampling/sampling_logp_difference/max": 0.38388776779174805, + "sampling/sampling_logp_difference/mean": 0.013905711472034454, + "step": 469 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00025374008691869676, + "clip_ratio/low_min": 0.00025374008691869676, + "clip_ratio/region_mean": 0.00025374008691869676, + "completions/clipped_ratio": 0.25, + "completions/max_length": 768.0, + "completions/max_terminated_length": 722.0, + "completions/mean_length": 561.1875, + "completions/mean_terminated_length": 492.25, + "completions/min_length": 178.0, + "completions/min_terminated_length": 178.0, + "entropy": 0.42329367622733116, + "epoch": 0.25160599571734477, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.010336724109947681, + "learning_rate": 1e-05, + "loss": 0.036, + "num_tokens": 10017960.0, + "reward": 0.65625, + "reward_std": 0.24511480331420898, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.559314489364624, + "sampling/importance_sampling_ratio/mean": 0.999816358089447, + "sampling/importance_sampling_ratio/min": 0.6663103699684143, + "sampling/sampling_logp_difference/max": 0.4442462921142578, + "sampling/sampling_logp_difference/mean": 0.013187477365136147, + "step": 470 + }, + { + "clip_ratio/high_max": 5.02008042531088e-05, + "clip_ratio/high_mean": 5.02008042531088e-05, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 5.02008042531088e-05, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 742.0, + "completions/mean_length": 537.3125, + "completions/mean_terminated_length": 504.357177734375, + "completions/min_length": 300.0, + "completions/min_terminated_length": 300.0, + "entropy": 0.3696693442761898, + "epoch": 0.25214132762312635, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.0031507830135524273, + "learning_rate": 1e-05, + "loss": 0.0262, + "num_tokens": 10038626.0, + "reward": 0.78125, + "reward_std": 0.2041158676147461, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.314389705657959, + "sampling/importance_sampling_ratio/mean": 0.9997232556343079, + "sampling/importance_sampling_ratio/min": 0.6920733451843262, + "sampling/sampling_logp_difference/max": 0.3680633306503296, + "sampling/sampling_logp_difference/mean": 0.011672900058329105, + "step": 471 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0002072493007290177, + "clip_ratio/low_min": 0.0002072493007290177, + "clip_ratio/region_mean": 0.0002072493007290177, + "completions/clipped_ratio": 0.40625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 707.0, + "completions/mean_length": 639.59375, + "completions/mean_terminated_length": 551.73681640625, + "completions/min_length": 250.0, + "completions/min_terminated_length": 250.0, + "entropy": 0.4280319772660732, + "epoch": 0.25267665952890794, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.004263668786734343, + "learning_rate": 1e-05, + "loss": 0.0685, + "num_tokens": 10062741.0, + "reward": 0.53125, + "reward_std": 0.3377465009689331, + "rewards/accuracy_reward/mean": 0.53125, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.4454926252365112, + "sampling/importance_sampling_ratio/mean": 1.0005178451538086, + "sampling/importance_sampling_ratio/min": 0.7545243501663208, + "sampling/sampling_logp_difference/max": 0.3684501647949219, + "sampling/sampling_logp_difference/mean": 0.01289775688201189, + "step": 472 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 6.917543214512989e-05, + "clip_ratio/low_min": 6.917543214512989e-05, + "clip_ratio/region_mean": 6.917543214512989e-05, + "completions/clipped_ratio": 0.0, + "completions/max_length": 636.0, + "completions/max_terminated_length": 636.0, + "completions/mean_length": 395.1875, + "completions/mean_terminated_length": 395.1875, + "completions/min_length": 248.0, + "completions/min_terminated_length": 248.0, + "entropy": 0.3105512037873268, + "epoch": 0.25321199143468953, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.011443409137427807, + "learning_rate": 1e-05, + "loss": -0.0437, + "num_tokens": 10078795.0, + "reward": 0.59375, + "reward_std": 0.3061639964580536, + "rewards/accuracy_reward/mean": 0.59375, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.2866857051849365, + "sampling/importance_sampling_ratio/mean": 0.999954879283905, + "sampling/importance_sampling_ratio/min": 0.7495657205581665, + "sampling/sampling_logp_difference/max": 0.2882612943649292, + "sampling/sampling_logp_difference/mean": 0.010995929129421711, + "step": 473 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.58534411538858e-05, + "clip_ratio/low_min": 5.58534411538858e-05, + "clip_ratio/region_mean": 5.58534411538858e-05, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 694.0, + "completions/mean_length": 481.34375, + "completions/mean_terminated_length": 462.2333679199219, + "completions/min_length": 304.0, + "completions/min_terminated_length": 304.0, + "entropy": 0.33481962233781815, + "epoch": 0.25374732334047106, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.009619650430977345, + "learning_rate": 1e-05, + "loss": 0.0067, + "num_tokens": 10097694.0, + "reward": 0.84375, + "reward_std": 0.24511480331420898, + "rewards/accuracy_reward/mean": 0.84375, + "rewards/accuracy_reward/std": 0.3689020276069641, + "sampling/importance_sampling_ratio/max": 1.4322324991226196, + "sampling/importance_sampling_ratio/mean": 0.9997403025627136, + "sampling/importance_sampling_ratio/min": 0.7026197910308838, + "sampling/sampling_logp_difference/max": 0.3592343330383301, + "sampling/sampling_logp_difference/mean": 0.011315671727061272, + "step": 474 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00023391545983031392, + "clip_ratio/low_min": 0.00023391545983031392, + "clip_ratio/region_mean": 0.00023391545983031392, + "completions/clipped_ratio": 0.25, + "completions/max_length": 768.0, + "completions/max_terminated_length": 610.0, + "completions/mean_length": 537.9375, + "completions/mean_terminated_length": 461.25, + "completions/min_length": 249.0, + "completions/min_terminated_length": 249.0, + "entropy": 0.4866362549364567, + "epoch": 0.25428265524625265, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.007133313920348883, + "learning_rate": 1e-05, + "loss": -0.0068, + "num_tokens": 10118892.0, + "reward": 0.71875, + "reward_std": 0.24511480331420898, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.4732807874679565, + "sampling/importance_sampling_ratio/mean": 1.0003376007080078, + "sampling/importance_sampling_ratio/min": 0.5954201817512512, + "sampling/sampling_logp_difference/max": 0.5184879302978516, + "sampling/sampling_logp_difference/mean": 0.015425093472003937, + "step": 475 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.645889905281365e-05, + "clip_ratio/low_min": 5.645889905281365e-05, + "clip_ratio/region_mean": 5.645889905281365e-05, + "completions/clipped_ratio": 0.0, + "completions/max_length": 763.0, + "completions/max_terminated_length": 763.0, + "completions/mean_length": 486.96875, + "completions/mean_terminated_length": 486.96875, + "completions/min_length": 278.0, + "completions/min_terminated_length": 278.0, + "entropy": 0.3146357908844948, + "epoch": 0.25481798715203424, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.012660901062190533, + "learning_rate": 1e-05, + "loss": 0.0086, + "num_tokens": 10138235.0, + "reward": 0.65625, + "reward_std": 0.4397946000099182, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.3446470499038696, + "sampling/importance_sampling_ratio/mean": 0.9998399019241333, + "sampling/importance_sampling_ratio/min": 0.6975634694099426, + "sampling/sampling_logp_difference/max": 0.36016178131103516, + "sampling/sampling_logp_difference/mean": 0.011733213439583778, + "step": 476 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 763.0, + "completions/max_terminated_length": 763.0, + "completions/mean_length": 520.625, + "completions/mean_terminated_length": 520.625, + "completions/min_length": 242.0, + "completions/min_terminated_length": 242.0, + "entropy": 0.33602092042565346, + "epoch": 0.25535331905781583, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.0043482608161866665, + "learning_rate": 1e-05, + "loss": -0.0085, + "num_tokens": 10158959.0, + "reward": 0.78125, + "reward_std": 0.2041158676147461, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.4700850248336792, + "sampling/importance_sampling_ratio/mean": 1.0000849962234497, + "sampling/importance_sampling_ratio/min": 0.6636093854904175, + "sampling/sampling_logp_difference/max": 0.4100615978240967, + "sampling/sampling_logp_difference/mean": 0.01105241198092699, + "step": 477 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 687.0, + "completions/mean_length": 485.6875, + "completions/mean_terminated_length": 420.5384826660156, + "completions/min_length": 303.0, + "completions/min_terminated_length": 303.0, + "entropy": 0.5346232987940311, + "epoch": 0.2558886509635974, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.01230322103947401, + "learning_rate": 1e-05, + "loss": 0.1196, + "num_tokens": 10178821.0, + "reward": 0.71875, + "reward_std": 0.3608423173427582, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.7887417078018188, + "sampling/importance_sampling_ratio/mean": 0.999797523021698, + "sampling/importance_sampling_ratio/min": 0.7177157402038574, + "sampling/sampling_logp_difference/max": 0.581512451171875, + "sampling/sampling_logp_difference/mean": 0.016022939234972, + "step": 478 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0001788809895515442, + "clip_ratio/low_min": 0.0001788809895515442, + "clip_ratio/region_mean": 0.0001788809895515442, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 745.0, + "completions/mean_length": 501.0625, + "completions/mean_terminated_length": 473.4482727050781, + "completions/min_length": 323.0, + "completions/min_terminated_length": 323.0, + "entropy": 0.402830358594656, + "epoch": 0.256423982869379, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": 0.0022, + "num_tokens": 10199175.0, + "reward": 0.6875, + "reward_std": 0.249358132481575, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.4302822351455688, + "sampling/importance_sampling_ratio/mean": 0.9999926090240479, + "sampling/importance_sampling_ratio/min": 0.5216460824012756, + "sampling/sampling_logp_difference/max": 0.6507658958435059, + "sampling/sampling_logp_difference/mean": 0.013747112825512886, + "step": 479 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 4.610844553099014e-05, + "clip_ratio/low_min": 4.610844553099014e-05, + "clip_ratio/region_mean": 4.610844553099014e-05, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 743.0, + "completions/mean_length": 561.1875, + "completions/mean_terminated_length": 522.888916015625, + "completions/min_length": 280.0, + "completions/min_terminated_length": 280.0, + "entropy": 0.33710160478949547, + "epoch": 0.2569593147751606, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.016739992424845695, + "learning_rate": 1e-05, + "loss": 0.0588, + "num_tokens": 10220837.0, + "reward": 0.78125, + "reward_std": 0.2041158676147461, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 2.0, + "sampling/importance_sampling_ratio/mean": 1.0000311136245728, + "sampling/importance_sampling_ratio/min": 0.72202467918396, + "sampling/sampling_logp_difference/max": 0.849064826965332, + "sampling/sampling_logp_difference/mean": 0.011132810264825821, + "step": 480 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 6.106497312430292e-05, + "clip_ratio/low_min": 6.106497312430292e-05, + "clip_ratio/region_mean": 6.106497312430292e-05, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 761.0, + "completions/mean_length": 488.09375, + "completions/mean_terminated_length": 469.433349609375, + "completions/min_length": 272.0, + "completions/min_terminated_length": 272.0, + "entropy": 0.4116736091673374, + "epoch": 0.2574946466809422, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.002297243569046259, + "learning_rate": 1e-05, + "loss": 0.008, + "num_tokens": 10239800.0, + "reward": 0.9375, + "reward_std": 0.1767766922712326, + "rewards/accuracy_reward/mean": 0.9375, + "rewards/accuracy_reward/std": 0.24593468010425568, + "sampling/importance_sampling_ratio/max": 1.332295536994934, + "sampling/importance_sampling_ratio/mean": 0.9997638463973999, + "sampling/importance_sampling_ratio/min": 0.6012037396430969, + "sampling/sampling_logp_difference/max": 0.5088214874267578, + "sampling/sampling_logp_difference/mean": 0.01294345036149025, + "step": 481 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 716.0, + "completions/mean_length": 497.78125, + "completions/mean_terminated_length": 489.06451416015625, + "completions/min_length": 320.0, + "completions/min_terminated_length": 320.0, + "entropy": 0.29102923534810543, + "epoch": 0.2580299785867238, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": 0.0257, + "num_tokens": 10259297.0, + "reward": 0.9375, + "reward_std": 0.1157275140285492, + "rewards/accuracy_reward/mean": 0.9375, + "rewards/accuracy_reward/std": 0.24593468010425568, + "sampling/importance_sampling_ratio/max": 1.4557756185531616, + "sampling/importance_sampling_ratio/mean": 1.0003036260604858, + "sampling/importance_sampling_ratio/min": 0.7349271774291992, + "sampling/sampling_logp_difference/max": 0.37553882598876953, + "sampling/sampling_logp_difference/mean": 0.010717620141804218, + "step": 482 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.463286652229726e-05, + "clip_ratio/low_min": 5.463286652229726e-05, + "clip_ratio/region_mean": 5.463286652229726e-05, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 719.0, + "completions/mean_length": 524.9375, + "completions/mean_terminated_length": 456.8800048828125, + "completions/min_length": 241.0, + "completions/min_terminated_length": 241.0, + "entropy": 0.3851854242384434, + "epoch": 0.25856531049250536, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.015280285850167274, + "learning_rate": 1e-05, + "loss": 0.0701, + "num_tokens": 10280359.0, + "reward": 0.71875, + "reward_std": 0.4534739851951599, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.4300639629364014, + "sampling/importance_sampling_ratio/mean": 1.0003355741500854, + "sampling/importance_sampling_ratio/min": 0.687623918056488, + "sampling/sampling_logp_difference/max": 0.3745131492614746, + "sampling/sampling_logp_difference/mean": 0.01244751363992691, + "step": 483 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 742.0, + "completions/mean_length": 550.53125, + "completions/mean_terminated_length": 519.4642944335938, + "completions/min_length": 328.0, + "completions/min_terminated_length": 328.0, + "entropy": 0.32115957140922546, + "epoch": 0.25910064239828695, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.02360318973660469, + "learning_rate": 1e-05, + "loss": 0.0242, + "num_tokens": 10301624.0, + "reward": 0.75, + "reward_std": 0.4261348247528076, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.4405555725097656, + "sampling/importance_sampling_ratio/mean": 1.0000886917114258, + "sampling/importance_sampling_ratio/min": 0.7017530798912048, + "sampling/sampling_logp_difference/max": 0.36502885818481445, + "sampling/sampling_logp_difference/mean": 0.011256097815930843, + "step": 484 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.610413063550368e-05, + "clip_ratio/low_min": 5.610413063550368e-05, + "clip_ratio/region_mean": 5.610413063550368e-05, + "completions/clipped_ratio": 0.25, + "completions/max_length": 768.0, + "completions/max_terminated_length": 707.0, + "completions/mean_length": 566.96875, + "completions/mean_terminated_length": 499.9583435058594, + "completions/min_length": 218.0, + "completions/min_terminated_length": 218.0, + "entropy": 0.4112725034356117, + "epoch": 0.25963597430406854, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.01185301411896944, + "learning_rate": 1e-05, + "loss": 0.0349, + "num_tokens": 10323711.0, + "reward": 0.75, + "reward_std": 0.3745020925998688, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.6268253326416016, + "sampling/importance_sampling_ratio/mean": 1.000152349472046, + "sampling/importance_sampling_ratio/min": 0.7719011902809143, + "sampling/sampling_logp_difference/max": 0.4866304397583008, + "sampling/sampling_logp_difference/mean": 0.012860638089478016, + "step": 485 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 6.52400849503465e-05, + "clip_ratio/low_min": 6.52400849503465e-05, + "clip_ratio/region_mean": 6.52400849503465e-05, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 671.0, + "completions/mean_length": 445.53125, + "completions/mean_terminated_length": 412.17242431640625, + "completions/min_length": 138.0, + "completions/min_terminated_length": 138.0, + "entropy": 0.4744582735002041, + "epoch": 0.26017130620985013, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.021493274718523026, + "learning_rate": 1e-05, + "loss": 0.0594, + "num_tokens": 10341472.0, + "reward": 0.71875, + "reward_std": 0.3471629321575165, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.3100839853286743, + "sampling/importance_sampling_ratio/mean": 0.9998884797096252, + "sampling/importance_sampling_ratio/min": 0.6793792843818665, + "sampling/sampling_logp_difference/max": 0.38657569885253906, + "sampling/sampling_logp_difference/mean": 0.013434859924018383, + "step": 486 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 761.0, + "completions/mean_length": 614.65625, + "completions/mean_terminated_length": 586.25927734375, + "completions/min_length": 364.0, + "completions/min_terminated_length": 364.0, + "entropy": 0.35626184195280075, + "epoch": 0.2607066381156317, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.004379448015242815, + "learning_rate": 1e-05, + "loss": 0.0517, + "num_tokens": 10365389.0, + "reward": 0.65625, + "reward_std": 0.2041158676147461, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.442468523979187, + "sampling/importance_sampling_ratio/mean": 1.000192403793335, + "sampling/importance_sampling_ratio/min": 0.741629958152771, + "sampling/sampling_logp_difference/max": 0.36635589599609375, + "sampling/sampling_logp_difference/mean": 0.012494973838329315, + "step": 487 + }, + { + "clip_ratio/high_max": 6.145526276668534e-05, + "clip_ratio/high_mean": 6.145526276668534e-05, + "clip_ratio/low_mean": 0.00017596149700693786, + "clip_ratio/low_min": 0.00017596149700693786, + "clip_ratio/region_mean": 0.0002374167597736232, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 760.0, + "completions/mean_length": 523.5625, + "completions/mean_terminated_length": 498.2758483886719, + "completions/min_length": 268.0, + "completions/min_terminated_length": 268.0, + "entropy": 0.42526591569185257, + "epoch": 0.26124197002141325, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.018124375492334366, + "learning_rate": 1e-05, + "loss": 0.0338, + "num_tokens": 10386135.0, + "reward": 0.6875, + "reward_std": 0.2925041913986206, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.6005200147628784, + "sampling/importance_sampling_ratio/mean": 1.0000367164611816, + "sampling/importance_sampling_ratio/min": 0.5470401048660278, + "sampling/sampling_logp_difference/max": 0.6032330989837646, + "sampling/sampling_logp_difference/mean": 0.013319337740540504, + "step": 488 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 768.0, + "completions/max_terminated_length": 719.0, + "completions/mean_length": 593.21875, + "completions/mean_terminated_length": 534.9583740234375, + "completions/min_length": 365.0, + "completions/min_terminated_length": 365.0, + "entropy": 0.444037351757288, + "epoch": 0.26177730192719484, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.014569836668670177, + "learning_rate": 1e-05, + "loss": 0.0437, + "num_tokens": 10409278.0, + "reward": 0.625, + "reward_std": 0.3745020925998688, + "rewards/accuracy_reward/mean": 0.625, + "rewards/accuracy_reward/std": 0.49186936020851135, + "sampling/importance_sampling_ratio/max": 1.433860421180725, + "sampling/importance_sampling_ratio/mean": 1.000381350517273, + "sampling/importance_sampling_ratio/min": 0.6514508128166199, + "sampling/sampling_logp_difference/max": 0.42855334281921387, + "sampling/sampling_logp_difference/mean": 0.013760481961071491, + "step": 489 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 739.0, + "completions/max_terminated_length": 739.0, + "completions/mean_length": 475.1875, + "completions/mean_terminated_length": 475.1875, + "completions/min_length": 227.0, + "completions/min_terminated_length": 227.0, + "entropy": 0.4105031192302704, + "epoch": 0.2623126338329764, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.011106204241514206, + "learning_rate": 1e-05, + "loss": -0.0591, + "num_tokens": 10428356.0, + "reward": 0.71875, + "reward_std": 0.2630178928375244, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.3552688360214233, + "sampling/importance_sampling_ratio/mean": 1.000112771987915, + "sampling/importance_sampling_ratio/min": 0.7021424174308777, + "sampling/sampling_logp_difference/max": 0.3536190986633301, + "sampling/sampling_logp_difference/mean": 0.013007367961108685, + "step": 490 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 768.0, + "completions/max_terminated_length": 750.0, + "completions/mean_length": 576.53125, + "completions/mean_terminated_length": 512.7083740234375, + "completions/min_length": 290.0, + "completions/min_terminated_length": 290.0, + "entropy": 0.40421775728464127, + "epoch": 0.262847965738758, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.007814164273440838, + "learning_rate": 1e-05, + "loss": 0.0464, + "num_tokens": 10451053.0, + "reward": 0.625, + "reward_std": 0.2177756428718567, + "rewards/accuracy_reward/mean": 0.625, + "rewards/accuracy_reward/std": 0.49186936020851135, + "sampling/importance_sampling_ratio/max": 1.4316725730895996, + "sampling/importance_sampling_ratio/mean": 1.0002444982528687, + "sampling/importance_sampling_ratio/min": 0.6939866542816162, + "sampling/sampling_logp_difference/max": 0.36530256271362305, + "sampling/sampling_logp_difference/mean": 0.012022517621517181, + "step": 491 + }, + { + "clip_ratio/high_max": 4.22582816099748e-05, + "clip_ratio/high_mean": 4.22582816099748e-05, + "clip_ratio/low_mean": 0.00010322048183297738, + "clip_ratio/low_min": 0.00010322048183297738, + "clip_ratio/region_mean": 0.00014547876344295219, + "completions/clipped_ratio": 0.34375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 733.0, + "completions/mean_length": 625.8125, + "completions/mean_terminated_length": 551.3333740234375, + "completions/min_length": 259.0, + "completions/min_terminated_length": 259.0, + "entropy": 0.3712773844599724, + "epoch": 0.2633832976445396, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.005484628025442362, + "learning_rate": 1e-05, + "loss": 0.0043, + "num_tokens": 10474983.0, + "reward": 0.5, + "reward_std": 0.4082317352294922, + "rewards/accuracy_reward/mean": 0.5, + "rewards/accuracy_reward/std": 0.5080004930496216, + "sampling/importance_sampling_ratio/max": 1.6351101398468018, + "sampling/importance_sampling_ratio/mean": 0.9999654293060303, + "sampling/importance_sampling_ratio/min": 0.6186308264732361, + "sampling/sampling_logp_difference/max": 0.49171018600463867, + "sampling/sampling_logp_difference/mean": 0.011516343802213669, + "step": 492 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00014466403081314638, + "clip_ratio/low_min": 0.00014466403081314638, + "clip_ratio/region_mean": 0.00014466403081314638, + "completions/clipped_ratio": 0.3125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 735.0, + "completions/mean_length": 602.125, + "completions/mean_terminated_length": 526.727294921875, + "completions/min_length": 395.0, + "completions/min_terminated_length": 395.0, + "entropy": 0.46679985895752907, + "epoch": 0.2639186295503212, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.009510909207165241, + "learning_rate": 1e-05, + "loss": 0.0668, + "num_tokens": 10497787.0, + "reward": 0.5625, + "reward_std": 0.3471825420856476, + "rewards/accuracy_reward/mean": 0.5625, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 1.4565629959106445, + "sampling/importance_sampling_ratio/mean": 1.0000561475753784, + "sampling/importance_sampling_ratio/min": 0.6113215684890747, + "sampling/sampling_logp_difference/max": 0.49213218688964844, + "sampling/sampling_logp_difference/mean": 0.014486799016594887, + "step": 493 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 4.402958802529611e-05, + "clip_ratio/low_min": 4.402958802529611e-05, + "clip_ratio/region_mean": 4.402958802529611e-05, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 710.0, + "completions/mean_length": 576.40625, + "completions/mean_terminated_length": 532.1923217773438, + "completions/min_length": 283.0, + "completions/min_terminated_length": 283.0, + "entropy": 0.4298512861132622, + "epoch": 0.2644539614561028, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.014777930453419685, + "learning_rate": 1e-05, + "loss": 0.0311, + "num_tokens": 10519952.0, + "reward": 0.46875, + "reward_std": 0.378745436668396, + "rewards/accuracy_reward/mean": 0.46875, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.420779824256897, + "sampling/importance_sampling_ratio/mean": 1.0001146793365479, + "sampling/importance_sampling_ratio/min": 0.5928531289100647, + "sampling/sampling_logp_difference/max": 0.5228085517883301, + "sampling/sampling_logp_difference/mean": 0.013614621013402939, + "step": 494 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 6.510417006211355e-05, + "clip_ratio/low_min": 6.510417006211355e-05, + "clip_ratio/region_mean": 6.510417006211355e-05, + "completions/clipped_ratio": 0.25, + "completions/max_length": 768.0, + "completions/max_terminated_length": 750.0, + "completions/mean_length": 543.71875, + "completions/mean_terminated_length": 468.9583435058594, + "completions/min_length": 288.0, + "completions/min_terminated_length": 288.0, + "entropy": 0.5003023743629456, + "epoch": 0.26498929336188437, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.013744396157562733, + "learning_rate": 1e-05, + "loss": 0.0594, + "num_tokens": 10541527.0, + "reward": 0.46875, + "reward_std": 0.35564959049224854, + "rewards/accuracy_reward/mean": 0.46875, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 2.0, + "sampling/importance_sampling_ratio/mean": 1.0005255937576294, + "sampling/importance_sampling_ratio/min": 0.6238965392112732, + "sampling/sampling_logp_difference/max": 0.695166826248169, + "sampling/sampling_logp_difference/mean": 0.015651416033506393, + "step": 495 + }, + { + "clip_ratio/high_max": 6.148548709461465e-05, + "clip_ratio/high_mean": 6.148548709461465e-05, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 6.148548709461465e-05, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 729.0, + "completions/mean_length": 455.9375, + "completions/mean_terminated_length": 445.8709411621094, + "completions/min_length": 161.0, + "completions/min_terminated_length": 161.0, + "entropy": 0.3635113351047039, + "epoch": 0.26552462526766596, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.015172747895121574, + "learning_rate": 1e-05, + "loss": -0.0157, + "num_tokens": 10559381.0, + "reward": 0.75, + "reward_std": 0.2925041913986206, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 2.0, + "sampling/importance_sampling_ratio/mean": 1.0002249479293823, + "sampling/importance_sampling_ratio/min": 0.19352464377880096, + "sampling/sampling_logp_difference/max": 1.642350435256958, + "sampling/sampling_logp_difference/mean": 0.012256532907485962, + "step": 496 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 4.6159526391420513e-05, + "clip_ratio/low_min": 4.6159526391420513e-05, + "clip_ratio/region_mean": 4.6159526391420513e-05, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 766.0, + "completions/mean_length": 635.875, + "completions/mean_terminated_length": 611.4074096679688, + "completions/min_length": 366.0, + "completions/min_terminated_length": 366.0, + "entropy": 0.32336436584591866, + "epoch": 0.26605995717344755, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.018259260803461075, + "learning_rate": 1e-05, + "loss": 0.0305, + "num_tokens": 10583457.0, + "reward": 0.78125, + "reward_std": 0.3061639964580536, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.597190260887146, + "sampling/importance_sampling_ratio/mean": 0.9999054074287415, + "sampling/importance_sampling_ratio/min": 0.5457773804664612, + "sampling/sampling_logp_difference/max": 0.6055440902709961, + "sampling/sampling_logp_difference/mean": 0.010763855651021004, + "step": 497 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 727.0, + "completions/mean_length": 559.3125, + "completions/mean_terminated_length": 552.5806274414062, + "completions/min_length": 287.0, + "completions/min_terminated_length": 287.0, + "entropy": 0.3208707682788372, + "epoch": 0.26659528907922914, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.003419401589781046, + "learning_rate": 1e-05, + "loss": -0.0228, + "num_tokens": 10604875.0, + "reward": 0.96875, + "reward_std": 0.0883883461356163, + "rewards/accuracy_reward/mean": 0.96875, + "rewards/accuracy_reward/std": 0.1767766922712326, + "sampling/importance_sampling_ratio/max": 1.5586161613464355, + "sampling/importance_sampling_ratio/mean": 1.0004326105117798, + "sampling/importance_sampling_ratio/min": 0.7551673650741577, + "sampling/sampling_logp_difference/max": 0.443798303604126, + "sampling/sampling_logp_difference/mean": 0.011351877823472023, + "step": 498 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 616.0, + "completions/mean_length": 436.875, + "completions/mean_terminated_length": 426.19354248046875, + "completions/min_length": 245.0, + "completions/min_terminated_length": 245.0, + "entropy": 0.3958047814667225, + "epoch": 0.2671306209850107, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.014426103793084621, + "learning_rate": 1e-05, + "loss": 0.0226, + "num_tokens": 10622375.0, + "reward": 0.75, + "reward_std": 0.26726123690605164, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.4606636762619019, + "sampling/importance_sampling_ratio/mean": 0.9991129040718079, + "sampling/importance_sampling_ratio/min": 0.6251304149627686, + "sampling/sampling_logp_difference/max": 0.46979498863220215, + "sampling/sampling_logp_difference/mean": 0.01428675651550293, + "step": 499 + }, + { + "clip_ratio/high_max": 4.4452346628531814e-05, + "clip_ratio/high_mean": 4.4452346628531814e-05, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 4.4452346628531814e-05, + "completions/clipped_ratio": 0.3125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 744.0, + "completions/mean_length": 583.375, + "completions/mean_terminated_length": 499.4545593261719, + "completions/min_length": 205.0, + "completions/min_terminated_length": 205.0, + "entropy": 0.6192383728921413, + "epoch": 0.2676659528907923, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.014534000307321548, + "learning_rate": 1e-05, + "loss": 0.1002, + "num_tokens": 10645627.0, + "reward": 0.59375, + "reward_std": 0.3061639666557312, + "rewards/accuracy_reward/mean": 0.59375, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.4215008020401, + "sampling/importance_sampling_ratio/mean": 1.0001673698425293, + "sampling/importance_sampling_ratio/min": 0.7219524383544922, + "sampling/sampling_logp_difference/max": 0.3517131805419922, + "sampling/sampling_logp_difference/mean": 0.016877198591828346, + "step": 500 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.530973430722952e-05, + "clip_ratio/low_min": 5.530973430722952e-05, + "clip_ratio/region_mean": 5.530973430722952e-05, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 640.0, + "completions/mean_length": 499.4375, + "completions/mean_terminated_length": 449.7037048339844, + "completions/min_length": 245.0, + "completions/min_terminated_length": 245.0, + "entropy": 0.5286917313933372, + "epoch": 0.2682012847965739, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.025257959961891174, + "learning_rate": 1e-05, + "loss": 0.0305, + "num_tokens": 10665641.0, + "reward": 0.6875, + "reward_std": 0.292504221200943, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.4534956216812134, + "sampling/importance_sampling_ratio/mean": 1.0004061460494995, + "sampling/importance_sampling_ratio/min": 0.6537541151046753, + "sampling/sampling_logp_difference/max": 0.42502403259277344, + "sampling/sampling_logp_difference/mean": 0.015922224149107933, + "step": 501 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 650.0, + "completions/max_terminated_length": 650.0, + "completions/mean_length": 443.59375, + "completions/mean_terminated_length": 443.59375, + "completions/min_length": 222.0, + "completions/min_terminated_length": 222.0, + "entropy": 0.34683436155319214, + "epoch": 0.26873661670235544, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.0054996381513774395, + "learning_rate": 1e-05, + "loss": -0.0156, + "num_tokens": 10683708.0, + "reward": 0.9375, + "reward_std": 0.1767766922712326, + "rewards/accuracy_reward/mean": 0.9375, + "rewards/accuracy_reward/std": 0.24593468010425568, + "sampling/importance_sampling_ratio/max": 1.4331287145614624, + "sampling/importance_sampling_ratio/mean": 0.999981164932251, + "sampling/importance_sampling_ratio/min": 0.6877760887145996, + "sampling/sampling_logp_difference/max": 0.37429189682006836, + "sampling/sampling_logp_difference/mean": 0.01251487247645855, + "step": 502 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 766.0, + "completions/mean_length": 509.375, + "completions/mean_terminated_length": 482.6206970214844, + "completions/min_length": 222.0, + "completions/min_terminated_length": 222.0, + "entropy": 0.41795216500759125, + "epoch": 0.269271948608137, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.03215835988521576, + "learning_rate": 1e-05, + "loss": 0.0994, + "num_tokens": 10704144.0, + "reward": 0.875, + "reward_std": 0.292504221200943, + "rewards/accuracy_reward/mean": 0.875, + "rewards/accuracy_reward/std": 0.33601075410842896, + "sampling/importance_sampling_ratio/max": 1.3911006450653076, + "sampling/importance_sampling_ratio/mean": 0.9998789429664612, + "sampling/importance_sampling_ratio/min": 0.7018213272094727, + "sampling/sampling_logp_difference/max": 0.3540763854980469, + "sampling/sampling_logp_difference/mean": 0.013545467518270016, + "step": 503 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0001050420178216882, + "clip_ratio/low_min": 0.0001050420178216882, + "clip_ratio/region_mean": 0.0001050420178216882, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 768.0, + "completions/mean_length": 517.96875, + "completions/mean_terminated_length": 482.2500305175781, + "completions/min_length": 282.0, + "completions/min_terminated_length": 282.0, + "entropy": 0.4228440932929516, + "epoch": 0.2698072805139186, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.013474730774760246, + "learning_rate": 1e-05, + "loss": -0.034, + "num_tokens": 10724279.0, + "reward": 0.75, + "reward_std": 0.2587745785713196, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.6665763854980469, + "sampling/importance_sampling_ratio/mean": 1.0000231266021729, + "sampling/importance_sampling_ratio/min": 0.702028214931488, + "sampling/sampling_logp_difference/max": 0.5107715129852295, + "sampling/sampling_logp_difference/mean": 0.013398643583059311, + "step": 504 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00020762170606758446, + "clip_ratio/low_min": 0.00020762170606758446, + "clip_ratio/region_mean": 0.00020762170606758446, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 763.0, + "completions/mean_length": 609.875, + "completions/mean_terminated_length": 565.5999755859375, + "completions/min_length": 255.0, + "completions/min_terminated_length": 255.0, + "entropy": 0.4561927057802677, + "epoch": 0.2703426124197002, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.012891552411019802, + "learning_rate": 1e-05, + "loss": 0.0592, + "num_tokens": 10747835.0, + "reward": 0.53125, + "reward_std": 0.521792471408844, + "rewards/accuracy_reward/mean": 0.53125, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.4030518531799316, + "sampling/importance_sampling_ratio/mean": 0.9998916983604431, + "sampling/importance_sampling_ratio/min": 0.6945270895957947, + "sampling/sampling_logp_difference/max": 0.36452412605285645, + "sampling/sampling_logp_difference/mean": 0.013256196863949299, + "step": 505 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.773672091891058e-05, + "clip_ratio/low_min": 5.773672091891058e-05, + "clip_ratio/region_mean": 5.773672091891058e-05, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 757.0, + "completions/mean_length": 523.375, + "completions/mean_terminated_length": 498.0689697265625, + "completions/min_length": 310.0, + "completions/min_terminated_length": 310.0, + "entropy": 0.4539306275546551, + "epoch": 0.2708779443254818, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.012112719006836414, + "learning_rate": 1e-05, + "loss": 0.0053, + "num_tokens": 10768615.0, + "reward": 0.625, + "reward_std": 0.3335031569004059, + "rewards/accuracy_reward/mean": 0.625, + "rewards/accuracy_reward/std": 0.49186936020851135, + "sampling/importance_sampling_ratio/max": 1.3568693399429321, + "sampling/importance_sampling_ratio/mean": 1.0000436305999756, + "sampling/importance_sampling_ratio/min": 0.6628824472427368, + "sampling/sampling_logp_difference/max": 0.41115760803222656, + "sampling/sampling_logp_difference/mean": 0.013466725125908852, + "step": 506 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 734.0, + "completions/mean_length": 534.6875, + "completions/mean_terminated_length": 501.357177734375, + "completions/min_length": 286.0, + "completions/min_terminated_length": 286.0, + "entropy": 0.35264989361166954, + "epoch": 0.2714132762312634, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.015589317306876183, + "learning_rate": 1e-05, + "loss": 0.052, + "num_tokens": 10789813.0, + "reward": 0.8125, + "reward_std": 0.2587745785713196, + "rewards/accuracy_reward/mean": 0.8125, + "rewards/accuracy_reward/std": 0.3965577781200409, + "sampling/importance_sampling_ratio/max": 1.425531029701233, + "sampling/importance_sampling_ratio/mean": 1.0001498460769653, + "sampling/importance_sampling_ratio/min": 0.6998605132102966, + "sampling/sampling_logp_difference/max": 0.3568742275238037, + "sampling/sampling_logp_difference/mean": 0.011807294562458992, + "step": 507 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0001117068823077716, + "clip_ratio/low_min": 0.0001117068823077716, + "clip_ratio/region_mean": 0.0001117068823077716, + "completions/clipped_ratio": 0.25, + "completions/max_length": 768.0, + "completions/max_terminated_length": 759.0, + "completions/mean_length": 581.4375, + "completions/mean_terminated_length": 519.25, + "completions/min_length": 277.0, + "completions/min_terminated_length": 277.0, + "entropy": 0.501917477697134, + "epoch": 0.27194860813704497, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.016425846144557, + "learning_rate": 1e-05, + "loss": 0.0573, + "num_tokens": 10811963.0, + "reward": 0.53125, + "reward_std": 0.35564959049224854, + "rewards/accuracy_reward/mean": 0.53125, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.439595341682434, + "sampling/importance_sampling_ratio/mean": 1.0000243186950684, + "sampling/importance_sampling_ratio/min": 0.7093459367752075, + "sampling/sampling_logp_difference/max": 0.3643620014190674, + "sampling/sampling_logp_difference/mean": 0.015107567422091961, + "step": 508 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 8.520791016053408e-05, + "clip_ratio/low_min": 8.520791016053408e-05, + "clip_ratio/region_mean": 8.520791016053408e-05, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 764.0, + "completions/mean_length": 475.34375, + "completions/mean_terminated_length": 465.9031982421875, + "completions/min_length": 238.0, + "completions/min_terminated_length": 238.0, + "entropy": 0.29503167793154716, + "epoch": 0.27248394004282656, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.005277770571410656, + "learning_rate": 1e-05, + "loss": -0.0383, + "num_tokens": 10830950.0, + "reward": 0.90625, + "reward_std": 0.1293872892856598, + "rewards/accuracy_reward/mean": 0.90625, + "rewards/accuracy_reward/std": 0.2961445748806, + "sampling/importance_sampling_ratio/max": 1.4213597774505615, + "sampling/importance_sampling_ratio/mean": 1.000784158706665, + "sampling/importance_sampling_ratio/min": 0.7571797370910645, + "sampling/sampling_logp_difference/max": 0.35161399841308594, + "sampling/sampling_logp_difference/mean": 0.010759200900793076, + "step": 509 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.390254591475241e-05, + "clip_ratio/low_min": 5.390254591475241e-05, + "clip_ratio/region_mean": 5.390254591475241e-05, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 705.0, + "completions/mean_length": 499.03125, + "completions/mean_terminated_length": 471.2069091796875, + "completions/min_length": 177.0, + "completions/min_terminated_length": 177.0, + "entropy": 0.3587932847440243, + "epoch": 0.27301927194860814, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.005858589429408312, + "learning_rate": 1e-05, + "loss": 0.0437, + "num_tokens": 10850343.0, + "reward": 0.8125, + "reward_std": 0.3104073107242584, + "rewards/accuracy_reward/mean": 0.8125, + "rewards/accuracy_reward/std": 0.3965577781200409, + "sampling/importance_sampling_ratio/max": 1.356927514076233, + "sampling/importance_sampling_ratio/mean": 1.0000473260879517, + "sampling/importance_sampling_ratio/min": 0.5159573554992676, + "sampling/sampling_logp_difference/max": 0.6617312431335449, + "sampling/sampling_logp_difference/mean": 0.011431219056248665, + "step": 510 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 6.076811041566543e-05, + "clip_ratio/low_min": 6.076811041566543e-05, + "clip_ratio/region_mean": 6.076811041566543e-05, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 690.0, + "completions/mean_length": 530.6875, + "completions/mean_terminated_length": 496.7857360839844, + "completions/min_length": 268.0, + "completions/min_terminated_length": 268.0, + "entropy": 0.3550248220562935, + "epoch": 0.27355460385438973, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.01730632595717907, + "learning_rate": 1e-05, + "loss": 0.0231, + "num_tokens": 10871165.0, + "reward": 0.78125, + "reward_std": 0.2630178928375244, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.4192852973937988, + "sampling/importance_sampling_ratio/mean": 0.9999898672103882, + "sampling/importance_sampling_ratio/min": 0.27570825815200806, + "sampling/sampling_logp_difference/max": 1.2884119749069214, + "sampling/sampling_logp_difference/mean": 0.011719655245542526, + "step": 511 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 641.0, + "completions/max_terminated_length": 641.0, + "completions/mean_length": 407.25, + "completions/mean_terminated_length": 407.25, + "completions/min_length": 248.0, + "completions/min_terminated_length": 248.0, + "entropy": 0.45111938565969467, + "epoch": 0.2740899357601713, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.015476170927286148, + "learning_rate": 1e-05, + "loss": -0.0452, + "num_tokens": 10887989.0, + "reward": 0.53125, + "reward_std": 0.3198433816432953, + "rewards/accuracy_reward/mean": 0.53125, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.4185147285461426, + "sampling/importance_sampling_ratio/mean": 1.0004252195358276, + "sampling/importance_sampling_ratio/min": 0.5482162833213806, + "sampling/sampling_logp_difference/max": 0.6010854244232178, + "sampling/sampling_logp_difference/mean": 0.012268926948308945, + "step": 512 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 678.0, + "completions/max_terminated_length": 678.0, + "completions/mean_length": 485.71875, + "completions/mean_terminated_length": 485.71875, + "completions/min_length": 258.0, + "completions/min_terminated_length": 258.0, + "entropy": 0.34177630580961704, + "epoch": 0.2746252676659529, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.008045018650591373, + "learning_rate": 1e-05, + "loss": 0.0137, + "num_tokens": 10907468.0, + "reward": 0.65625, + "reward_std": 0.3787454068660736, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.4314159154891968, + "sampling/importance_sampling_ratio/mean": 0.9998471736907959, + "sampling/importance_sampling_ratio/min": 0.4723207354545593, + "sampling/sampling_logp_difference/max": 0.7500970363616943, + "sampling/sampling_logp_difference/mean": 0.012011357583105564, + "step": 513 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 707.0, + "completions/mean_length": 567.15625, + "completions/mean_terminated_length": 520.8077392578125, + "completions/min_length": 306.0, + "completions/min_terminated_length": 306.0, + "entropy": 0.3598812371492386, + "epoch": 0.2751605995717345, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": 0.0, + "num_tokens": 10929745.0, + "reward": 0.5, + "reward_std": 0.0, + "rewards/accuracy_reward/mean": 0.5, + "rewards/accuracy_reward/std": 0.5080004930496216, + "sampling/importance_sampling_ratio/max": 1.4048995971679688, + "sampling/importance_sampling_ratio/mean": 0.9999309182167053, + "sampling/importance_sampling_ratio/min": 0.7220192551612854, + "sampling/sampling_logp_difference/max": 0.3399658203125, + "sampling/sampling_logp_difference/mean": 0.010898095555603504, + "step": 514 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 9.611687710275874e-05, + "clip_ratio/low_min": 9.611687710275874e-05, + "clip_ratio/region_mean": 9.611687710275874e-05, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 676.0, + "completions/mean_length": 573.25, + "completions/mean_terminated_length": 518.719970703125, + "completions/min_length": 253.0, + "completions/min_terminated_length": 253.0, + "entropy": 0.2508980128914118, + "epoch": 0.2756959314775161, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": 0.0214, + "num_tokens": 10952049.0, + "reward": 0.71875, + "reward_std": 0.2041158676147461, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.2707433700561523, + "sampling/importance_sampling_ratio/mean": 0.9999803900718689, + "sampling/importance_sampling_ratio/min": 0.4754341244697571, + "sampling/sampling_logp_difference/max": 0.7435269355773926, + "sampling/sampling_logp_difference/mean": 0.008386613801121712, + "step": 515 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00019170560699421912, + "clip_ratio/low_min": 0.00019170560699421912, + "clip_ratio/region_mean": 0.00019170560699421912, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 753.0, + "completions/mean_length": 594.9375, + "completions/mean_terminated_length": 562.888916015625, + "completions/min_length": 257.0, + "completions/min_terminated_length": 257.0, + "entropy": 0.3323214463889599, + "epoch": 0.2762312633832976, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.019251735880970955, + "learning_rate": 1e-05, + "loss": 0.0146, + "num_tokens": 10975007.0, + "reward": 0.65625, + "reward_std": 0.3608423173427582, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.4540327787399292, + "sampling/importance_sampling_ratio/mean": 1.0000197887420654, + "sampling/importance_sampling_ratio/min": 0.7002866864204407, + "sampling/sampling_logp_difference/max": 0.37434089183807373, + "sampling/sampling_logp_difference/mean": 0.011816645041108131, + "step": 516 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 693.0, + "completions/mean_length": 507.28125, + "completions/mean_terminated_length": 459.0, + "completions/min_length": 182.0, + "completions/min_terminated_length": 182.0, + "entropy": 0.44671597331762314, + "epoch": 0.2767665952890792, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": 0.0322, + "num_tokens": 10995560.0, + "reward": 0.71875, + "reward_std": 0.0883883461356163, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.6457420587539673, + "sampling/importance_sampling_ratio/mean": 1.000201940536499, + "sampling/importance_sampling_ratio/min": 0.6064935922622681, + "sampling/sampling_logp_difference/max": 0.50006103515625, + "sampling/sampling_logp_difference/mean": 0.014169696718454361, + "step": 517 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.3125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 747.0, + "completions/mean_length": 580.875, + "completions/mean_terminated_length": 495.8182067871094, + "completions/min_length": 314.0, + "completions/min_terminated_length": 314.0, + "entropy": 0.38640883564949036, + "epoch": 0.2773019271948608, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.025809966027736664, + "learning_rate": 1e-05, + "loss": 0.0468, + "num_tokens": 11018028.0, + "reward": 0.5, + "reward_std": 0.4671337604522705, + "rewards/accuracy_reward/mean": 0.5, + "rewards/accuracy_reward/std": 0.5080004930496216, + "sampling/importance_sampling_ratio/max": 1.3842450380325317, + "sampling/importance_sampling_ratio/mean": 1.0000641345977783, + "sampling/importance_sampling_ratio/min": 0.7114458680152893, + "sampling/sampling_logp_difference/max": 0.3404560089111328, + "sampling/sampling_logp_difference/mean": 0.012557556852698326, + "step": 518 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00011339053526171483, + "clip_ratio/low_min": 0.00011339053526171483, + "clip_ratio/region_mean": 0.00011339053526171483, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 758.0, + "completions/mean_length": 542.625, + "completions/mean_terminated_length": 519.3103637695312, + "completions/min_length": 293.0, + "completions/min_terminated_length": 293.0, + "entropy": 0.426570650190115, + "epoch": 0.2778372591006424, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.010524033568799496, + "learning_rate": 1e-05, + "loss": -0.0037, + "num_tokens": 11039184.0, + "reward": 0.65625, + "reward_std": 0.3471629321575165, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.4324995279312134, + "sampling/importance_sampling_ratio/mean": 0.9998242855072021, + "sampling/importance_sampling_ratio/min": 0.6445524096488953, + "sampling/sampling_logp_difference/max": 0.43919920921325684, + "sampling/sampling_logp_difference/mean": 0.013481113128364086, + "step": 519 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.4229934903560206e-05, + "clip_ratio/low_min": 5.4229934903560206e-05, + "clip_ratio/region_mean": 5.4229934903560206e-05, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 698.0, + "completions/mean_length": 452.375, + "completions/mean_terminated_length": 442.19354248046875, + "completions/min_length": 234.0, + "completions/min_terminated_length": 234.0, + "entropy": 0.3255032114684582, + "epoch": 0.278372591006424, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.012102417647838593, + "learning_rate": 1e-05, + "loss": 0.0121, + "num_tokens": 11057236.0, + "reward": 0.6875, + "reward_std": 0.2177756428718567, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.5126256942749023, + "sampling/importance_sampling_ratio/mean": 0.9996980428695679, + "sampling/importance_sampling_ratio/min": 0.6459639668464661, + "sampling/sampling_logp_difference/max": 0.43701159954071045, + "sampling/sampling_logp_difference/mean": 0.010542369447648525, + "step": 520 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.091649654787034e-05, + "clip_ratio/low_min": 5.091649654787034e-05, + "clip_ratio/region_mean": 5.091649654787034e-05, + "completions/clipped_ratio": 0.25, + "completions/max_length": 768.0, + "completions/max_terminated_length": 700.0, + "completions/mean_length": 560.28125, + "completions/mean_terminated_length": 491.04168701171875, + "completions/min_length": 235.0, + "completions/min_terminated_length": 235.0, + "entropy": 0.3828450106084347, + "epoch": 0.27890792291220556, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.006895342841744423, + "learning_rate": 1e-05, + "loss": 0.01, + "num_tokens": 11079325.0, + "reward": 0.6875, + "reward_std": 0.292504221200943, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.5072721242904663, + "sampling/importance_sampling_ratio/mean": 1.000172734260559, + "sampling/importance_sampling_ratio/min": 0.6726121306419373, + "sampling/sampling_logp_difference/max": 0.41030144691467285, + "sampling/sampling_logp_difference/mean": 0.011244766414165497, + "step": 521 + }, + { + "clip_ratio/high_max": 4.6227811253629625e-05, + "clip_ratio/high_mean": 4.6227811253629625e-05, + "clip_ratio/low_mean": 0.0002374210707785096, + "clip_ratio/low_min": 0.0002374210707785096, + "clip_ratio/region_mean": 0.0002836488820321392, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 766.0, + "completions/mean_length": 612.96875, + "completions/mean_terminated_length": 569.5599975585938, + "completions/min_length": 383.0, + "completions/min_terminated_length": 383.0, + "entropy": 0.4224571939557791, + "epoch": 0.27944325481798715, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.02829921804368496, + "learning_rate": 1e-05, + "loss": 0.0335, + "num_tokens": 11102820.0, + "reward": 0.6875, + "reward_std": 0.3335031569004059, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.4662220478057861, + "sampling/importance_sampling_ratio/mean": 0.9996917843818665, + "sampling/importance_sampling_ratio/min": 0.6035193204879761, + "sampling/sampling_logp_difference/max": 0.5049772262573242, + "sampling/sampling_logp_difference/mean": 0.01334192045032978, + "step": 522 + }, + { + "clip_ratio/high_max": 7.212925265775993e-05, + "clip_ratio/high_mean": 7.212925265775993e-05, + "clip_ratio/low_mean": 7.851758709875867e-05, + "clip_ratio/low_min": 7.851758709875867e-05, + "clip_ratio/region_mean": 0.0001506468397565186, + "completions/clipped_ratio": 0.0, + "completions/max_length": 676.0, + "completions/max_terminated_length": 676.0, + "completions/mean_length": 452.46875, + "completions/mean_terminated_length": 452.46875, + "completions/min_length": 287.0, + "completions/min_terminated_length": 287.0, + "entropy": 0.34968700259923935, + "epoch": 0.27997858672376874, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.014668166637420654, + "learning_rate": 1e-05, + "loss": 0.0144, + "num_tokens": 11120483.0, + "reward": 0.65625, + "reward_std": 0.3061639964580536, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.4449950456619263, + "sampling/importance_sampling_ratio/mean": 0.9998252987861633, + "sampling/importance_sampling_ratio/min": 0.7312772274017334, + "sampling/sampling_logp_difference/max": 0.3681058883666992, + "sampling/sampling_logp_difference/mean": 0.011569187045097351, + "step": 523 + }, + { + "clip_ratio/high_max": 0.00010992674651788548, + "clip_ratio/high_mean": 0.00010992674651788548, + "clip_ratio/low_mean": 4.39831092080567e-05, + "clip_ratio/low_min": 4.39831092080567e-05, + "clip_ratio/region_mean": 0.00015390985572594218, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 757.0, + "completions/mean_length": 553.1875, + "completions/mean_terminated_length": 522.5, + "completions/min_length": 240.0, + "completions/min_terminated_length": 240.0, + "entropy": 0.33637315034866333, + "epoch": 0.28051391862955033, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.010086177848279476, + "learning_rate": 1e-05, + "loss": 0.0157, + "num_tokens": 11142033.0, + "reward": 0.78125, + "reward_std": 0.4218915104866028, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.4344085454940796, + "sampling/importance_sampling_ratio/mean": 1.0000633001327515, + "sampling/importance_sampling_ratio/min": 0.7605924606323242, + "sampling/sampling_logp_difference/max": 0.36075258255004883, + "sampling/sampling_logp_difference/mean": 0.011049482971429825, + "step": 524 + }, + { + "clip_ratio/high_max": 7.122506940504536e-05, + "clip_ratio/high_mean": 7.122506940504536e-05, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 7.122506940504536e-05, + "completions/clipped_ratio": 0.0, + "completions/max_length": 703.0, + "completions/max_terminated_length": 703.0, + "completions/mean_length": 424.875, + "completions/mean_terminated_length": 424.875, + "completions/min_length": 97.0, + "completions/min_terminated_length": 97.0, + "entropy": 0.3426404930651188, + "epoch": 0.2810492505353319, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.0028623396065086126, + "learning_rate": 1e-05, + "loss": -0.0547, + "num_tokens": 11158925.0, + "reward": 0.90625, + "reward_std": 0.2041158676147461, + "rewards/accuracy_reward/mean": 0.90625, + "rewards/accuracy_reward/std": 0.2961445748806, + "sampling/importance_sampling_ratio/max": 1.6245464086532593, + "sampling/importance_sampling_ratio/mean": 0.9997588396072388, + "sampling/importance_sampling_ratio/min": 0.5904847979545593, + "sampling/sampling_logp_difference/max": 0.5268113613128662, + "sampling/sampling_logp_difference/mean": 0.011864609085023403, + "step": 525 + }, + { + "clip_ratio/high_max": 5.600358417723328e-05, + "clip_ratio/high_mean": 5.600358417723328e-05, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 5.600358417723328e-05, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 755.0, + "completions/mean_length": 551.84375, + "completions/mean_terminated_length": 529.4827270507812, + "completions/min_length": 318.0, + "completions/min_terminated_length": 318.0, + "entropy": 0.48494572564959526, + "epoch": 0.2815845824411135, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.018498221412301064, + "learning_rate": 1e-05, + "loss": 0.0008, + "num_tokens": 11180128.0, + "reward": 0.59375, + "reward_std": 0.4628904461860657, + "rewards/accuracy_reward/mean": 0.59375, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.4667741060256958, + "sampling/importance_sampling_ratio/mean": 1.0001482963562012, + "sampling/importance_sampling_ratio/min": 0.6934671998023987, + "sampling/sampling_logp_difference/max": 0.38306546211242676, + "sampling/sampling_logp_difference/mean": 0.014432166703045368, + "step": 526 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 714.0, + "completions/mean_length": 474.1875, + "completions/mean_terminated_length": 419.77777099609375, + "completions/min_length": 214.0, + "completions/min_terminated_length": 214.0, + "entropy": 0.31750617176294327, + "epoch": 0.2821199143468951, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.007369263097643852, + "learning_rate": 1e-05, + "loss": -0.0133, + "num_tokens": 11198910.0, + "reward": 0.8125, + "reward_std": 0.2177756428718567, + "rewards/accuracy_reward/mean": 0.8125, + "rewards/accuracy_reward/std": 0.3965577781200409, + "sampling/importance_sampling_ratio/max": 1.366921067237854, + "sampling/importance_sampling_ratio/mean": 0.9999237656593323, + "sampling/importance_sampling_ratio/min": 0.747006893157959, + "sampling/sampling_logp_difference/max": 0.3125607967376709, + "sampling/sampling_logp_difference/mean": 0.010578572750091553, + "step": 527 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 722.0, + "completions/mean_length": 485.90625, + "completions/mean_terminated_length": 476.8064270019531, + "completions/min_length": 239.0, + "completions/min_terminated_length": 239.0, + "entropy": 0.29819491878151894, + "epoch": 0.2826552462526767, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.006596745923161507, + "learning_rate": 1e-05, + "loss": 0.0212, + "num_tokens": 11217907.0, + "reward": 0.96875, + "reward_std": 0.0883883461356163, + "rewards/accuracy_reward/mean": 0.96875, + "rewards/accuracy_reward/std": 0.1767766922712326, + "sampling/importance_sampling_ratio/max": 1.779435396194458, + "sampling/importance_sampling_ratio/mean": 0.9997535347938538, + "sampling/importance_sampling_ratio/min": 0.7113785147666931, + "sampling/sampling_logp_difference/max": 0.5762960910797119, + "sampling/sampling_logp_difference/mean": 0.01006353460252285, + "step": 528 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00015349980094470084, + "clip_ratio/low_min": 0.00015349980094470084, + "clip_ratio/region_mean": 0.00015349980094470084, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 734.0, + "completions/mean_length": 504.9375, + "completions/mean_terminated_length": 477.72412109375, + "completions/min_length": 251.0, + "completions/min_terminated_length": 251.0, + "entropy": 0.33065625093877316, + "epoch": 0.2831905781584582, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.01009507104754448, + "learning_rate": 1e-05, + "loss": 0.0059, + "num_tokens": 11237785.0, + "reward": 0.75, + "reward_std": 0.2587745785713196, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.397469162940979, + "sampling/importance_sampling_ratio/mean": 1.0005205869674683, + "sampling/importance_sampling_ratio/min": 0.7454022169113159, + "sampling/sampling_logp_difference/max": 0.33466291427612305, + "sampling/sampling_logp_difference/mean": 0.010764190927147865, + "step": 529 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00011261971667408943, + "clip_ratio/low_min": 0.00011261971667408943, + "clip_ratio/region_mean": 0.00011261971667408943, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 735.0, + "completions/mean_length": 540.59375, + "completions/mean_terminated_length": 498.4814758300781, + "completions/min_length": 219.0, + "completions/min_terminated_length": 219.0, + "entropy": 0.3277864959090948, + "epoch": 0.2837259100642398, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.01308476086705923, + "learning_rate": 1e-05, + "loss": 0.0069, + "num_tokens": 11259068.0, + "reward": 0.84375, + "reward_std": 0.1293872892856598, + "rewards/accuracy_reward/mean": 0.84375, + "rewards/accuracy_reward/std": 0.3689020276069641, + "sampling/importance_sampling_ratio/max": 1.7836689949035645, + "sampling/importance_sampling_ratio/mean": 1.0001953840255737, + "sampling/importance_sampling_ratio/min": 0.731069803237915, + "sampling/sampling_logp_difference/max": 0.5786724090576172, + "sampling/sampling_logp_difference/mean": 0.010434354655444622, + "step": 530 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00013345943807507865, + "clip_ratio/low_min": 0.00013345943807507865, + "clip_ratio/region_mean": 0.00013345943807507865, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 640.0, + "completions/mean_length": 448.125, + "completions/mean_terminated_length": 437.8064270019531, + "completions/min_length": 263.0, + "completions/min_terminated_length": 263.0, + "entropy": 0.340240228921175, + "epoch": 0.2842612419700214, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.008644849993288517, + "learning_rate": 1e-05, + "loss": 0.0528, + "num_tokens": 11277008.0, + "reward": 0.78125, + "reward_std": 0.4218915104866028, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.3213413953781128, + "sampling/importance_sampling_ratio/mean": 1.000316858291626, + "sampling/importance_sampling_ratio/min": 0.7618153691291809, + "sampling/sampling_logp_difference/max": 0.27864742279052734, + "sampling/sampling_logp_difference/mean": 0.010548721067607403, + "step": 531 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 761.0, + "completions/mean_length": 534.40625, + "completions/mean_terminated_length": 501.0357360839844, + "completions/min_length": 271.0, + "completions/min_terminated_length": 271.0, + "entropy": 0.37937711365520954, + "epoch": 0.284796573875803, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.007510210853070021, + "learning_rate": 1e-05, + "loss": 0.0653, + "num_tokens": 11297765.0, + "reward": 0.5625, + "reward_std": 0.4261348247528076, + "rewards/accuracy_reward/mean": 0.5625, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 1.3444803953170776, + "sampling/importance_sampling_ratio/mean": 0.9998617768287659, + "sampling/importance_sampling_ratio/min": 0.7237727642059326, + "sampling/sampling_logp_difference/max": 0.3232778310775757, + "sampling/sampling_logp_difference/mean": 0.011948785744607449, + "step": 532 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00010255300730932504, + "clip_ratio/low_min": 0.00010255300730932504, + "clip_ratio/region_mean": 0.00010255300730932504, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 761.0, + "completions/mean_length": 542.84375, + "completions/mean_terminated_length": 510.6785888671875, + "completions/min_length": 92.0, + "completions/min_terminated_length": 92.0, + "entropy": 0.33837086148560047, + "epoch": 0.28533190578158457, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.019465498626232147, + "learning_rate": 1e-05, + "loss": 0.0268, + "num_tokens": 11318960.0, + "reward": 0.75, + "reward_std": 0.3650856614112854, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.383101463317871, + "sampling/importance_sampling_ratio/mean": 1.000052571296692, + "sampling/importance_sampling_ratio/min": 0.726313591003418, + "sampling/sampling_logp_difference/max": 0.3243284225463867, + "sampling/sampling_logp_difference/mean": 0.011019178666174412, + "step": 533 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 748.0, + "completions/mean_length": 462.71875, + "completions/mean_terminated_length": 431.137939453125, + "completions/min_length": 253.0, + "completions/min_terminated_length": 253.0, + "entropy": 0.38455117493867874, + "epoch": 0.28586723768736616, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.011966320686042309, + "learning_rate": 1e-05, + "loss": 0.0413, + "num_tokens": 11337511.0, + "reward": 0.71875, + "reward_std": 0.2630178928375244, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.394614338874817, + "sampling/importance_sampling_ratio/mean": 1.0001449584960938, + "sampling/importance_sampling_ratio/min": 0.6992799043655396, + "sampling/sampling_logp_difference/max": 0.35770416259765625, + "sampling/sampling_logp_difference/mean": 0.01251339539885521, + "step": 534 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 749.0, + "completions/mean_length": 455.375, + "completions/mean_terminated_length": 423.03448486328125, + "completions/min_length": 183.0, + "completions/min_terminated_length": 183.0, + "entropy": 0.3394015394151211, + "epoch": 0.28640256959314775, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.003924597520381212, + "learning_rate": 1e-05, + "loss": -0.0078, + "num_tokens": 11355939.0, + "reward": 0.71875, + "reward_std": 0.0883883461356163, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.4565744400024414, + "sampling/importance_sampling_ratio/mean": 1.0002892017364502, + "sampling/importance_sampling_ratio/min": 0.7144251465797424, + "sampling/sampling_logp_difference/max": 0.3760874271392822, + "sampling/sampling_logp_difference/mean": 0.011010273359715939, + "step": 535 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 768.0, + "completions/max_terminated_length": 768.0, + "completions/mean_length": 473.28125, + "completions/mean_terminated_length": 473.28125, + "completions/min_length": 209.0, + "completions/min_terminated_length": 209.0, + "entropy": 0.35326141864061356, + "epoch": 0.28693790149892934, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": 0.0, + "num_tokens": 11374788.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/accuracy_reward/mean": 1.0, + "rewards/accuracy_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.2930437326431274, + "sampling/importance_sampling_ratio/mean": 1.0008374452590942, + "sampling/importance_sampling_ratio/min": 0.6965340971946716, + "sampling/sampling_logp_difference/max": 0.36163854598999023, + "sampling/sampling_logp_difference/mean": 0.011547897942364216, + "step": 536 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.451373726828024e-05, + "clip_ratio/low_min": 5.451373726828024e-05, + "clip_ratio/region_mean": 5.451373726828024e-05, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 706.0, + "completions/mean_length": 603.65625, + "completions/mean_terminated_length": 557.6400146484375, + "completions/min_length": 393.0, + "completions/min_terminated_length": 393.0, + "entropy": 0.2894803434610367, + "epoch": 0.2874732334047109, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.012875039130449295, + "learning_rate": 1e-05, + "loss": 0.0792, + "num_tokens": 11398169.0, + "reward": 0.4375, + "reward_std": 0.49022960662841797, + "rewards/accuracy_reward/mean": 0.4375, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 1.3899998664855957, + "sampling/importance_sampling_ratio/mean": 0.9999532103538513, + "sampling/importance_sampling_ratio/min": 0.697039783000946, + "sampling/sampling_logp_difference/max": 0.3609127998352051, + "sampling/sampling_logp_difference/mean": 0.010038269683718681, + "step": 537 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00015962265024427325, + "clip_ratio/low_min": 0.00015962265024427325, + "clip_ratio/region_mean": 0.00015962265024427325, + "completions/clipped_ratio": 0.40625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 734.0, + "completions/mean_length": 614.4375, + "completions/mean_terminated_length": 509.3684387207031, + "completions/min_length": 359.0, + "completions/min_terminated_length": 359.0, + "entropy": 0.576258696615696, + "epoch": 0.2880085653104925, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.013417059555649757, + "learning_rate": 1e-05, + "loss": -0.0048, + "num_tokens": 11422311.0, + "reward": 0.40625, + "reward_std": 0.3198433816432953, + "rewards/accuracy_reward/mean": 0.40625, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.3797985315322876, + "sampling/importance_sampling_ratio/mean": 0.9999678134918213, + "sampling/importance_sampling_ratio/min": 0.6583792567253113, + "sampling/sampling_logp_difference/max": 0.41797423362731934, + "sampling/sampling_logp_difference/mean": 0.015053062699735165, + "step": 538 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 9.074409899767488e-05, + "clip_ratio/low_min": 9.074409899767488e-05, + "clip_ratio/region_mean": 9.074409899767488e-05, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 750.0, + "completions/mean_length": 506.21875, + "completions/mean_terminated_length": 445.8077087402344, + "completions/min_length": 214.0, + "completions/min_terminated_length": 214.0, + "entropy": 0.5533407405018806, + "epoch": 0.2885438972162741, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.014826291240751743, + "learning_rate": 1e-05, + "loss": 0.0122, + "num_tokens": 11442222.0, + "reward": 0.59375, + "reward_std": 0.3377464711666107, + "rewards/accuracy_reward/mean": 0.59375, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.3939472436904907, + "sampling/importance_sampling_ratio/mean": 1.0001685619354248, + "sampling/importance_sampling_ratio/min": 0.7790518999099731, + "sampling/sampling_logp_difference/max": 0.3321394920349121, + "sampling/sampling_logp_difference/mean": 0.014903251081705093, + "step": 539 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.4375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 688.0, + "completions/mean_length": 641.8125, + "completions/mean_terminated_length": 543.6666870117188, + "completions/min_length": 312.0, + "completions/min_terminated_length": 312.0, + "entropy": 0.41299524903297424, + "epoch": 0.2890792291220557, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.005552849732339382, + "learning_rate": 1e-05, + "loss": 0.0707, + "num_tokens": 11466568.0, + "reward": 0.53125, + "reward_std": 0.3471629321575165, + "rewards/accuracy_reward/mean": 0.53125, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.4220200777053833, + "sampling/importance_sampling_ratio/mean": 0.9999431371688843, + "sampling/importance_sampling_ratio/min": 0.6992934346199036, + "sampling/sampling_logp_difference/max": 0.357684850692749, + "sampling/sampling_logp_difference/mean": 0.012839055620133877, + "step": 540 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 731.0, + "completions/mean_length": 528.96875, + "completions/mean_terminated_length": 484.7037048339844, + "completions/min_length": 287.0, + "completions/min_terminated_length": 287.0, + "entropy": 0.36412297561764717, + "epoch": 0.2896145610278373, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.003376058069989085, + "learning_rate": 1e-05, + "loss": -0.021, + "num_tokens": 11487335.0, + "reward": 0.6875, + "reward_std": 0.1157275140285492, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.3501732349395752, + "sampling/importance_sampling_ratio/mean": 0.999661922454834, + "sampling/importance_sampling_ratio/min": 0.7226549983024597, + "sampling/sampling_logp_difference/max": 0.32482337951660156, + "sampling/sampling_logp_difference/mean": 0.011531899683177471, + "step": 541 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.430060991784558e-05, + "clip_ratio/low_min": 5.430060991784558e-05, + "clip_ratio/region_mean": 5.430060991784558e-05, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 758.0, + "completions/mean_length": 505.3125, + "completions/mean_terminated_length": 456.6666564941406, + "completions/min_length": 282.0, + "completions/min_terminated_length": 282.0, + "entropy": 0.4022515006363392, + "epoch": 0.29014989293361887, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.019753659144043922, + "learning_rate": 1e-05, + "loss": -0.0232, + "num_tokens": 11507697.0, + "reward": 0.625, + "reward_std": 0.3335031569004059, + "rewards/accuracy_reward/mean": 0.625, + "rewards/accuracy_reward/std": 0.49186936020851135, + "sampling/importance_sampling_ratio/max": 2.0, + "sampling/importance_sampling_ratio/mean": 1.0000683069229126, + "sampling/importance_sampling_ratio/min": 0.7496616840362549, + "sampling/sampling_logp_difference/max": 0.6967096328735352, + "sampling/sampling_logp_difference/mean": 0.012148507870733738, + "step": 542 + }, + { + "clip_ratio/high_max": 4.701015495811589e-05, + "clip_ratio/high_mean": 4.701015495811589e-05, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 4.701015495811589e-05, + "completions/clipped_ratio": 0.3125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 760.0, + "completions/mean_length": 566.90625, + "completions/mean_terminated_length": 475.5, + "completions/min_length": 187.0, + "completions/min_terminated_length": 187.0, + "entropy": 0.3782684691250324, + "epoch": 0.2906852248394004, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.013792258687317371, + "learning_rate": 1e-05, + "loss": 0.044, + "num_tokens": 11529982.0, + "reward": 0.53125, + "reward_std": 0.2630178928375244, + "rewards/accuracy_reward/mean": 0.53125, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.336808204650879, + "sampling/importance_sampling_ratio/mean": 1.0000829696655273, + "sampling/importance_sampling_ratio/min": 0.6733843088150024, + "sampling/sampling_logp_difference/max": 0.39543914794921875, + "sampling/sampling_logp_difference/mean": 0.012462123297154903, + "step": 543 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 6.42673549009487e-05, + "clip_ratio/low_min": 6.42673549009487e-05, + "clip_ratio/region_mean": 6.42673549009487e-05, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 728.0, + "completions/mean_length": 423.59375, + "completions/mean_terminated_length": 412.4838562011719, + "completions/min_length": 240.0, + "completions/min_terminated_length": 240.0, + "entropy": 0.3379148058593273, + "epoch": 0.291220556745182, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.040512192994356155, + "learning_rate": 1e-05, + "loss": 0.0553, + "num_tokens": 11546929.0, + "reward": 0.90625, + "reward_std": 0.2041158676147461, + "rewards/accuracy_reward/mean": 0.90625, + "rewards/accuracy_reward/std": 0.2961445748806, + "sampling/importance_sampling_ratio/max": 1.434513807296753, + "sampling/importance_sampling_ratio/mean": 1.000313401222229, + "sampling/importance_sampling_ratio/min": 0.6895756125450134, + "sampling/sampling_logp_difference/max": 0.37167888879776, + "sampling/sampling_logp_difference/mean": 0.012057242915034294, + "step": 544 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 723.0, + "completions/mean_length": 558.34375, + "completions/mean_terminated_length": 509.9615478515625, + "completions/min_length": 335.0, + "completions/min_terminated_length": 335.0, + "entropy": 0.3759257663041353, + "epoch": 0.2917558886509636, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": 0.0, + "num_tokens": 11568764.0, + "reward": 0.75, + "reward_std": 0.0, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.4700766801834106, + "sampling/importance_sampling_ratio/mean": 0.9996271133422852, + "sampling/importance_sampling_ratio/min": 0.536020040512085, + "sampling/sampling_logp_difference/max": 0.6235837936401367, + "sampling/sampling_logp_difference/mean": 0.011745463125407696, + "step": 545 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00014392330922419205, + "clip_ratio/low_min": 0.00014392330922419205, + "clip_ratio/region_mean": 0.00014392330922419205, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 756.0, + "completions/mean_length": 561.84375, + "completions/mean_terminated_length": 540.5172119140625, + "completions/min_length": 258.0, + "completions/min_terminated_length": 258.0, + "entropy": 0.37757671996951103, + "epoch": 0.29229122055674517, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.033179812133312225, + "learning_rate": 1e-05, + "loss": 0.0605, + "num_tokens": 11590695.0, + "reward": 0.78125, + "reward_std": 0.3377464711666107, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.7618918418884277, + "sampling/importance_sampling_ratio/mean": 1.0002686977386475, + "sampling/importance_sampling_ratio/min": 0.7163848280906677, + "sampling/sampling_logp_difference/max": 0.5663881301879883, + "sampling/sampling_logp_difference/mean": 0.011982995085418224, + "step": 546 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 750.0, + "completions/mean_length": 550.46875, + "completions/mean_terminated_length": 527.9655151367188, + "completions/min_length": 377.0, + "completions/min_terminated_length": 377.0, + "entropy": 0.30198042280972004, + "epoch": 0.29282655246252676, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.010587194934487343, + "learning_rate": 1e-05, + "loss": 0.0073, + "num_tokens": 11611958.0, + "reward": 0.875, + "reward_std": 0.2177756428718567, + "rewards/accuracy_reward/mean": 0.875, + "rewards/accuracy_reward/std": 0.33601075410842896, + "sampling/importance_sampling_ratio/max": 1.424277901649475, + "sampling/importance_sampling_ratio/mean": 0.9997079372406006, + "sampling/importance_sampling_ratio/min": 0.68252032995224, + "sampling/sampling_logp_difference/max": 0.38196301460266113, + "sampling/sampling_logp_difference/mean": 0.01032177358865738, + "step": 547 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 6.29089045105502e-05, + "clip_ratio/low_min": 6.29089045105502e-05, + "clip_ratio/region_mean": 6.29089045105502e-05, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 728.0, + "completions/mean_length": 533.25, + "completions/mean_terminated_length": 525.6774291992188, + "completions/min_length": 319.0, + "completions/min_terminated_length": 319.0, + "entropy": 0.34730110689997673, + "epoch": 0.29336188436830835, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.0331089049577713, + "learning_rate": 1e-05, + "loss": 0.0456, + "num_tokens": 11633478.0, + "reward": 0.9375, + "reward_std": 0.1767766922712326, + "rewards/accuracy_reward/mean": 0.9375, + "rewards/accuracy_reward/std": 0.24593468010425568, + "sampling/importance_sampling_ratio/max": 1.6550542116165161, + "sampling/importance_sampling_ratio/mean": 1.0001535415649414, + "sampling/importance_sampling_ratio/min": 0.6979007720947266, + "sampling/sampling_logp_difference/max": 0.5038337707519531, + "sampling/sampling_logp_difference/mean": 0.011588075198233128, + "step": 548 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.760368730989285e-05, + "clip_ratio/low_min": 5.760368730989285e-05, + "clip_ratio/region_mean": 5.760368730989285e-05, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 579.0, + "completions/mean_length": 417.65625, + "completions/mean_terminated_length": 394.3000183105469, + "completions/min_length": 211.0, + "completions/min_terminated_length": 211.0, + "entropy": 0.3705006167292595, + "epoch": 0.29389721627408993, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.023140978068113327, + "learning_rate": 1e-05, + "loss": 0.0602, + "num_tokens": 11650819.0, + "reward": 0.75, + "reward_std": 0.3335031569004059, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.3773125410079956, + "sampling/importance_sampling_ratio/mean": 1.0000128746032715, + "sampling/importance_sampling_ratio/min": 0.7603497505187988, + "sampling/sampling_logp_difference/max": 0.32013416290283203, + "sampling/sampling_logp_difference/mean": 0.012140178121626377, + "step": 549 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.545696694753133e-05, + "clip_ratio/low_min": 5.545696694753133e-05, + "clip_ratio/region_mean": 5.545696694753133e-05, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 715.0, + "completions/mean_length": 449.4375, + "completions/mean_terminated_length": 439.1612854003906, + "completions/min_length": 231.0, + "completions/min_terminated_length": 231.0, + "entropy": 0.29957092367112637, + "epoch": 0.2944325481798715, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.004742546007037163, + "learning_rate": 1e-05, + "loss": 0.0437, + "num_tokens": 11668473.0, + "reward": 0.875, + "reward_std": 0.2314550280570984, + "rewards/accuracy_reward/mean": 0.875, + "rewards/accuracy_reward/std": 0.33601075410842896, + "sampling/importance_sampling_ratio/max": 1.457090973854065, + "sampling/importance_sampling_ratio/mean": 0.9997395873069763, + "sampling/importance_sampling_ratio/min": 0.7257310748100281, + "sampling/sampling_logp_difference/max": 0.37644195556640625, + "sampling/sampling_logp_difference/mean": 0.011099728755652905, + "step": 550 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 6.894649413879961e-05, + "clip_ratio/low_min": 6.894649413879961e-05, + "clip_ratio/region_mean": 6.894649413879961e-05, + "completions/clipped_ratio": 0.0, + "completions/max_length": 766.0, + "completions/max_terminated_length": 766.0, + "completions/mean_length": 447.0625, + "completions/mean_terminated_length": 447.0625, + "completions/min_length": 222.0, + "completions/min_terminated_length": 222.0, + "entropy": 0.33931807056069374, + "epoch": 0.2949678800856531, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.02971675433218479, + "learning_rate": 1e-05, + "loss": 0.0353, + "num_tokens": 11686419.0, + "reward": 0.6875, + "reward_std": 0.4355512857437134, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.3912973403930664, + "sampling/importance_sampling_ratio/mean": 1.0001190900802612, + "sampling/importance_sampling_ratio/min": 0.7627151608467102, + "sampling/sampling_logp_difference/max": 0.33023667335510254, + "sampling/sampling_logp_difference/mean": 0.0111058559268713, + "step": 551 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 768.0, + "completions/mean_length": 483.09375, + "completions/mean_terminated_length": 473.9031982421875, + "completions/min_length": 251.0, + "completions/min_terminated_length": 251.0, + "entropy": 0.31367006339132786, + "epoch": 0.2955032119914347, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.0038670250214636326, + "learning_rate": 1e-05, + "loss": 0.0014, + "num_tokens": 11705734.0, + "reward": 0.6875, + "reward_std": 0.2177756428718567, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.4247621297836304, + "sampling/importance_sampling_ratio/mean": 0.9998100996017456, + "sampling/importance_sampling_ratio/min": 0.6891435980796814, + "sampling/sampling_logp_difference/max": 0.37230563163757324, + "sampling/sampling_logp_difference/mean": 0.010242590680718422, + "step": 552 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 701.0, + "completions/mean_length": 486.46875, + "completions/mean_terminated_length": 477.3870849609375, + "completions/min_length": 258.0, + "completions/min_terminated_length": 258.0, + "entropy": 0.33838013373315334, + "epoch": 0.2960385438972163, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.005681267008185387, + "learning_rate": 1e-05, + "loss": 0.0715, + "num_tokens": 11724821.0, + "reward": 0.90625, + "reward_std": 0.2041158676147461, + "rewards/accuracy_reward/mean": 0.90625, + "rewards/accuracy_reward/std": 0.2961445748806, + "sampling/importance_sampling_ratio/max": 1.4111241102218628, + "sampling/importance_sampling_ratio/mean": 1.0002086162567139, + "sampling/importance_sampling_ratio/min": 0.7292938232421875, + "sampling/sampling_logp_difference/max": 0.34438657760620117, + "sampling/sampling_logp_difference/mean": 0.0109095498919487, + "step": 553 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 700.0, + "completions/mean_length": 453.09375, + "completions/mean_terminated_length": 432.10003662109375, + "completions/min_length": 239.0, + "completions/min_terminated_length": 239.0, + "entropy": 0.3436109106987715, + "epoch": 0.2965738758029979, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.006001697387546301, + "learning_rate": 1e-05, + "loss": 0.0345, + "num_tokens": 11742920.0, + "reward": 0.8125, + "reward_std": 0.249358132481575, + "rewards/accuracy_reward/mean": 0.8125, + "rewards/accuracy_reward/std": 0.3965577781200409, + "sampling/importance_sampling_ratio/max": 1.3577940464019775, + "sampling/importance_sampling_ratio/mean": 1.0000940561294556, + "sampling/importance_sampling_ratio/min": 0.6864410042762756, + "sampling/sampling_logp_difference/max": 0.3762350082397461, + "sampling/sampling_logp_difference/mean": 0.010374966077506542, + "step": 554 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.885122300242074e-05, + "clip_ratio/low_min": 5.885122300242074e-05, + "clip_ratio/region_mean": 5.885122300242074e-05, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 750.0, + "completions/mean_length": 474.84375, + "completions/mean_terminated_length": 465.3870849609375, + "completions/min_length": 330.0, + "completions/min_terminated_length": 330.0, + "entropy": 0.38593094795942307, + "epoch": 0.29710920770877947, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.017718249931931496, + "learning_rate": 1e-05, + "loss": 0.0331, + "num_tokens": 11762171.0, + "reward": 0.625, + "reward_std": 0.3335031569004059, + "rewards/accuracy_reward/mean": 0.625, + "rewards/accuracy_reward/std": 0.49186936020851135, + "sampling/importance_sampling_ratio/max": 1.4624276161193848, + "sampling/importance_sampling_ratio/mean": 0.9998676180839539, + "sampling/importance_sampling_ratio/min": 0.711609423160553, + "sampling/sampling_logp_difference/max": 0.3800978660583496, + "sampling/sampling_logp_difference/mean": 0.011882642284035683, + "step": 555 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 762.0, + "completions/mean_length": 518.40625, + "completions/mean_terminated_length": 460.8077087402344, + "completions/min_length": 270.0, + "completions/min_terminated_length": 270.0, + "entropy": 0.3598509766161442, + "epoch": 0.29764453961456105, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.01213877834379673, + "learning_rate": 1e-05, + "loss": 0.031, + "num_tokens": 11782272.0, + "reward": 0.71875, + "reward_std": 0.4628904461860657, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.3928390741348267, + "sampling/importance_sampling_ratio/mean": 1.000434398651123, + "sampling/importance_sampling_ratio/min": 0.7674993276596069, + "sampling/sampling_logp_difference/max": 0.3313441276550293, + "sampling/sampling_logp_difference/mean": 0.011637666262686253, + "step": 556 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00012170137415523641, + "clip_ratio/low_min": 0.00012170137415523641, + "clip_ratio/region_mean": 0.00012170137415523641, + "completions/clipped_ratio": 0.25, + "completions/max_length": 768.0, + "completions/max_terminated_length": 704.0, + "completions/mean_length": 542.25, + "completions/mean_terminated_length": 467.0, + "completions/min_length": 267.0, + "completions/min_terminated_length": 267.0, + "entropy": 0.3468864634633064, + "epoch": 0.2981798715203426, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.026100486516952515, + "learning_rate": 1e-05, + "loss": 0.0751, + "num_tokens": 11803352.0, + "reward": 0.59375, + "reward_std": 0.3061639964580536, + "rewards/accuracy_reward/mean": 0.59375, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.4118132591247559, + "sampling/importance_sampling_ratio/mean": 1.0002919435501099, + "sampling/importance_sampling_ratio/min": 0.7072765231132507, + "sampling/sampling_logp_difference/max": 0.34633350372314453, + "sampling/sampling_logp_difference/mean": 0.011029123328626156, + "step": 557 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.3125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 742.0, + "completions/mean_length": 568.6875, + "completions/mean_terminated_length": 478.0909118652344, + "completions/min_length": 263.0, + "completions/min_terminated_length": 263.0, + "entropy": 0.33586900494992733, + "epoch": 0.2987152034261242, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.007690643426030874, + "learning_rate": 1e-05, + "loss": 0.0607, + "num_tokens": 11825174.0, + "reward": 0.375, + "reward_std": 0.3104073107242584, + "rewards/accuracy_reward/mean": 0.375, + "rewards/accuracy_reward/std": 0.49186936020851135, + "sampling/importance_sampling_ratio/max": 1.4109917879104614, + "sampling/importance_sampling_ratio/mean": 1.0002493858337402, + "sampling/importance_sampling_ratio/min": 0.6766687035560608, + "sampling/sampling_logp_difference/max": 0.39057350158691406, + "sampling/sampling_logp_difference/mean": 0.010549957863986492, + "step": 558 + }, + { + "clip_ratio/high_max": 5.171700468054041e-05, + "clip_ratio/high_mean": 5.171700468054041e-05, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 5.171700468054041e-05, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 696.0, + "completions/mean_length": 549.125, + "completions/mean_terminated_length": 526.4827270507812, + "completions/min_length": 302.0, + "completions/min_terminated_length": 302.0, + "entropy": 0.29706745594739914, + "epoch": 0.29925053533190576, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.012323442846536636, + "learning_rate": 1e-05, + "loss": 0.042, + "num_tokens": 11846578.0, + "reward": 0.875, + "reward_std": 0.2177756428718567, + "rewards/accuracy_reward/mean": 0.875, + "rewards/accuracy_reward/std": 0.33601075410842896, + "sampling/importance_sampling_ratio/max": 1.4247862100601196, + "sampling/importance_sampling_ratio/mean": 1.0000526905059814, + "sampling/importance_sampling_ratio/min": 0.6969889402389526, + "sampling/sampling_logp_difference/max": 0.36098575592041016, + "sampling/sampling_logp_difference/mean": 0.01032998226583004, + "step": 559 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 7.327080675167963e-05, + "clip_ratio/low_min": 7.327080675167963e-05, + "clip_ratio/region_mean": 7.327080675167963e-05, + "completions/clipped_ratio": 0.0, + "completions/max_length": 600.0, + "completions/max_terminated_length": 600.0, + "completions/mean_length": 401.125, + "completions/mean_terminated_length": 401.125, + "completions/min_length": 164.0, + "completions/min_terminated_length": 164.0, + "entropy": 0.28526004403829575, + "epoch": 0.29978586723768735, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.010400772094726562, + "learning_rate": 1e-05, + "loss": -0.0236, + "num_tokens": 11862974.0, + "reward": 0.84375, + "reward_std": 0.22201895713806152, + "rewards/accuracy_reward/mean": 0.84375, + "rewards/accuracy_reward/std": 0.3689020276069641, + "sampling/importance_sampling_ratio/max": 1.4232308864593506, + "sampling/importance_sampling_ratio/mean": 0.9998368620872498, + "sampling/importance_sampling_ratio/min": 0.763550341129303, + "sampling/sampling_logp_difference/max": 0.35292959213256836, + "sampling/sampling_logp_difference/mean": 0.009960007853806019, + "step": 560 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 4.493170490604825e-05, + "clip_ratio/low_min": 4.493170490604825e-05, + "clip_ratio/region_mean": 4.493170490604825e-05, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 732.0, + "completions/mean_length": 575.34375, + "completions/mean_terminated_length": 499.95654296875, + "completions/min_length": 345.0, + "completions/min_terminated_length": 345.0, + "entropy": 0.3350704815238714, + "epoch": 0.30032119914346894, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.01137653086334467, + "learning_rate": 1e-05, + "loss": 0.0114, + "num_tokens": 11884577.0, + "reward": 0.65625, + "reward_std": 0.3377464711666107, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.2981727123260498, + "sampling/importance_sampling_ratio/mean": 0.999843418598175, + "sampling/importance_sampling_ratio/min": 0.7069149613380432, + "sampling/sampling_logp_difference/max": 0.3468449115753174, + "sampling/sampling_logp_difference/mean": 0.010699411854147911, + "step": 561 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00011851915041916072, + "clip_ratio/low_min": 0.00011851915041916072, + "clip_ratio/region_mean": 0.00011851915041916072, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 706.0, + "completions/mean_length": 506.1875, + "completions/mean_terminated_length": 488.7333679199219, + "completions/min_length": 312.0, + "completions/min_terminated_length": 312.0, + "entropy": 0.2681577932089567, + "epoch": 0.30085653104925053, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.008579129353165627, + "learning_rate": 1e-05, + "loss": 0.0102, + "num_tokens": 11904759.0, + "reward": 0.65625, + "reward_std": 0.22201895713806152, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.4129170179367065, + "sampling/importance_sampling_ratio/mean": 1.000343680381775, + "sampling/importance_sampling_ratio/min": 0.7034191489219666, + "sampling/sampling_logp_difference/max": 0.35180234909057617, + "sampling/sampling_logp_difference/mean": 0.009591000154614449, + "step": 562 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00010635163926053792, + "clip_ratio/low_min": 0.00010635163926053792, + "clip_ratio/region_mean": 0.00010635163926053792, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 717.0, + "completions/mean_length": 560.59375, + "completions/mean_terminated_length": 502.5199890136719, + "completions/min_length": 282.0, + "completions/min_terminated_length": 282.0, + "entropy": 0.3293144442141056, + "epoch": 0.3013918629550321, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.003007086692377925, + "learning_rate": 1e-05, + "loss": -0.0021, + "num_tokens": 11926234.0, + "reward": 0.625, + "reward_std": 0.2177756428718567, + "rewards/accuracy_reward/mean": 0.625, + "rewards/accuracy_reward/std": 0.49186936020851135, + "sampling/importance_sampling_ratio/max": 1.4711403846740723, + "sampling/importance_sampling_ratio/mean": 1.000432014465332, + "sampling/importance_sampling_ratio/min": 0.6421256065368652, + "sampling/sampling_logp_difference/max": 0.4429713487625122, + "sampling/sampling_logp_difference/mean": 0.010263259522616863, + "step": 563 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00010347682109568268, + "clip_ratio/low_min": 0.00010347682109568268, + "clip_ratio/region_mean": 0.00010347682109568268, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 609.0, + "completions/mean_length": 565.90625, + "completions/mean_terminated_length": 486.82611083984375, + "completions/min_length": 344.0, + "completions/min_terminated_length": 344.0, + "entropy": 0.5083322878926992, + "epoch": 0.3019271948608137, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.009941553696990013, + "learning_rate": 1e-05, + "loss": 0.0005, + "num_tokens": 11948487.0, + "reward": 0.59375, + "reward_std": 0.22201895713806152, + "rewards/accuracy_reward/mean": 0.59375, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.7080460786819458, + "sampling/importance_sampling_ratio/mean": 1.0003758668899536, + "sampling/importance_sampling_ratio/min": 0.6820323467254639, + "sampling/sampling_logp_difference/max": 0.5353500843048096, + "sampling/sampling_logp_difference/mean": 0.015410225838422775, + "step": 564 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 681.0, + "completions/mean_length": 476.21875, + "completions/mean_terminated_length": 456.7666931152344, + "completions/min_length": 227.0, + "completions/min_terminated_length": 227.0, + "entropy": 0.35852690041065216, + "epoch": 0.3024625267665953, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.012777484022080898, + "learning_rate": 1e-05, + "loss": 0.0032, + "num_tokens": 11967254.0, + "reward": 0.59375, + "reward_std": 0.24511480331420898, + "rewards/accuracy_reward/mean": 0.59375, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.363776445388794, + "sampling/importance_sampling_ratio/mean": 1.0000510215759277, + "sampling/importance_sampling_ratio/min": 0.6584330201148987, + "sampling/sampling_logp_difference/max": 0.4178924560546875, + "sampling/sampling_logp_difference/mean": 0.011805204674601555, + "step": 565 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 678.0, + "completions/max_terminated_length": 678.0, + "completions/mean_length": 474.25, + "completions/mean_terminated_length": 474.25, + "completions/min_length": 217.0, + "completions/min_terminated_length": 217.0, + "entropy": 0.27754843421280384, + "epoch": 0.3029978586723769, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.004616685211658478, + "learning_rate": 1e-05, + "loss": -0.0059, + "num_tokens": 11986078.0, + "reward": 0.875, + "reward_std": 0.2177756428718567, + "rewards/accuracy_reward/mean": 0.875, + "rewards/accuracy_reward/std": 0.33601075410842896, + "sampling/importance_sampling_ratio/max": 1.4319286346435547, + "sampling/importance_sampling_ratio/mean": 1.0002179145812988, + "sampling/importance_sampling_ratio/min": 0.7719475626945496, + "sampling/sampling_logp_difference/max": 0.3590221405029297, + "sampling/sampling_logp_difference/mean": 0.009731479920446873, + "step": 566 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 9.335325012216344e-05, + "clip_ratio/low_min": 9.335325012216344e-05, + "clip_ratio/region_mean": 9.335325012216344e-05, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 626.0, + "completions/mean_length": 438.65625, + "completions/mean_terminated_length": 416.70001220703125, + "completions/min_length": 214.0, + "completions/min_terminated_length": 214.0, + "entropy": 0.2740330286324024, + "epoch": 0.3035331905781585, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.004365598317235708, + "learning_rate": 1e-05, + "loss": 0.0706, + "num_tokens": 12003803.0, + "reward": 0.6875, + "reward_std": 0.2177756428718567, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.346114993095398, + "sampling/importance_sampling_ratio/mean": 1.0001251697540283, + "sampling/importance_sampling_ratio/min": 0.7477982640266418, + "sampling/sampling_logp_difference/max": 0.2972226142883301, + "sampling/sampling_logp_difference/mean": 0.009908981621265411, + "step": 567 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 660.0, + "completions/max_terminated_length": 660.0, + "completions/mean_length": 435.59375, + "completions/mean_terminated_length": 435.59375, + "completions/min_length": 101.0, + "completions/min_terminated_length": 101.0, + "entropy": 0.37362296879291534, + "epoch": 0.30406852248394006, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.022444570437073708, + "learning_rate": 1e-05, + "loss": -0.0767, + "num_tokens": 12020990.0, + "reward": 0.875, + "reward_std": 0.2177756428718567, + "rewards/accuracy_reward/mean": 0.875, + "rewards/accuracy_reward/std": 0.33601075410842896, + "sampling/importance_sampling_ratio/max": 1.4383749961853027, + "sampling/importance_sampling_ratio/mean": 1.0005102157592773, + "sampling/importance_sampling_ratio/min": 0.619283139705658, + "sampling/sampling_logp_difference/max": 0.47919273376464844, + "sampling/sampling_logp_difference/mean": 0.010805340483784676, + "step": 568 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.577866977546364e-05, + "clip_ratio/low_min": 5.577866977546364e-05, + "clip_ratio/region_mean": 5.577866977546364e-05, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 723.0, + "completions/mean_length": 474.78125, + "completions/mean_terminated_length": 465.32257080078125, + "completions/min_length": 259.0, + "completions/min_terminated_length": 259.0, + "entropy": 0.3136964626610279, + "epoch": 0.30460385438972165, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.00746798375621438, + "learning_rate": 1e-05, + "loss": 0.0394, + "num_tokens": 12039719.0, + "reward": 0.6875, + "reward_std": 0.2587745785713196, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.4146543741226196, + "sampling/importance_sampling_ratio/mean": 1.0001848936080933, + "sampling/importance_sampling_ratio/min": 0.7112087607383728, + "sampling/sampling_logp_difference/max": 0.34688520431518555, + "sampling/sampling_logp_difference/mean": 0.010922006331384182, + "step": 569 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 7.171543256845325e-05, + "clip_ratio/low_min": 7.171543256845325e-05, + "clip_ratio/region_mean": 7.171543256845325e-05, + "completions/clipped_ratio": 0.0, + "completions/max_length": 711.0, + "completions/max_terminated_length": 711.0, + "completions/mean_length": 420.5625, + "completions/mean_terminated_length": 420.5625, + "completions/min_length": 231.0, + "completions/min_terminated_length": 231.0, + "entropy": 0.3946829177439213, + "epoch": 0.30513918629550324, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.027273882180452347, + "learning_rate": 1e-05, + "loss": 0.0292, + "num_tokens": 12057401.0, + "reward": 0.65625, + "reward_std": 0.3808925747871399, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.4265540838241577, + "sampling/importance_sampling_ratio/mean": 1.0003025531768799, + "sampling/importance_sampling_ratio/min": 0.699528694152832, + "sampling/sampling_logp_difference/max": 0.3573484420776367, + "sampling/sampling_logp_difference/mean": 0.013109739869832993, + "step": 570 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 743.0, + "completions/mean_length": 509.6875, + "completions/mean_terminated_length": 501.3548278808594, + "completions/min_length": 238.0, + "completions/min_terminated_length": 238.0, + "entropy": 0.3093193285167217, + "epoch": 0.3056745182012848, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.0035189527552574873, + "learning_rate": 1e-05, + "loss": -0.0725, + "num_tokens": 12077711.0, + "reward": 0.75, + "reward_std": 0.1767766922712326, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.3714594841003418, + "sampling/importance_sampling_ratio/mean": 1.000036358833313, + "sampling/importance_sampling_ratio/min": 0.6436813473701477, + "sampling/sampling_logp_difference/max": 0.4405515193939209, + "sampling/sampling_logp_difference/mean": 0.010766547173261642, + "step": 571 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00021212662977632135, + "clip_ratio/low_min": 0.00021212662977632135, + "clip_ratio/region_mean": 0.00021212662977632135, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 660.0, + "completions/mean_length": 479.53125, + "completions/mean_terminated_length": 460.3000183105469, + "completions/min_length": 171.0, + "completions/min_terminated_length": 171.0, + "entropy": 0.3884589597582817, + "epoch": 0.30620985010706636, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.006360316649079323, + "learning_rate": 1e-05, + "loss": -0.0034, + "num_tokens": 12096544.0, + "reward": 0.71875, + "reward_std": 0.3198433816432953, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.4243212938308716, + "sampling/importance_sampling_ratio/mean": 0.9998013973236084, + "sampling/importance_sampling_ratio/min": 0.7178239226341248, + "sampling/sampling_logp_difference/max": 0.3536953926086426, + "sampling/sampling_logp_difference/mean": 0.01242065615952015, + "step": 572 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 7.066139369271696e-05, + "clip_ratio/low_min": 7.066139369271696e-05, + "clip_ratio/region_mean": 7.066139369271696e-05, + "completions/clipped_ratio": 0.0, + "completions/max_length": 700.0, + "completions/max_terminated_length": 700.0, + "completions/mean_length": 416.84375, + "completions/mean_terminated_length": 416.84375, + "completions/min_length": 245.0, + "completions/min_terminated_length": 245.0, + "entropy": 0.32096610218286514, + "epoch": 0.30674518201284795, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.026198146864771843, + "learning_rate": 1e-05, + "loss": -0.0354, + "num_tokens": 12113083.0, + "reward": 0.5, + "reward_std": 0.1767766922712326, + "rewards/accuracy_reward/mean": 0.5, + "rewards/accuracy_reward/std": 0.5080004930496216, + "sampling/importance_sampling_ratio/max": 1.3706988096237183, + "sampling/importance_sampling_ratio/mean": 1.0001565217971802, + "sampling/importance_sampling_ratio/min": 0.7378323078155518, + "sampling/sampling_logp_difference/max": 0.3153207302093506, + "sampling/sampling_logp_difference/mean": 0.011015393771231174, + "step": 573 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 762.0, + "completions/mean_length": 495.1875, + "completions/mean_terminated_length": 456.21429443359375, + "completions/min_length": 248.0, + "completions/min_terminated_length": 248.0, + "entropy": 0.34507349506020546, + "epoch": 0.30728051391862954, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.014050091616809368, + "learning_rate": 1e-05, + "loss": 0.0363, + "num_tokens": 12132921.0, + "reward": 0.75, + "reward_std": 0.2587745785713196, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.451440453529358, + "sampling/importance_sampling_ratio/mean": 0.9998599290847778, + "sampling/importance_sampling_ratio/min": 0.7007318735122681, + "sampling/sampling_logp_difference/max": 0.3725564479827881, + "sampling/sampling_logp_difference/mean": 0.011650852859020233, + "step": 574 + }, + { + "clip_ratio/high_max": 8.350033749593422e-05, + "clip_ratio/high_mean": 8.350033749593422e-05, + "clip_ratio/low_mean": 6.551362457685173e-05, + "clip_ratio/low_min": 6.551362457685173e-05, + "clip_ratio/region_mean": 0.00014901396207278594, + "completions/clipped_ratio": 0.0, + "completions/max_length": 659.0, + "completions/max_terminated_length": 659.0, + "completions/mean_length": 412.4375, + "completions/mean_terminated_length": 412.4375, + "completions/min_length": 151.0, + "completions/min_terminated_length": 151.0, + "entropy": 0.40968939661979675, + "epoch": 0.3078158458244111, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.014298057183623314, + "learning_rate": 1e-05, + "loss": -0.0156, + "num_tokens": 12149927.0, + "reward": 0.65625, + "reward_std": 0.4807935357093811, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.3289611339569092, + "sampling/importance_sampling_ratio/mean": 0.9999315142631531, + "sampling/importance_sampling_ratio/min": 0.6963590383529663, + "sampling/sampling_logp_difference/max": 0.3618898391723633, + "sampling/sampling_logp_difference/mean": 0.013523918576538563, + "step": 575 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00021885034948354587, + "clip_ratio/low_min": 0.00021885034948354587, + "clip_ratio/region_mean": 0.00021885034948354587, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 610.0, + "completions/mean_length": 454.34375, + "completions/mean_terminated_length": 433.433349609375, + "completions/min_length": 293.0, + "completions/min_terminated_length": 293.0, + "entropy": 0.5596118047833443, + "epoch": 0.3083511777301927, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.017974786460399628, + "learning_rate": 1e-05, + "loss": 0.0508, + "num_tokens": 12168386.0, + "reward": 0.71875, + "reward_std": 0.3198433816432953, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.6629281044006348, + "sampling/importance_sampling_ratio/mean": 1.000468134880066, + "sampling/importance_sampling_ratio/min": 0.6651910543441772, + "sampling/sampling_logp_difference/max": 0.5085799694061279, + "sampling/sampling_logp_difference/mean": 0.015394064597785473, + "step": 576 + }, + { + "clip_ratio/high_max": 4.936808909405954e-05, + "clip_ratio/high_mean": 4.936808909405954e-05, + "clip_ratio/low_mean": 8.355615136679262e-05, + "clip_ratio/low_min": 8.355615136679262e-05, + "clip_ratio/region_mean": 0.00013292424046085216, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 681.0, + "completions/mean_length": 494.8125, + "completions/mean_terminated_length": 455.7857360839844, + "completions/min_length": 234.0, + "completions/min_terminated_length": 234.0, + "entropy": 0.41458289697766304, + "epoch": 0.3088865096359743, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.010435909032821655, + "learning_rate": 1e-05, + "loss": -0.0022, + "num_tokens": 12188092.0, + "reward": 0.625, + "reward_std": 0.13363061845302582, + "rewards/accuracy_reward/mean": 0.625, + "rewards/accuracy_reward/std": 0.49186936020851135, + "sampling/importance_sampling_ratio/max": 2.0, + "sampling/importance_sampling_ratio/mean": 0.9999361634254456, + "sampling/importance_sampling_ratio/min": 0.651833176612854, + "sampling/sampling_logp_difference/max": 0.710848331451416, + "sampling/sampling_logp_difference/mean": 0.013841825537383556, + "step": 577 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 752.0, + "completions/mean_length": 445.75, + "completions/mean_terminated_length": 435.3548278808594, + "completions/min_length": 235.0, + "completions/min_terminated_length": 235.0, + "entropy": 0.2802911829203367, + "epoch": 0.3094218415417559, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.009748955257236958, + "learning_rate": 1e-05, + "loss": 0.0593, + "num_tokens": 12205708.0, + "reward": 0.78125, + "reward_std": 0.3061639964580536, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.4513030052185059, + "sampling/importance_sampling_ratio/mean": 1.0002527236938477, + "sampling/importance_sampling_ratio/min": 0.7443873882293701, + "sampling/sampling_logp_difference/max": 0.37246179580688477, + "sampling/sampling_logp_difference/mean": 0.01012650690972805, + "step": 578 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 618.0, + "completions/max_terminated_length": 618.0, + "completions/mean_length": 433.96875, + "completions/mean_terminated_length": 433.96875, + "completions/min_length": 264.0, + "completions/min_terminated_length": 264.0, + "entropy": 0.2798552177846432, + "epoch": 0.3099571734475375, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.007197749800980091, + "learning_rate": 1e-05, + "loss": 0.0012, + "num_tokens": 12223843.0, + "reward": 0.9375, + "reward_std": 0.1157275140285492, + "rewards/accuracy_reward/mean": 0.9375, + "rewards/accuracy_reward/std": 0.24593468010425568, + "sampling/importance_sampling_ratio/max": 1.3898836374282837, + "sampling/importance_sampling_ratio/mean": 0.9999532103538513, + "sampling/importance_sampling_ratio/min": 0.6809445023536682, + "sampling/sampling_logp_difference/max": 0.3842744827270508, + "sampling/sampling_logp_difference/mean": 0.010100327432155609, + "step": 579 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 694.0, + "completions/max_terminated_length": 694.0, + "completions/mean_length": 477.6875, + "completions/mean_terminated_length": 477.6875, + "completions/min_length": 329.0, + "completions/min_terminated_length": 329.0, + "entropy": 0.3741901181638241, + "epoch": 0.31049250535331907, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": 0.0116, + "num_tokens": 12243473.0, + "reward": 0.96875, + "reward_std": 0.0883883461356163, + "rewards/accuracy_reward/mean": 0.96875, + "rewards/accuracy_reward/std": 0.1767766922712326, + "sampling/importance_sampling_ratio/max": 1.6115895509719849, + "sampling/importance_sampling_ratio/mean": 0.9998815655708313, + "sampling/importance_sampling_ratio/min": 0.7135924100875854, + "sampling/sampling_logp_difference/max": 0.4772210121154785, + "sampling/sampling_logp_difference/mean": 0.012397648766636848, + "step": 580 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00014245013881009072, + "clip_ratio/low_min": 0.00014245013881009072, + "clip_ratio/region_mean": 0.00014245013881009072, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 700.0, + "completions/mean_length": 458.46875, + "completions/mean_terminated_length": 437.8333435058594, + "completions/min_length": 224.0, + "completions/min_terminated_length": 224.0, + "entropy": 0.36436559818685055, + "epoch": 0.31102783725910066, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.008065787144005299, + "learning_rate": 1e-05, + "loss": 0.0445, + "num_tokens": 12261968.0, + "reward": 0.90625, + "reward_std": 0.1293872892856598, + "rewards/accuracy_reward/mean": 0.90625, + "rewards/accuracy_reward/std": 0.2961445748806, + "sampling/importance_sampling_ratio/max": 1.4186723232269287, + "sampling/importance_sampling_ratio/mean": 0.9999932646751404, + "sampling/importance_sampling_ratio/min": 0.5788215398788452, + "sampling/sampling_logp_difference/max": 0.5467610359191895, + "sampling/sampling_logp_difference/mean": 0.011901299469172955, + "step": 581 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.628095459542237e-05, + "clip_ratio/low_min": 5.628095459542237e-05, + "clip_ratio/region_mean": 5.628095459542237e-05, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 703.0, + "completions/mean_length": 489.34375, + "completions/mean_terminated_length": 470.7666931152344, + "completions/min_length": 313.0, + "completions/min_terminated_length": 313.0, + "entropy": 0.31481700018048286, + "epoch": 0.31156316916488225, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.009348850697278976, + "learning_rate": 1e-05, + "loss": 0.0264, + "num_tokens": 12280947.0, + "reward": 0.84375, + "reward_std": 0.24511480331420898, + "rewards/accuracy_reward/mean": 0.84375, + "rewards/accuracy_reward/std": 0.3689020276069641, + "sampling/importance_sampling_ratio/max": 1.4497952461242676, + "sampling/importance_sampling_ratio/mean": 0.9998577237129211, + "sampling/importance_sampling_ratio/min": 0.6553128957748413, + "sampling/sampling_logp_difference/max": 0.42264246940612793, + "sampling/sampling_logp_difference/mean": 0.010949268937110901, + "step": 582 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 660.0, + "completions/mean_length": 483.28125, + "completions/mean_terminated_length": 417.5769348144531, + "completions/min_length": 163.0, + "completions/min_terminated_length": 163.0, + "entropy": 0.4111689142882824, + "epoch": 0.31209850107066384, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.006937266327440739, + "learning_rate": 1e-05, + "loss": 0.0841, + "num_tokens": 12299828.0, + "reward": 0.625, + "reward_std": 0.2314550280570984, + "rewards/accuracy_reward/mean": 0.625, + "rewards/accuracy_reward/std": 0.49186936020851135, + "sampling/importance_sampling_ratio/max": 1.3981070518493652, + "sampling/importance_sampling_ratio/mean": 1.0002576112747192, + "sampling/importance_sampling_ratio/min": 0.6941964626312256, + "sampling/sampling_logp_difference/max": 0.36500024795532227, + "sampling/sampling_logp_difference/mean": 0.012879115529358387, + "step": 583 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 580.0, + "completions/max_terminated_length": 580.0, + "completions/mean_length": 362.75, + "completions/mean_terminated_length": 362.75, + "completions/min_length": 194.0, + "completions/min_terminated_length": 194.0, + "entropy": 0.3672071844339371, + "epoch": 0.31263383297644537, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.0040225074626505375, + "learning_rate": 1e-05, + "loss": 0.0381, + "num_tokens": 12314796.0, + "reward": 0.9375, + "reward_std": 0.1767766922712326, + "rewards/accuracy_reward/mean": 0.9375, + "rewards/accuracy_reward/std": 0.24593468010425568, + "sampling/importance_sampling_ratio/max": 2.0, + "sampling/importance_sampling_ratio/mean": 1.0000457763671875, + "sampling/importance_sampling_ratio/min": 0.685818076133728, + "sampling/sampling_logp_difference/max": 0.959202766418457, + "sampling/sampling_logp_difference/mean": 0.01274156291037798, + "step": 584 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 700.0, + "completions/max_terminated_length": 700.0, + "completions/mean_length": 425.46875, + "completions/mean_terminated_length": 425.46875, + "completions/min_length": 229.0, + "completions/min_terminated_length": 229.0, + "entropy": 0.3684750981628895, + "epoch": 0.31316916488222696, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.006314178463071585, + "learning_rate": 1e-05, + "loss": -0.0178, + "num_tokens": 12331947.0, + "reward": 0.9375, + "reward_std": 0.1767766922712326, + "rewards/accuracy_reward/mean": 0.9375, + "rewards/accuracy_reward/std": 0.24593468010425568, + "sampling/importance_sampling_ratio/max": 1.5478490591049194, + "sampling/importance_sampling_ratio/mean": 1.0000567436218262, + "sampling/importance_sampling_ratio/min": 0.608217716217041, + "sampling/sampling_logp_difference/max": 0.4972224235534668, + "sampling/sampling_logp_difference/mean": 0.012741255573928356, + "step": 585 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.7313158322358504e-05, + "clip_ratio/low_min": 5.7313158322358504e-05, + "clip_ratio/region_mean": 5.7313158322358504e-05, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 763.0, + "completions/mean_length": 495.5, + "completions/mean_terminated_length": 486.70965576171875, + "completions/min_length": 305.0, + "completions/min_terminated_length": 305.0, + "entropy": 0.2861919142305851, + "epoch": 0.31370449678800855, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.01361022423952818, + "learning_rate": 1e-05, + "loss": -0.0097, + "num_tokens": 12351307.0, + "reward": 0.84375, + "reward_std": 0.3061639964580536, + "rewards/accuracy_reward/mean": 0.84375, + "rewards/accuracy_reward/std": 0.3689020276069641, + "sampling/importance_sampling_ratio/max": 1.4768742322921753, + "sampling/importance_sampling_ratio/mean": 0.9997875690460205, + "sampling/importance_sampling_ratio/min": 0.6761652231216431, + "sampling/sampling_logp_difference/max": 0.39131784439086914, + "sampling/sampling_logp_difference/mean": 0.010119804181158543, + "step": 586 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 4.276428444427438e-05, + "clip_ratio/low_min": 4.276428444427438e-05, + "clip_ratio/region_mean": 4.276428444427438e-05, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 730.0, + "completions/mean_length": 541.78125, + "completions/mean_terminated_length": 478.44000244140625, + "completions/min_length": 301.0, + "completions/min_terminated_length": 301.0, + "entropy": 0.3428086116909981, + "epoch": 0.31423982869379014, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.010882562957704067, + "learning_rate": 1e-05, + "loss": 0.081, + "num_tokens": 12372084.0, + "reward": 0.53125, + "reward_std": 0.378745436668396, + "rewards/accuracy_reward/mean": 0.53125, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.6300252676010132, + "sampling/importance_sampling_ratio/mean": 1.0001062154769897, + "sampling/importance_sampling_ratio/min": 0.6990644931793213, + "sampling/sampling_logp_difference/max": 0.48859548568725586, + "sampling/sampling_logp_difference/mean": 0.010662877932190895, + "step": 587 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 595.0, + "completions/max_terminated_length": 595.0, + "completions/mean_length": 408.125, + "completions/mean_terminated_length": 408.125, + "completions/min_length": 240.0, + "completions/min_terminated_length": 240.0, + "entropy": 0.2970775030553341, + "epoch": 0.3147751605995717, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.0053651840426027775, + "learning_rate": 1e-05, + "loss": 0.0052, + "num_tokens": 12388208.0, + "reward": 0.90625, + "reward_std": 0.1293872892856598, + "rewards/accuracy_reward/mean": 0.90625, + "rewards/accuracy_reward/std": 0.2961445748806, + "sampling/importance_sampling_ratio/max": 1.4733755588531494, + "sampling/importance_sampling_ratio/mean": 0.9997366070747375, + "sampling/importance_sampling_ratio/min": 0.7308172583580017, + "sampling/sampling_logp_difference/max": 0.3875560760498047, + "sampling/sampling_logp_difference/mean": 0.009974067099392414, + "step": 588 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00017644630133872852, + "clip_ratio/low_min": 0.00017644630133872852, + "clip_ratio/region_mean": 0.00017644630133872852, + "completions/clipped_ratio": 0.0, + "completions/max_length": 574.0, + "completions/max_terminated_length": 574.0, + "completions/mean_length": 352.3125, + "completions/mean_terminated_length": 352.3125, + "completions/min_length": 204.0, + "completions/min_terminated_length": 204.0, + "entropy": 0.3529447540640831, + "epoch": 0.3153104925053533, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.006927943788468838, + "learning_rate": 1e-05, + "loss": -0.0377, + "num_tokens": 12403074.0, + "reward": 0.875, + "reward_std": 0.292504221200943, + "rewards/accuracy_reward/mean": 0.875, + "rewards/accuracy_reward/std": 0.33601075410842896, + "sampling/importance_sampling_ratio/max": 1.2885662317276, + "sampling/importance_sampling_ratio/mean": 0.9999823570251465, + "sampling/importance_sampling_ratio/min": 0.727958619594574, + "sampling/sampling_logp_difference/max": 0.31751108169555664, + "sampling/sampling_logp_difference/mean": 0.011859443970024586, + "step": 589 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00014966265007387847, + "clip_ratio/low_min": 0.00014966265007387847, + "clip_ratio/region_mean": 0.00014966265007387847, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 726.0, + "completions/mean_length": 456.875, + "completions/mean_terminated_length": 436.13336181640625, + "completions/min_length": 260.0, + "completions/min_terminated_length": 260.0, + "entropy": 0.302200585603714, + "epoch": 0.3158458244111349, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.008140325546264648, + "learning_rate": 1e-05, + "loss": -0.0021, + "num_tokens": 12421270.0, + "reward": 0.71875, + "reward_std": 0.2041158676147461, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.3672969341278076, + "sampling/importance_sampling_ratio/mean": 0.999929666519165, + "sampling/importance_sampling_ratio/min": 0.7372152805328369, + "sampling/sampling_logp_difference/max": 0.312835693359375, + "sampling/sampling_logp_difference/mean": 0.010572618804872036, + "step": 590 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 697.0, + "completions/mean_length": 456.34375, + "completions/mean_terminated_length": 435.5666809082031, + "completions/min_length": 223.0, + "completions/min_terminated_length": 223.0, + "entropy": 0.3139870762825012, + "epoch": 0.3163811563169165, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.00583472428843379, + "learning_rate": 1e-05, + "loss": 0.0876, + "num_tokens": 12439761.0, + "reward": 0.6875, + "reward_std": 0.1767766922712326, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.4433567523956299, + "sampling/importance_sampling_ratio/mean": 0.9998515844345093, + "sampling/importance_sampling_ratio/min": 0.5484488606452942, + "sampling/sampling_logp_difference/max": 0.6006612777709961, + "sampling/sampling_logp_difference/mean": 0.011254210956394672, + "step": 591 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 628.0, + "completions/max_terminated_length": 628.0, + "completions/mean_length": 406.25, + "completions/mean_terminated_length": 406.25, + "completions/min_length": 267.0, + "completions/min_terminated_length": 267.0, + "entropy": 0.3209523782134056, + "epoch": 0.3169164882226981, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.0074307601898908615, + "learning_rate": 1e-05, + "loss": 0.0107, + "num_tokens": 12456321.0, + "reward": 0.90625, + "reward_std": 0.2041158676147461, + "rewards/accuracy_reward/mean": 0.90625, + "rewards/accuracy_reward/std": 0.2961445748806, + "sampling/importance_sampling_ratio/max": 1.3655701875686646, + "sampling/importance_sampling_ratio/mean": 0.9996328949928284, + "sampling/importance_sampling_ratio/min": 0.6671118140220642, + "sampling/sampling_logp_difference/max": 0.4047975540161133, + "sampling/sampling_logp_difference/mean": 0.011223303154110909, + "step": 592 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 678.0, + "completions/mean_length": 434.53125, + "completions/mean_terminated_length": 386.89288330078125, + "completions/min_length": 229.0, + "completions/min_terminated_length": 229.0, + "entropy": 0.5003904700279236, + "epoch": 0.31745182012847967, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.03328745439648628, + "learning_rate": 1e-05, + "loss": -0.0125, + "num_tokens": 12474082.0, + "reward": 0.53125, + "reward_std": 0.2041158676147461, + "rewards/accuracy_reward/mean": 0.53125, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.4050369262695312, + "sampling/importance_sampling_ratio/mean": 1.0001511573791504, + "sampling/importance_sampling_ratio/min": 0.6716787219047546, + "sampling/sampling_logp_difference/max": 0.39797520637512207, + "sampling/sampling_logp_difference/mean": 0.015175948850810528, + "step": 593 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00012303800394874997, + "clip_ratio/low_min": 0.00012303800394874997, + "clip_ratio/region_mean": 0.00012303800394874997, + "completions/clipped_ratio": 0.25, + "completions/max_length": 768.0, + "completions/max_terminated_length": 751.0, + "completions/mean_length": 544.3125, + "completions/mean_terminated_length": 469.75, + "completions/min_length": 297.0, + "completions/min_terminated_length": 297.0, + "entropy": 0.36914557218551636, + "epoch": 0.31798715203426126, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.017193464562296867, + "learning_rate": 1e-05, + "loss": 0.0734, + "num_tokens": 12495924.0, + "reward": 0.53125, + "reward_std": 0.4397946000099182, + "rewards/accuracy_reward/mean": 0.53125, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.4910573959350586, + "sampling/importance_sampling_ratio/mean": 0.9997447729110718, + "sampling/importance_sampling_ratio/min": 0.6404514908790588, + "sampling/sampling_logp_difference/max": 0.44558191299438477, + "sampling/sampling_logp_difference/mean": 0.012314929626882076, + "step": 594 + }, + { + "clip_ratio/high_max": 6.256256165215746e-05, + "clip_ratio/high_mean": 6.256256165215746e-05, + "clip_ratio/low_mean": 5.827505810884759e-05, + "clip_ratio/low_min": 5.827505810884759e-05, + "clip_ratio/region_mean": 0.00012083761976100504, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 727.0, + "completions/mean_length": 478.6875, + "completions/mean_terminated_length": 437.3571472167969, + "completions/min_length": 171.0, + "completions/min_terminated_length": 171.0, + "entropy": 0.36259258911013603, + "epoch": 0.31852248394004284, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.019357332959771156, + "learning_rate": 1e-05, + "loss": 0.0489, + "num_tokens": 12515138.0, + "reward": 0.8125, + "reward_std": 0.3945523500442505, + "rewards/accuracy_reward/mean": 0.8125, + "rewards/accuracy_reward/std": 0.3965577781200409, + "sampling/importance_sampling_ratio/max": 1.5570584535598755, + "sampling/importance_sampling_ratio/mean": 0.9998414516448975, + "sampling/importance_sampling_ratio/min": 0.317865252494812, + "sampling/sampling_logp_difference/max": 1.146127700805664, + "sampling/sampling_logp_difference/mean": 0.012102233245968819, + "step": 595 + }, + { + "clip_ratio/high_max": 8.05931631475687e-05, + "clip_ratio/high_mean": 8.05931631475687e-05, + "clip_ratio/low_mean": 0.00011983118019998074, + "clip_ratio/low_min": 0.00011983118019998074, + "clip_ratio/region_mean": 0.00020042434334754944, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 754.0, + "completions/mean_length": 471.0625, + "completions/mean_terminated_length": 440.3448181152344, + "completions/min_length": 202.0, + "completions/min_terminated_length": 202.0, + "entropy": 0.43530962988734245, + "epoch": 0.31905781584582443, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.010029680095613003, + "learning_rate": 1e-05, + "loss": 0.0822, + "num_tokens": 12534300.0, + "reward": 0.625, + "reward_std": 0.3745020925998688, + "rewards/accuracy_reward/mean": 0.625, + "rewards/accuracy_reward/std": 0.49186936020851135, + "sampling/importance_sampling_ratio/max": 1.3254101276397705, + "sampling/importance_sampling_ratio/mean": 1.0000773668289185, + "sampling/importance_sampling_ratio/min": 0.6802136301994324, + "sampling/sampling_logp_difference/max": 0.3853483200073242, + "sampling/sampling_logp_difference/mean": 0.014276028610765934, + "step": 596 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 670.0, + "completions/mean_length": 455.8125, + "completions/mean_terminated_length": 423.5172424316406, + "completions/min_length": 176.0, + "completions/min_terminated_length": 176.0, + "entropy": 0.3696868233382702, + "epoch": 0.319593147751606, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.0108862966299057, + "learning_rate": 1e-05, + "loss": 0.0345, + "num_tokens": 12552694.0, + "reward": 0.59375, + "reward_std": 0.1293872892856598, + "rewards/accuracy_reward/mean": 0.59375, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.4264345169067383, + "sampling/importance_sampling_ratio/mean": 0.999952495098114, + "sampling/importance_sampling_ratio/min": 0.6310193538665771, + "sampling/sampling_logp_difference/max": 0.460418701171875, + "sampling/sampling_logp_difference/mean": 0.012025072239339352, + "step": 597 + }, + { + "clip_ratio/high_max": 8.591065125074238e-05, + "clip_ratio/high_mean": 8.591065125074238e-05, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 8.591065125074238e-05, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 683.0, + "completions/mean_length": 485.125, + "completions/mean_terminated_length": 466.2666931152344, + "completions/min_length": 180.0, + "completions/min_terminated_length": 180.0, + "entropy": 0.3785792216658592, + "epoch": 0.32012847965738755, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.02048811875283718, + "learning_rate": 1e-05, + "loss": -0.0037, + "num_tokens": 12572498.0, + "reward": 0.53125, + "reward_std": 0.4628904461860657, + "rewards/accuracy_reward/mean": 0.53125, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.4059467315673828, + "sampling/importance_sampling_ratio/mean": 0.9998643398284912, + "sampling/importance_sampling_ratio/min": 0.6858024001121521, + "sampling/sampling_logp_difference/max": 0.3771657943725586, + "sampling/sampling_logp_difference/mean": 0.012960119172930717, + "step": 598 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 6.234413740457967e-05, + "clip_ratio/low_min": 6.234413740457967e-05, + "clip_ratio/region_mean": 6.234413740457967e-05, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 698.0, + "completions/mean_length": 445.625, + "completions/mean_terminated_length": 435.2257995605469, + "completions/min_length": 212.0, + "completions/min_terminated_length": 212.0, + "entropy": 0.36770792305469513, + "epoch": 0.32066381156316914, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.00507653784006834, + "learning_rate": 1e-05, + "loss": -0.0499, + "num_tokens": 12591094.0, + "reward": 0.6875, + "reward_std": 0.2925041913986206, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.405523419380188, + "sampling/importance_sampling_ratio/mean": 0.9998577833175659, + "sampling/importance_sampling_ratio/min": 0.5965756773948669, + "sampling/sampling_logp_difference/max": 0.5165491104125977, + "sampling/sampling_logp_difference/mean": 0.012210875749588013, + "step": 599 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 571.0, + "completions/max_terminated_length": 571.0, + "completions/mean_length": 367.21875, + "completions/mean_terminated_length": 367.21875, + "completions/min_length": 236.0, + "completions/min_terminated_length": 236.0, + "entropy": 0.28999911807477474, + "epoch": 0.32119914346895073, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": 0.0, + "num_tokens": 12606389.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/accuracy_reward/mean": 1.0, + "rewards/accuracy_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.4321156740188599, + "sampling/importance_sampling_ratio/mean": 0.9997718334197998, + "sampling/importance_sampling_ratio/min": 0.6788029074668884, + "sampling/sampling_logp_difference/max": 0.3874244689941406, + "sampling/sampling_logp_difference/mean": 0.01094890758395195, + "step": 600 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 720.0, + "completions/mean_length": 502.46875, + "completions/mean_terminated_length": 464.5357360839844, + "completions/min_length": 270.0, + "completions/min_terminated_length": 270.0, + "entropy": 0.37995409965515137, + "epoch": 0.3217344753747323, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.010627315379679203, + "learning_rate": 1e-05, + "loss": 0.0361, + "num_tokens": 12626204.0, + "reward": 0.6875, + "reward_std": 0.4671337604522705, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.3667848110198975, + "sampling/importance_sampling_ratio/mean": 1.0004326105117798, + "sampling/importance_sampling_ratio/min": 0.548538327217102, + "sampling/sampling_logp_difference/max": 0.6004981398582458, + "sampling/sampling_logp_difference/mean": 0.011942821554839611, + "step": 601 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 7.749535143375397e-05, + "clip_ratio/low_min": 7.749535143375397e-05, + "clip_ratio/region_mean": 7.749535143375397e-05, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 691.0, + "completions/mean_length": 455.09375, + "completions/mean_terminated_length": 445.0, + "completions/min_length": 220.0, + "completions/min_terminated_length": 220.0, + "entropy": 0.3100219089537859, + "epoch": 0.3222698072805139, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.01845543086528778, + "learning_rate": 1e-05, + "loss": 0.0445, + "num_tokens": 12644647.0, + "reward": 0.71875, + "reward_std": 0.3198433816432953, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.4430947303771973, + "sampling/importance_sampling_ratio/mean": 1.0000067949295044, + "sampling/importance_sampling_ratio/min": 0.699577271938324, + "sampling/sampling_logp_difference/max": 0.36678993701934814, + "sampling/sampling_logp_difference/mean": 0.010655754245817661, + "step": 602 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 615.0, + "completions/mean_length": 445.53125, + "completions/mean_terminated_length": 435.1290283203125, + "completions/min_length": 274.0, + "completions/min_terminated_length": 274.0, + "entropy": 0.3358000721782446, + "epoch": 0.3228051391862955, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.0026316565927118063, + "learning_rate": 1e-05, + "loss": -0.0196, + "num_tokens": 12662824.0, + "reward": 0.8125, + "reward_std": 0.2177756428718567, + "rewards/accuracy_reward/mean": 0.8125, + "rewards/accuracy_reward/std": 0.3965577781200409, + "sampling/importance_sampling_ratio/max": 1.4125378131866455, + "sampling/importance_sampling_ratio/mean": 0.999963104724884, + "sampling/importance_sampling_ratio/min": 0.704192042350769, + "sampling/sampling_logp_difference/max": 0.3507041931152344, + "sampling/sampling_logp_difference/mean": 0.011649207212030888, + "step": 603 + }, + { + "clip_ratio/high_max": 5.3717230912297964e-05, + "clip_ratio/high_mean": 5.3717230912297964e-05, + "clip_ratio/low_mean": 0.00017549217227497138, + "clip_ratio/low_min": 0.00017549217227497138, + "clip_ratio/region_mean": 0.00022920940318726934, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 674.0, + "completions/mean_length": 523.59375, + "completions/mean_terminated_length": 467.19232177734375, + "completions/min_length": 278.0, + "completions/min_terminated_length": 278.0, + "entropy": 0.5017894580960274, + "epoch": 0.3233404710920771, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.007616211660206318, + "learning_rate": 1e-05, + "loss": 0.009, + "num_tokens": 12683963.0, + "reward": 0.5, + "reward_std": 0.2314550280570984, + "rewards/accuracy_reward/mean": 0.5, + "rewards/accuracy_reward/std": 0.5080004930496216, + "sampling/importance_sampling_ratio/max": 1.6693872213363647, + "sampling/importance_sampling_ratio/mean": 0.9996455311775208, + "sampling/importance_sampling_ratio/min": 0.6767191886901855, + "sampling/sampling_logp_difference/max": 0.5124566555023193, + "sampling/sampling_logp_difference/mean": 0.014794141054153442, + "step": 604 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 756.0, + "completions/mean_length": 505.15625, + "completions/mean_terminated_length": 487.63336181640625, + "completions/min_length": 275.0, + "completions/min_terminated_length": 275.0, + "entropy": 0.37051649391651154, + "epoch": 0.3238758029978587, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.017546482384204865, + "learning_rate": 1e-05, + "loss": 0.0457, + "num_tokens": 12703776.0, + "reward": 0.75, + "reward_std": 0.2587745785713196, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.2899664640426636, + "sampling/importance_sampling_ratio/mean": 0.9997596144676208, + "sampling/importance_sampling_ratio/min": 0.6594014167785645, + "sampling/sampling_logp_difference/max": 0.41642284393310547, + "sampling/sampling_logp_difference/mean": 0.012441151775419712, + "step": 605 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 748.0, + "completions/mean_length": 490.15625, + "completions/mean_terminated_length": 450.46429443359375, + "completions/min_length": 249.0, + "completions/min_terminated_length": 249.0, + "entropy": 0.39272466488182545, + "epoch": 0.32441113490364026, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.011300858110189438, + "learning_rate": 1e-05, + "loss": 0.0577, + "num_tokens": 12722837.0, + "reward": 0.8125, + "reward_std": 0.3335031569004059, + "rewards/accuracy_reward/mean": 0.8125, + "rewards/accuracy_reward/std": 0.3965577781200409, + "sampling/importance_sampling_ratio/max": 1.4034194946289062, + "sampling/importance_sampling_ratio/mean": 1.0000683069229126, + "sampling/importance_sampling_ratio/min": 0.6582999229431152, + "sampling/sampling_logp_difference/max": 0.4180946350097656, + "sampling/sampling_logp_difference/mean": 0.01232908759266138, + "step": 606 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 678.0, + "completions/mean_length": 505.625, + "completions/mean_terminated_length": 457.03704833984375, + "completions/min_length": 242.0, + "completions/min_terminated_length": 242.0, + "entropy": 0.4399935882538557, + "epoch": 0.32494646680942185, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": 0.0, + "num_tokens": 12742385.0, + "reward": 0.75, + "reward_std": 0.0, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.3815072774887085, + "sampling/importance_sampling_ratio/mean": 0.9998868107795715, + "sampling/importance_sampling_ratio/min": 0.6501041054725647, + "sampling/sampling_logp_difference/max": 0.4306226968765259, + "sampling/sampling_logp_difference/mean": 0.013430907391011715, + "step": 607 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 4.693954178947024e-05, + "clip_ratio/low_min": 4.693954178947024e-05, + "clip_ratio/region_mean": 4.693954178947024e-05, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 752.0, + "completions/mean_length": 494.875, + "completions/mean_terminated_length": 444.2962951660156, + "completions/min_length": 268.0, + "completions/min_terminated_length": 268.0, + "entropy": 0.3446064256131649, + "epoch": 0.32548179871520344, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.02724466659128666, + "learning_rate": 1e-05, + "loss": -0.0147, + "num_tokens": 12761701.0, + "reward": 0.71875, + "reward_std": 0.24511480331420898, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.4556853771209717, + "sampling/importance_sampling_ratio/mean": 0.9998579025268555, + "sampling/importance_sampling_ratio/min": 0.7118197679519653, + "sampling/sampling_logp_difference/max": 0.3754768371582031, + "sampling/sampling_logp_difference/mean": 0.01101600844413042, + "step": 608 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 733.0, + "completions/mean_length": 490.375, + "completions/mean_terminated_length": 461.6551818847656, + "completions/min_length": 280.0, + "completions/min_terminated_length": 280.0, + "entropy": 0.4092586301267147, + "epoch": 0.32601713062098503, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.006393893621861935, + "learning_rate": 1e-05, + "loss": 0.0006, + "num_tokens": 12781009.0, + "reward": 0.59375, + "reward_std": 0.1293872892856598, + "rewards/accuracy_reward/mean": 0.59375, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.5612653493881226, + "sampling/importance_sampling_ratio/mean": 0.999735951423645, + "sampling/importance_sampling_ratio/min": 0.7179129123687744, + "sampling/sampling_logp_difference/max": 0.4454965591430664, + "sampling/sampling_logp_difference/mean": 0.012787237763404846, + "step": 609 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 722.0, + "completions/mean_length": 398.625, + "completions/mean_terminated_length": 386.70965576171875, + "completions/min_length": 191.0, + "completions/min_terminated_length": 191.0, + "entropy": 0.314617732539773, + "epoch": 0.3265524625267666, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.0067096371203660965, + "learning_rate": 1e-05, + "loss": 0.0434, + "num_tokens": 12797181.0, + "reward": 0.96875, + "reward_std": 0.0883883461356163, + "rewards/accuracy_reward/mean": 0.96875, + "rewards/accuracy_reward/std": 0.1767766922712326, + "sampling/importance_sampling_ratio/max": 1.3729749917984009, + "sampling/importance_sampling_ratio/mean": 0.9996647834777832, + "sampling/importance_sampling_ratio/min": 0.7131001949310303, + "sampling/sampling_logp_difference/max": 0.3381333351135254, + "sampling/sampling_logp_difference/mean": 0.010695677250623703, + "step": 610 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00012370113108772784, + "clip_ratio/low_min": 0.00012370113108772784, + "clip_ratio/region_mean": 0.00012370113108772784, + "completions/clipped_ratio": 0.0, + "completions/max_length": 736.0, + "completions/max_terminated_length": 736.0, + "completions/mean_length": 428.125, + "completions/mean_terminated_length": 428.125, + "completions/min_length": 258.0, + "completions/min_terminated_length": 258.0, + "entropy": 0.2924865297973156, + "epoch": 0.3270877944325482, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": 0.0315, + "num_tokens": 12814377.0, + "reward": 0.6875, + "reward_std": 0.2587745785713196, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.4198427200317383, + "sampling/importance_sampling_ratio/mean": 0.9998385906219482, + "sampling/importance_sampling_ratio/min": 0.6952728033065796, + "sampling/sampling_logp_difference/max": 0.3634510040283203, + "sampling/sampling_logp_difference/mean": 0.010244255885481834, + "step": 611 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 634.0, + "completions/mean_length": 473.34375, + "completions/mean_terminated_length": 453.70001220703125, + "completions/min_length": 237.0, + "completions/min_terminated_length": 237.0, + "entropy": 0.3630869872868061, + "epoch": 0.32762312633832974, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.012424632906913757, + "learning_rate": 1e-05, + "loss": 0.0484, + "num_tokens": 12833252.0, + "reward": 0.46875, + "reward_std": 0.2630178928375244, + "rewards/accuracy_reward/mean": 0.46875, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.430388331413269, + "sampling/importance_sampling_ratio/mean": 0.999660074710846, + "sampling/importance_sampling_ratio/min": 0.6966915130615234, + "sampling/sampling_logp_difference/max": 0.36141252517700195, + "sampling/sampling_logp_difference/mean": 0.010973338969051838, + "step": 612 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 9.038322605192661e-05, + "clip_ratio/low_min": 9.038322605192661e-05, + "clip_ratio/region_mean": 9.038322605192661e-05, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 598.0, + "completions/mean_length": 396.25, + "completions/mean_terminated_length": 343.14288330078125, + "completions/min_length": 226.0, + "completions/min_terminated_length": 226.0, + "entropy": 0.39421503990888596, + "epoch": 0.32815845824411133, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.015410852618515491, + "learning_rate": 1e-05, + "loss": -0.0137, + "num_tokens": 12849428.0, + "reward": 0.5625, + "reward_std": 0.2177756428718567, + "rewards/accuracy_reward/mean": 0.5625, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 1.4275332689285278, + "sampling/importance_sampling_ratio/mean": 1.0005714893341064, + "sampling/importance_sampling_ratio/min": 0.7691166400909424, + "sampling/sampling_logp_difference/max": 0.35594797134399414, + "sampling/sampling_logp_difference/mean": 0.012892412021756172, + "step": 613 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00013157616558601148, + "clip_ratio/low_min": 0.00013157616558601148, + "clip_ratio/region_mean": 0.00013157616558601148, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 716.0, + "completions/mean_length": 493.5625, + "completions/mean_terminated_length": 454.357177734375, + "completions/min_length": 184.0, + "completions/min_terminated_length": 184.0, + "entropy": 0.38468278013169765, + "epoch": 0.3286937901498929, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.03001777082681656, + "learning_rate": 1e-05, + "loss": 0.0024, + "num_tokens": 12868870.0, + "reward": 0.46875, + "reward_std": 0.4397946000099182, + "rewards/accuracy_reward/mean": 0.46875, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.455602765083313, + "sampling/importance_sampling_ratio/mean": 1.0003761053085327, + "sampling/importance_sampling_ratio/min": 0.6040632128715515, + "sampling/sampling_logp_difference/max": 0.5040764808654785, + "sampling/sampling_logp_difference/mean": 0.012294515036046505, + "step": 614 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 8.017960499273613e-05, + "clip_ratio/low_min": 8.017960499273613e-05, + "clip_ratio/region_mean": 8.017960499273613e-05, + "completions/clipped_ratio": 0.0, + "completions/max_length": 661.0, + "completions/max_terminated_length": 661.0, + "completions/mean_length": 385.34375, + "completions/mean_terminated_length": 385.34375, + "completions/min_length": 258.0, + "completions/min_terminated_length": 258.0, + "entropy": 0.3371674194931984, + "epoch": 0.3292291220556745, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.009930756874382496, + "learning_rate": 1e-05, + "loss": 0.0399, + "num_tokens": 12884737.0, + "reward": 0.90625, + "reward_std": 0.1293872892856598, + "rewards/accuracy_reward/mean": 0.90625, + "rewards/accuracy_reward/std": 0.2961445748806, + "sampling/importance_sampling_ratio/max": 1.4359911680221558, + "sampling/importance_sampling_ratio/mean": 1.0005505084991455, + "sampling/importance_sampling_ratio/min": 0.7238682508468628, + "sampling/sampling_logp_difference/max": 0.36185526847839355, + "sampling/sampling_logp_difference/mean": 0.01123226061463356, + "step": 615 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00019472106214379892, + "clip_ratio/low_min": 0.00019472106214379892, + "clip_ratio/region_mean": 0.00019472106214379892, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 647.0, + "completions/mean_length": 483.875, + "completions/mean_terminated_length": 474.70965576171875, + "completions/min_length": 284.0, + "completions/min_terminated_length": 284.0, + "entropy": 0.3057300020009279, + "epoch": 0.3297644539614561, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.026799941435456276, + "learning_rate": 1e-05, + "loss": 0.0185, + "num_tokens": 12904085.0, + "reward": 0.75, + "reward_std": 0.1767766922712326, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.359426736831665, + "sampling/importance_sampling_ratio/mean": 0.9996913075447083, + "sampling/importance_sampling_ratio/min": 0.6805739402770996, + "sampling/sampling_logp_difference/max": 0.38481879234313965, + "sampling/sampling_logp_difference/mean": 0.009956633672118187, + "step": 616 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 741.0, + "completions/mean_length": 495.125, + "completions/mean_terminated_length": 486.32257080078125, + "completions/min_length": 276.0, + "completions/min_terminated_length": 276.0, + "entropy": 0.33366139978170395, + "epoch": 0.3302997858672377, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.008611881174147129, + "learning_rate": 1e-05, + "loss": 0.0004, + "num_tokens": 12924049.0, + "reward": 0.875, + "reward_std": 0.292504221200943, + "rewards/accuracy_reward/mean": 0.875, + "rewards/accuracy_reward/std": 0.33601075410842896, + "sampling/importance_sampling_ratio/max": 1.3542838096618652, + "sampling/importance_sampling_ratio/mean": 1.0000187158584595, + "sampling/importance_sampling_ratio/min": 0.5206681489944458, + "sampling/sampling_logp_difference/max": 0.6526424884796143, + "sampling/sampling_logp_difference/mean": 0.011231090873479843, + "step": 617 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 557.0, + "completions/max_terminated_length": 557.0, + "completions/mean_length": 367.75, + "completions/mean_terminated_length": 367.75, + "completions/min_length": 220.0, + "completions/min_terminated_length": 220.0, + "entropy": 0.25993393547832966, + "epoch": 0.33083511777301927, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": -0.0004, + "num_tokens": 12939537.0, + "reward": 0.96875, + "reward_std": 0.0883883461356163, + "rewards/accuracy_reward/mean": 0.96875, + "rewards/accuracy_reward/std": 0.1767766922712326, + "sampling/importance_sampling_ratio/max": 1.6012578010559082, + "sampling/importance_sampling_ratio/mean": 1.000769019126892, + "sampling/importance_sampling_ratio/min": 0.6878544688224792, + "sampling/sampling_logp_difference/max": 0.47078943252563477, + "sampling/sampling_logp_difference/mean": 0.010146456770598888, + "step": 618 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 738.0, + "completions/max_terminated_length": 738.0, + "completions/mean_length": 414.75, + "completions/mean_terminated_length": 414.75, + "completions/min_length": 151.0, + "completions/min_terminated_length": 151.0, + "entropy": 0.29927484318614006, + "epoch": 0.33137044967880086, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": -0.0156, + "num_tokens": 12956089.0, + "reward": 0.9375, + "reward_std": 0.1767766922712326, + "rewards/accuracy_reward/mean": 0.9375, + "rewards/accuracy_reward/std": 0.24593468010425568, + "sampling/importance_sampling_ratio/max": 1.3900020122528076, + "sampling/importance_sampling_ratio/mean": 0.9998085498809814, + "sampling/importance_sampling_ratio/min": 0.6901939511299133, + "sampling/sampling_logp_difference/max": 0.37078261375427246, + "sampling/sampling_logp_difference/mean": 0.009291633032262325, + "step": 619 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00012921850793645717, + "clip_ratio/low_min": 0.00012921850793645717, + "clip_ratio/region_mean": 0.00012921850793645717, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 734.0, + "completions/mean_length": 503.5, + "completions/mean_terminated_length": 465.71429443359375, + "completions/min_length": 278.0, + "completions/min_terminated_length": 278.0, + "entropy": 0.29361248947679996, + "epoch": 0.33190578158458245, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.007103252224624157, + "learning_rate": 1e-05, + "loss": 0.0687, + "num_tokens": 12976161.0, + "reward": 0.75, + "reward_std": 0.292504221200943, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.4360511302947998, + "sampling/importance_sampling_ratio/mean": 1.0004138946533203, + "sampling/importance_sampling_ratio/min": 0.7299440503120422, + "sampling/sampling_logp_difference/max": 0.36189699172973633, + "sampling/sampling_logp_difference/mean": 0.009897296316921711, + "step": 620 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 762.0, + "completions/max_terminated_length": 762.0, + "completions/mean_length": 415.625, + "completions/mean_terminated_length": 415.625, + "completions/min_length": 254.0, + "completions/min_terminated_length": 254.0, + "entropy": 0.3415104281157255, + "epoch": 0.33244111349036404, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.00380335608497262, + "learning_rate": 1e-05, + "loss": -0.0231, + "num_tokens": 12993245.0, + "reward": 0.71875, + "reward_std": 0.0883883461356163, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.4889551401138306, + "sampling/importance_sampling_ratio/mean": 0.9999030828475952, + "sampling/importance_sampling_ratio/min": 0.7551504969596863, + "sampling/sampling_logp_difference/max": 0.3980746269226074, + "sampling/sampling_logp_difference/mean": 0.011102134361863136, + "step": 621 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 8.778089977568015e-05, + "clip_ratio/low_min": 8.778089977568015e-05, + "clip_ratio/region_mean": 8.778089977568015e-05, + "completions/clipped_ratio": 0.0, + "completions/max_length": 536.0, + "completions/max_terminated_length": 536.0, + "completions/mean_length": 375.71875, + "completions/mean_terminated_length": 375.71875, + "completions/min_length": 234.0, + "completions/min_terminated_length": 234.0, + "entropy": 0.33459382504224777, + "epoch": 0.3329764453961456, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.020825320854783058, + "learning_rate": 1e-05, + "loss": -0.0145, + "num_tokens": 13008980.0, + "reward": 0.6875, + "reward_std": 0.3335031569004059, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.4972962141036987, + "sampling/importance_sampling_ratio/mean": 1.000253438949585, + "sampling/importance_sampling_ratio/min": 0.6408749222755432, + "sampling/sampling_logp_difference/max": 0.44492101669311523, + "sampling/sampling_logp_difference/mean": 0.011420239694416523, + "step": 622 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 510.0, + "completions/max_terminated_length": 510.0, + "completions/mean_length": 312.71875, + "completions/mean_terminated_length": 312.71875, + "completions/min_length": 142.0, + "completions/min_terminated_length": 142.0, + "entropy": 0.29183613508939743, + "epoch": 0.3335117773019272, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": 0.0, + "num_tokens": 13022371.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/accuracy_reward/mean": 1.0, + "rewards/accuracy_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.3589096069335938, + "sampling/importance_sampling_ratio/mean": 1.000159740447998, + "sampling/importance_sampling_ratio/min": 0.7225852012634277, + "sampling/sampling_logp_difference/max": 0.3249199390411377, + "sampling/sampling_logp_difference/mean": 0.010989728383719921, + "step": 623 + }, + { + "clip_ratio/high_max": 0.00012599398905877024, + "clip_ratio/high_mean": 0.00012599398905877024, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.00012599398905877024, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 712.0, + "completions/mean_length": 474.375, + "completions/mean_terminated_length": 444.0, + "completions/min_length": 229.0, + "completions/min_terminated_length": 229.0, + "entropy": 0.3476267158985138, + "epoch": 0.3340471092077088, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.03480792045593262, + "learning_rate": 1e-05, + "loss": 0.0719, + "num_tokens": 13041231.0, + "reward": 0.75, + "reward_std": 0.3335031569004059, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.4345088005065918, + "sampling/importance_sampling_ratio/mean": 0.9998278021812439, + "sampling/importance_sampling_ratio/min": 0.5451315641403198, + "sampling/sampling_logp_difference/max": 0.6067280769348145, + "sampling/sampling_logp_difference/mean": 0.011186945252120495, + "step": 624 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 756.0, + "completions/mean_length": 505.90625, + "completions/mean_terminated_length": 497.45159912109375, + "completions/min_length": 239.0, + "completions/min_terminated_length": 239.0, + "entropy": 0.292398190125823, + "epoch": 0.33458244111349034, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": 0.0, + "num_tokens": 13061828.0, + "reward": 0.75, + "reward_std": 0.0, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.5679001808166504, + "sampling/importance_sampling_ratio/mean": 0.9997112154960632, + "sampling/importance_sampling_ratio/min": 0.7010811567306519, + "sampling/sampling_logp_difference/max": 0.4497373104095459, + "sampling/sampling_logp_difference/mean": 0.010309663601219654, + "step": 625 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 624.0, + "completions/mean_length": 376.25, + "completions/mean_terminated_length": 363.6128845214844, + "completions/min_length": 186.0, + "completions/min_terminated_length": 186.0, + "entropy": 0.36883340403437614, + "epoch": 0.3351177730192719, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.00871721189469099, + "learning_rate": 1e-05, + "loss": -0.0147, + "num_tokens": 13077196.0, + "reward": 0.5625, + "reward_std": 0.3335031569004059, + "rewards/accuracy_reward/mean": 0.5625, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 1.686894178390503, + "sampling/importance_sampling_ratio/mean": 0.9995442628860474, + "sampling/importance_sampling_ratio/min": 0.6272304654121399, + "sampling/sampling_logp_difference/max": 0.5228891372680664, + "sampling/sampling_logp_difference/mean": 0.013311648741364479, + "step": 626 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0001392210542690009, + "clip_ratio/low_min": 0.0001392210542690009, + "clip_ratio/region_mean": 0.0001392210542690009, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 756.0, + "completions/mean_length": 485.75, + "completions/mean_terminated_length": 420.6153869628906, + "completions/min_length": 108.0, + "completions/min_terminated_length": 108.0, + "entropy": 0.446913231164217, + "epoch": 0.3356531049250535, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.008604291826486588, + "learning_rate": 1e-05, + "loss": 0.034, + "num_tokens": 13096524.0, + "reward": 0.65625, + "reward_std": 0.3808925747871399, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.5247750282287598, + "sampling/importance_sampling_ratio/mean": 1.0002942085266113, + "sampling/importance_sampling_ratio/min": 0.7407239675521851, + "sampling/sampling_logp_difference/max": 0.421846866607666, + "sampling/sampling_logp_difference/mean": 0.013524254783987999, + "step": 627 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 8.573388186050579e-05, + "clip_ratio/low_min": 8.573388186050579e-05, + "clip_ratio/region_mean": 8.573388186050579e-05, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 713.0, + "completions/mean_length": 456.59375, + "completions/mean_terminated_length": 446.5483703613281, + "completions/min_length": 236.0, + "completions/min_terminated_length": 236.0, + "entropy": 0.3660719469189644, + "epoch": 0.3361884368308351, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.0075892615132033825, + "learning_rate": 1e-05, + "loss": 0.0429, + "num_tokens": 13114503.0, + "reward": 0.875, + "reward_std": 0.2925041913986206, + "rewards/accuracy_reward/mean": 0.875, + "rewards/accuracy_reward/std": 0.33601075410842896, + "sampling/importance_sampling_ratio/max": 1.3400059938430786, + "sampling/importance_sampling_ratio/mean": 0.9996907711029053, + "sampling/importance_sampling_ratio/min": 0.4545471668243408, + "sampling/sampling_logp_difference/max": 0.7884535789489746, + "sampling/sampling_logp_difference/mean": 0.011270992457866669, + "step": 628 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.000156465066538658, + "clip_ratio/low_min": 0.000156465066538658, + "clip_ratio/region_mean": 0.000156465066538658, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 736.0, + "completions/mean_length": 529.25, + "completions/mean_terminated_length": 495.14288330078125, + "completions/min_length": 266.0, + "completions/min_terminated_length": 266.0, + "entropy": 0.2408799361437559, + "epoch": 0.3367237687366167, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.003707155818119645, + "learning_rate": 1e-05, + "loss": 0.0326, + "num_tokens": 13135175.0, + "reward": 0.75, + "reward_std": 0.292504221200943, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.9237333536148071, + "sampling/importance_sampling_ratio/mean": 1.0000091791152954, + "sampling/importance_sampling_ratio/min": 0.7001087069511414, + "sampling/sampling_logp_difference/max": 0.6542677879333496, + "sampling/sampling_logp_difference/mean": 0.008334740065038204, + "step": 629 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 7.301401637960225e-05, + "clip_ratio/low_min": 7.301401637960225e-05, + "clip_ratio/region_mean": 7.301401637960225e-05, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 560.0, + "completions/mean_length": 410.5625, + "completions/mean_terminated_length": 399.0322570800781, + "completions/min_length": 210.0, + "completions/min_terminated_length": 210.0, + "entropy": 0.23404156975448132, + "epoch": 0.3372591006423983, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.013698897324502468, + "learning_rate": 1e-05, + "loss": -0.007, + "num_tokens": 13151961.0, + "reward": 0.8125, + "reward_std": 0.2177756428718567, + "rewards/accuracy_reward/mean": 0.8125, + "rewards/accuracy_reward/std": 0.3965577781200409, + "sampling/importance_sampling_ratio/max": 1.6546179056167603, + "sampling/importance_sampling_ratio/mean": 0.9999274611473083, + "sampling/importance_sampling_ratio/min": 0.7193414568901062, + "sampling/sampling_logp_difference/max": 0.5035700798034668, + "sampling/sampling_logp_difference/mean": 0.008526804856956005, + "step": 630 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 503.0, + "completions/max_terminated_length": 503.0, + "completions/mean_length": 345.625, + "completions/mean_terminated_length": 345.625, + "completions/min_length": 220.0, + "completions/min_terminated_length": 220.0, + "entropy": 0.3241981565952301, + "epoch": 0.33779443254817987, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": 0.0, + "num_tokens": 13166741.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/accuracy_reward/mean": 1.0, + "rewards/accuracy_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.339516520500183, + "sampling/importance_sampling_ratio/mean": 0.9997711181640625, + "sampling/importance_sampling_ratio/min": 0.5709845423698425, + "sampling/sampling_logp_difference/max": 0.5603930950164795, + "sampling/sampling_logp_difference/mean": 0.011668842285871506, + "step": 631 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 504.0, + "completions/max_terminated_length": 504.0, + "completions/mean_length": 408.96875, + "completions/mean_terminated_length": 408.96875, + "completions/min_length": 249.0, + "completions/min_terminated_length": 249.0, + "entropy": 0.2496287040412426, + "epoch": 0.33832976445396146, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.0036416437942534685, + "learning_rate": 1e-05, + "loss": -0.0081, + "num_tokens": 13183428.0, + "reward": 0.96875, + "reward_std": 0.0883883461356163, + "rewards/accuracy_reward/mean": 0.96875, + "rewards/accuracy_reward/std": 0.1767766922712326, + "sampling/importance_sampling_ratio/max": 1.3984626531600952, + "sampling/importance_sampling_ratio/mean": 0.9999477863311768, + "sampling/importance_sampling_ratio/min": 0.7157694101333618, + "sampling/sampling_logp_difference/max": 0.3353736400604248, + "sampling/sampling_logp_difference/mean": 0.008716152049601078, + "step": 632 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 754.0, + "completions/mean_length": 431.5, + "completions/mean_terminated_length": 409.0666809082031, + "completions/min_length": 184.0, + "completions/min_terminated_length": 184.0, + "entropy": 0.5070165768265724, + "epoch": 0.33886509635974305, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.022825445979833603, + "learning_rate": 1e-05, + "loss": -0.0167, + "num_tokens": 13201492.0, + "reward": 0.625, + "reward_std": 0.2177756428718567, + "rewards/accuracy_reward/mean": 0.625, + "rewards/accuracy_reward/std": 0.49186936020851135, + "sampling/importance_sampling_ratio/max": 1.369990587234497, + "sampling/importance_sampling_ratio/mean": 0.9995102882385254, + "sampling/importance_sampling_ratio/min": 0.6855331063270569, + "sampling/sampling_logp_difference/max": 0.37755846977233887, + "sampling/sampling_logp_difference/mean": 0.01544769573956728, + "step": 633 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 658.0, + "completions/max_terminated_length": 658.0, + "completions/mean_length": 337.65625, + "completions/mean_terminated_length": 337.65625, + "completions/min_length": 153.0, + "completions/min_terminated_length": 153.0, + "entropy": 0.39696030132472515, + "epoch": 0.33940042826552463, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": 0.0322, + "num_tokens": 13216009.0, + "reward": 0.8125, + "reward_std": 0.249358132481575, + "rewards/accuracy_reward/mean": 0.8125, + "rewards/accuracy_reward/std": 0.3965577781200409, + "sampling/importance_sampling_ratio/max": 1.6422919034957886, + "sampling/importance_sampling_ratio/mean": 1.0000416040420532, + "sampling/importance_sampling_ratio/min": 0.6801885962486267, + "sampling/sampling_logp_difference/max": 0.4960927963256836, + "sampling/sampling_logp_difference/mean": 0.012387676164507866, + "step": 634 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00012491943198256195, + "clip_ratio/low_min": 0.00012491943198256195, + "clip_ratio/region_mean": 0.00012491943198256195, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 763.0, + "completions/mean_length": 494.375, + "completions/mean_terminated_length": 431.23077392578125, + "completions/min_length": 243.0, + "completions/min_terminated_length": 243.0, + "entropy": 0.38411999866366386, + "epoch": 0.3399357601713062, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.017900027334690094, + "learning_rate": 1e-05, + "loss": 0.0241, + "num_tokens": 13235501.0, + "reward": 0.6875, + "reward_std": 0.3924051821231842, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.5086416006088257, + "sampling/importance_sampling_ratio/mean": 1.0000834465026855, + "sampling/importance_sampling_ratio/min": 0.5438719391822815, + "sampling/sampling_logp_difference/max": 0.6090414524078369, + "sampling/sampling_logp_difference/mean": 0.012571083381772041, + "step": 635 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 596.0, + "completions/max_terminated_length": 596.0, + "completions/mean_length": 348.5625, + "completions/mean_terminated_length": 348.5625, + "completions/min_length": 96.0, + "completions/min_terminated_length": 96.0, + "entropy": 0.2587822787463665, + "epoch": 0.3404710920770878, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.0038941828534007072, + "learning_rate": 1e-05, + "loss": -0.0019, + "num_tokens": 13250751.0, + "reward": 0.96875, + "reward_std": 0.0883883461356163, + "rewards/accuracy_reward/mean": 0.96875, + "rewards/accuracy_reward/std": 0.1767766922712326, + "sampling/importance_sampling_ratio/max": 1.6068605184555054, + "sampling/importance_sampling_ratio/mean": 1.0001121759414673, + "sampling/importance_sampling_ratio/min": 0.6504940390586853, + "sampling/sampling_logp_difference/max": 0.47428226470947266, + "sampling/sampling_logp_difference/mean": 0.008922860026359558, + "step": 636 + }, + { + "clip_ratio/high_max": 5.816658813273534e-05, + "clip_ratio/high_mean": 5.816658813273534e-05, + "clip_ratio/low_mean": 0.00019885157598764636, + "clip_ratio/low_min": 0.00019885157598764636, + "clip_ratio/region_mean": 0.0002570181641203817, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 662.0, + "completions/mean_length": 478.53125, + "completions/mean_terminated_length": 437.1785888671875, + "completions/min_length": 223.0, + "completions/min_terminated_length": 223.0, + "entropy": 0.4376033619046211, + "epoch": 0.3410064239828694, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.022219238802790642, + "learning_rate": 1e-05, + "loss": 0.0423, + "num_tokens": 13270656.0, + "reward": 0.5, + "reward_std": 0.5081326961517334, + "rewards/accuracy_reward/mean": 0.5, + "rewards/accuracy_reward/std": 0.5080004930496216, + "sampling/importance_sampling_ratio/max": 1.4648514986038208, + "sampling/importance_sampling_ratio/mean": 0.9998618960380554, + "sampling/importance_sampling_ratio/min": 0.6825798749923706, + "sampling/sampling_logp_difference/max": 0.38187575340270996, + "sampling/sampling_logp_difference/mean": 0.013110549189150333, + "step": 637 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 761.0, + "completions/mean_length": 486.96875, + "completions/mean_terminated_length": 446.8214416503906, + "completions/min_length": 328.0, + "completions/min_terminated_length": 328.0, + "entropy": 0.46911646053195, + "epoch": 0.341541755888651, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.030147511512041092, + "learning_rate": 1e-05, + "loss": 0.028, + "num_tokens": 13290447.0, + "reward": 0.5625, + "reward_std": 0.3104073107242584, + "rewards/accuracy_reward/mean": 0.5625, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 1.4902180433273315, + "sampling/importance_sampling_ratio/mean": 0.9998611211776733, + "sampling/importance_sampling_ratio/min": 0.658909797668457, + "sampling/sampling_logp_difference/max": 0.41716861724853516, + "sampling/sampling_logp_difference/mean": 0.01516705472022295, + "step": 638 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 686.0, + "completions/max_terminated_length": 686.0, + "completions/mean_length": 418.5625, + "completions/mean_terminated_length": 418.5625, + "completions/min_length": 245.0, + "completions/min_terminated_length": 245.0, + "entropy": 0.2731906659901142, + "epoch": 0.3420770877944325, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.02774636074900627, + "learning_rate": 1e-05, + "loss": -0.0124, + "num_tokens": 13307305.0, + "reward": 0.5625, + "reward_std": 0.2925041913986206, + "rewards/accuracy_reward/mean": 0.5625, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 1.4035072326660156, + "sampling/importance_sampling_ratio/mean": 1.0002001523971558, + "sampling/importance_sampling_ratio/min": 0.6635493040084839, + "sampling/sampling_logp_difference/max": 0.4101520776748657, + "sampling/sampling_logp_difference/mean": 0.010155044496059418, + "step": 639 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 8.491847984259948e-05, + "clip_ratio/low_min": 8.491847984259948e-05, + "clip_ratio/region_mean": 8.491847984259948e-05, + "completions/clipped_ratio": 0.0, + "completions/max_length": 582.0, + "completions/max_terminated_length": 582.0, + "completions/mean_length": 336.75, + "completions/mean_terminated_length": 336.75, + "completions/min_length": 115.0, + "completions/min_terminated_length": 115.0, + "entropy": 0.3048718683421612, + "epoch": 0.3426124197002141, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.018667301163077354, + "learning_rate": 1e-05, + "loss": -0.0075, + "num_tokens": 13321665.0, + "reward": 0.90625, + "reward_std": 0.1293872892856598, + "rewards/accuracy_reward/mean": 0.90625, + "rewards/accuracy_reward/std": 0.2961445748806, + "sampling/importance_sampling_ratio/max": 1.9191588163375854, + "sampling/importance_sampling_ratio/mean": 0.9999879598617554, + "sampling/importance_sampling_ratio/min": 0.7230131030082703, + "sampling/sampling_logp_difference/max": 0.6518869400024414, + "sampling/sampling_logp_difference/mean": 0.010768414475023746, + "step": 640 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 574.0, + "completions/max_terminated_length": 574.0, + "completions/mean_length": 409.96875, + "completions/mean_terminated_length": 409.96875, + "completions/min_length": 239.0, + "completions/min_terminated_length": 239.0, + "entropy": 0.27836749143898487, + "epoch": 0.3431477516059957, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": 0.0146, + "num_tokens": 13338288.0, + "reward": 0.9375, + "reward_std": 0.1157275140285492, + "rewards/accuracy_reward/mean": 0.9375, + "rewards/accuracy_reward/std": 0.24593468010425568, + "sampling/importance_sampling_ratio/max": 1.4447996616363525, + "sampling/importance_sampling_ratio/mean": 0.9999314546585083, + "sampling/importance_sampling_ratio/min": 0.7261752486228943, + "sampling/sampling_logp_difference/max": 0.36797070503234863, + "sampling/sampling_logp_difference/mean": 0.010056580416858196, + "step": 641 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 9.667439735494554e-05, + "clip_ratio/low_min": 9.667439735494554e-05, + "clip_ratio/region_mean": 9.667439735494554e-05, + "completions/clipped_ratio": 0.0, + "completions/max_length": 592.0, + "completions/max_terminated_length": 592.0, + "completions/mean_length": 362.3125, + "completions/mean_terminated_length": 362.3125, + "completions/min_length": 205.0, + "completions/min_terminated_length": 205.0, + "entropy": 0.33984828367829323, + "epoch": 0.3436830835117773, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.009369701147079468, + "learning_rate": 1e-05, + "loss": -0.0089, + "num_tokens": 13353946.0, + "reward": 0.9375, + "reward_std": 0.1157275140285492, + "rewards/accuracy_reward/mean": 0.9375, + "rewards/accuracy_reward/std": 0.24593468010425568, + "sampling/importance_sampling_ratio/max": 1.4577983617782593, + "sampling/importance_sampling_ratio/mean": 0.9998650550842285, + "sampling/importance_sampling_ratio/min": 0.7225463390350342, + "sampling/sampling_logp_difference/max": 0.37692737579345703, + "sampling/sampling_logp_difference/mean": 0.011347470805048943, + "step": 642 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00012793374844477512, + "clip_ratio/low_min": 0.00012793374844477512, + "clip_ratio/region_mean": 0.00012793374844477512, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 648.0, + "completions/mean_length": 473.6875, + "completions/mean_terminated_length": 405.7692565917969, + "completions/min_length": 228.0, + "completions/min_terminated_length": 228.0, + "entropy": 0.48161710426211357, + "epoch": 0.3442184154175589, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.051371026784181595, + "learning_rate": 1e-05, + "loss": 0.0165, + "num_tokens": 13373248.0, + "reward": 0.71875, + "reward_std": 0.2651650309562683, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.6313735246658325, + "sampling/importance_sampling_ratio/mean": 1.0000813007354736, + "sampling/importance_sampling_ratio/min": 0.48223310708999634, + "sampling/sampling_logp_difference/max": 0.7293276786804199, + "sampling/sampling_logp_difference/mean": 0.014763213694095612, + "step": 643 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00018642803479451686, + "clip_ratio/low_min": 0.00018642803479451686, + "clip_ratio/region_mean": 0.00018642803479451686, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 705.0, + "completions/mean_length": 392.8125, + "completions/mean_terminated_length": 354.0, + "completions/min_length": 115.0, + "completions/min_terminated_length": 115.0, + "entropy": 0.40540508180856705, + "epoch": 0.34475374732334046, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.015619847923517227, + "learning_rate": 1e-05, + "loss": 0.0966, + "num_tokens": 13389274.0, + "reward": 0.84375, + "reward_std": 0.3198433816432953, + "rewards/accuracy_reward/mean": 0.84375, + "rewards/accuracy_reward/std": 0.3689020276069641, + "sampling/importance_sampling_ratio/max": 1.7861274480819702, + "sampling/importance_sampling_ratio/mean": 1.0002065896987915, + "sampling/importance_sampling_ratio/min": 0.7051683068275452, + "sampling/sampling_logp_difference/max": 0.5800497531890869, + "sampling/sampling_logp_difference/mean": 0.012819117866456509, + "step": 644 + }, + { + "clip_ratio/high_max": 5.6407941883662716e-05, + "clip_ratio/high_mean": 5.6407941883662716e-05, + "clip_ratio/low_mean": 0.00013486891839420423, + "clip_ratio/low_min": 0.00013486891839420423, + "clip_ratio/region_mean": 0.00019127686027786694, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 738.0, + "completions/mean_length": 427.0, + "completions/mean_terminated_length": 391.7241516113281, + "completions/min_length": 207.0, + "completions/min_terminated_length": 207.0, + "entropy": 0.3380007464438677, + "epoch": 0.34528907922912205, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.018019359558820724, + "learning_rate": 1e-05, + "loss": 0.0676, + "num_tokens": 13406490.0, + "reward": 0.75, + "reward_std": 0.2587745785713196, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.3706324100494385, + "sampling/importance_sampling_ratio/mean": 0.9995419979095459, + "sampling/importance_sampling_ratio/min": 0.7309109568595886, + "sampling/sampling_logp_difference/max": 0.31527233123779297, + "sampling/sampling_logp_difference/mean": 0.011513370089232922, + "step": 645 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 6.902263703523204e-05, + "clip_ratio/low_min": 6.902263703523204e-05, + "clip_ratio/region_mean": 6.902263703523204e-05, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 651.0, + "completions/mean_length": 441.53125, + "completions/mean_terminated_length": 407.75860595703125, + "completions/min_length": 239.0, + "completions/min_terminated_length": 239.0, + "entropy": 0.3893450815230608, + "epoch": 0.34582441113490364, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.01541195809841156, + "learning_rate": 1e-05, + "loss": 0.0126, + "num_tokens": 13424499.0, + "reward": 0.625, + "reward_std": 0.3335031569004059, + "rewards/accuracy_reward/mean": 0.625, + "rewards/accuracy_reward/std": 0.49186936020851135, + "sampling/importance_sampling_ratio/max": 1.5599128007888794, + "sampling/importance_sampling_ratio/mean": 1.000036358833313, + "sampling/importance_sampling_ratio/min": 0.7086088061332703, + "sampling/sampling_logp_difference/max": 0.4446299076080322, + "sampling/sampling_logp_difference/mean": 0.01274517085403204, + "step": 646 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00019918624457204714, + "clip_ratio/low_min": 0.00019918624457204714, + "clip_ratio/region_mean": 0.00019918624457204714, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 679.0, + "completions/mean_length": 492.40625, + "completions/mean_terminated_length": 441.370361328125, + "completions/min_length": 183.0, + "completions/min_terminated_length": 183.0, + "entropy": 0.5329282619059086, + "epoch": 0.34635974304068523, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.01550175342708826, + "learning_rate": 1e-05, + "loss": 0.074, + "num_tokens": 13444560.0, + "reward": 0.34375, + "reward_std": 0.4944729208946228, + "rewards/accuracy_reward/mean": 0.34375, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.900555968284607, + "sampling/importance_sampling_ratio/mean": 1.0000826120376587, + "sampling/importance_sampling_ratio/min": 0.3233708143234253, + "sampling/sampling_logp_difference/max": 1.128955602645874, + "sampling/sampling_logp_difference/mean": 0.016209932044148445, + "step": 647 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 712.0, + "completions/mean_length": 478.375, + "completions/mean_terminated_length": 469.0322570800781, + "completions/min_length": 296.0, + "completions/min_terminated_length": 296.0, + "entropy": 0.28995392099022865, + "epoch": 0.3468950749464668, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.007992844097316265, + "learning_rate": 1e-05, + "loss": 0.0169, + "num_tokens": 13463636.0, + "reward": 0.75, + "reward_std": 0.292504221200943, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.9551899433135986, + "sampling/importance_sampling_ratio/mean": 0.99978107213974, + "sampling/importance_sampling_ratio/min": 0.6795781850814819, + "sampling/sampling_logp_difference/max": 0.6704874038696289, + "sampling/sampling_logp_difference/mean": 0.010118094272911549, + "step": 648 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.487269663717598e-05, + "clip_ratio/low_min": 5.487269663717598e-05, + "clip_ratio/region_mean": 5.487269663717598e-05, + "completions/clipped_ratio": 0.0, + "completions/max_length": 688.0, + "completions/max_terminated_length": 688.0, + "completions/mean_length": 428.0625, + "completions/mean_terminated_length": 428.0625, + "completions/min_length": 244.0, + "completions/min_terminated_length": 244.0, + "entropy": 0.26039893738925457, + "epoch": 0.3474304068522484, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.01623428985476494, + "learning_rate": 1e-05, + "loss": 0.0077, + "num_tokens": 13480558.0, + "reward": 0.8125, + "reward_std": 0.1157275140285492, + "rewards/accuracy_reward/mean": 0.8125, + "rewards/accuracy_reward/std": 0.3965577781200409, + "sampling/importance_sampling_ratio/max": 1.528709053993225, + "sampling/importance_sampling_ratio/mean": 1.0003135204315186, + "sampling/importance_sampling_ratio/min": 0.7110331654548645, + "sampling/sampling_logp_difference/max": 0.4244236946105957, + "sampling/sampling_logp_difference/mean": 0.009109492413699627, + "step": 649 + }, + { + "clip_ratio/high_max": 9.984025382436812e-05, + "clip_ratio/high_mean": 9.984025382436812e-05, + "clip_ratio/low_mean": 0.00010296540131093934, + "clip_ratio/low_min": 0.00010296540131093934, + "clip_ratio/region_mean": 0.00020280565513530746, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 588.0, + "completions/mean_length": 381.125, + "completions/mean_terminated_length": 325.8571472167969, + "completions/min_length": 147.0, + "completions/min_terminated_length": 147.0, + "entropy": 0.41817745566368103, + "epoch": 0.34796573875803, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.020147062838077545, + "learning_rate": 1e-05, + "loss": 0.0932, + "num_tokens": 13496930.0, + "reward": 0.8125, + "reward_std": 0.1157275140285492, + "rewards/accuracy_reward/mean": 0.8125, + "rewards/accuracy_reward/std": 0.3965577781200409, + "sampling/importance_sampling_ratio/max": 1.4589701890945435, + "sampling/importance_sampling_ratio/mean": 1.000463843345642, + "sampling/importance_sampling_ratio/min": 0.6899737119674683, + "sampling/sampling_logp_difference/max": 0.3777308464050293, + "sampling/sampling_logp_difference/mean": 0.014919880777597427, + "step": 650 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 743.0, + "completions/mean_length": 482.9375, + "completions/mean_terminated_length": 442.21429443359375, + "completions/min_length": 177.0, + "completions/min_terminated_length": 177.0, + "entropy": 0.4246320314705372, + "epoch": 0.3485010706638116, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.00700872577726841, + "learning_rate": 1e-05, + "loss": -0.0015, + "num_tokens": 13516568.0, + "reward": 0.40625, + "reward_std": 0.1293872892856598, + "rewards/accuracy_reward/mean": 0.40625, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.6444509029388428, + "sampling/importance_sampling_ratio/mean": 0.9996163845062256, + "sampling/importance_sampling_ratio/min": 0.6946247816085815, + "sampling/sampling_logp_difference/max": 0.4974064826965332, + "sampling/sampling_logp_difference/mean": 0.014024922624230385, + "step": 651 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00015512068785028532, + "clip_ratio/low_min": 0.00015512068785028532, + "clip_ratio/region_mean": 0.00015512068785028532, + "completions/clipped_ratio": 0.0, + "completions/max_length": 551.0, + "completions/max_terminated_length": 551.0, + "completions/mean_length": 348.03125, + "completions/mean_terminated_length": 348.03125, + "completions/min_length": 190.0, + "completions/min_terminated_length": 190.0, + "entropy": 0.35516560450196266, + "epoch": 0.3490364025695932, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.0200100876390934, + "learning_rate": 1e-05, + "loss": -0.004, + "num_tokens": 13531425.0, + "reward": 0.6875, + "reward_std": 0.249358132481575, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.5035443305969238, + "sampling/importance_sampling_ratio/mean": 0.9996904730796814, + "sampling/importance_sampling_ratio/min": 0.413336843252182, + "sampling/sampling_logp_difference/max": 0.8834924697875977, + "sampling/sampling_logp_difference/mean": 0.012606313452124596, + "step": 652 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00016290302301058546, + "clip_ratio/low_min": 0.00016290302301058546, + "clip_ratio/region_mean": 0.00016290302301058546, + "completions/clipped_ratio": 0.0, + "completions/max_length": 564.0, + "completions/max_terminated_length": 564.0, + "completions/mean_length": 436.4375, + "completions/mean_terminated_length": 436.4375, + "completions/min_length": 307.0, + "completions/min_terminated_length": 307.0, + "entropy": 0.259181784465909, + "epoch": 0.3495717344753747, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.014700076542794704, + "learning_rate": 1e-05, + "loss": 0.0024, + "num_tokens": 13549503.0, + "reward": 0.59375, + "reward_std": 0.4534739851951599, + "rewards/accuracy_reward/mean": 0.59375, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.2649004459381104, + "sampling/importance_sampling_ratio/mean": 0.9999638795852661, + "sampling/importance_sampling_ratio/min": 0.6338332891464233, + "sampling/sampling_logp_difference/max": 0.45596933364868164, + "sampling/sampling_logp_difference/mean": 0.008763880468904972, + "step": 653 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 760.0, + "completions/mean_length": 393.5625, + "completions/mean_terminated_length": 381.4838562011719, + "completions/min_length": 118.0, + "completions/min_terminated_length": 118.0, + "entropy": 0.460235420614481, + "epoch": 0.3501070663811563, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.004804051946848631, + "learning_rate": 1e-05, + "loss": -0.0044, + "num_tokens": 13565617.0, + "reward": 0.75, + "reward_std": 0.1767766922712326, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.5459589958190918, + "sampling/importance_sampling_ratio/mean": 1.0002037286758423, + "sampling/importance_sampling_ratio/min": 0.7705621719360352, + "sampling/sampling_logp_difference/max": 0.43564438819885254, + "sampling/sampling_logp_difference/mean": 0.013562281616032124, + "step": 654 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0001592706612427719, + "clip_ratio/low_min": 0.0001592706612427719, + "clip_ratio/region_mean": 0.0001592706612427719, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 693.0, + "completions/mean_length": 445.21875, + "completions/mean_terminated_length": 399.1071472167969, + "completions/min_length": 186.0, + "completions/min_terminated_length": 186.0, + "entropy": 0.39120227843523026, + "epoch": 0.3506423982869379, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.01896318420767784, + "learning_rate": 1e-05, + "loss": 0.0684, + "num_tokens": 13583144.0, + "reward": 0.6875, + "reward_std": 0.3471825420856476, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.3875007629394531, + "sampling/importance_sampling_ratio/mean": 0.9998114109039307, + "sampling/importance_sampling_ratio/min": 0.6009103655815125, + "sampling/sampling_logp_difference/max": 0.5093095302581787, + "sampling/sampling_logp_difference/mean": 0.012231100350618362, + "step": 655 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0001298701245104894, + "clip_ratio/low_min": 0.0001298701245104894, + "clip_ratio/region_mean": 0.0001298701245104894, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 695.0, + "completions/mean_length": 502.5625, + "completions/mean_terminated_length": 484.86669921875, + "completions/min_length": 317.0, + "completions/min_terminated_length": 317.0, + "entropy": 0.2781757991760969, + "epoch": 0.3511777301927195, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.007967548444867134, + "learning_rate": 1e-05, + "loss": 0.0895, + "num_tokens": 13602442.0, + "reward": 0.78125, + "reward_std": 0.3377464711666107, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.4814249277114868, + "sampling/importance_sampling_ratio/mean": 0.9999395608901978, + "sampling/importance_sampling_ratio/min": 0.6002770066261292, + "sampling/sampling_logp_difference/max": 0.5103640556335449, + "sampling/sampling_logp_difference/mean": 0.009514091536402702, + "step": 656 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00014287418889580294, + "clip_ratio/low_min": 0.00014287418889580294, + "clip_ratio/region_mean": 0.00014287418889580294, + "completions/clipped_ratio": 0.0, + "completions/max_length": 744.0, + "completions/max_terminated_length": 744.0, + "completions/mean_length": 344.28125, + "completions/mean_terminated_length": 344.28125, + "completions/min_length": 157.0, + "completions/min_terminated_length": 157.0, + "entropy": 0.4778929203748703, + "epoch": 0.35171306209850106, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.03561371564865112, + "learning_rate": 1e-05, + "loss": -0.0209, + "num_tokens": 13618115.0, + "reward": 0.75, + "reward_std": 0.3745020925998688, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 2.0, + "sampling/importance_sampling_ratio/mean": 0.9999615550041199, + "sampling/importance_sampling_ratio/min": 0.6989265084266663, + "sampling/sampling_logp_difference/max": 0.8352646827697754, + "sampling/sampling_logp_difference/mean": 0.014193162322044373, + "step": 657 + }, + { + "clip_ratio/high_max": 8.043758134590462e-05, + "clip_ratio/high_mean": 8.043758134590462e-05, + "clip_ratio/low_mean": 5.885122300242074e-05, + "clip_ratio/low_min": 5.885122300242074e-05, + "clip_ratio/region_mean": 0.00013928880434832536, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 669.0, + "completions/mean_length": 422.8125, + "completions/mean_terminated_length": 411.6773986816406, + "completions/min_length": 236.0, + "completions/min_terminated_length": 236.0, + "entropy": 0.27686809934675694, + "epoch": 0.35224839400428265, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.010238280519843102, + "learning_rate": 1e-05, + "loss": 0.0128, + "num_tokens": 13635869.0, + "reward": 0.875, + "reward_std": 0.2177756428718567, + "rewards/accuracy_reward/mean": 0.875, + "rewards/accuracy_reward/std": 0.33601075410842896, + "sampling/importance_sampling_ratio/max": 1.4051257371902466, + "sampling/importance_sampling_ratio/mean": 1.0004974603652954, + "sampling/importance_sampling_ratio/min": 0.5414208173751831, + "sampling/sampling_logp_difference/max": 0.6135585308074951, + "sampling/sampling_logp_difference/mean": 0.010490876622498035, + "step": 658 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 743.0, + "completions/mean_length": 496.59375, + "completions/mean_terminated_length": 420.5999755859375, + "completions/min_length": 228.0, + "completions/min_terminated_length": 228.0, + "entropy": 0.4050780236721039, + "epoch": 0.35278372591006424, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.01573796197772026, + "learning_rate": 1e-05, + "loss": 0.0283, + "num_tokens": 13655752.0, + "reward": 0.53125, + "reward_std": 0.2041158676147461, + "rewards/accuracy_reward/mean": 0.53125, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.6682548522949219, + "sampling/importance_sampling_ratio/mean": 0.9996892809867859, + "sampling/importance_sampling_ratio/min": 0.6459711194038391, + "sampling/sampling_logp_difference/max": 0.5117781162261963, + "sampling/sampling_logp_difference/mean": 0.01291722059249878, + "step": 659 + }, + { + "clip_ratio/high_max": 6.909894727868959e-05, + "clip_ratio/high_mean": 6.909894727868959e-05, + "clip_ratio/low_mean": 0.00014629126235377043, + "clip_ratio/low_min": 0.00014629126235377043, + "clip_ratio/region_mean": 0.00021539020963246003, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 611.0, + "completions/mean_length": 446.09375, + "completions/mean_terminated_length": 424.63336181640625, + "completions/min_length": 191.0, + "completions/min_terminated_length": 191.0, + "entropy": 0.3119290601462126, + "epoch": 0.3533190578158458, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.04319221153855324, + "learning_rate": 1e-05, + "loss": 0.0081, + "num_tokens": 13674315.0, + "reward": 0.59375, + "reward_std": 0.3987956643104553, + "rewards/accuracy_reward/mean": 0.59375, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.6307077407836914, + "sampling/importance_sampling_ratio/mean": 1.0006777048110962, + "sampling/importance_sampling_ratio/min": 0.6531321406364441, + "sampling/sampling_logp_difference/max": 0.4890141487121582, + "sampling/sampling_logp_difference/mean": 0.010623999871313572, + "step": 660 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 8.729050023248419e-05, + "clip_ratio/low_min": 8.729050023248419e-05, + "clip_ratio/region_mean": 8.729050023248419e-05, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 576.0, + "completions/mean_length": 379.75, + "completions/mean_terminated_length": 367.2257995605469, + "completions/min_length": 235.0, + "completions/min_terminated_length": 235.0, + "entropy": 0.3697916939854622, + "epoch": 0.3538543897216274, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.04080353304743767, + "learning_rate": 1e-05, + "loss": 0.0464, + "num_tokens": 13690123.0, + "reward": 0.53125, + "reward_std": 0.4355708956718445, + "rewards/accuracy_reward/mean": 0.53125, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.438925862312317, + "sampling/importance_sampling_ratio/mean": 0.9999911189079285, + "sampling/importance_sampling_ratio/min": 0.6053933501243591, + "sampling/sampling_logp_difference/max": 0.5018768310546875, + "sampling/sampling_logp_difference/mean": 0.01327377837151289, + "step": 661 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 681.0, + "completions/mean_length": 397.125, + "completions/mean_terminated_length": 385.1612854003906, + "completions/min_length": 150.0, + "completions/min_terminated_length": 150.0, + "entropy": 0.26181492395699024, + "epoch": 0.354389721627409, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.0094644445925951, + "learning_rate": 1e-05, + "loss": -0.0471, + "num_tokens": 13705999.0, + "reward": 0.9375, + "reward_std": 0.1157275140285492, + "rewards/accuracy_reward/mean": 0.9375, + "rewards/accuracy_reward/std": 0.24593468010425568, + "sampling/importance_sampling_ratio/max": 1.5801801681518555, + "sampling/importance_sampling_ratio/mean": 0.9998408555984497, + "sampling/importance_sampling_ratio/min": 0.3525858521461487, + "sampling/sampling_logp_difference/max": 1.0424611568450928, + "sampling/sampling_logp_difference/mean": 0.009918587282299995, + "step": 662 + }, + { + "clip_ratio/high_max": 9.110787505051121e-05, + "clip_ratio/high_mean": 9.110787505051121e-05, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 9.110787505051121e-05, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 672.0, + "completions/mean_length": 495.75, + "completions/mean_terminated_length": 419.5199890136719, + "completions/min_length": 275.0, + "completions/min_terminated_length": 275.0, + "entropy": 0.5153794698417187, + "epoch": 0.3549250535331906, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.027701010927557945, + "learning_rate": 1e-05, + "loss": 0.0769, + "num_tokens": 13725967.0, + "reward": 0.46875, + "reward_std": 0.3198433816432953, + "rewards/accuracy_reward/mean": 0.46875, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.5489026308059692, + "sampling/importance_sampling_ratio/mean": 0.9996196627616882, + "sampling/importance_sampling_ratio/min": 0.6261435747146606, + "sampling/sampling_logp_difference/max": 0.4681755304336548, + "sampling/sampling_logp_difference/mean": 0.015272910706698895, + "step": 663 + }, + { + "clip_ratio/high_max": 6.789788312744349e-05, + "clip_ratio/high_mean": 6.789788312744349e-05, + "clip_ratio/low_mean": 6.789788312744349e-05, + "clip_ratio/low_min": 6.789788312744349e-05, + "clip_ratio/region_mean": 0.00013579576625488698, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 614.0, + "completions/mean_length": 449.65625, + "completions/mean_terminated_length": 404.1785888671875, + "completions/min_length": 246.0, + "completions/min_terminated_length": 246.0, + "entropy": 0.5399555154144764, + "epoch": 0.3554603854389722, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.012038396671414375, + "learning_rate": 1e-05, + "loss": 0.1164, + "num_tokens": 13744804.0, + "reward": 0.59375, + "reward_std": 0.4807935357093811, + "rewards/accuracy_reward/mean": 0.59375, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 2.0, + "sampling/importance_sampling_ratio/mean": 0.999747097492218, + "sampling/importance_sampling_ratio/min": 0.4197999835014343, + "sampling/sampling_logp_difference/max": 0.8679769039154053, + "sampling/sampling_logp_difference/mean": 0.016079533845186234, + "step": 664 + }, + { + "clip_ratio/high_max": 7.494004967156798e-05, + "clip_ratio/high_mean": 7.494004967156798e-05, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 7.494004967156798e-05, + "completions/clipped_ratio": 0.0, + "completions/max_length": 637.0, + "completions/max_terminated_length": 637.0, + "completions/mean_length": 377.65625, + "completions/mean_terminated_length": 377.65625, + "completions/min_length": 108.0, + "completions/min_terminated_length": 108.0, + "entropy": 0.2771514691412449, + "epoch": 0.35599571734475377, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.005888278130441904, + "learning_rate": 1e-05, + "loss": 0.0078, + "num_tokens": 13760321.0, + "reward": 0.75, + "reward_std": 0.1767766922712326, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.4778823852539062, + "sampling/importance_sampling_ratio/mean": 0.9998041391372681, + "sampling/importance_sampling_ratio/min": 0.6845996379852295, + "sampling/sampling_logp_difference/max": 0.3906102180480957, + "sampling/sampling_logp_difference/mean": 0.010204792954027653, + "step": 665 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.000129915468278341, + "clip_ratio/low_min": 0.000129915468278341, + "clip_ratio/region_mean": 0.000129915468278341, + "completions/clipped_ratio": 0.0, + "completions/max_length": 714.0, + "completions/max_terminated_length": 714.0, + "completions/mean_length": 444.40625, + "completions/mean_terminated_length": 444.40625, + "completions/min_length": 207.0, + "completions/min_terminated_length": 207.0, + "entropy": 0.3105376996099949, + "epoch": 0.35653104925053536, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.016817567870020866, + "learning_rate": 1e-05, + "loss": -0.007, + "num_tokens": 13778270.0, + "reward": 0.78125, + "reward_std": 0.3471629321575165, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.6711481809616089, + "sampling/importance_sampling_ratio/mean": 0.999639630317688, + "sampling/importance_sampling_ratio/min": 0.6574479341506958, + "sampling/sampling_logp_difference/max": 0.5135109424591064, + "sampling/sampling_logp_difference/mean": 0.011440515518188477, + "step": 666 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 568.0, + "completions/max_terminated_length": 568.0, + "completions/mean_length": 364.625, + "completions/mean_terminated_length": 364.625, + "completions/min_length": 176.0, + "completions/min_terminated_length": 176.0, + "entropy": 0.3822487108409405, + "epoch": 0.3570663811563169, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.022095561027526855, + "learning_rate": 1e-05, + "loss": -0.0609, + "num_tokens": 13793434.0, + "reward": 0.65625, + "reward_std": 0.3377464711666107, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.9472275972366333, + "sampling/importance_sampling_ratio/mean": 1.0000711679458618, + "sampling/importance_sampling_ratio/min": 0.6074678897857666, + "sampling/sampling_logp_difference/max": 0.6664066314697266, + "sampling/sampling_logp_difference/mean": 0.01337414886802435, + "step": 667 + }, + { + "clip_ratio/high_max": 6.537656736327335e-05, + "clip_ratio/high_mean": 6.537656736327335e-05, + "clip_ratio/low_mean": 0.00015136247384361923, + "clip_ratio/low_min": 0.00015136247384361923, + "clip_ratio/region_mean": 0.00021673904120689258, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 740.0, + "completions/mean_length": 434.9375, + "completions/mean_terminated_length": 424.19354248046875, + "completions/min_length": 163.0, + "completions/min_terminated_length": 163.0, + "entropy": 0.2582597993314266, + "epoch": 0.3576017130620985, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.048923395574092865, + "learning_rate": 1e-05, + "loss": 0.0356, + "num_tokens": 13810568.0, + "reward": 0.65625, + "reward_std": 0.3808925747871399, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.4425057172775269, + "sampling/importance_sampling_ratio/mean": 1.0000734329223633, + "sampling/importance_sampling_ratio/min": 0.5596014857292175, + "sampling/sampling_logp_difference/max": 0.5805304050445557, + "sampling/sampling_logp_difference/mean": 0.009786554612219334, + "step": 668 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 546.0, + "completions/max_terminated_length": 546.0, + "completions/mean_length": 367.53125, + "completions/mean_terminated_length": 367.53125, + "completions/min_length": 201.0, + "completions/min_terminated_length": 201.0, + "entropy": 0.315692201256752, + "epoch": 0.35813704496788007, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.00875961035490036, + "learning_rate": 1e-05, + "loss": 0.0145, + "num_tokens": 13826417.0, + "reward": 0.8125, + "reward_std": 0.2587745785713196, + "rewards/accuracy_reward/mean": 0.8125, + "rewards/accuracy_reward/std": 0.3965577781200409, + "sampling/importance_sampling_ratio/max": 1.6791738271713257, + "sampling/importance_sampling_ratio/mean": 0.9998831748962402, + "sampling/importance_sampling_ratio/min": 0.6887439489364624, + "sampling/sampling_logp_difference/max": 0.5183019638061523, + "sampling/sampling_logp_difference/mean": 0.012204647995531559, + "step": 669 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 7.678132533328608e-05, + "clip_ratio/low_min": 7.678132533328608e-05, + "clip_ratio/region_mean": 7.678132533328608e-05, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 754.0, + "completions/mean_length": 431.78125, + "completions/mean_terminated_length": 409.36669921875, + "completions/min_length": 198.0, + "completions/min_terminated_length": 198.0, + "entropy": 0.3540220931172371, + "epoch": 0.35867237687366166, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.012857015244662762, + "learning_rate": 1e-05, + "loss": 0.0082, + "num_tokens": 13844394.0, + "reward": 0.6875, + "reward_std": 0.249358132481575, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.58338463306427, + "sampling/importance_sampling_ratio/mean": 1.00005042552948, + "sampling/importance_sampling_ratio/min": 0.64851313829422, + "sampling/sampling_logp_difference/max": 0.4595646858215332, + "sampling/sampling_logp_difference/mean": 0.011811037547886372, + "step": 670 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 613.0, + "completions/mean_length": 414.375, + "completions/mean_terminated_length": 402.9677429199219, + "completions/min_length": 202.0, + "completions/min_terminated_length": 202.0, + "entropy": 0.2561605256050825, + "epoch": 0.35920770877944325, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.005706072319298983, + "learning_rate": 1e-05, + "loss": 0.078, + "num_tokens": 13861166.0, + "reward": 0.8125, + "reward_std": 0.2177756428718567, + "rewards/accuracy_reward/mean": 0.8125, + "rewards/accuracy_reward/std": 0.3965577781200409, + "sampling/importance_sampling_ratio/max": 1.402472972869873, + "sampling/importance_sampling_ratio/mean": 0.999974250793457, + "sampling/importance_sampling_ratio/min": 0.5753530263900757, + "sampling/sampling_logp_difference/max": 0.5527715682983398, + "sampling/sampling_logp_difference/mean": 0.00912991538643837, + "step": 671 + }, + { + "clip_ratio/high_max": 0.00017741198098519817, + "clip_ratio/high_mean": 0.00017741198098519817, + "clip_ratio/low_mean": 0.00018907566845882684, + "clip_ratio/low_min": 0.00018907566845882684, + "clip_ratio/region_mean": 0.000366487649444025, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 666.0, + "completions/mean_length": 401.0, + "completions/mean_terminated_length": 389.1612854003906, + "completions/min_length": 118.0, + "completions/min_terminated_length": 118.0, + "entropy": 0.38735750690102577, + "epoch": 0.35974304068522484, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.008876451291143894, + "learning_rate": 1e-05, + "loss": -0.0949, + "num_tokens": 13878022.0, + "reward": 0.84375, + "reward_std": 0.3808925747871399, + "rewards/accuracy_reward/mean": 0.84375, + "rewards/accuracy_reward/std": 0.3689020276069641, + "sampling/importance_sampling_ratio/max": 1.535284161567688, + "sampling/importance_sampling_ratio/mean": 1.0001300573349, + "sampling/importance_sampling_ratio/min": 0.45783501863479614, + "sampling/sampling_logp_difference/max": 0.7812464237213135, + "sampling/sampling_logp_difference/mean": 0.01291007548570633, + "step": 672 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 526.0, + "completions/max_terminated_length": 526.0, + "completions/mean_length": 359.6875, + "completions/mean_terminated_length": 359.6875, + "completions/min_length": 128.0, + "completions/min_terminated_length": 128.0, + "entropy": 0.29211121797561646, + "epoch": 0.3602783725910064, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": 0.0, + "num_tokens": 13893076.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/accuracy_reward/mean": 1.0, + "rewards/accuracy_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.832126498222351, + "sampling/importance_sampling_ratio/mean": 0.9998286366462708, + "sampling/importance_sampling_ratio/min": 0.6737604737281799, + "sampling/sampling_logp_difference/max": 0.6054773330688477, + "sampling/sampling_logp_difference/mean": 0.010949667543172836, + "step": 673 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00013384085468715057, + "clip_ratio/low_min": 0.00013384085468715057, + "clip_ratio/region_mean": 0.00013384085468715057, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 764.0, + "completions/mean_length": 478.375, + "completions/mean_terminated_length": 459.0666809082031, + "completions/min_length": 296.0, + "completions/min_terminated_length": 296.0, + "entropy": 0.3839053250849247, + "epoch": 0.360813704496788, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.015960123389959335, + "learning_rate": 1e-05, + "loss": 0.1086, + "num_tokens": 13912512.0, + "reward": 0.6875, + "reward_std": 0.3335031569004059, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.4055489301681519, + "sampling/importance_sampling_ratio/mean": 0.9999480843544006, + "sampling/importance_sampling_ratio/min": 0.5603016018867493, + "sampling/sampling_logp_difference/max": 0.5792800188064575, + "sampling/sampling_logp_difference/mean": 0.013557146303355694, + "step": 674 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00015738436923129484, + "clip_ratio/low_min": 0.00015738436923129484, + "clip_ratio/region_mean": 0.00015738436923129484, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 742.0, + "completions/mean_length": 375.71875, + "completions/mean_terminated_length": 363.06451416015625, + "completions/min_length": 232.0, + "completions/min_terminated_length": 232.0, + "entropy": 0.3089521825313568, + "epoch": 0.3613490364025696, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": -0.0096, + "num_tokens": 13928359.0, + "reward": 0.90625, + "reward_std": 0.1293872892856598, + "rewards/accuracy_reward/mean": 0.90625, + "rewards/accuracy_reward/std": 0.2961445748806, + "sampling/importance_sampling_ratio/max": 1.8923866748809814, + "sampling/importance_sampling_ratio/mean": 1.0001062154769897, + "sampling/importance_sampling_ratio/min": 0.6327048540115356, + "sampling/sampling_logp_difference/max": 0.6378388404846191, + "sampling/sampling_logp_difference/mean": 0.012162160128355026, + "step": 675 + }, + { + "clip_ratio/high_max": 0.00012065636838087812, + "clip_ratio/high_mean": 0.00012065636838087812, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.00012065636838087812, + "completions/clipped_ratio": 0.0, + "completions/max_length": 727.0, + "completions/max_terminated_length": 727.0, + "completions/mean_length": 314.125, + "completions/mean_terminated_length": 314.125, + "completions/min_length": 120.0, + "completions/min_terminated_length": 120.0, + "entropy": 0.34653982892632484, + "epoch": 0.3618843683083512, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.004590500611811876, + "learning_rate": 1e-05, + "loss": 0.0698, + "num_tokens": 13942379.0, + "reward": 0.9375, + "reward_std": 0.1767766922712326, + "rewards/accuracy_reward/mean": 0.9375, + "rewards/accuracy_reward/std": 0.24593468010425568, + "sampling/importance_sampling_ratio/max": 1.4619866609573364, + "sampling/importance_sampling_ratio/mean": 0.9997506141662598, + "sampling/importance_sampling_ratio/min": 0.546380877494812, + "sampling/sampling_logp_difference/max": 0.6044389009475708, + "sampling/sampling_logp_difference/mean": 0.011684027500450611, + "step": 676 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 605.0, + "completions/mean_length": 433.34375, + "completions/mean_terminated_length": 385.5357360839844, + "completions/min_length": 239.0, + "completions/min_terminated_length": 239.0, + "entropy": 0.40856448002159595, + "epoch": 0.3624197002141328, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.003538036486133933, + "learning_rate": 1e-05, + "loss": -0.0164, + "num_tokens": 13960094.0, + "reward": 0.71875, + "reward_std": 0.0883883461356163, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.4316827058792114, + "sampling/importance_sampling_ratio/mean": 0.9999025464057922, + "sampling/importance_sampling_ratio/min": 0.5827119946479797, + "sampling/sampling_logp_difference/max": 0.5400621891021729, + "sampling/sampling_logp_difference/mean": 0.013678331859409809, + "step": 677 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 7.78331232140772e-05, + "clip_ratio/low_min": 7.78331232140772e-05, + "clip_ratio/region_mean": 7.78331232140772e-05, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 755.0, + "completions/mean_length": 416.875, + "completions/mean_terminated_length": 393.4666748046875, + "completions/min_length": 79.0, + "completions/min_terminated_length": 79.0, + "entropy": 0.279329190030694, + "epoch": 0.36295503211991437, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.009009398519992828, + "learning_rate": 1e-05, + "loss": 0.0378, + "num_tokens": 13976618.0, + "reward": 0.84375, + "reward_std": 0.1293872892856598, + "rewards/accuracy_reward/mean": 0.84375, + "rewards/accuracy_reward/std": 0.3689020276069641, + "sampling/importance_sampling_ratio/max": 1.7500592470169067, + "sampling/importance_sampling_ratio/mean": 1.0001730918884277, + "sampling/importance_sampling_ratio/min": 0.5929864645004272, + "sampling/sampling_logp_difference/max": 0.5596495866775513, + "sampling/sampling_logp_difference/mean": 0.009725116193294525, + "step": 678 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00016687424067640677, + "clip_ratio/low_min": 0.00016687424067640677, + "clip_ratio/region_mean": 0.00016687424067640677, + "completions/clipped_ratio": 0.0, + "completions/max_length": 628.0, + "completions/max_terminated_length": 628.0, + "completions/mean_length": 373.75, + "completions/mean_terminated_length": 373.75, + "completions/min_length": 110.0, + "completions/min_terminated_length": 110.0, + "entropy": 0.2835135981440544, + "epoch": 0.36349036402569596, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": -0.1194, + "num_tokens": 13992162.0, + "reward": 0.75, + "reward_std": 0.1767766922712326, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.4620192050933838, + "sampling/importance_sampling_ratio/mean": 0.9996680021286011, + "sampling/importance_sampling_ratio/min": 0.5569161772727966, + "sampling/sampling_logp_difference/max": 0.5853404998779297, + "sampling/sampling_logp_difference/mean": 0.011053714901208878, + "step": 679 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00014245013881009072, + "clip_ratio/low_min": 0.00014245013881009072, + "clip_ratio/region_mean": 0.00014245013881009072, + "completions/clipped_ratio": 0.0, + "completions/max_length": 633.0, + "completions/max_terminated_length": 633.0, + "completions/mean_length": 419.59375, + "completions/mean_terminated_length": 419.59375, + "completions/min_length": 272.0, + "completions/min_terminated_length": 272.0, + "entropy": 0.26943989656865597, + "epoch": 0.3640256959314775, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.022552641108632088, + "learning_rate": 1e-05, + "loss": 0.0257, + "num_tokens": 14009085.0, + "reward": 0.625, + "reward_std": 0.3745020925998688, + "rewards/accuracy_reward/mean": 0.625, + "rewards/accuracy_reward/std": 0.49186936020851135, + "sampling/importance_sampling_ratio/max": 2.0, + "sampling/importance_sampling_ratio/mean": 0.9997143745422363, + "sampling/importance_sampling_ratio/min": 0.4835348427295685, + "sampling/sampling_logp_difference/max": 0.7421181201934814, + "sampling/sampling_logp_difference/mean": 0.010526628233492374, + "step": 680 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00015492347301915288, + "clip_ratio/low_min": 0.00015492347301915288, + "clip_ratio/region_mean": 0.00015492347301915288, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 723.0, + "completions/mean_length": 451.34375, + "completions/mean_terminated_length": 418.5862121582031, + "completions/min_length": 244.0, + "completions/min_terminated_length": 244.0, + "entropy": 0.47729120403528214, + "epoch": 0.3645610278372591, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.016579551622271538, + "learning_rate": 1e-05, + "loss": 0.0787, + "num_tokens": 14027240.0, + "reward": 0.53125, + "reward_std": 0.4397946000099182, + "rewards/accuracy_reward/mean": 0.53125, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.8460532426834106, + "sampling/importance_sampling_ratio/mean": 1.0000616312026978, + "sampling/importance_sampling_ratio/min": 0.6330141425132751, + "sampling/sampling_logp_difference/max": 0.6130499839782715, + "sampling/sampling_logp_difference/mean": 0.014952778816223145, + "step": 681 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 586.0, + "completions/max_terminated_length": 586.0, + "completions/mean_length": 394.84375, + "completions/mean_terminated_length": 394.84375, + "completions/min_length": 183.0, + "completions/min_terminated_length": 183.0, + "entropy": 0.27810865454375744, + "epoch": 0.36509635974304067, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.012250752188265324, + "learning_rate": 1e-05, + "loss": -0.0525, + "num_tokens": 14043987.0, + "reward": 0.78125, + "reward_std": 0.3061639666557312, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.499852180480957, + "sampling/importance_sampling_ratio/mean": 1.0003774166107178, + "sampling/importance_sampling_ratio/min": 0.46095797419548035, + "sampling/sampling_logp_difference/max": 0.7744483947753906, + "sampling/sampling_logp_difference/mean": 0.010378003120422363, + "step": 682 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 479.0, + "completions/max_terminated_length": 479.0, + "completions/mean_length": 305.1875, + "completions/mean_terminated_length": 305.1875, + "completions/min_length": 118.0, + "completions/min_terminated_length": 118.0, + "entropy": 0.3248937167227268, + "epoch": 0.36563169164882225, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.02616657130420208, + "learning_rate": 1e-05, + "loss": 0.0288, + "num_tokens": 14057617.0, + "reward": 0.625, + "reward_std": 0.2314550280570984, + "rewards/accuracy_reward/mean": 0.625, + "rewards/accuracy_reward/std": 0.49186936020851135, + "sampling/importance_sampling_ratio/max": 1.5338889360427856, + "sampling/importance_sampling_ratio/mean": 1.0000650882720947, + "sampling/importance_sampling_ratio/min": 0.6569048762321472, + "sampling/sampling_logp_difference/max": 0.4278062582015991, + "sampling/sampling_logp_difference/mean": 0.012229588814079762, + "step": 683 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 637.0, + "completions/mean_length": 374.53125, + "completions/mean_terminated_length": 361.83868408203125, + "completions/min_length": 175.0, + "completions/min_terminated_length": 175.0, + "entropy": 0.3952745087444782, + "epoch": 0.36616702355460384, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.0198307354003191, + "learning_rate": 1e-05, + "loss": 0.028, + "num_tokens": 14073274.0, + "reward": 0.71875, + "reward_std": 0.378745436668396, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.6379166841506958, + "sampling/importance_sampling_ratio/mean": 0.9999538064002991, + "sampling/importance_sampling_ratio/min": 0.6694729924201965, + "sampling/sampling_logp_difference/max": 0.4934251308441162, + "sampling/sampling_logp_difference/mean": 0.013427930884063244, + "step": 684 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00014134941739030182, + "clip_ratio/low_min": 0.00014134941739030182, + "clip_ratio/region_mean": 0.00014134941739030182, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 754.0, + "completions/mean_length": 445.09375, + "completions/mean_terminated_length": 423.5666809082031, + "completions/min_length": 146.0, + "completions/min_terminated_length": 146.0, + "entropy": 0.37205213122069836, + "epoch": 0.36670235546038543, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.02514619193971157, + "learning_rate": 1e-05, + "loss": -0.0186, + "num_tokens": 14091501.0, + "reward": 0.625, + "reward_std": 0.3104073107242584, + "rewards/accuracy_reward/mean": 0.625, + "rewards/accuracy_reward/std": 0.49186936020851135, + "sampling/importance_sampling_ratio/max": 2.0, + "sampling/importance_sampling_ratio/mean": 1.000280737876892, + "sampling/importance_sampling_ratio/min": 0.3590173125267029, + "sampling/sampling_logp_difference/max": 1.0243847370147705, + "sampling/sampling_logp_difference/mean": 0.013068465515971184, + "step": 685 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 586.0, + "completions/mean_length": 360.78125, + "completions/mean_terminated_length": 347.6451416015625, + "completions/min_length": 225.0, + "completions/min_terminated_length": 225.0, + "entropy": 0.34020672738552094, + "epoch": 0.367237687366167, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.0432632714509964, + "learning_rate": 1e-05, + "loss": 0.0745, + "num_tokens": 14106646.0, + "reward": 0.6875, + "reward_std": 0.3335031569004059, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.7226487398147583, + "sampling/importance_sampling_ratio/mean": 1.0001485347747803, + "sampling/importance_sampling_ratio/min": 0.6301517486572266, + "sampling/sampling_logp_difference/max": 0.54386305809021, + "sampling/sampling_logp_difference/mean": 0.01216810755431652, + "step": 686 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 8.366800466319546e-05, + "clip_ratio/low_min": 8.366800466319546e-05, + "clip_ratio/region_mean": 8.366800466319546e-05, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 630.0, + "completions/mean_length": 417.0, + "completions/mean_terminated_length": 380.6896667480469, + "completions/min_length": 220.0, + "completions/min_terminated_length": 220.0, + "entropy": 0.3689681924879551, + "epoch": 0.3677730192719486, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.017116356641054153, + "learning_rate": 1e-05, + "loss": 0.0792, + "num_tokens": 14123014.0, + "reward": 0.84375, + "reward_std": 0.22201895713806152, + "rewards/accuracy_reward/mean": 0.84375, + "rewards/accuracy_reward/std": 0.3689020276069641, + "sampling/importance_sampling_ratio/max": 1.6711039543151855, + "sampling/importance_sampling_ratio/mean": 0.9996310472488403, + "sampling/importance_sampling_ratio/min": 0.5510715842247009, + "sampling/sampling_logp_difference/max": 0.5958905220031738, + "sampling/sampling_logp_difference/mean": 0.012507694773375988, + "step": 687 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 730.0, + "completions/mean_length": 429.4375, + "completions/mean_terminated_length": 406.86669921875, + "completions/min_length": 234.0, + "completions/min_terminated_length": 234.0, + "entropy": 0.36085673049092293, + "epoch": 0.3683083511777302, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.026025017723441124, + "learning_rate": 1e-05, + "loss": 0.0327, + "num_tokens": 14140332.0, + "reward": 0.65625, + "reward_std": 0.1293872892856598, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.9497008323669434, + "sampling/importance_sampling_ratio/mean": 0.9995465874671936, + "sampling/importance_sampling_ratio/min": 0.5762607455253601, + "sampling/sampling_logp_difference/max": 0.6676759719848633, + "sampling/sampling_logp_difference/mean": 0.01313733495771885, + "step": 688 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 7.263218867592514e-05, + "clip_ratio/low_min": 7.263218867592514e-05, + "clip_ratio/region_mean": 7.263218867592514e-05, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 722.0, + "completions/mean_length": 487.40625, + "completions/mean_terminated_length": 435.4444580078125, + "completions/min_length": 219.0, + "completions/min_terminated_length": 219.0, + "entropy": 0.382045628502965, + "epoch": 0.3688436830835118, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.019538233056664467, + "learning_rate": 1e-05, + "loss": 0.0505, + "num_tokens": 14160465.0, + "reward": 0.5625, + "reward_std": 0.292504221200943, + "rewards/accuracy_reward/mean": 0.5625, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 1.953310489654541, + "sampling/importance_sampling_ratio/mean": 1.0001777410507202, + "sampling/importance_sampling_ratio/min": 0.5836070775985718, + "sampling/sampling_logp_difference/max": 0.6695256233215332, + "sampling/sampling_logp_difference/mean": 0.014216558076441288, + "step": 689 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 746.0, + "completions/mean_length": 377.59375, + "completions/mean_terminated_length": 351.5666809082031, + "completions/min_length": 154.0, + "completions/min_terminated_length": 154.0, + "entropy": 0.35210950300097466, + "epoch": 0.3693790149892934, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.00846430379897356, + "learning_rate": 1e-05, + "loss": 0.0782, + "num_tokens": 14176300.0, + "reward": 0.65625, + "reward_std": 0.2041158676147461, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 2.0, + "sampling/importance_sampling_ratio/mean": 0.9999990463256836, + "sampling/importance_sampling_ratio/min": 0.5328687429428101, + "sampling/sampling_logp_difference/max": 0.7396030426025391, + "sampling/sampling_logp_difference/mean": 0.01267185527831316, + "step": 690 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 7.53012063796632e-05, + "clip_ratio/low_min": 7.53012063796632e-05, + "clip_ratio/region_mean": 7.53012063796632e-05, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 722.0, + "completions/mean_length": 502.0, + "completions/mean_terminated_length": 464.0000305175781, + "completions/min_length": 222.0, + "completions/min_terminated_length": 222.0, + "entropy": 0.482187956571579, + "epoch": 0.36991434689507496, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.0195754524320364, + "learning_rate": 1e-05, + "loss": 0.0041, + "num_tokens": 14196060.0, + "reward": 0.5, + "reward_std": 0.3514062762260437, + "rewards/accuracy_reward/mean": 0.5, + "rewards/accuracy_reward/std": 0.5080004930496216, + "sampling/importance_sampling_ratio/max": 2.0, + "sampling/importance_sampling_ratio/mean": 0.9999594688415527, + "sampling/importance_sampling_ratio/min": 0.5215132832527161, + "sampling/sampling_logp_difference/max": 0.8177294731140137, + "sampling/sampling_logp_difference/mean": 0.0157401692122221, + "step": 691 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 597.0, + "completions/max_terminated_length": 597.0, + "completions/mean_length": 447.1875, + "completions/mean_terminated_length": 447.1875, + "completions/min_length": 276.0, + "completions/min_terminated_length": 276.0, + "entropy": 0.23954479582607746, + "epoch": 0.37044967880085655, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.009266071952879429, + "learning_rate": 1e-05, + "loss": 0.0032, + "num_tokens": 14214330.0, + "reward": 0.6875, + "reward_std": 0.249358132481575, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.872075080871582, + "sampling/importance_sampling_ratio/mean": 0.9997952580451965, + "sampling/importance_sampling_ratio/min": 0.6269657611846924, + "sampling/sampling_logp_difference/max": 0.6270475387573242, + "sampling/sampling_logp_difference/mean": 0.009122824296355247, + "step": 692 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00010674636723706499, + "clip_ratio/low_min": 0.00010674636723706499, + "clip_ratio/region_mean": 0.00010674636723706499, + "completions/clipped_ratio": 0.0, + "completions/max_length": 562.0, + "completions/max_terminated_length": 562.0, + "completions/mean_length": 325.84375, + "completions/mean_terminated_length": 325.84375, + "completions/min_length": 72.0, + "completions/min_terminated_length": 72.0, + "entropy": 0.30397594533860683, + "epoch": 0.37098501070663814, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.027942130342125893, + "learning_rate": 1e-05, + "loss": -0.0269, + "num_tokens": 14228373.0, + "reward": 0.6875, + "reward_std": 0.38298875093460083, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.5171500444412231, + "sampling/importance_sampling_ratio/mean": 1.000351071357727, + "sampling/importance_sampling_ratio/min": 0.5740833282470703, + "sampling/sampling_logp_difference/max": 0.5549807548522949, + "sampling/sampling_logp_difference/mean": 0.011213074438273907, + "step": 693 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 762.0, + "completions/mean_length": 511.625, + "completions/mean_terminated_length": 475.0000305175781, + "completions/min_length": 282.0, + "completions/min_terminated_length": 282.0, + "entropy": 0.376805879175663, + "epoch": 0.3715203426124197, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.00635635107755661, + "learning_rate": 1e-05, + "loss": 0.0007, + "num_tokens": 14248377.0, + "reward": 0.75, + "reward_std": 0.1767766922712326, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.5212767124176025, + "sampling/importance_sampling_ratio/mean": 0.9996983408927917, + "sampling/importance_sampling_ratio/min": 0.7199479937553406, + "sampling/sampling_logp_difference/max": 0.41954994201660156, + "sampling/sampling_logp_difference/mean": 0.01242767833173275, + "step": 694 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 658.0, + "completions/mean_length": 414.53125, + "completions/mean_terminated_length": 403.1290283203125, + "completions/min_length": 168.0, + "completions/min_terminated_length": 168.0, + "entropy": 0.32878440991044044, + "epoch": 0.37205567451820126, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.014503064565360546, + "learning_rate": 1e-05, + "loss": -0.0431, + "num_tokens": 14264962.0, + "reward": 0.71875, + "reward_std": 0.3787454068660736, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.730616807937622, + "sampling/importance_sampling_ratio/mean": 0.9998878240585327, + "sampling/importance_sampling_ratio/min": 0.5685059428215027, + "sampling/sampling_logp_difference/max": 0.5647435188293457, + "sampling/sampling_logp_difference/mean": 0.011785061098635197, + "step": 695 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0001320291485171765, + "clip_ratio/low_min": 0.0001320291485171765, + "clip_ratio/region_mean": 0.0001320291485171765, + "completions/clipped_ratio": 0.0, + "completions/max_length": 700.0, + "completions/max_terminated_length": 700.0, + "completions/mean_length": 455.59375, + "completions/mean_terminated_length": 455.59375, + "completions/min_length": 271.0, + "completions/min_terminated_length": 271.0, + "entropy": 0.2597401551902294, + "epoch": 0.37259100642398285, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.01776432991027832, + "learning_rate": 1e-05, + "loss": -0.0185, + "num_tokens": 14283445.0, + "reward": 0.78125, + "reward_std": 0.3608423173427582, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.6622546911239624, + "sampling/importance_sampling_ratio/mean": 1.000230312347412, + "sampling/importance_sampling_ratio/min": 0.5818818807601929, + "sampling/sampling_logp_difference/max": 0.5414879322052002, + "sampling/sampling_logp_difference/mean": 0.010266934521496296, + "step": 696 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 655.0, + "completions/max_terminated_length": 655.0, + "completions/mean_length": 371.25, + "completions/mean_terminated_length": 371.25, + "completions/min_length": 153.0, + "completions/min_terminated_length": 153.0, + "entropy": 0.3413393497467041, + "epoch": 0.37312633832976444, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.021115630865097046, + "learning_rate": 1e-05, + "loss": -0.0336, + "num_tokens": 14298893.0, + "reward": 0.59375, + "reward_std": 0.4534739851951599, + "rewards/accuracy_reward/mean": 0.59375, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.779329776763916, + "sampling/importance_sampling_ratio/mean": 1.000458836555481, + "sampling/importance_sampling_ratio/min": 0.5514275431632996, + "sampling/sampling_logp_difference/max": 0.5952447652816772, + "sampling/sampling_logp_difference/mean": 0.012435673736035824, + "step": 697 + }, + { + "clip_ratio/high_max": 6.800870323786512e-05, + "clip_ratio/high_mean": 6.800870323786512e-05, + "clip_ratio/low_mean": 0.00015918692224659026, + "clip_ratio/low_min": 0.00015918692224659026, + "clip_ratio/region_mean": 0.00022719562548445538, + "completions/clipped_ratio": 0.0, + "completions/max_length": 562.0, + "completions/max_terminated_length": 562.0, + "completions/mean_length": 422.375, + "completions/mean_terminated_length": 422.375, + "completions/min_length": 293.0, + "completions/min_terminated_length": 293.0, + "entropy": 0.3052702955901623, + "epoch": 0.37366167023554603, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.021168548613786697, + "learning_rate": 1e-05, + "loss": -0.019, + "num_tokens": 14316409.0, + "reward": 0.71875, + "reward_std": 0.378745436668396, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.7156918048858643, + "sampling/importance_sampling_ratio/mean": 1.0006873607635498, + "sampling/importance_sampling_ratio/min": 0.6708534955978394, + "sampling/sampling_logp_difference/max": 0.5398163795471191, + "sampling/sampling_logp_difference/mean": 0.010774184949696064, + "step": 698 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 656.0, + "completions/max_terminated_length": 656.0, + "completions/mean_length": 353.90625, + "completions/mean_terminated_length": 353.90625, + "completions/min_length": 200.0, + "completions/min_terminated_length": 200.0, + "entropy": 0.2485798615962267, + "epoch": 0.3741970021413276, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": 0.0, + "num_tokens": 14331086.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/accuracy_reward/mean": 1.0, + "rewards/accuracy_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.2952086925506592, + "sampling/importance_sampling_ratio/mean": 0.999834418296814, + "sampling/importance_sampling_ratio/min": 0.46038660407066345, + "sampling/sampling_logp_difference/max": 0.775688648223877, + "sampling/sampling_logp_difference/mean": 0.009568484500050545, + "step": 699 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 480.0, + "completions/max_terminated_length": 480.0, + "completions/mean_length": 271.09375, + "completions/mean_terminated_length": 271.09375, + "completions/min_length": 58.0, + "completions/min_terminated_length": 58.0, + "entropy": 0.3374805971980095, + "epoch": 0.3747323340471092, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.03927936777472496, + "learning_rate": 1e-05, + "loss": -0.034, + "num_tokens": 14342969.0, + "reward": 0.75, + "reward_std": 0.3650856614112854, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.6612154245376587, + "sampling/importance_sampling_ratio/mean": 0.9998925924301147, + "sampling/importance_sampling_ratio/min": 0.4506168067455292, + "sampling/sampling_logp_difference/max": 0.797137975692749, + "sampling/sampling_logp_difference/mean": 0.012434404343366623, + "step": 700 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 642.0, + "completions/max_terminated_length": 642.0, + "completions/mean_length": 398.375, + "completions/mean_terminated_length": 398.375, + "completions/min_length": 181.0, + "completions/min_terminated_length": 181.0, + "entropy": 0.2815327737480402, + "epoch": 0.3752676659528908, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.005384574178606272, + "learning_rate": 1e-05, + "loss": -0.0201, + "num_tokens": 14359309.0, + "reward": 0.6875, + "reward_std": 0.1767766922712326, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.9044970273971558, + "sampling/importance_sampling_ratio/mean": 1.0000734329223633, + "sampling/importance_sampling_ratio/min": 0.4690447151660919, + "sampling/sampling_logp_difference/max": 0.7570571899414062, + "sampling/sampling_logp_difference/mean": 0.010723521932959557, + "step": 701 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 659.0, + "completions/mean_length": 435.71875, + "completions/mean_terminated_length": 359.0384826660156, + "completions/min_length": 93.0, + "completions/min_terminated_length": 93.0, + "entropy": 0.49586478620767593, + "epoch": 0.3758029978586724, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": 0.0055, + "num_tokens": 14377364.0, + "reward": 0.59375, + "reward_std": 0.1293872892856598, + "rewards/accuracy_reward/mean": 0.59375, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 2.0, + "sampling/importance_sampling_ratio/mean": 1.000350832939148, + "sampling/importance_sampling_ratio/min": 0.6016424894332886, + "sampling/sampling_logp_difference/max": 0.7710986137390137, + "sampling/sampling_logp_difference/mean": 0.014931955374777317, + "step": 702 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 469.0, + "completions/max_terminated_length": 469.0, + "completions/mean_length": 303.875, + "completions/mean_terminated_length": 303.875, + "completions/min_length": 126.0, + "completions/min_terminated_length": 126.0, + "entropy": 0.37582309544086456, + "epoch": 0.37633832976445397, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.016746819019317627, + "learning_rate": 1e-05, + "loss": 0.0071, + "num_tokens": 14390600.0, + "reward": 0.78125, + "reward_std": 0.3471629321575165, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.5888361930847168, + "sampling/importance_sampling_ratio/mean": 1.0002955198287964, + "sampling/importance_sampling_ratio/min": 0.4946345090866089, + "sampling/sampling_logp_difference/max": 0.7039362192153931, + "sampling/sampling_logp_difference/mean": 0.01304656732827425, + "step": 703 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 726.0, + "completions/mean_length": 506.15625, + "completions/mean_terminated_length": 445.73077392578125, + "completions/min_length": 200.0, + "completions/min_terminated_length": 200.0, + "entropy": 0.5743923224508762, + "epoch": 0.37687366167023556, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.01048553641885519, + "learning_rate": 1e-05, + "loss": -0.0655, + "num_tokens": 14410789.0, + "reward": 0.375, + "reward_std": 0.2177756428718567, + "rewards/accuracy_reward/mean": 0.375, + "rewards/accuracy_reward/std": 0.49186936020851135, + "sampling/importance_sampling_ratio/max": 1.9519473314285278, + "sampling/importance_sampling_ratio/mean": 1.0001193284988403, + "sampling/importance_sampling_ratio/min": 0.6970666646957397, + "sampling/sampling_logp_difference/max": 0.6688275337219238, + "sampling/sampling_logp_difference/mean": 0.016540801152586937, + "step": 704 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 8.305647497763857e-05, + "clip_ratio/low_min": 8.305647497763857e-05, + "clip_ratio/region_mean": 8.305647497763857e-05, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 766.0, + "completions/mean_length": 408.5, + "completions/mean_terminated_length": 384.5333557128906, + "completions/min_length": 225.0, + "completions/min_terminated_length": 225.0, + "entropy": 0.31146340630948544, + "epoch": 0.37740899357601715, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.020676637068390846, + "learning_rate": 1e-05, + "loss": 0.0654, + "num_tokens": 14427789.0, + "reward": 0.6875, + "reward_std": 0.3514062464237213, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.3920773267745972, + "sampling/importance_sampling_ratio/mean": 1.0001630783081055, + "sampling/importance_sampling_ratio/min": 0.4027249217033386, + "sampling/sampling_logp_difference/max": 0.9095015525817871, + "sampling/sampling_logp_difference/mean": 0.011563870124518871, + "step": 705 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 8.96057317731902e-05, + "clip_ratio/low_min": 8.96057317731902e-05, + "clip_ratio/region_mean": 8.96057317731902e-05, + "completions/clipped_ratio": 0.0, + "completions/max_length": 615.0, + "completions/max_terminated_length": 615.0, + "completions/mean_length": 338.0625, + "completions/mean_terminated_length": 338.0625, + "completions/min_length": 144.0, + "completions/min_terminated_length": 144.0, + "entropy": 0.3190981522202492, + "epoch": 0.37794432548179874, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.011464577168226242, + "learning_rate": 1e-05, + "loss": -0.024, + "num_tokens": 14442351.0, + "reward": 0.6875, + "reward_std": 0.2925041913986206, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.728134036064148, + "sampling/importance_sampling_ratio/mean": 1.0003695487976074, + "sampling/importance_sampling_ratio/min": 0.6184782385826111, + "sampling/sampling_logp_difference/max": 0.5470422506332397, + "sampling/sampling_logp_difference/mean": 0.010759817436337471, + "step": 706 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00011438337605795823, + "clip_ratio/low_min": 0.00011438337605795823, + "clip_ratio/region_mean": 0.00011438337605795823, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 725.0, + "completions/mean_length": 482.375, + "completions/mean_terminated_length": 452.82757568359375, + "completions/min_length": 276.0, + "completions/min_terminated_length": 276.0, + "entropy": 0.2971290107816458, + "epoch": 0.3784796573875803, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.0201345793902874, + "learning_rate": 1e-05, + "loss": 0.0308, + "num_tokens": 14461387.0, + "reward": 0.71875, + "reward_std": 0.3787454068660736, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.3662681579589844, + "sampling/importance_sampling_ratio/mean": 0.9997867345809937, + "sampling/importance_sampling_ratio/min": 0.5406734943389893, + "sampling/sampling_logp_difference/max": 0.6149396896362305, + "sampling/sampling_logp_difference/mean": 0.010508444160223007, + "step": 707 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 7.977025961736217e-05, + "clip_ratio/low_min": 7.977025961736217e-05, + "clip_ratio/region_mean": 7.977025961736217e-05, + "completions/clipped_ratio": 0.0, + "completions/max_length": 584.0, + "completions/max_terminated_length": 584.0, + "completions/mean_length": 363.0625, + "completions/mean_terminated_length": 363.0625, + "completions/min_length": 177.0, + "completions/min_terminated_length": 177.0, + "entropy": 0.26614492386579514, + "epoch": 0.37901498929336186, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.028473904356360435, + "learning_rate": 1e-05, + "loss": -0.0015, + "num_tokens": 14476405.0, + "reward": 0.90625, + "reward_std": 0.2651650309562683, + "rewards/accuracy_reward/mean": 0.90625, + "rewards/accuracy_reward/std": 0.2961445748806, + "sampling/importance_sampling_ratio/max": 1.7788512706756592, + "sampling/importance_sampling_ratio/mean": 1.0002089738845825, + "sampling/importance_sampling_ratio/min": 0.30601656436920166, + "sampling/sampling_logp_difference/max": 1.1841161251068115, + "sampling/sampling_logp_difference/mean": 0.010216321796178818, + "step": 708 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 673.0, + "completions/max_terminated_length": 673.0, + "completions/mean_length": 346.21875, + "completions/mean_terminated_length": 346.21875, + "completions/min_length": 213.0, + "completions/min_terminated_length": 213.0, + "entropy": 0.37835797667503357, + "epoch": 0.37955032119914345, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.007868225686252117, + "learning_rate": 1e-05, + "loss": -0.0044, + "num_tokens": 14491556.0, + "reward": 0.84375, + "reward_std": 0.3061639964580536, + "rewards/accuracy_reward/mean": 0.84375, + "rewards/accuracy_reward/std": 0.3689020276069641, + "sampling/importance_sampling_ratio/max": 1.4122024774551392, + "sampling/importance_sampling_ratio/mean": 0.9999848008155823, + "sampling/importance_sampling_ratio/min": 0.3929634988307953, + "sampling/sampling_logp_difference/max": 0.9340386390686035, + "sampling/sampling_logp_difference/mean": 0.01364780031144619, + "step": 709 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00013705813034903258, + "clip_ratio/low_min": 0.00013705813034903258, + "clip_ratio/region_mean": 0.00013705813034903258, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 649.0, + "completions/mean_length": 417.03125, + "completions/mean_terminated_length": 405.70965576171875, + "completions/min_length": 208.0, + "completions/min_terminated_length": 208.0, + "entropy": 0.3463163934648037, + "epoch": 0.38008565310492504, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.01991078443825245, + "learning_rate": 1e-05, + "loss": 0.0074, + "num_tokens": 14509581.0, + "reward": 0.65625, + "reward_std": 0.3377465009689331, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.5266883373260498, + "sampling/importance_sampling_ratio/mean": 0.9995376467704773, + "sampling/importance_sampling_ratio/min": 0.4936709403991699, + "sampling/sampling_logp_difference/max": 0.7058861255645752, + "sampling/sampling_logp_difference/mean": 0.013335020281374454, + "step": 710 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00023627230984857306, + "clip_ratio/low_min": 0.00023627230984857306, + "clip_ratio/region_mean": 0.00023627230984857306, + "completions/clipped_ratio": 0.0, + "completions/max_length": 605.0, + "completions/max_terminated_length": 605.0, + "completions/mean_length": 379.28125, + "completions/mean_terminated_length": 379.28125, + "completions/min_length": 230.0, + "completions/min_terminated_length": 230.0, + "entropy": 0.29287885315716267, + "epoch": 0.3806209850107066, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.00403599301353097, + "learning_rate": 1e-05, + "loss": 0.0136, + "num_tokens": 14525126.0, + "reward": 0.84375, + "reward_std": 0.22201895713806152, + "rewards/accuracy_reward/mean": 0.84375, + "rewards/accuracy_reward/std": 0.3689020276069641, + "sampling/importance_sampling_ratio/max": 1.4173802137374878, + "sampling/importance_sampling_ratio/mean": 1.0004808902740479, + "sampling/importance_sampling_ratio/min": 0.7535046935081482, + "sampling/sampling_logp_difference/max": 0.34881019592285156, + "sampling/sampling_logp_difference/mean": 0.010657152161002159, + "step": 711 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.7313158322358504e-05, + "clip_ratio/low_min": 5.7313158322358504e-05, + "clip_ratio/region_mean": 5.7313158322358504e-05, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 664.0, + "completions/mean_length": 449.125, + "completions/mean_terminated_length": 403.5714416503906, + "completions/min_length": 268.0, + "completions/min_terminated_length": 268.0, + "entropy": 0.42011337354779243, + "epoch": 0.3811563169164882, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.02013416960835457, + "learning_rate": 1e-05, + "loss": 0.0405, + "num_tokens": 14543698.0, + "reward": 0.625, + "reward_std": 0.2177756428718567, + "rewards/accuracy_reward/mean": 0.625, + "rewards/accuracy_reward/std": 0.49186936020851135, + "sampling/importance_sampling_ratio/max": 1.5694878101348877, + "sampling/importance_sampling_ratio/mean": 1.000623345375061, + "sampling/importance_sampling_ratio/min": 0.685395359992981, + "sampling/sampling_logp_difference/max": 0.45074939727783203, + "sampling/sampling_logp_difference/mean": 0.013381236232817173, + "step": 712 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0001274974856642075, + "clip_ratio/low_min": 0.0001274974856642075, + "clip_ratio/region_mean": 0.0001274974856642075, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 707.0, + "completions/mean_length": 415.0, + "completions/mean_terminated_length": 364.5714416503906, + "completions/min_length": 196.0, + "completions/min_terminated_length": 196.0, + "entropy": 0.33021325804293156, + "epoch": 0.3816916488222698, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.03616506606340408, + "learning_rate": 1e-05, + "loss": 0.1167, + "num_tokens": 14560242.0, + "reward": 0.59375, + "reward_std": 0.378745436668396, + "rewards/accuracy_reward/mean": 0.59375, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.5868172645568848, + "sampling/importance_sampling_ratio/mean": 1.0000605583190918, + "sampling/importance_sampling_ratio/min": 0.6655277609825134, + "sampling/sampling_logp_difference/max": 0.4617302417755127, + "sampling/sampling_logp_difference/mean": 0.011609706096351147, + "step": 713 + }, + { + "clip_ratio/high_max": 0.000141934011480771, + "clip_ratio/high_mean": 0.000141934011480771, + "clip_ratio/low_mean": 6.433350790757686e-05, + "clip_ratio/low_min": 6.433350790757686e-05, + "clip_ratio/region_mean": 0.00020626751938834786, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 762.0, + "completions/mean_length": 497.1875, + "completions/mean_terminated_length": 479.13336181640625, + "completions/min_length": 213.0, + "completions/min_terminated_length": 213.0, + "entropy": 0.2680256310850382, + "epoch": 0.3822269807280514, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.018306700512766838, + "learning_rate": 1e-05, + "loss": 0.0252, + "num_tokens": 14579952.0, + "reward": 0.75, + "reward_std": 0.4261348247528076, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.8021780252456665, + "sampling/importance_sampling_ratio/mean": 1.0002920627593994, + "sampling/importance_sampling_ratio/min": 0.6692163944244385, + "sampling/sampling_logp_difference/max": 0.5889959335327148, + "sampling/sampling_logp_difference/mean": 0.010151496157050133, + "step": 714 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00035480190490488894, + "clip_ratio/low_min": 0.00035480190490488894, + "clip_ratio/region_mean": 0.00035480190490488894, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 768.0, + "completions/mean_length": 491.53125, + "completions/mean_terminated_length": 427.73077392578125, + "completions/min_length": 179.0, + "completions/min_terminated_length": 179.0, + "entropy": 0.4604201167821884, + "epoch": 0.382762312633833, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.020163601264357567, + "learning_rate": 1e-05, + "loss": 0.0457, + "num_tokens": 14599729.0, + "reward": 0.59375, + "reward_std": 0.2041158676147461, + "rewards/accuracy_reward/mean": 0.59375, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 2.0, + "sampling/importance_sampling_ratio/mean": 1.0004055500030518, + "sampling/importance_sampling_ratio/min": 0.6129925847053528, + "sampling/sampling_logp_difference/max": 0.7964284420013428, + "sampling/sampling_logp_difference/mean": 0.013950590044260025, + "step": 715 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 766.0, + "completions/max_terminated_length": 766.0, + "completions/mean_length": 446.25, + "completions/mean_terminated_length": 446.25, + "completions/min_length": 258.0, + "completions/min_terminated_length": 258.0, + "entropy": 0.2816047314554453, + "epoch": 0.38329764453961457, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.016802972182631493, + "learning_rate": 1e-05, + "loss": 0.0159, + "num_tokens": 14617649.0, + "reward": 0.875, + "reward_std": 0.13363061845302582, + "rewards/accuracy_reward/mean": 0.875, + "rewards/accuracy_reward/std": 0.33601075410842896, + "sampling/importance_sampling_ratio/max": 1.6031620502471924, + "sampling/importance_sampling_ratio/mean": 0.9994713068008423, + "sampling/importance_sampling_ratio/min": 0.5475712418556213, + "sampling/sampling_logp_difference/max": 0.6022627353668213, + "sampling/sampling_logp_difference/mean": 0.01150565966963768, + "step": 716 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 755.0, + "completions/mean_length": 473.0625, + "completions/mean_terminated_length": 442.5517272949219, + "completions/min_length": 197.0, + "completions/min_terminated_length": 197.0, + "entropy": 0.32550000585615635, + "epoch": 0.38383297644539616, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.018743062391877174, + "learning_rate": 1e-05, + "loss": 0.0594, + "num_tokens": 14636843.0, + "reward": 0.84375, + "reward_std": 0.3061639666557312, + "rewards/accuracy_reward/mean": 0.84375, + "rewards/accuracy_reward/std": 0.3689020276069641, + "sampling/importance_sampling_ratio/max": 1.4701733589172363, + "sampling/importance_sampling_ratio/mean": 0.9999250173568726, + "sampling/importance_sampling_ratio/min": 0.6543622612953186, + "sampling/sampling_logp_difference/max": 0.42409420013427734, + "sampling/sampling_logp_difference/mean": 0.011421573348343372, + "step": 717 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00010977495912811719, + "clip_ratio/low_min": 0.00010977495912811719, + "clip_ratio/region_mean": 0.00010977495912811719, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 740.0, + "completions/mean_length": 517.25, + "completions/mean_terminated_length": 481.4285888671875, + "completions/min_length": 348.0, + "completions/min_terminated_length": 348.0, + "entropy": 0.2791203670203686, + "epoch": 0.38436830835117775, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.016800623387098312, + "learning_rate": 1e-05, + "loss": -0.0087, + "num_tokens": 14657019.0, + "reward": 0.71875, + "reward_std": 0.3808925747871399, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.4731851816177368, + "sampling/importance_sampling_ratio/mean": 1.000221848487854, + "sampling/importance_sampling_ratio/min": 0.6232567429542542, + "sampling/sampling_logp_difference/max": 0.4727966785430908, + "sampling/sampling_logp_difference/mean": 0.009981920011341572, + "step": 718 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 724.0, + "completions/max_terminated_length": 724.0, + "completions/mean_length": 447.59375, + "completions/mean_terminated_length": 447.59375, + "completions/min_length": 194.0, + "completions/min_terminated_length": 194.0, + "entropy": 0.3053096607327461, + "epoch": 0.38490364025695933, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.016528841108083725, + "learning_rate": 1e-05, + "loss": 0.0071, + "num_tokens": 14675134.0, + "reward": 0.75, + "reward_std": 0.2587745785713196, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.2990198135375977, + "sampling/importance_sampling_ratio/mean": 0.9999493956565857, + "sampling/importance_sampling_ratio/min": 0.6751003861427307, + "sampling/sampling_logp_difference/max": 0.39289385080337524, + "sampling/sampling_logp_difference/mean": 0.011058874428272247, + "step": 719 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 582.0, + "completions/max_terminated_length": 582.0, + "completions/mean_length": 375.625, + "completions/mean_terminated_length": 375.625, + "completions/min_length": 44.0, + "completions/min_terminated_length": 44.0, + "entropy": 0.28763924166560173, + "epoch": 0.3854389721627409, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.007499333005398512, + "learning_rate": 1e-05, + "loss": 0.0178, + "num_tokens": 14690354.0, + "reward": 0.75, + "reward_std": 0.3335031569004059, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.4215821027755737, + "sampling/importance_sampling_ratio/mean": 1.000325083732605, + "sampling/importance_sampling_ratio/min": 0.6132378578186035, + "sampling/sampling_logp_difference/max": 0.4890024662017822, + "sampling/sampling_logp_difference/mean": 0.010743832215666771, + "step": 720 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 542.0, + "completions/mean_length": 366.25, + "completions/mean_terminated_length": 353.2903137207031, + "completions/min_length": 180.0, + "completions/min_terminated_length": 180.0, + "entropy": 0.3741515651345253, + "epoch": 0.3859743040685225, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.054097432643175125, + "learning_rate": 1e-05, + "loss": 0.0758, + "num_tokens": 14705674.0, + "reward": 0.875, + "reward_std": 0.2177756428718567, + "rewards/accuracy_reward/mean": 0.875, + "rewards/accuracy_reward/std": 0.33601075410842896, + "sampling/importance_sampling_ratio/max": 2.0, + "sampling/importance_sampling_ratio/mean": 0.9993348121643066, + "sampling/importance_sampling_ratio/min": 0.6482955813407898, + "sampling/sampling_logp_difference/max": 0.7959752082824707, + "sampling/sampling_logp_difference/mean": 0.012856943532824516, + "step": 721 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 7.876496238168329e-05, + "clip_ratio/low_min": 7.876496238168329e-05, + "clip_ratio/region_mean": 7.876496238168329e-05, + "completions/clipped_ratio": 0.0, + "completions/max_length": 640.0, + "completions/max_terminated_length": 640.0, + "completions/mean_length": 384.59375, + "completions/mean_terminated_length": 384.59375, + "completions/min_length": 240.0, + "completions/min_terminated_length": 240.0, + "entropy": 0.3284129276871681, + "epoch": 0.38650963597430404, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.04252352938055992, + "learning_rate": 1e-05, + "loss": 0.0663, + "num_tokens": 14721941.0, + "reward": 0.65625, + "reward_std": 0.2041158676147461, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.455653429031372, + "sampling/importance_sampling_ratio/mean": 1.000124454498291, + "sampling/importance_sampling_ratio/min": 0.7095494270324707, + "sampling/sampling_logp_difference/max": 0.3754549026489258, + "sampling/sampling_logp_difference/mean": 0.01177291665226221, + "step": 722 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 718.0, + "completions/max_terminated_length": 718.0, + "completions/mean_length": 446.90625, + "completions/mean_terminated_length": 446.90625, + "completions/min_length": 182.0, + "completions/min_terminated_length": 182.0, + "entropy": 0.33111138828098774, + "epoch": 0.38704496788008563, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.03499797731637955, + "learning_rate": 1e-05, + "loss": 0.0411, + "num_tokens": 14740330.0, + "reward": 0.8125, + "reward_std": 0.3335031569004059, + "rewards/accuracy_reward/mean": 0.8125, + "rewards/accuracy_reward/std": 0.3965577781200409, + "sampling/importance_sampling_ratio/max": 1.595922589302063, + "sampling/importance_sampling_ratio/mean": 0.9998600482940674, + "sampling/importance_sampling_ratio/min": 0.6625149250030518, + "sampling/sampling_logp_difference/max": 0.4674520492553711, + "sampling/sampling_logp_difference/mean": 0.011430527083575726, + "step": 723 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 8.37801635498181e-05, + "clip_ratio/low_min": 8.37801635498181e-05, + "clip_ratio/region_mean": 8.37801635498181e-05, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 693.0, + "completions/mean_length": 456.625, + "completions/mean_terminated_length": 435.86669921875, + "completions/min_length": 226.0, + "completions/min_terminated_length": 226.0, + "entropy": 0.3971633203327656, + "epoch": 0.3875802997858672, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.024260831996798515, + "learning_rate": 1e-05, + "loss": 0.0843, + "num_tokens": 14758574.0, + "reward": 0.59375, + "reward_std": 0.3987956643104553, + "rewards/accuracy_reward/mean": 0.59375, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 2.0, + "sampling/importance_sampling_ratio/mean": 1.0007160902023315, + "sampling/importance_sampling_ratio/min": 0.6169441938400269, + "sampling/sampling_logp_difference/max": 0.7076516151428223, + "sampling/sampling_logp_difference/mean": 0.013493540696799755, + "step": 724 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 745.0, + "completions/max_terminated_length": 745.0, + "completions/mean_length": 378.5, + "completions/mean_terminated_length": 378.5, + "completions/min_length": 201.0, + "completions/min_terminated_length": 201.0, + "entropy": 0.31588367745280266, + "epoch": 0.3881156316916488, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.0039367713034152985, + "learning_rate": 1e-05, + "loss": 0.016, + "num_tokens": 14774078.0, + "reward": 0.96875, + "reward_std": 0.0883883461356163, + "rewards/accuracy_reward/mean": 0.96875, + "rewards/accuracy_reward/std": 0.1767766922712326, + "sampling/importance_sampling_ratio/max": 1.409155249595642, + "sampling/importance_sampling_ratio/mean": 0.999642550945282, + "sampling/importance_sampling_ratio/min": 0.5600169897079468, + "sampling/sampling_logp_difference/max": 0.5797882080078125, + "sampling/sampling_logp_difference/mean": 0.01088521908968687, + "step": 725 + }, + { + "clip_ratio/high_max": 7.413997809635475e-05, + "clip_ratio/high_mean": 7.413997809635475e-05, + "clip_ratio/low_mean": 5.592841262114234e-05, + "clip_ratio/low_min": 5.592841262114234e-05, + "clip_ratio/region_mean": 0.0001300683907174971, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 642.0, + "completions/mean_length": 407.03125, + "completions/mean_terminated_length": 355.46429443359375, + "completions/min_length": 177.0, + "completions/min_terminated_length": 177.0, + "entropy": 0.3510789033025503, + "epoch": 0.3886509635974304, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.026060735806822777, + "learning_rate": 1e-05, + "loss": 0.1434, + "num_tokens": 14790591.0, + "reward": 0.78125, + "reward_std": 0.3608423173427582, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.9262347221374512, + "sampling/importance_sampling_ratio/mean": 0.9992910027503967, + "sampling/importance_sampling_ratio/min": 0.5453320145606995, + "sampling/sampling_logp_difference/max": 0.6555671691894531, + "sampling/sampling_logp_difference/mean": 0.01253750640898943, + "step": 726 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 9.335325012216344e-05, + "clip_ratio/low_min": 9.335325012216344e-05, + "clip_ratio/region_mean": 9.335325012216344e-05, + "completions/clipped_ratio": 0.0, + "completions/max_length": 537.0, + "completions/max_terminated_length": 537.0, + "completions/mean_length": 336.75, + "completions/mean_terminated_length": 336.75, + "completions/min_length": 123.0, + "completions/min_terminated_length": 123.0, + "entropy": 0.3309592194855213, + "epoch": 0.389186295503212, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.03679654002189636, + "learning_rate": 1e-05, + "loss": -0.0065, + "num_tokens": 14805127.0, + "reward": 0.625, + "reward_std": 0.2177756428718567, + "rewards/accuracy_reward/mean": 0.625, + "rewards/accuracy_reward/std": 0.49186936020851135, + "sampling/importance_sampling_ratio/max": 2.0, + "sampling/importance_sampling_ratio/mean": 0.9996213912963867, + "sampling/importance_sampling_ratio/min": 0.7002161741256714, + "sampling/sampling_logp_difference/max": 0.8192439079284668, + "sampling/sampling_logp_difference/mean": 0.01202657725661993, + "step": 727 + }, + { + "clip_ratio/high_max": 7.598783849971369e-05, + "clip_ratio/high_mean": 7.598783849971369e-05, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 7.598783849971369e-05, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 646.0, + "completions/mean_length": 479.09375, + "completions/mean_terminated_length": 449.2069091796875, + "completions/min_length": 245.0, + "completions/min_terminated_length": 245.0, + "entropy": 0.29027774184942245, + "epoch": 0.3897216274089936, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.01727694272994995, + "learning_rate": 1e-05, + "loss": 0.0732, + "num_tokens": 14824298.0, + "reward": 0.8125, + "reward_std": 0.2587745785713196, + "rewards/accuracy_reward/mean": 0.8125, + "rewards/accuracy_reward/std": 0.3965577781200409, + "sampling/importance_sampling_ratio/max": 1.7989857196807861, + "sampling/importance_sampling_ratio/mean": 1.0000174045562744, + "sampling/importance_sampling_ratio/min": 0.56607586145401, + "sampling/sampling_logp_difference/max": 0.5872230529785156, + "sampling/sampling_logp_difference/mean": 0.010853583924472332, + "step": 728 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 735.0, + "completions/mean_length": 506.6875, + "completions/mean_terminated_length": 479.6551818847656, + "completions/min_length": 311.0, + "completions/min_terminated_length": 311.0, + "entropy": 0.418092992156744, + "epoch": 0.39025695931477516, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.03450678288936615, + "learning_rate": 1e-05, + "loss": 0.0174, + "num_tokens": 14844528.0, + "reward": 0.40625, + "reward_std": 0.4628904461860657, + "rewards/accuracy_reward/mean": 0.40625, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.4291409254074097, + "sampling/importance_sampling_ratio/mean": 1.0002530813217163, + "sampling/importance_sampling_ratio/min": 0.4035452604293823, + "sampling/sampling_logp_difference/max": 0.9074666500091553, + "sampling/sampling_logp_difference/mean": 0.01486382819712162, + "step": 729 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 540.0, + "completions/mean_length": 368.375, + "completions/mean_terminated_length": 355.4838562011719, + "completions/min_length": 174.0, + "completions/min_terminated_length": 174.0, + "entropy": 0.268840491771698, + "epoch": 0.39079229122055675, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.015092047862708569, + "learning_rate": 1e-05, + "loss": 0.0295, + "num_tokens": 14860092.0, + "reward": 0.90625, + "reward_std": 0.1293872892856598, + "rewards/accuracy_reward/mean": 0.90625, + "rewards/accuracy_reward/std": 0.2961445748806, + "sampling/importance_sampling_ratio/max": 1.7625010013580322, + "sampling/importance_sampling_ratio/mean": 1.0003862380981445, + "sampling/importance_sampling_ratio/min": 0.5371314883232117, + "sampling/sampling_logp_difference/max": 0.6215124130249023, + "sampling/sampling_logp_difference/mean": 0.01023824792355299, + "step": 730 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0001708475756458938, + "clip_ratio/low_min": 0.0001708475756458938, + "clip_ratio/region_mean": 0.0001708475756458938, + "completions/clipped_ratio": 0.0, + "completions/max_length": 676.0, + "completions/max_terminated_length": 676.0, + "completions/mean_length": 406.59375, + "completions/mean_terminated_length": 406.59375, + "completions/min_length": 161.0, + "completions/min_terminated_length": 161.0, + "entropy": 0.305848591029644, + "epoch": 0.39132762312633834, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.010290650650858879, + "learning_rate": 1e-05, + "loss": -0.0165, + "num_tokens": 14876815.0, + "reward": 0.84375, + "reward_std": 0.24511480331420898, + "rewards/accuracy_reward/mean": 0.84375, + "rewards/accuracy_reward/std": 0.3689020276069641, + "sampling/importance_sampling_ratio/max": 1.9217058420181274, + "sampling/importance_sampling_ratio/mean": 1.0004491806030273, + "sampling/importance_sampling_ratio/min": 0.6322147846221924, + "sampling/sampling_logp_difference/max": 0.6532132625579834, + "sampling/sampling_logp_difference/mean": 0.012023229151964188, + "step": 731 + }, + { + "clip_ratio/high_max": 7.246376480907202e-05, + "clip_ratio/high_mean": 7.246376480907202e-05, + "clip_ratio/low_mean": 0.000138184055685997, + "clip_ratio/low_min": 0.000138184055685997, + "clip_ratio/region_mean": 0.00021064782049506903, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 567.0, + "completions/mean_length": 426.53125, + "completions/mean_terminated_length": 377.7500305175781, + "completions/min_length": 225.0, + "completions/min_terminated_length": 225.0, + "entropy": 0.3874700292944908, + "epoch": 0.39186295503211993, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.030863050371408463, + "learning_rate": 1e-05, + "loss": 0.1072, + "num_tokens": 14894096.0, + "reward": 0.53125, + "reward_std": 0.3471629321575165, + "rewards/accuracy_reward/mean": 0.53125, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.5078504085540771, + "sampling/importance_sampling_ratio/mean": 0.9998968243598938, + "sampling/importance_sampling_ratio/min": 0.5072235465049744, + "sampling/sampling_logp_difference/max": 0.6788034439086914, + "sampling/sampling_logp_difference/mean": 0.012878618203103542, + "step": 732 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 659.0, + "completions/mean_length": 415.5, + "completions/mean_terminated_length": 392.0000305175781, + "completions/min_length": 234.0, + "completions/min_terminated_length": 234.0, + "entropy": 0.406641460955143, + "epoch": 0.3923982869379015, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.01745034195482731, + "learning_rate": 1e-05, + "loss": 0.0481, + "num_tokens": 14911208.0, + "reward": 0.5, + "reward_std": 0.3514062464237213, + "rewards/accuracy_reward/mean": 0.5, + "rewards/accuracy_reward/std": 0.5080004930496216, + "sampling/importance_sampling_ratio/max": 1.880895972251892, + "sampling/importance_sampling_ratio/mean": 1.0001474618911743, + "sampling/importance_sampling_ratio/min": 0.4862619638442993, + "sampling/sampling_logp_difference/max": 0.7210078239440918, + "sampling/sampling_logp_difference/mean": 0.01395708043128252, + "step": 733 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 572.0, + "completions/mean_length": 393.875, + "completions/mean_terminated_length": 381.8064270019531, + "completions/min_length": 213.0, + "completions/min_terminated_length": 213.0, + "entropy": 0.30655495449900627, + "epoch": 0.3929336188436831, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": -0.0009, + "num_tokens": 14927036.0, + "reward": 0.78125, + "reward_std": 0.0883883461356163, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 2.0, + "sampling/importance_sampling_ratio/mean": 1.0001674890518188, + "sampling/importance_sampling_ratio/min": 0.5404631495475769, + "sampling/sampling_logp_difference/max": 0.9599714279174805, + "sampling/sampling_logp_difference/mean": 0.01090756431221962, + "step": 734 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 8.256275032181293e-05, + "clip_ratio/low_min": 8.256275032181293e-05, + "clip_ratio/region_mean": 8.256275032181293e-05, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 691.0, + "completions/mean_length": 480.03125, + "completions/mean_terminated_length": 438.89288330078125, + "completions/min_length": 196.0, + "completions/min_terminated_length": 196.0, + "entropy": 0.32466258853673935, + "epoch": 0.39346895074946464, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.004461785312741995, + "learning_rate": 1e-05, + "loss": 0.0427, + "num_tokens": 14946549.0, + "reward": 0.59375, + "reward_std": 0.22201895713806152, + "rewards/accuracy_reward/mean": 0.59375, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.7895437479019165, + "sampling/importance_sampling_ratio/mean": 1.000295877456665, + "sampling/importance_sampling_ratio/min": 0.5543748140335083, + "sampling/sampling_logp_difference/max": 0.5899143218994141, + "sampling/sampling_logp_difference/mean": 0.011947096325457096, + "step": 735 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00014100589760346338, + "clip_ratio/low_min": 0.00014100589760346338, + "clip_ratio/region_mean": 0.00014100589760346338, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 646.0, + "completions/mean_length": 438.15625, + "completions/mean_terminated_length": 427.51611328125, + "completions/min_length": 193.0, + "completions/min_terminated_length": 193.0, + "entropy": 0.2634274195879698, + "epoch": 0.39400428265524623, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.017593378201127052, + "learning_rate": 1e-05, + "loss": 0.0109, + "num_tokens": 14964250.0, + "reward": 0.53125, + "reward_std": 0.47137707471847534, + "rewards/accuracy_reward/mean": 0.53125, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.4653797149658203, + "sampling/importance_sampling_ratio/mean": 0.9998362064361572, + "sampling/importance_sampling_ratio/min": 0.6434476375579834, + "sampling/sampling_logp_difference/max": 0.4409146308898926, + "sampling/sampling_logp_difference/mean": 0.010042332112789154, + "step": 736 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 618.0, + "completions/max_terminated_length": 618.0, + "completions/mean_length": 379.1875, + "completions/mean_terminated_length": 379.1875, + "completions/min_length": 195.0, + "completions/min_terminated_length": 195.0, + "entropy": 0.31994888186454773, + "epoch": 0.3945396145610278, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.008601024746894836, + "learning_rate": 1e-05, + "loss": 0.0103, + "num_tokens": 14979848.0, + "reward": 0.90625, + "reward_std": 0.2041158676147461, + "rewards/accuracy_reward/mean": 0.90625, + "rewards/accuracy_reward/std": 0.2961445748806, + "sampling/importance_sampling_ratio/max": 1.6797938346862793, + "sampling/importance_sampling_ratio/mean": 1.0005078315734863, + "sampling/importance_sampling_ratio/min": 0.617961585521698, + "sampling/sampling_logp_difference/max": 0.5186710357666016, + "sampling/sampling_logp_difference/mean": 0.011722034774720669, + "step": 737 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 524.0, + "completions/max_terminated_length": 524.0, + "completions/mean_length": 369.28125, + "completions/mean_terminated_length": 369.28125, + "completions/min_length": 183.0, + "completions/min_terminated_length": 183.0, + "entropy": 0.26390823535621166, + "epoch": 0.3950749464668094, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": -0.006, + "num_tokens": 14995105.0, + "reward": 0.78125, + "reward_std": 0.24511480331420898, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.510011076927185, + "sampling/importance_sampling_ratio/mean": 1.0000381469726562, + "sampling/importance_sampling_ratio/min": 0.5908969044685364, + "sampling/sampling_logp_difference/max": 0.5261136293411255, + "sampling/sampling_logp_difference/mean": 0.010361729189753532, + "step": 738 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 565.0, + "completions/max_terminated_length": 565.0, + "completions/mean_length": 337.65625, + "completions/mean_terminated_length": 337.65625, + "completions/min_length": 161.0, + "completions/min_terminated_length": 161.0, + "entropy": 0.3128814436495304, + "epoch": 0.395610278372591, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.009559151716530323, + "learning_rate": 1e-05, + "loss": 0.0669, + "num_tokens": 15009862.0, + "reward": 0.71875, + "reward_std": 0.2041158676147461, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.8868142366409302, + "sampling/importance_sampling_ratio/mean": 1.0002555847167969, + "sampling/importance_sampling_ratio/min": 0.5747727751731873, + "sampling/sampling_logp_difference/max": 0.6348898410797119, + "sampling/sampling_logp_difference/mean": 0.01171938981860876, + "step": 739 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 686.0, + "completions/mean_length": 440.875, + "completions/mean_terminated_length": 419.0666809082031, + "completions/min_length": 225.0, + "completions/min_terminated_length": 225.0, + "entropy": 0.31444643810391426, + "epoch": 0.3961456102783726, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.021820830181241035, + "learning_rate": 1e-05, + "loss": 0.0243, + "num_tokens": 15027850.0, + "reward": 0.625, + "reward_std": 0.3335031569004059, + "rewards/accuracy_reward/mean": 0.625, + "rewards/accuracy_reward/std": 0.49186936020851135, + "sampling/importance_sampling_ratio/max": 1.4483706951141357, + "sampling/importance_sampling_ratio/mean": 0.999547004699707, + "sampling/importance_sampling_ratio/min": 0.33352625370025635, + "sampling/sampling_logp_difference/max": 1.0980336666107178, + "sampling/sampling_logp_difference/mean": 0.011443336494266987, + "step": 740 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00018385963267064653, + "clip_ratio/low_min": 0.00018385963267064653, + "clip_ratio/region_mean": 0.00018385963267064653, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 679.0, + "completions/mean_length": 470.875, + "completions/mean_terminated_length": 451.0666809082031, + "completions/min_length": 229.0, + "completions/min_terminated_length": 229.0, + "entropy": 0.3229607306420803, + "epoch": 0.3966809421841542, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.034369517117738724, + "learning_rate": 1e-05, + "loss": 0.0112, + "num_tokens": 15047286.0, + "reward": 0.75, + "reward_std": 0.1767766922712326, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.4677540063858032, + "sampling/importance_sampling_ratio/mean": 1.0001025199890137, + "sampling/importance_sampling_ratio/min": 0.43193694949150085, + "sampling/sampling_logp_difference/max": 0.8394756317138672, + "sampling/sampling_logp_difference/mean": 0.012847675010561943, + "step": 741 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 717.0, + "completions/max_terminated_length": 717.0, + "completions/mean_length": 410.4375, + "completions/mean_terminated_length": 410.4375, + "completions/min_length": 106.0, + "completions/min_terminated_length": 106.0, + "entropy": 0.2223656103014946, + "epoch": 0.39721627408993576, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.00397397531196475, + "learning_rate": 1e-05, + "loss": -0.0189, + "num_tokens": 15063660.0, + "reward": 0.9375, + "reward_std": 0.1767766922712326, + "rewards/accuracy_reward/mean": 0.9375, + "rewards/accuracy_reward/std": 0.24593468010425568, + "sampling/importance_sampling_ratio/max": 1.66285240650177, + "sampling/importance_sampling_ratio/mean": 0.999877393245697, + "sampling/importance_sampling_ratio/min": 0.6022452116012573, + "sampling/sampling_logp_difference/max": 0.5085344314575195, + "sampling/sampling_logp_difference/mean": 0.008522205986082554, + "step": 742 + }, + { + "clip_ratio/high_max": 6.742178811691701e-05, + "clip_ratio/high_mean": 6.742178811691701e-05, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 6.742178811691701e-05, + "completions/clipped_ratio": 0.0, + "completions/max_length": 578.0, + "completions/max_terminated_length": 578.0, + "completions/mean_length": 378.65625, + "completions/mean_terminated_length": 378.65625, + "completions/min_length": 143.0, + "completions/min_terminated_length": 143.0, + "entropy": 0.27840827964246273, + "epoch": 0.39775160599571735, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.02370659075677395, + "learning_rate": 1e-05, + "loss": -0.0534, + "num_tokens": 15079385.0, + "reward": 0.75, + "reward_std": 0.3514062464237213, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.4989700317382812, + "sampling/importance_sampling_ratio/mean": 0.9998046159744263, + "sampling/importance_sampling_ratio/min": 0.4175216853618622, + "sampling/sampling_logp_difference/max": 0.8734188079833984, + "sampling/sampling_logp_difference/mean": 0.010278995148837566, + "step": 743 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 757.0, + "completions/max_terminated_length": 757.0, + "completions/mean_length": 387.59375, + "completions/mean_terminated_length": 387.59375, + "completions/min_length": 209.0, + "completions/min_terminated_length": 209.0, + "entropy": 0.2853036727756262, + "epoch": 0.39828693790149894, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.036839559674263, + "learning_rate": 1e-05, + "loss": -0.0005, + "num_tokens": 15095156.0, + "reward": 0.96875, + "reward_std": 0.0883883461356163, + "rewards/accuracy_reward/mean": 0.96875, + "rewards/accuracy_reward/std": 0.1767766922712326, + "sampling/importance_sampling_ratio/max": 1.6023297309875488, + "sampling/importance_sampling_ratio/mean": 0.999885082244873, + "sampling/importance_sampling_ratio/min": 0.5930861234664917, + "sampling/sampling_logp_difference/max": 0.5224156379699707, + "sampling/sampling_logp_difference/mean": 0.010929422453045845, + "step": 744 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00016292092186631635, + "clip_ratio/low_min": 0.00016292092186631635, + "clip_ratio/region_mean": 0.00016292092186631635, + "completions/clipped_ratio": 0.0, + "completions/max_length": 539.0, + "completions/max_terminated_length": 539.0, + "completions/mean_length": 363.09375, + "completions/mean_terminated_length": 363.09375, + "completions/min_length": 208.0, + "completions/min_terminated_length": 208.0, + "entropy": 0.26418436504900455, + "epoch": 0.3988222698072805, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.035045281052589417, + "learning_rate": 1e-05, + "loss": 0.0432, + "num_tokens": 15110391.0, + "reward": 0.8125, + "reward_std": 0.3104073107242584, + "rewards/accuracy_reward/mean": 0.8125, + "rewards/accuracy_reward/std": 0.3965577781200409, + "sampling/importance_sampling_ratio/max": 1.4381076097488403, + "sampling/importance_sampling_ratio/mean": 0.9996638298034668, + "sampling/importance_sampling_ratio/min": 0.6747323870658875, + "sampling/sampling_logp_difference/max": 0.3934391736984253, + "sampling/sampling_logp_difference/mean": 0.010694892145693302, + "step": 745 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 468.0, + "completions/max_terminated_length": 468.0, + "completions/mean_length": 323.21875, + "completions/mean_terminated_length": 323.21875, + "completions/min_length": 193.0, + "completions/min_terminated_length": 193.0, + "entropy": 0.31976547837257385, + "epoch": 0.3993576017130621, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.028641268610954285, + "learning_rate": 1e-05, + "loss": 0.0052, + "num_tokens": 15124222.0, + "reward": 0.84375, + "reward_std": 0.3198433816432953, + "rewards/accuracy_reward/mean": 0.84375, + "rewards/accuracy_reward/std": 0.3689020276069641, + "sampling/importance_sampling_ratio/max": 2.0, + "sampling/importance_sampling_ratio/mean": 1.0001282691955566, + "sampling/importance_sampling_ratio/min": 0.601650595664978, + "sampling/sampling_logp_difference/max": 0.8825466632843018, + "sampling/sampling_logp_difference/mean": 0.012408490292727947, + "step": 746 + }, + { + "clip_ratio/high_max": 0.00011574073869269341, + "clip_ratio/high_mean": 0.00011574073869269341, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.00011574073869269341, + "completions/clipped_ratio": 0.0, + "completions/max_length": 490.0, + "completions/max_terminated_length": 490.0, + "completions/mean_length": 346.59375, + "completions/mean_terminated_length": 346.59375, + "completions/min_length": 162.0, + "completions/min_terminated_length": 162.0, + "entropy": 0.38410582952201366, + "epoch": 0.3998929336188437, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.05293239653110504, + "learning_rate": 1e-05, + "loss": -0.0311, + "num_tokens": 15139073.0, + "reward": 0.875, + "reward_std": 0.292504221200943, + "rewards/accuracy_reward/mean": 0.875, + "rewards/accuracy_reward/std": 0.33601075410842896, + "sampling/importance_sampling_ratio/max": 1.3349459171295166, + "sampling/importance_sampling_ratio/mean": 0.9997342228889465, + "sampling/importance_sampling_ratio/min": 0.6193897128105164, + "sampling/sampling_logp_difference/max": 0.4790205955505371, + "sampling/sampling_logp_difference/mean": 0.011812268756330013, + "step": 747 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00016867216618265957, + "clip_ratio/low_min": 0.00016867216618265957, + "clip_ratio/region_mean": 0.00016867216618265957, + "completions/clipped_ratio": 0.0, + "completions/max_length": 619.0, + "completions/max_terminated_length": 619.0, + "completions/mean_length": 384.25, + "completions/mean_terminated_length": 384.25, + "completions/min_length": 256.0, + "completions/min_terminated_length": 256.0, + "entropy": 0.2777410428971052, + "epoch": 0.4004282655246253, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.03175799176096916, + "learning_rate": 1e-05, + "loss": 0.017, + "num_tokens": 15154881.0, + "reward": 0.6875, + "reward_std": 0.2177756428718567, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.5922999382019043, + "sampling/importance_sampling_ratio/mean": 0.9996979236602783, + "sampling/importance_sampling_ratio/min": 0.6333731412887573, + "sampling/sampling_logp_difference/max": 0.465179443359375, + "sampling/sampling_logp_difference/mean": 0.010911596938967705, + "step": 748 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 9.938473522197455e-05, + "clip_ratio/low_min": 9.938473522197455e-05, + "clip_ratio/region_mean": 9.938473522197455e-05, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 742.0, + "completions/mean_length": 500.90625, + "completions/mean_terminated_length": 462.7500305175781, + "completions/min_length": 195.0, + "completions/min_terminated_length": 195.0, + "entropy": 0.39744829572737217, + "epoch": 0.4009635974304068, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.016104180365800858, + "learning_rate": 1e-05, + "loss": 0.0094, + "num_tokens": 15175198.0, + "reward": 0.59375, + "reward_std": 0.3608423173427582, + "rewards/accuracy_reward/mean": 0.59375, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.7563401460647583, + "sampling/importance_sampling_ratio/mean": 1.0001014471054077, + "sampling/importance_sampling_ratio/min": 0.32650068402290344, + "sampling/sampling_logp_difference/max": 1.1193232536315918, + "sampling/sampling_logp_difference/mean": 0.013287602923810482, + "step": 749 + }, + { + "clip_ratio/high_max": 0.00015699722280260175, + "clip_ratio/high_mean": 0.00015699722280260175, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.00015699722280260175, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 630.0, + "completions/mean_length": 432.28125, + "completions/mean_terminated_length": 409.9000244140625, + "completions/min_length": 216.0, + "completions/min_terminated_length": 216.0, + "entropy": 0.25118602253496647, + "epoch": 0.4014989293361884, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.00824292004108429, + "learning_rate": 1e-05, + "loss": 0.0091, + "num_tokens": 15192655.0, + "reward": 0.6875, + "reward_std": 0.4082317352294922, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 2.0, + "sampling/importance_sampling_ratio/mean": 1.0002760887145996, + "sampling/importance_sampling_ratio/min": 0.7011640667915344, + "sampling/sampling_logp_difference/max": 0.707486629486084, + "sampling/sampling_logp_difference/mean": 0.0095582390204072, + "step": 750 + }, + { + "clip_ratio/high_max": 7.911392458481714e-05, + "clip_ratio/high_mean": 7.911392458481714e-05, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 7.911392458481714e-05, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 654.0, + "completions/mean_length": 436.78125, + "completions/mean_terminated_length": 389.46429443359375, + "completions/min_length": 236.0, + "completions/min_terminated_length": 236.0, + "entropy": 0.3411795999854803, + "epoch": 0.40203426124197, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.037053268402814865, + "learning_rate": 1e-05, + "loss": 0.1044, + "num_tokens": 15210064.0, + "reward": 0.71875, + "reward_std": 0.2041158676147461, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.9501639604568481, + "sampling/importance_sampling_ratio/mean": 0.9999677538871765, + "sampling/importance_sampling_ratio/min": 0.6075533032417297, + "sampling/sampling_logp_difference/max": 0.6679134368896484, + "sampling/sampling_logp_difference/mean": 0.013313813135027885, + "step": 751 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 740.0, + "completions/max_terminated_length": 740.0, + "completions/mean_length": 421.78125, + "completions/mean_terminated_length": 421.78125, + "completions/min_length": 274.0, + "completions/min_terminated_length": 274.0, + "entropy": 0.3252240139991045, + "epoch": 0.4025695931477516, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.017057444900274277, + "learning_rate": 1e-05, + "loss": -0.0399, + "num_tokens": 15227369.0, + "reward": 0.53125, + "reward_std": 0.2041158676147461, + "rewards/accuracy_reward/mean": 0.53125, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 2.0, + "sampling/importance_sampling_ratio/mean": 0.9996662735939026, + "sampling/importance_sampling_ratio/min": 0.22936411201953888, + "sampling/sampling_logp_difference/max": 1.4724445343017578, + "sampling/sampling_logp_difference/mean": 0.01190988253802061, + "step": 752 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 705.0, + "completions/max_terminated_length": 705.0, + "completions/mean_length": 394.625, + "completions/mean_terminated_length": 394.625, + "completions/min_length": 246.0, + "completions/min_terminated_length": 246.0, + "entropy": 0.28018102422356606, + "epoch": 0.4031049250535332, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.008426283486187458, + "learning_rate": 1e-05, + "loss": -0.0432, + "num_tokens": 15243709.0, + "reward": 0.84375, + "reward_std": 0.24511480331420898, + "rewards/accuracy_reward/mean": 0.84375, + "rewards/accuracy_reward/std": 0.3689020276069641, + "sampling/importance_sampling_ratio/max": 1.3750056028366089, + "sampling/importance_sampling_ratio/mean": 0.9998921751976013, + "sampling/importance_sampling_ratio/min": 0.5485638976097107, + "sampling/sampling_logp_difference/max": 0.6004514694213867, + "sampling/sampling_logp_difference/mean": 0.010837670415639877, + "step": 753 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 766.0, + "completions/mean_length": 509.65625, + "completions/mean_terminated_length": 461.8148193359375, + "completions/min_length": 239.0, + "completions/min_terminated_length": 239.0, + "entropy": 0.39862199127674103, + "epoch": 0.40364025695931477, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.018544510006904602, + "learning_rate": 1e-05, + "loss": 0.0106, + "num_tokens": 15264162.0, + "reward": 0.5625, + "reward_std": 0.2587745785713196, + "rewards/accuracy_reward/mean": 0.5625, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 1.8872061967849731, + "sampling/importance_sampling_ratio/mean": 1.000205159187317, + "sampling/importance_sampling_ratio/min": 0.4344595670700073, + "sampling/sampling_logp_difference/max": 0.8336523771286011, + "sampling/sampling_logp_difference/mean": 0.01450402569025755, + "step": 754 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00021248032862786204, + "clip_ratio/low_min": 0.00021248032862786204, + "clip_ratio/region_mean": 0.00021248032862786204, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 749.0, + "completions/mean_length": 392.21875, + "completions/mean_terminated_length": 338.5357360839844, + "completions/min_length": 132.0, + "completions/min_terminated_length": 132.0, + "entropy": 0.3706481456756592, + "epoch": 0.40417558886509636, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.01635662093758583, + "learning_rate": 1e-05, + "loss": 0.0605, + "num_tokens": 15279929.0, + "reward": 0.4375, + "reward_std": 0.3335031569004059, + "rewards/accuracy_reward/mean": 0.4375, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 1.7147916555404663, + "sampling/importance_sampling_ratio/mean": 0.9998767971992493, + "sampling/importance_sampling_ratio/min": 0.502638041973114, + "sampling/sampling_logp_difference/max": 0.687885046005249, + "sampling/sampling_logp_difference/mean": 0.01205374300479889, + "step": 755 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0001334980333922431, + "clip_ratio/low_min": 0.0001334980333922431, + "clip_ratio/region_mean": 0.0001334980333922431, + "completions/clipped_ratio": 0.0, + "completions/max_length": 702.0, + "completions/max_terminated_length": 702.0, + "completions/mean_length": 411.90625, + "completions/mean_terminated_length": 411.90625, + "completions/min_length": 163.0, + "completions/min_terminated_length": 163.0, + "entropy": 0.37051665410399437, + "epoch": 0.40471092077087795, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.01385048869997263, + "learning_rate": 1e-05, + "loss": 0.0404, + "num_tokens": 15297390.0, + "reward": 0.71875, + "reward_std": 0.24511480331420898, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 2.0, + "sampling/importance_sampling_ratio/mean": 1.0005110502243042, + "sampling/importance_sampling_ratio/min": 0.43829840421676636, + "sampling/sampling_logp_difference/max": 0.8248553276062012, + "sampling/sampling_logp_difference/mean": 0.011706370860338211, + "step": 756 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 605.0, + "completions/max_terminated_length": 605.0, + "completions/mean_length": 383.75, + "completions/mean_terminated_length": 383.75, + "completions/min_length": 176.0, + "completions/min_terminated_length": 176.0, + "entropy": 0.2585982959717512, + "epoch": 0.40524625267665954, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.01750893145799637, + "learning_rate": 1e-05, + "loss": -0.0102, + "num_tokens": 15313430.0, + "reward": 0.78125, + "reward_std": 0.3377465009689331, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.4770658016204834, + "sampling/importance_sampling_ratio/mean": 1.0001264810562134, + "sampling/importance_sampling_ratio/min": 0.6391497254371643, + "sampling/sampling_logp_difference/max": 0.4476165771484375, + "sampling/sampling_logp_difference/mean": 0.010347745381295681, + "step": 757 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 638.0, + "completions/mean_length": 366.125, + "completions/mean_terminated_length": 353.1612854003906, + "completions/min_length": 230.0, + "completions/min_terminated_length": 230.0, + "entropy": 0.26789403706789017, + "epoch": 0.4057815845824411, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.02607472613453865, + "learning_rate": 1e-05, + "loss": 0.0282, + "num_tokens": 15328610.0, + "reward": 0.71875, + "reward_std": 0.4218915104866028, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 2.0, + "sampling/importance_sampling_ratio/mean": 1.0002455711364746, + "sampling/importance_sampling_ratio/min": 0.3001675307750702, + "sampling/sampling_logp_difference/max": 1.2034145593643188, + "sampling/sampling_logp_difference/mean": 0.009937835857272148, + "step": 758 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 698.0, + "completions/mean_length": 476.0, + "completions/mean_terminated_length": 456.5333557128906, + "completions/min_length": 232.0, + "completions/min_terminated_length": 232.0, + "entropy": 0.2855139411985874, + "epoch": 0.4063169164882227, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.02170407772064209, + "learning_rate": 1e-05, + "loss": 0.0117, + "num_tokens": 15348266.0, + "reward": 0.65625, + "reward_std": 0.38816186785697937, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.4933019876480103, + "sampling/importance_sampling_ratio/mean": 1.0000791549682617, + "sampling/importance_sampling_ratio/min": 0.516124427318573, + "sampling/sampling_logp_difference/max": 0.661407470703125, + "sampling/sampling_logp_difference/mean": 0.01078470703214407, + "step": 759 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.765682726632804e-05, + "clip_ratio/low_min": 5.765682726632804e-05, + "clip_ratio/region_mean": 5.765682726632804e-05, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 753.0, + "completions/mean_length": 503.5, + "completions/mean_terminated_length": 485.86669921875, + "completions/min_length": 257.0, + "completions/min_terminated_length": 257.0, + "entropy": 0.25144183821976185, + "epoch": 0.4068522483940043, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.00593197625130415, + "learning_rate": 1e-05, + "loss": 0.008, + "num_tokens": 15367922.0, + "reward": 0.5, + "reward_std": 0.3514062762260437, + "rewards/accuracy_reward/mean": 0.5, + "rewards/accuracy_reward/std": 0.5080004930496216, + "sampling/importance_sampling_ratio/max": 1.6604912281036377, + "sampling/importance_sampling_ratio/mean": 1.000133991241455, + "sampling/importance_sampling_ratio/min": 0.4977162480354309, + "sampling/sampling_logp_difference/max": 0.6977251768112183, + "sampling/sampling_logp_difference/mean": 0.00899930763989687, + "step": 760 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 561.0, + "completions/mean_length": 429.46875, + "completions/mean_terminated_length": 418.5483703613281, + "completions/min_length": 246.0, + "completions/min_terminated_length": 246.0, + "entropy": 0.24014792777597904, + "epoch": 0.4073875802997859, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.01803966425359249, + "learning_rate": 1e-05, + "loss": 0.0309, + "num_tokens": 15385553.0, + "reward": 0.6875, + "reward_std": 0.38298875093460083, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 2.0, + "sampling/importance_sampling_ratio/mean": 1.0003598928451538, + "sampling/importance_sampling_ratio/min": 0.5966231822967529, + "sampling/sampling_logp_difference/max": 0.7804055213928223, + "sampling/sampling_logp_difference/mean": 0.009317105636000633, + "step": 761 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00011831928350147791, + "clip_ratio/low_min": 0.00011831928350147791, + "clip_ratio/region_mean": 0.00011831928350147791, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 745.0, + "completions/mean_length": 540.46875, + "completions/mean_terminated_length": 507.96429443359375, + "completions/min_length": 128.0, + "completions/min_terminated_length": 128.0, + "entropy": 0.3541956767439842, + "epoch": 0.4079229122055675, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.007222883403301239, + "learning_rate": 1e-05, + "loss": 0.0316, + "num_tokens": 15406136.0, + "reward": 0.78125, + "reward_std": 0.24511480331420898, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.6312808990478516, + "sampling/importance_sampling_ratio/mean": 1.000046730041504, + "sampling/importance_sampling_ratio/min": 0.5512207746505737, + "sampling/sampling_logp_difference/max": 0.5956199169158936, + "sampling/sampling_logp_difference/mean": 0.01018043328076601, + "step": 762 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 584.0, + "completions/mean_length": 420.5625, + "completions/mean_terminated_length": 409.3548278808594, + "completions/min_length": 249.0, + "completions/min_terminated_length": 249.0, + "entropy": 0.244696456938982, + "epoch": 0.408458244111349, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.0164913572371006, + "learning_rate": 1e-05, + "loss": -0.0001, + "num_tokens": 15423402.0, + "reward": 0.71875, + "reward_std": 0.24511480331420898, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.335079312324524, + "sampling/importance_sampling_ratio/mean": 0.9999603629112244, + "sampling/importance_sampling_ratio/min": 0.5473047494888306, + "sampling/sampling_logp_difference/max": 0.6027494668960571, + "sampling/sampling_logp_difference/mean": 0.009655633009970188, + "step": 763 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 699.0, + "completions/mean_length": 454.375, + "completions/mean_terminated_length": 433.4666748046875, + "completions/min_length": 205.0, + "completions/min_terminated_length": 205.0, + "entropy": 0.32273755595088005, + "epoch": 0.4089935760171306, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.029607856646180153, + "learning_rate": 1e-05, + "loss": 0.0472, + "num_tokens": 15441606.0, + "reward": 0.8125, + "reward_std": 0.2177756428718567, + "rewards/accuracy_reward/mean": 0.8125, + "rewards/accuracy_reward/std": 0.3965577781200409, + "sampling/importance_sampling_ratio/max": 1.4689022302627563, + "sampling/importance_sampling_ratio/mean": 1.0001521110534668, + "sampling/importance_sampling_ratio/min": 0.33022794127464294, + "sampling/sampling_logp_difference/max": 1.1079721450805664, + "sampling/sampling_logp_difference/mean": 0.010980737395584583, + "step": 764 + }, + { + "clip_ratio/high_max": 9.384384611621499e-05, + "clip_ratio/high_mean": 9.384384611621499e-05, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 9.384384611621499e-05, + "completions/clipped_ratio": 0.0, + "completions/max_length": 674.0, + "completions/max_terminated_length": 674.0, + "completions/mean_length": 377.90625, + "completions/mean_terminated_length": 377.90625, + "completions/min_length": 202.0, + "completions/min_terminated_length": 202.0, + "entropy": 0.24666313268244267, + "epoch": 0.4095289079229122, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.0074622477404773235, + "learning_rate": 1e-05, + "loss": 0.0023, + "num_tokens": 15457755.0, + "reward": 0.9375, + "reward_std": 0.1767766922712326, + "rewards/accuracy_reward/mean": 0.9375, + "rewards/accuracy_reward/std": 0.24593468010425568, + "sampling/importance_sampling_ratio/max": 1.3826043605804443, + "sampling/importance_sampling_ratio/mean": 0.9999831914901733, + "sampling/importance_sampling_ratio/min": 0.4515019953250885, + "sampling/sampling_logp_difference/max": 0.7951755523681641, + "sampling/sampling_logp_difference/mean": 0.009976961649954319, + "step": 765 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00010403341366327368, + "clip_ratio/low_min": 0.00010403341366327368, + "clip_ratio/region_mean": 0.00010403341366327368, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 682.0, + "completions/mean_length": 537.21875, + "completions/mean_terminated_length": 494.4814758300781, + "completions/min_length": 250.0, + "completions/min_terminated_length": 250.0, + "entropy": 0.23389310017228127, + "epoch": 0.4100642398286938, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.01662544719874859, + "learning_rate": 1e-05, + "loss": 0.0479, + "num_tokens": 15479738.0, + "reward": 0.6875, + "reward_std": 0.249358132481575, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.617555856704712, + "sampling/importance_sampling_ratio/mean": 0.9997934103012085, + "sampling/importance_sampling_ratio/min": 0.6282559037208557, + "sampling/sampling_logp_difference/max": 0.48091626167297363, + "sampling/sampling_logp_difference/mean": 0.009437416680157185, + "step": 766 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 8.305647497763857e-05, + "clip_ratio/low_min": 8.305647497763857e-05, + "clip_ratio/region_mean": 8.305647497763857e-05, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 675.0, + "completions/mean_length": 437.34375, + "completions/mean_terminated_length": 426.6773986816406, + "completions/min_length": 220.0, + "completions/min_terminated_length": 220.0, + "entropy": 0.305824164301157, + "epoch": 0.41059957173447537, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.01872457191348076, + "learning_rate": 1e-05, + "loss": 0.0059, + "num_tokens": 15497637.0, + "reward": 0.6875, + "reward_std": 0.292504221200943, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 2.0, + "sampling/importance_sampling_ratio/mean": 1.000535249710083, + "sampling/importance_sampling_ratio/min": 0.4695589542388916, + "sampling/sampling_logp_difference/max": 0.7767643928527832, + "sampling/sampling_logp_difference/mean": 0.012080066837370396, + "step": 767 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 613.0, + "completions/max_terminated_length": 613.0, + "completions/mean_length": 355.78125, + "completions/mean_terminated_length": 355.78125, + "completions/min_length": 191.0, + "completions/min_terminated_length": 191.0, + "entropy": 0.25859714299440384, + "epoch": 0.41113490364025695, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": 0.0, + "num_tokens": 15512774.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/accuracy_reward/mean": 1.0, + "rewards/accuracy_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 2.0, + "sampling/importance_sampling_ratio/mean": 1.000051736831665, + "sampling/importance_sampling_ratio/min": 0.39346182346343994, + "sampling/sampling_logp_difference/max": 0.9327712059020996, + "sampling/sampling_logp_difference/mean": 0.010906939394772053, + "step": 768 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 693.0, + "completions/max_terminated_length": 693.0, + "completions/mean_length": 387.84375, + "completions/mean_terminated_length": 387.84375, + "completions/min_length": 146.0, + "completions/min_terminated_length": 146.0, + "entropy": 0.27067269943654537, + "epoch": 0.41167023554603854, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.024706391617655754, + "learning_rate": 1e-05, + "loss": 0.0289, + "num_tokens": 15528361.0, + "reward": 0.71875, + "reward_std": 0.3198433816432953, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.6198508739471436, + "sampling/importance_sampling_ratio/mean": 1.0004866123199463, + "sampling/importance_sampling_ratio/min": 0.36960989236831665, + "sampling/sampling_logp_difference/max": 0.995307207107544, + "sampling/sampling_logp_difference/mean": 0.010347813367843628, + "step": 769 + }, + { + "clip_ratio/high_max": 6.433350790757686e-05, + "clip_ratio/high_mean": 6.433350790757686e-05, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 6.433350790757686e-05, + "completions/clipped_ratio": 0.0, + "completions/max_length": 543.0, + "completions/max_terminated_length": 543.0, + "completions/mean_length": 384.5625, + "completions/mean_terminated_length": 384.5625, + "completions/min_length": 104.0, + "completions/min_terminated_length": 104.0, + "entropy": 0.3696562983095646, + "epoch": 0.41220556745182013, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.007952449843287468, + "learning_rate": 1e-05, + "loss": -0.0314, + "num_tokens": 15544723.0, + "reward": 0.53125, + "reward_std": 0.3198433816432953, + "rewards/accuracy_reward/mean": 0.53125, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.5049047470092773, + "sampling/importance_sampling_ratio/mean": 0.9992790818214417, + "sampling/importance_sampling_ratio/min": 0.45941370725631714, + "sampling/sampling_logp_difference/max": 0.7778041362762451, + "sampling/sampling_logp_difference/mean": 0.012590371072292328, + "step": 770 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 697.0, + "completions/mean_length": 444.5, + "completions/mean_terminated_length": 422.933349609375, + "completions/min_length": 205.0, + "completions/min_terminated_length": 205.0, + "entropy": 0.304232407361269, + "epoch": 0.4127408993576017, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.007635242305696011, + "learning_rate": 1e-05, + "loss": 0.0027, + "num_tokens": 15563411.0, + "reward": 0.53125, + "reward_std": 0.2041158676147461, + "rewards/accuracy_reward/mean": 0.53125, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.6617324352264404, + "sampling/importance_sampling_ratio/mean": 0.9998767375946045, + "sampling/importance_sampling_ratio/min": 0.5821569561958313, + "sampling/sampling_logp_difference/max": 0.5410151481628418, + "sampling/sampling_logp_difference/mean": 0.010919096879661083, + "step": 771 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 704.0, + "completions/max_terminated_length": 704.0, + "completions/mean_length": 442.40625, + "completions/mean_terminated_length": 442.40625, + "completions/min_length": 191.0, + "completions/min_terminated_length": 191.0, + "entropy": 0.23302766494452953, + "epoch": 0.4132762312633833, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.004053059499710798, + "learning_rate": 1e-05, + "loss": -0.0086, + "num_tokens": 15581904.0, + "reward": 0.625, + "reward_std": 0.2177756428718567, + "rewards/accuracy_reward/mean": 0.625, + "rewards/accuracy_reward/std": 0.49186936020851135, + "sampling/importance_sampling_ratio/max": 2.0, + "sampling/importance_sampling_ratio/mean": 0.9999101161956787, + "sampling/importance_sampling_ratio/min": 0.5091020464897156, + "sampling/sampling_logp_difference/max": 0.7373270988464355, + "sampling/sampling_logp_difference/mean": 0.009034702554345131, + "step": 772 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00020220431179041043, + "clip_ratio/low_min": 0.00020220431179041043, + "clip_ratio/region_mean": 0.00020220431179041043, + "completions/clipped_ratio": 0.0, + "completions/max_length": 519.0, + "completions/max_terminated_length": 519.0, + "completions/mean_length": 345.46875, + "completions/mean_terminated_length": 345.46875, + "completions/min_length": 182.0, + "completions/min_terminated_length": 182.0, + "entropy": 0.2553048860281706, + "epoch": 0.4138115631691649, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.03436039760708809, + "learning_rate": 1e-05, + "loss": -0.0183, + "num_tokens": 15596871.0, + "reward": 0.9375, + "reward_std": 0.1767766922712326, + "rewards/accuracy_reward/mean": 0.9375, + "rewards/accuracy_reward/std": 0.24593468010425568, + "sampling/importance_sampling_ratio/max": 1.488649845123291, + "sampling/importance_sampling_ratio/mean": 0.9997662901878357, + "sampling/importance_sampling_ratio/min": 0.4369298219680786, + "sampling/sampling_logp_difference/max": 0.8279826641082764, + "sampling/sampling_logp_difference/mean": 0.010252413339912891, + "step": 773 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00018368846212979406, + "clip_ratio/low_min": 0.00018368846212979406, + "clip_ratio/region_mean": 0.00018368846212979406, + "completions/clipped_ratio": 0.0, + "completions/max_length": 500.0, + "completions/max_terminated_length": 500.0, + "completions/mean_length": 328.0625, + "completions/mean_terminated_length": 328.0625, + "completions/min_length": 127.0, + "completions/min_terminated_length": 127.0, + "entropy": 0.27277201041579247, + "epoch": 0.4143468950749465, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.016949793323874474, + "learning_rate": 1e-05, + "loss": -0.0561, + "num_tokens": 15610977.0, + "reward": 0.71875, + "reward_std": 0.2630178928375244, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.9013302326202393, + "sampling/importance_sampling_ratio/mean": 0.9998557567596436, + "sampling/importance_sampling_ratio/min": 0.6245912313461304, + "sampling/sampling_logp_difference/max": 0.6425538063049316, + "sampling/sampling_logp_difference/mean": 0.010486333630979061, + "step": 774 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00037325148878153414, + "clip_ratio/low_min": 0.00037325148878153414, + "clip_ratio/region_mean": 0.00037325148878153414, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 636.0, + "completions/mean_length": 413.875, + "completions/mean_terminated_length": 402.45159912109375, + "completions/min_length": 129.0, + "completions/min_terminated_length": 129.0, + "entropy": 0.40199044346809387, + "epoch": 0.4148822269807281, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.024073319509625435, + "learning_rate": 1e-05, + "loss": 0.0021, + "num_tokens": 15628629.0, + "reward": 0.65625, + "reward_std": 0.3608423173427582, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 2.0, + "sampling/importance_sampling_ratio/mean": 0.9997939467430115, + "sampling/importance_sampling_ratio/min": 0.38455668091773987, + "sampling/sampling_logp_difference/max": 0.9556640982627869, + "sampling/sampling_logp_difference/mean": 0.01387203112244606, + "step": 775 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00013112008309690282, + "clip_ratio/low_min": 0.00013112008309690282, + "clip_ratio/region_mean": 0.00013112008309690282, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 763.0, + "completions/mean_length": 529.0, + "completions/mean_terminated_length": 494.857177734375, + "completions/min_length": 322.0, + "completions/min_terminated_length": 322.0, + "entropy": 0.2845477182418108, + "epoch": 0.41541755888650966, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.0260250773280859, + "learning_rate": 1e-05, + "loss": 0.0167, + "num_tokens": 15649469.0, + "reward": 0.59375, + "reward_std": 0.378745436668396, + "rewards/accuracy_reward/mean": 0.59375, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.4169169664382935, + "sampling/importance_sampling_ratio/mean": 0.9998788833618164, + "sampling/importance_sampling_ratio/min": 0.36439773440361023, + "sampling/sampling_logp_difference/max": 1.0095093250274658, + "sampling/sampling_logp_difference/mean": 0.010882158763706684, + "step": 776 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0002542619840824045, + "clip_ratio/low_min": 0.0002542619840824045, + "clip_ratio/region_mean": 0.0002542619840824045, + "completions/clipped_ratio": 0.0, + "completions/max_length": 569.0, + "completions/max_terminated_length": 569.0, + "completions/mean_length": 376.15625, + "completions/mean_terminated_length": 376.15625, + "completions/min_length": 220.0, + "completions/min_terminated_length": 220.0, + "entropy": 0.2752981185913086, + "epoch": 0.4159528907922912, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.021978862583637238, + "learning_rate": 1e-05, + "loss": 0.0158, + "num_tokens": 15664810.0, + "reward": 0.65625, + "reward_std": 0.38816186785697937, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.4845292568206787, + "sampling/importance_sampling_ratio/mean": 1.0002071857452393, + "sampling/importance_sampling_ratio/min": 0.6175705194473267, + "sampling/sampling_logp_difference/max": 0.4819619655609131, + "sampling/sampling_logp_difference/mean": 0.010415537282824516, + "step": 777 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.383290408644825e-05, + "clip_ratio/low_min": 5.383290408644825e-05, + "clip_ratio/region_mean": 5.383290408644825e-05, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 722.0, + "completions/mean_length": 488.78125, + "completions/mean_terminated_length": 437.0740661621094, + "completions/min_length": 142.0, + "completions/min_terminated_length": 142.0, + "entropy": 0.3771573305130005, + "epoch": 0.4164882226980728, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.0047351461835205555, + "learning_rate": 1e-05, + "loss": 0.0318, + "num_tokens": 15684283.0, + "reward": 0.46875, + "reward_std": 0.2651650309562683, + "rewards/accuracy_reward/mean": 0.46875, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.717313289642334, + "sampling/importance_sampling_ratio/mean": 0.999986469745636, + "sampling/importance_sampling_ratio/min": 0.593487560749054, + "sampling/sampling_logp_difference/max": 0.5407609939575195, + "sampling/sampling_logp_difference/mean": 0.012063194066286087, + "step": 778 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 579.0, + "completions/mean_length": 358.0, + "completions/mean_terminated_length": 330.66668701171875, + "completions/min_length": 115.0, + "completions/min_terminated_length": 115.0, + "entropy": 0.3367639631032944, + "epoch": 0.4170235546038544, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.025062229484319687, + "learning_rate": 1e-05, + "loss": -0.0248, + "num_tokens": 15699363.0, + "reward": 0.65625, + "reward_std": 0.3966485261917114, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.511836051940918, + "sampling/importance_sampling_ratio/mean": 1.0002409219741821, + "sampling/importance_sampling_ratio/min": 0.5363574624061584, + "sampling/sampling_logp_difference/max": 0.6229543685913086, + "sampling/sampling_logp_difference/mean": 0.0124586820602417, + "step": 779 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00021610707335639745, + "clip_ratio/low_min": 0.00021610707335639745, + "clip_ratio/region_mean": 0.00021610707335639745, + "completions/clipped_ratio": 0.0, + "completions/max_length": 681.0, + "completions/max_terminated_length": 681.0, + "completions/mean_length": 373.28125, + "completions/mean_terminated_length": 373.28125, + "completions/min_length": 244.0, + "completions/min_terminated_length": 244.0, + "entropy": 0.25080471858382225, + "epoch": 0.41755888650963596, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.005736921913921833, + "learning_rate": 1e-05, + "loss": 0.008, + "num_tokens": 15714508.0, + "reward": 0.75, + "reward_std": 0.1767766922712326, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 2.0, + "sampling/importance_sampling_ratio/mean": 0.9998818635940552, + "sampling/importance_sampling_ratio/min": 0.5307590365409851, + "sampling/sampling_logp_difference/max": 0.7510523796081543, + "sampling/sampling_logp_difference/mean": 0.010422402992844582, + "step": 780 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 583.0, + "completions/mean_length": 482.5, + "completions/mean_terminated_length": 429.629638671875, + "completions/min_length": 255.0, + "completions/min_terminated_length": 255.0, + "entropy": 0.4434059262275696, + "epoch": 0.41809421841541755, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.0073394631035625935, + "learning_rate": 1e-05, + "loss": 0.0235, + "num_tokens": 15733892.0, + "reward": 0.59375, + "reward_std": 0.22201895713806152, + "rewards/accuracy_reward/mean": 0.59375, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.9157443046569824, + "sampling/importance_sampling_ratio/mean": 0.9997337460517883, + "sampling/importance_sampling_ratio/min": 0.38598206639289856, + "sampling/sampling_logp_difference/max": 0.9519643783569336, + "sampling/sampling_logp_difference/mean": 0.013192393817007542, + "step": 781 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 7.571168680442497e-05, + "clip_ratio/low_min": 7.571168680442497e-05, + "clip_ratio/region_mean": 7.571168680442497e-05, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 734.0, + "completions/mean_length": 471.0625, + "completions/mean_terminated_length": 461.4838562011719, + "completions/min_length": 330.0, + "completions/min_terminated_length": 330.0, + "entropy": 0.33475570380687714, + "epoch": 0.41862955032119914, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.007119338493794203, + "learning_rate": 1e-05, + "loss": -0.0001, + "num_tokens": 15752534.0, + "reward": 0.5625, + "reward_std": 0.249358132481575, + "rewards/accuracy_reward/mean": 0.5625, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 1.4272743463516235, + "sampling/importance_sampling_ratio/mean": 0.9999619722366333, + "sampling/importance_sampling_ratio/min": 0.5658502578735352, + "sampling/sampling_logp_difference/max": 0.5694258213043213, + "sampling/sampling_logp_difference/mean": 0.011955409310758114, + "step": 782 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 659.0, + "completions/max_terminated_length": 659.0, + "completions/mean_length": 384.8125, + "completions/mean_terminated_length": 384.8125, + "completions/min_length": 132.0, + "completions/min_terminated_length": 132.0, + "entropy": 0.2584788240492344, + "epoch": 0.41916488222698073, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.0038310440722852945, + "learning_rate": 1e-05, + "loss": -0.0908, + "num_tokens": 15768888.0, + "reward": 0.75, + "reward_std": 0.1767766922712326, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.5195432901382446, + "sampling/importance_sampling_ratio/mean": 0.9996142983436584, + "sampling/importance_sampling_ratio/min": 0.5287886261940002, + "sampling/sampling_logp_difference/max": 0.6371665000915527, + "sampling/sampling_logp_difference/mean": 0.010382593609392643, + "step": 783 + }, + { + "clip_ratio/high_max": 0.0001403668211423792, + "clip_ratio/high_mean": 0.0001403668211423792, + "clip_ratio/low_mean": 5.643340773531236e-05, + "clip_ratio/low_min": 5.643340773531236e-05, + "clip_ratio/region_mean": 0.00019680022887769155, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 609.0, + "completions/mean_length": 424.15625, + "completions/mean_terminated_length": 375.0357360839844, + "completions/min_length": 149.0, + "completions/min_terminated_length": 149.0, + "entropy": 0.39270056039094925, + "epoch": 0.4197002141327623, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.02223135344684124, + "learning_rate": 1e-05, + "loss": -0.0231, + "num_tokens": 15786613.0, + "reward": 0.65625, + "reward_std": 0.3198433816432953, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 2.0, + "sampling/importance_sampling_ratio/mean": 1.0002459287643433, + "sampling/importance_sampling_ratio/min": 0.5257071256637573, + "sampling/sampling_logp_difference/max": 0.9976482391357422, + "sampling/sampling_logp_difference/mean": 0.014230569824576378, + "step": 784 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 7.896398892626166e-05, + "clip_ratio/low_min": 7.896398892626166e-05, + "clip_ratio/region_mean": 7.896398892626166e-05, + "completions/clipped_ratio": 0.0, + "completions/max_length": 646.0, + "completions/max_terminated_length": 646.0, + "completions/mean_length": 415.34375, + "completions/mean_terminated_length": 415.34375, + "completions/min_length": 227.0, + "completions/min_terminated_length": 227.0, + "entropy": 0.35414634086191654, + "epoch": 0.4202355460385439, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.0093699861317873, + "learning_rate": 1e-05, + "loss": 0.0066, + "num_tokens": 15804024.0, + "reward": 0.84375, + "reward_std": 0.3061639964580536, + "rewards/accuracy_reward/mean": 0.84375, + "rewards/accuracy_reward/std": 0.3689020276069641, + "sampling/importance_sampling_ratio/max": 2.0, + "sampling/importance_sampling_ratio/mean": 0.9995676279067993, + "sampling/importance_sampling_ratio/min": 0.5625743269920349, + "sampling/sampling_logp_difference/max": 1.251774787902832, + "sampling/sampling_logp_difference/mean": 0.0111850555986166, + "step": 785 + }, + { + "clip_ratio/high_max": 9.689922444522381e-05, + "clip_ratio/high_mean": 9.689922444522381e-05, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 9.689922444522381e-05, + "completions/clipped_ratio": 0.0, + "completions/max_length": 589.0, + "completions/max_terminated_length": 589.0, + "completions/mean_length": 378.53125, + "completions/mean_terminated_length": 378.53125, + "completions/min_length": 156.0, + "completions/min_terminated_length": 156.0, + "entropy": 0.26351975463330746, + "epoch": 0.4207708779443255, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.008030323311686516, + "learning_rate": 1e-05, + "loss": -0.0131, + "num_tokens": 15819897.0, + "reward": 0.90625, + "reward_std": 0.1293872892856598, + "rewards/accuracy_reward/mean": 0.90625, + "rewards/accuracy_reward/std": 0.2961445748806, + "sampling/importance_sampling_ratio/max": 1.9409581422805786, + "sampling/importance_sampling_ratio/mean": 0.9995185732841492, + "sampling/importance_sampling_ratio/min": 0.5583049654960632, + "sampling/sampling_logp_difference/max": 0.6631817817687988, + "sampling/sampling_logp_difference/mean": 0.011047407053411007, + "step": 786 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 714.0, + "completions/max_terminated_length": 714.0, + "completions/mean_length": 380.34375, + "completions/mean_terminated_length": 380.34375, + "completions/min_length": 103.0, + "completions/min_terminated_length": 103.0, + "entropy": 0.26448206044733524, + "epoch": 0.4213062098501071, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.002527264878153801, + "learning_rate": 1e-05, + "loss": 0.0025, + "num_tokens": 15835484.0, + "reward": 0.8125, + "reward_std": 0.3335031569004059, + "rewards/accuracy_reward/mean": 0.8125, + "rewards/accuracy_reward/std": 0.3965577781200409, + "sampling/importance_sampling_ratio/max": 1.9102119207382202, + "sampling/importance_sampling_ratio/mean": 1.0003339052200317, + "sampling/importance_sampling_ratio/min": 0.5570401549339294, + "sampling/sampling_logp_difference/max": 0.6472141742706299, + "sampling/sampling_logp_difference/mean": 0.011137889698147774, + "step": 787 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00015303726104320958, + "clip_ratio/low_min": 0.00015303726104320958, + "clip_ratio/region_mean": 0.00015303726104320958, + "completions/clipped_ratio": 0.0, + "completions/max_length": 633.0, + "completions/max_terminated_length": 633.0, + "completions/mean_length": 377.53125, + "completions/mean_terminated_length": 377.53125, + "completions/min_length": 221.0, + "completions/min_terminated_length": 221.0, + "entropy": 0.2805671710520983, + "epoch": 0.42184154175588867, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.025459816679358482, + "learning_rate": 1e-05, + "loss": -0.0408, + "num_tokens": 15851317.0, + "reward": 0.71875, + "reward_std": 0.2630178928375244, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.5567599534988403, + "sampling/importance_sampling_ratio/mean": 0.9995025396347046, + "sampling/importance_sampling_ratio/min": 0.6074600219726562, + "sampling/sampling_logp_difference/max": 0.49846887588500977, + "sampling/sampling_logp_difference/mean": 0.011317256838083267, + "step": 788 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 8.153946691891178e-05, + "clip_ratio/low_min": 8.153946691891178e-05, + "clip_ratio/region_mean": 8.153946691891178e-05, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 698.0, + "completions/mean_length": 454.96875, + "completions/mean_terminated_length": 422.5862121582031, + "completions/min_length": 221.0, + "completions/min_terminated_length": 221.0, + "entropy": 0.3912490103393793, + "epoch": 0.42237687366167026, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.004201194737106562, + "learning_rate": 1e-05, + "loss": 0.0112, + "num_tokens": 15870156.0, + "reward": 0.3125, + "reward_std": 0.1767766922712326, + "rewards/accuracy_reward/mean": 0.3125, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.5440092086791992, + "sampling/importance_sampling_ratio/mean": 0.9996799230575562, + "sampling/importance_sampling_ratio/min": 0.5852885246276855, + "sampling/sampling_logp_difference/max": 0.5356502532958984, + "sampling/sampling_logp_difference/mean": 0.013936889357864857, + "step": 789 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 8.445946150459349e-05, + "clip_ratio/low_min": 8.445946150459349e-05, + "clip_ratio/region_mean": 8.445946150459349e-05, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 753.0, + "completions/mean_length": 429.375, + "completions/mean_terminated_length": 406.8000183105469, + "completions/min_length": 233.0, + "completions/min_terminated_length": 233.0, + "entropy": 0.28259735368192196, + "epoch": 0.4229122055674518, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.013709502294659615, + "learning_rate": 1e-05, + "loss": 0.0427, + "num_tokens": 15887672.0, + "reward": 0.84375, + "reward_std": 0.24511480331420898, + "rewards/accuracy_reward/mean": 0.84375, + "rewards/accuracy_reward/std": 0.3689020276069641, + "sampling/importance_sampling_ratio/max": 1.6652288436889648, + "sampling/importance_sampling_ratio/mean": 1.0003029108047485, + "sampling/importance_sampling_ratio/min": 0.5105920433998108, + "sampling/sampling_logp_difference/max": 0.6721844673156738, + "sampling/sampling_logp_difference/mean": 0.011160780675709248, + "step": 790 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00018738366634352133, + "clip_ratio/low_min": 0.00018738366634352133, + "clip_ratio/region_mean": 0.00018738366634352133, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 628.0, + "completions/mean_length": 462.78125, + "completions/mean_terminated_length": 442.433349609375, + "completions/min_length": 265.0, + "completions/min_terminated_length": 265.0, + "entropy": 0.3385087884962559, + "epoch": 0.4234475374732334, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.023096540942788124, + "learning_rate": 1e-05, + "loss": 0.0255, + "num_tokens": 15906681.0, + "reward": 0.71875, + "reward_std": 0.2041158676147461, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.5910266637802124, + "sampling/importance_sampling_ratio/mean": 1.0004183053970337, + "sampling/importance_sampling_ratio/min": 0.5628833174705505, + "sampling/sampling_logp_difference/max": 0.5746829509735107, + "sampling/sampling_logp_difference/mean": 0.011910232715308666, + "step": 791 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 7.485030073439702e-05, + "clip_ratio/low_min": 7.485030073439702e-05, + "clip_ratio/region_mean": 7.485030073439702e-05, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 675.0, + "completions/mean_length": 431.625, + "completions/mean_terminated_length": 409.20001220703125, + "completions/min_length": 230.0, + "completions/min_terminated_length": 230.0, + "entropy": 0.3437661435455084, + "epoch": 0.42398286937901497, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.03732145577669144, + "learning_rate": 1e-05, + "loss": 0.0649, + "num_tokens": 15924069.0, + "reward": 0.78125, + "reward_std": 0.4218915104866028, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 2.0, + "sampling/importance_sampling_ratio/mean": 0.9997829794883728, + "sampling/importance_sampling_ratio/min": 0.5115315318107605, + "sampling/sampling_logp_difference/max": 1.006821632385254, + "sampling/sampling_logp_difference/mean": 0.012319444678723812, + "step": 792 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 575.0, + "completions/max_terminated_length": 575.0, + "completions/mean_length": 340.375, + "completions/mean_terminated_length": 340.375, + "completions/min_length": 33.0, + "completions/min_terminated_length": 33.0, + "entropy": 0.2199815083295107, + "epoch": 0.42451820128479656, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.022471919655799866, + "learning_rate": 1e-05, + "loss": -0.005, + "num_tokens": 15938449.0, + "reward": 0.9375, + "reward_std": 0.1157275140285492, + "rewards/accuracy_reward/mean": 0.9375, + "rewards/accuracy_reward/std": 0.24593468010425568, + "sampling/importance_sampling_ratio/max": 1.6085669994354248, + "sampling/importance_sampling_ratio/mean": 1.0001497268676758, + "sampling/importance_sampling_ratio/min": 0.500328540802002, + "sampling/sampling_logp_difference/max": 0.6924903392791748, + "sampling/sampling_logp_difference/mean": 0.008921113796532154, + "step": 793 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00010339123400626704, + "clip_ratio/low_min": 0.00010339123400626704, + "clip_ratio/region_mean": 0.00010339123400626704, + "completions/clipped_ratio": 0.0, + "completions/max_length": 482.0, + "completions/max_terminated_length": 482.0, + "completions/mean_length": 292.40625, + "completions/mean_terminated_length": 292.40625, + "completions/min_length": 137.0, + "completions/min_terminated_length": 137.0, + "entropy": 0.24353780783712864, + "epoch": 0.42505353319057815, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": -0.0156, + "num_tokens": 15951358.0, + "reward": 0.84375, + "reward_std": 0.1293872892856598, + "rewards/accuracy_reward/mean": 0.84375, + "rewards/accuracy_reward/std": 0.3689020276069641, + "sampling/importance_sampling_ratio/max": 1.457561731338501, + "sampling/importance_sampling_ratio/mean": 1.0000132322311401, + "sampling/importance_sampling_ratio/min": 0.5762859582901001, + "sampling/sampling_logp_difference/max": 0.5511512756347656, + "sampling/sampling_logp_difference/mean": 0.009628204628825188, + "step": 794 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 644.0, + "completions/max_terminated_length": 644.0, + "completions/mean_length": 434.03125, + "completions/mean_terminated_length": 434.03125, + "completions/min_length": 223.0, + "completions/min_terminated_length": 223.0, + "entropy": 0.3120315596461296, + "epoch": 0.42558886509635974, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.01217488944530487, + "learning_rate": 1e-05, + "loss": 0.0066, + "num_tokens": 15969039.0, + "reward": 0.4375, + "reward_std": 0.4261348247528076, + "rewards/accuracy_reward/mean": 0.4375, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 1.461539387702942, + "sampling/importance_sampling_ratio/mean": 1.0000052452087402, + "sampling/importance_sampling_ratio/min": 0.5028045177459717, + "sampling/sampling_logp_difference/max": 0.687553882598877, + "sampling/sampling_logp_difference/mean": 0.012451590970158577, + "step": 795 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 6.656016921624541e-05, + "clip_ratio/low_min": 6.656016921624541e-05, + "clip_ratio/region_mean": 6.656016921624541e-05, + "completions/clipped_ratio": 0.0, + "completions/max_length": 615.0, + "completions/max_terminated_length": 615.0, + "completions/mean_length": 401.78125, + "completions/mean_terminated_length": 401.78125, + "completions/min_length": 247.0, + "completions/min_terminated_length": 247.0, + "entropy": 0.28384484723210335, + "epoch": 0.4261241970021413, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.003345826640725136, + "learning_rate": 1e-05, + "loss": -0.015, + "num_tokens": 15985536.0, + "reward": 0.875, + "reward_std": 0.2177756428718567, + "rewards/accuracy_reward/mean": 0.875, + "rewards/accuracy_reward/std": 0.33601075410842896, + "sampling/importance_sampling_ratio/max": 1.8771435022354126, + "sampling/importance_sampling_ratio/mean": 0.9998907446861267, + "sampling/importance_sampling_ratio/min": 0.615212619304657, + "sampling/sampling_logp_difference/max": 0.6297512054443359, + "sampling/sampling_logp_difference/mean": 0.011010140180587769, + "step": 796 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 588.0, + "completions/mean_length": 318.53125, + "completions/mean_terminated_length": 304.0322570800781, + "completions/min_length": 132.0, + "completions/min_terminated_length": 132.0, + "entropy": 0.2827566247433424, + "epoch": 0.4266595289079229, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": -0.0147, + "num_tokens": 15999217.0, + "reward": 0.90625, + "reward_std": 0.1293872892856598, + "rewards/accuracy_reward/mean": 0.90625, + "rewards/accuracy_reward/std": 0.2961445748806, + "sampling/importance_sampling_ratio/max": 1.7602216005325317, + "sampling/importance_sampling_ratio/mean": 1.0001087188720703, + "sampling/importance_sampling_ratio/min": 0.44405969977378845, + "sampling/sampling_logp_difference/max": 0.8117963075637817, + "sampling/sampling_logp_difference/mean": 0.011242852546274662, + "step": 797 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 596.0, + "completions/max_terminated_length": 596.0, + "completions/mean_length": 385.71875, + "completions/mean_terminated_length": 385.71875, + "completions/min_length": 134.0, + "completions/min_terminated_length": 134.0, + "entropy": 0.33430335484445095, + "epoch": 0.4271948608137045, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.019825590774416924, + "learning_rate": 1e-05, + "loss": -0.0319, + "num_tokens": 16015280.0, + "reward": 0.65625, + "reward_std": 0.3808925747871399, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.4260501861572266, + "sampling/importance_sampling_ratio/mean": 1.0001707077026367, + "sampling/importance_sampling_ratio/min": 0.511563777923584, + "sampling/sampling_logp_difference/max": 0.6702830791473389, + "sampling/sampling_logp_difference/mean": 0.011605370789766312, + "step": 798 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.755064557888545e-05, + "clip_ratio/low_min": 5.755064557888545e-05, + "clip_ratio/region_mean": 5.755064557888545e-05, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 759.0, + "completions/mean_length": 500.46875, + "completions/mean_terminated_length": 491.83868408203125, + "completions/min_length": 251.0, + "completions/min_terminated_length": 251.0, + "entropy": 0.28260005451738834, + "epoch": 0.4277301927194861, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.009162155911326408, + "learning_rate": 1e-05, + "loss": 0.0544, + "num_tokens": 16035271.0, + "reward": 0.78125, + "reward_std": 0.3377465009689331, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.4117162227630615, + "sampling/importance_sampling_ratio/mean": 0.9999385476112366, + "sampling/importance_sampling_ratio/min": 0.3373129665851593, + "sampling/sampling_logp_difference/max": 1.0867441892623901, + "sampling/sampling_logp_difference/mean": 0.011142052710056305, + "step": 799 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 8.778089977568015e-05, + "clip_ratio/low_min": 8.778089977568015e-05, + "clip_ratio/region_mean": 8.778089977568015e-05, + "completions/clipped_ratio": 0.0, + "completions/max_length": 604.0, + "completions/max_terminated_length": 604.0, + "completions/mean_length": 387.125, + "completions/mean_terminated_length": 387.125, + "completions/min_length": 140.0, + "completions/min_terminated_length": 140.0, + "entropy": 0.27057322300970554, + "epoch": 0.4282655246252677, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.028597332537174225, + "learning_rate": 1e-05, + "loss": 0.0163, + "num_tokens": 16051723.0, + "reward": 0.8125, + "reward_std": 0.3335031569004059, + "rewards/accuracy_reward/mean": 0.8125, + "rewards/accuracy_reward/std": 0.3965577781200409, + "sampling/importance_sampling_ratio/max": 2.0, + "sampling/importance_sampling_ratio/mean": 1.0005011558532715, + "sampling/importance_sampling_ratio/min": 0.6890622973442078, + "sampling/sampling_logp_difference/max": 0.8228888511657715, + "sampling/sampling_logp_difference/mean": 0.011211972683668137, + "step": 800 + }, + { + "clip_ratio/high_max": 7.00672680977732e-05, + "clip_ratio/high_mean": 7.00672680977732e-05, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 7.00672680977732e-05, + "completions/clipped_ratio": 0.0, + "completions/max_length": 640.0, + "completions/max_terminated_length": 640.0, + "completions/mean_length": 444.125, + "completions/mean_terminated_length": 444.125, + "completions/min_length": 279.0, + "completions/min_terminated_length": 279.0, + "entropy": 0.2429835107177496, + "epoch": 0.42880085653104927, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.015095447190105915, + "learning_rate": 1e-05, + "loss": -0.039, + "num_tokens": 16069567.0, + "reward": 0.6875, + "reward_std": 0.3335031569004059, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.426932454109192, + "sampling/importance_sampling_ratio/mean": 1.0000156164169312, + "sampling/importance_sampling_ratio/min": 0.3773273229598999, + "sampling/sampling_logp_difference/max": 0.974642276763916, + "sampling/sampling_logp_difference/mean": 0.009842971339821815, + "step": 801 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.827505810884759e-05, + "clip_ratio/low_min": 5.827505810884759e-05, + "clip_ratio/region_mean": 5.827505810884759e-05, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 702.0, + "completions/mean_length": 450.90625, + "completions/mean_terminated_length": 440.6773986816406, + "completions/min_length": 218.0, + "completions/min_terminated_length": 218.0, + "entropy": 0.26054478995501995, + "epoch": 0.42933618843683086, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.0036774608306586742, + "learning_rate": 1e-05, + "loss": 0.0426, + "num_tokens": 16088276.0, + "reward": 0.6875, + "reward_std": 0.249358132481575, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.48768949508667, + "sampling/importance_sampling_ratio/mean": 0.9999830722808838, + "sampling/importance_sampling_ratio/min": 0.6813663840293884, + "sampling/sampling_logp_difference/max": 0.39722418785095215, + "sampling/sampling_logp_difference/mean": 0.009339271113276482, + "step": 802 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 7.17565999366343e-05, + "clip_ratio/low_min": 7.17565999366343e-05, + "clip_ratio/region_mean": 7.17565999366343e-05, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 585.0, + "completions/mean_length": 406.53125, + "completions/mean_terminated_length": 394.8709411621094, + "completions/min_length": 215.0, + "completions/min_terminated_length": 215.0, + "entropy": 0.2608866672962904, + "epoch": 0.42987152034261245, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.014327667653560638, + "learning_rate": 1e-05, + "loss": -0.0132, + "num_tokens": 16104981.0, + "reward": 0.8125, + "reward_std": 0.249358132481575, + "rewards/accuracy_reward/mean": 0.8125, + "rewards/accuracy_reward/std": 0.3965577781200409, + "sampling/importance_sampling_ratio/max": 1.6868841648101807, + "sampling/importance_sampling_ratio/mean": 0.9999310374259949, + "sampling/importance_sampling_ratio/min": 0.4192970097064972, + "sampling/sampling_logp_difference/max": 0.8691757917404175, + "sampling/sampling_logp_difference/mean": 0.01003206241875887, + "step": 803 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 589.0, + "completions/mean_length": 396.625, + "completions/mean_terminated_length": 327.85186767578125, + "completions/min_length": 216.0, + "completions/min_terminated_length": 216.0, + "entropy": 0.31929980032145977, + "epoch": 0.430406852248394, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.004386443179100752, + "learning_rate": 1e-05, + "loss": 0.0304, + "num_tokens": 16121689.0, + "reward": 0.78125, + "reward_std": 0.0883883461356163, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.8314975500106812, + "sampling/importance_sampling_ratio/mean": 1.000288963317871, + "sampling/importance_sampling_ratio/min": 0.6134749054908752, + "sampling/sampling_logp_difference/max": 0.6051340103149414, + "sampling/sampling_logp_difference/mean": 0.01133585162460804, + "step": 804 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 7.720815483480692e-05, + "clip_ratio/low_min": 7.720815483480692e-05, + "clip_ratio/region_mean": 7.720815483480692e-05, + "completions/clipped_ratio": 0.0, + "completions/max_length": 623.0, + "completions/max_terminated_length": 623.0, + "completions/mean_length": 379.34375, + "completions/mean_terminated_length": 379.34375, + "completions/min_length": 135.0, + "completions/min_terminated_length": 135.0, + "entropy": 0.25638870522379875, + "epoch": 0.43094218415417557, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.020009981468319893, + "learning_rate": 1e-05, + "loss": -0.0083, + "num_tokens": 16137164.0, + "reward": 0.75, + "reward_std": 0.3514062762260437, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.7594571113586426, + "sampling/importance_sampling_ratio/mean": 1.0000542402267456, + "sampling/importance_sampling_ratio/min": 0.5219526290893555, + "sampling/sampling_logp_difference/max": 0.6501784324645996, + "sampling/sampling_logp_difference/mean": 0.010279352776706219, + "step": 805 + }, + { + "clip_ratio/high_max": 9.65996878221631e-05, + "clip_ratio/high_mean": 9.65996878221631e-05, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 9.65996878221631e-05, + "completions/clipped_ratio": 0.0, + "completions/max_length": 575.0, + "completions/max_terminated_length": 575.0, + "completions/mean_length": 345.78125, + "completions/mean_terminated_length": 345.78125, + "completions/min_length": 195.0, + "completions/min_terminated_length": 195.0, + "entropy": 0.24040366522967815, + "epoch": 0.43147751605995716, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.038052044808864594, + "learning_rate": 1e-05, + "loss": 0.0042, + "num_tokens": 16151581.0, + "reward": 0.71875, + "reward_std": 0.3808925747871399, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.432822346687317, + "sampling/importance_sampling_ratio/mean": 0.9999960064888, + "sampling/importance_sampling_ratio/min": 0.5406076908111572, + "sampling/sampling_logp_difference/max": 0.6150614023208618, + "sampling/sampling_logp_difference/mean": 0.00955849140882492, + "step": 806 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 8.127438195515424e-05, + "clip_ratio/low_min": 8.127438195515424e-05, + "clip_ratio/region_mean": 8.127438195515424e-05, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 695.0, + "completions/mean_length": 404.4375, + "completions/mean_terminated_length": 392.70965576171875, + "completions/min_length": 252.0, + "completions/min_terminated_length": 252.0, + "entropy": 0.2859781552106142, + "epoch": 0.43201284796573874, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.006455859635025263, + "learning_rate": 1e-05, + "loss": 0.0495, + "num_tokens": 16168123.0, + "reward": 0.6875, + "reward_std": 0.2925041913986206, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.4351640939712524, + "sampling/importance_sampling_ratio/mean": 1.0002063512802124, + "sampling/importance_sampling_ratio/min": 0.5675336122512817, + "sampling/sampling_logp_difference/max": 0.5664552450180054, + "sampling/sampling_logp_difference/mean": 0.011134625412523746, + "step": 807 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 684.0, + "completions/mean_length": 405.0, + "completions/mean_terminated_length": 380.8000183105469, + "completions/min_length": 213.0, + "completions/min_terminated_length": 213.0, + "entropy": 0.35887262411415577, + "epoch": 0.43254817987152033, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.005303828977048397, + "learning_rate": 1e-05, + "loss": 0.0038, + "num_tokens": 16184803.0, + "reward": 0.78125, + "reward_std": 0.2041158676147461, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.824461817741394, + "sampling/importance_sampling_ratio/mean": 0.9998227953910828, + "sampling/importance_sampling_ratio/min": 0.5102828741073608, + "sampling/sampling_logp_difference/max": 0.6727900505065918, + "sampling/sampling_logp_difference/mean": 0.012198110111057758, + "step": 808 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 768.0, + "completions/mean_length": 500.8125, + "completions/mean_terminated_length": 462.64288330078125, + "completions/min_length": 220.0, + "completions/min_terminated_length": 220.0, + "entropy": 0.2566366661339998, + "epoch": 0.4330835117773019, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.0034935621079057455, + "learning_rate": 1e-05, + "loss": 0.0774, + "num_tokens": 16204661.0, + "reward": 0.875, + "reward_std": 0.2177756428718567, + "rewards/accuracy_reward/mean": 0.875, + "rewards/accuracy_reward/std": 0.33601075410842896, + "sampling/importance_sampling_ratio/max": 1.5814470052719116, + "sampling/importance_sampling_ratio/mean": 0.999999463558197, + "sampling/importance_sampling_ratio/min": 0.6377719640731812, + "sampling/sampling_logp_difference/max": 0.4583401679992676, + "sampling/sampling_logp_difference/mean": 0.0101484265178442, + "step": 809 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 668.0, + "completions/mean_length": 435.875, + "completions/mean_terminated_length": 425.1612854003906, + "completions/min_length": 163.0, + "completions/min_terminated_length": 163.0, + "entropy": 0.2633406352251768, + "epoch": 0.4336188436830835, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.021491779014468193, + "learning_rate": 1e-05, + "loss": 0.0484, + "num_tokens": 16222193.0, + "reward": 0.71875, + "reward_std": 0.35564959049224854, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.5119787454605103, + "sampling/importance_sampling_ratio/mean": 0.9998757243156433, + "sampling/importance_sampling_ratio/min": 0.30166324973106384, + "sampling/sampling_logp_difference/max": 1.19844388961792, + "sampling/sampling_logp_difference/mean": 0.01047497894614935, + "step": 810 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 8.207485370803624e-05, + "clip_ratio/low_min": 8.207485370803624e-05, + "clip_ratio/region_mean": 8.207485370803624e-05, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 738.0, + "completions/mean_length": 428.21875, + "completions/mean_terminated_length": 405.5666809082031, + "completions/min_length": 139.0, + "completions/min_terminated_length": 139.0, + "entropy": 0.25248922407627106, + "epoch": 0.4341541755888651, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.01177243608981371, + "learning_rate": 1e-05, + "loss": 0.0085, + "num_tokens": 16239528.0, + "reward": 0.65625, + "reward_std": 0.22201895713806152, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 2.0, + "sampling/importance_sampling_ratio/mean": 1.000215768814087, + "sampling/importance_sampling_ratio/min": 0.7016726136207581, + "sampling/sampling_logp_difference/max": 0.7899539470672607, + "sampling/sampling_logp_difference/mean": 0.009390268474817276, + "step": 811 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 6.0444875998655334e-05, + "clip_ratio/low_min": 6.0444875998655334e-05, + "clip_ratio/region_mean": 6.0444875998655334e-05, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 703.0, + "completions/mean_length": 489.96875, + "completions/mean_terminated_length": 471.433349609375, + "completions/min_length": 277.0, + "completions/min_terminated_length": 277.0, + "entropy": 0.28769651986658573, + "epoch": 0.4346895074946467, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.0207393579185009, + "learning_rate": 1e-05, + "loss": 0.0106, + "num_tokens": 16258999.0, + "reward": 0.75, + "reward_std": 0.2587745785713196, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.388028860092163, + "sampling/importance_sampling_ratio/mean": 0.9997342228889465, + "sampling/importance_sampling_ratio/min": 0.5272652506828308, + "sampling/sampling_logp_difference/max": 0.6400516033172607, + "sampling/sampling_logp_difference/mean": 0.01168107334524393, + "step": 812 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 734.0, + "completions/mean_length": 503.59375, + "completions/mean_terminated_length": 465.8214416503906, + "completions/min_length": 252.0, + "completions/min_terminated_length": 252.0, + "entropy": 0.32064057886600494, + "epoch": 0.4352248394004283, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.025505701079964638, + "learning_rate": 1e-05, + "loss": -0.0651, + "num_tokens": 16278738.0, + "reward": 0.40625, + "reward_std": 0.3061639666557312, + "rewards/accuracy_reward/mean": 0.40625, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.4701932668685913, + "sampling/importance_sampling_ratio/mean": 1.000252366065979, + "sampling/importance_sampling_ratio/min": 0.4715697467327118, + "sampling/sampling_logp_difference/max": 0.7516883611679077, + "sampling/sampling_logp_difference/mean": 0.011873779818415642, + "step": 813 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 9.342301927972585e-05, + "clip_ratio/low_min": 9.342301927972585e-05, + "clip_ratio/region_mean": 9.342301927972585e-05, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 525.0, + "completions/mean_length": 366.25, + "completions/mean_terminated_length": 353.2903137207031, + "completions/min_length": 117.0, + "completions/min_terminated_length": 117.0, + "entropy": 0.2732634488493204, + "epoch": 0.43576017130620986, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.012402699328958988, + "learning_rate": 1e-05, + "loss": 0.0694, + "num_tokens": 16294066.0, + "reward": 0.59375, + "reward_std": 0.4534739851951599, + "rewards/accuracy_reward/mean": 0.59375, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.687206745147705, + "sampling/importance_sampling_ratio/mean": 1.0001044273376465, + "sampling/importance_sampling_ratio/min": 0.4808214604854584, + "sampling/sampling_logp_difference/max": 0.7322592735290527, + "sampling/sampling_logp_difference/mean": 0.01016717217862606, + "step": 814 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00013087536353850737, + "clip_ratio/low_min": 0.00013087536353850737, + "clip_ratio/region_mean": 0.00013087536353850737, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 656.0, + "completions/mean_length": 473.28125, + "completions/mean_terminated_length": 442.7930908203125, + "completions/min_length": 280.0, + "completions/min_terminated_length": 280.0, + "entropy": 0.3047125283628702, + "epoch": 0.43629550321199145, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.015324367210268974, + "learning_rate": 1e-05, + "loss": 0.0015, + "num_tokens": 16312779.0, + "reward": 0.65625, + "reward_std": 0.4397946000099182, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.4569432735443115, + "sampling/importance_sampling_ratio/mean": 1.0001065731048584, + "sampling/importance_sampling_ratio/min": 0.5850861668586731, + "sampling/sampling_logp_difference/max": 0.5359960794448853, + "sampling/sampling_logp_difference/mean": 0.010756208561360836, + "step": 815 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 9.117432637140155e-05, + "clip_ratio/low_min": 9.117432637140155e-05, + "clip_ratio/region_mean": 9.117432637140155e-05, + "completions/clipped_ratio": 0.0, + "completions/max_length": 496.0, + "completions/max_terminated_length": 496.0, + "completions/mean_length": 350.40625, + "completions/mean_terminated_length": 350.40625, + "completions/min_length": 239.0, + "completions/min_terminated_length": 239.0, + "entropy": 0.3057396877557039, + "epoch": 0.43683083511777304, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.015559912659227848, + "learning_rate": 1e-05, + "loss": 0.03, + "num_tokens": 16327880.0, + "reward": 0.875, + "reward_std": 0.2177756428718567, + "rewards/accuracy_reward/mean": 0.875, + "rewards/accuracy_reward/std": 0.33601075410842896, + "sampling/importance_sampling_ratio/max": 1.5554462671279907, + "sampling/importance_sampling_ratio/mean": 1.000300407409668, + "sampling/importance_sampling_ratio/min": 0.6858586072921753, + "sampling/sampling_logp_difference/max": 0.44176244735717773, + "sampling/sampling_logp_difference/mean": 0.011645578779280186, + "step": 816 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 7.866582745919004e-05, + "clip_ratio/low_min": 7.866582745919004e-05, + "clip_ratio/region_mean": 7.866582745919004e-05, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 734.0, + "completions/mean_length": 362.53125, + "completions/mean_terminated_length": 335.5000305175781, + "completions/min_length": 122.0, + "completions/min_terminated_length": 122.0, + "entropy": 0.32446580566465855, + "epoch": 0.43736616702355463, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.028071025386452675, + "learning_rate": 1e-05, + "loss": 0.058, + "num_tokens": 16343377.0, + "reward": 0.90625, + "reward_std": 0.2041158676147461, + "rewards/accuracy_reward/mean": 0.90625, + "rewards/accuracy_reward/std": 0.2961445748806, + "sampling/importance_sampling_ratio/max": 1.6221219301223755, + "sampling/importance_sampling_ratio/mean": 1.0005759000778198, + "sampling/importance_sampling_ratio/min": 0.5040833353996277, + "sampling/sampling_logp_difference/max": 0.6850136518478394, + "sampling/sampling_logp_difference/mean": 0.011989017017185688, + "step": 817 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00027901858265977353, + "clip_ratio/low_min": 0.00027901858265977353, + "clip_ratio/region_mean": 0.00027901858265977353, + "completions/clipped_ratio": 0.0, + "completions/max_length": 664.0, + "completions/max_terminated_length": 664.0, + "completions/mean_length": 462.9375, + "completions/mean_terminated_length": 462.9375, + "completions/min_length": 327.0, + "completions/min_terminated_length": 327.0, + "entropy": 0.3302745521068573, + "epoch": 0.43790149892933616, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.010575388558208942, + "learning_rate": 1e-05, + "loss": -0.0575, + "num_tokens": 16362679.0, + "reward": 0.65625, + "reward_std": 0.3198433816432953, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.608583927154541, + "sampling/importance_sampling_ratio/mean": 1.000242829322815, + "sampling/importance_sampling_ratio/min": 0.6750907897949219, + "sampling/sampling_logp_difference/max": 0.4753541946411133, + "sampling/sampling_logp_difference/mean": 0.011667050421237946, + "step": 818 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.71037016925402e-05, + "clip_ratio/low_min": 5.71037016925402e-05, + "clip_ratio/region_mean": 5.71037016925402e-05, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 764.0, + "completions/mean_length": 489.4375, + "completions/mean_terminated_length": 470.86669921875, + "completions/min_length": 272.0, + "completions/min_terminated_length": 272.0, + "entropy": 0.3521044850349426, + "epoch": 0.43843683083511775, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.012330119498074055, + "learning_rate": 1e-05, + "loss": 0.0152, + "num_tokens": 16382245.0, + "reward": 0.75, + "reward_std": 0.2587745785713196, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.6827696561813354, + "sampling/importance_sampling_ratio/mean": 0.9994314908981323, + "sampling/importance_sampling_ratio/min": 0.6177808046340942, + "sampling/sampling_logp_difference/max": 0.5204410552978516, + "sampling/sampling_logp_difference/mean": 0.012086994014680386, + "step": 819 + }, + { + "clip_ratio/high_max": 8.164597966242582e-05, + "clip_ratio/high_mean": 8.164597966242582e-05, + "clip_ratio/low_mean": 0.00023453583708032966, + "clip_ratio/low_min": 0.00023453583708032966, + "clip_ratio/region_mean": 0.00031618181674275547, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 708.0, + "completions/mean_length": 459.09375, + "completions/mean_terminated_length": 427.137939453125, + "completions/min_length": 212.0, + "completions/min_terminated_length": 212.0, + "entropy": 0.3209436070173979, + "epoch": 0.43897216274089934, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.018147796392440796, + "learning_rate": 1e-05, + "loss": 0.114, + "num_tokens": 16400768.0, + "reward": 0.65625, + "reward_std": 0.4628904461860657, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.606744647026062, + "sampling/importance_sampling_ratio/mean": 1.0000828504562378, + "sampling/importance_sampling_ratio/min": 0.5109875202178955, + "sampling/sampling_logp_difference/max": 0.671410083770752, + "sampling/sampling_logp_difference/mean": 0.011897324584424496, + "step": 820 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0001959901928785257, + "clip_ratio/low_min": 0.0001959901928785257, + "clip_ratio/region_mean": 0.0001959901928785257, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 737.0, + "completions/mean_length": 576.0, + "completions/mean_terminated_length": 522.239990234375, + "completions/min_length": 203.0, + "completions/min_terminated_length": 203.0, + "entropy": 0.3952139653265476, + "epoch": 0.43950749464668093, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.01596885733306408, + "learning_rate": 1e-05, + "loss": 0.038, + "num_tokens": 16424760.0, + "reward": 0.46875, + "reward_std": 0.47137707471847534, + "rewards/accuracy_reward/mean": 0.46875, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.8614606857299805, + "sampling/importance_sampling_ratio/mean": 1.0003068447113037, + "sampling/importance_sampling_ratio/min": 0.613844633102417, + "sampling/sampling_logp_difference/max": 0.621361494064331, + "sampling/sampling_logp_difference/mean": 0.014272034168243408, + "step": 821 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 619.0, + "completions/mean_length": 380.3125, + "completions/mean_terminated_length": 367.8064270019531, + "completions/min_length": 158.0, + "completions/min_terminated_length": 158.0, + "entropy": 0.27511613443493843, + "epoch": 0.4400428265524625, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.023855295032262802, + "learning_rate": 1e-05, + "loss": 0.0241, + "num_tokens": 16440602.0, + "reward": 0.90625, + "reward_std": 0.1293872892856598, + "rewards/accuracy_reward/mean": 0.90625, + "rewards/accuracy_reward/std": 0.2961445748806, + "sampling/importance_sampling_ratio/max": 1.6222468614578247, + "sampling/importance_sampling_ratio/mean": 1.000023365020752, + "sampling/importance_sampling_ratio/min": 0.40764185786247253, + "sampling/sampling_logp_difference/max": 0.8973662853240967, + "sampling/sampling_logp_difference/mean": 0.010873417370021343, + "step": 822 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00024445212693535723, + "clip_ratio/low_min": 0.00024445212693535723, + "clip_ratio/region_mean": 0.00024445212693535723, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 682.0, + "completions/mean_length": 501.6875, + "completions/mean_terminated_length": 452.370361328125, + "completions/min_length": 233.0, + "completions/min_terminated_length": 233.0, + "entropy": 0.3429740257561207, + "epoch": 0.4405781584582441, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.0056623490527272224, + "learning_rate": 1e-05, + "loss": 0.0578, + "num_tokens": 16460352.0, + "reward": 0.5625, + "reward_std": 0.292504221200943, + "rewards/accuracy_reward/mean": 0.5625, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 1.5929806232452393, + "sampling/importance_sampling_ratio/mean": 0.999823808670044, + "sampling/importance_sampling_ratio/min": 0.554802656173706, + "sampling/sampling_logp_difference/max": 0.5891427993774414, + "sampling/sampling_logp_difference/mean": 0.012048034928739071, + "step": 823 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 6.259389192564413e-05, + "clip_ratio/low_min": 6.259389192564413e-05, + "clip_ratio/region_mean": 6.259389192564413e-05, + "completions/clipped_ratio": 0.0, + "completions/max_length": 605.0, + "completions/max_terminated_length": 605.0, + "completions/mean_length": 408.71875, + "completions/mean_terminated_length": 408.71875, + "completions/min_length": 120.0, + "completions/min_terminated_length": 120.0, + "entropy": 0.2218430433422327, + "epoch": 0.4411134903640257, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.008542133495211601, + "learning_rate": 1e-05, + "loss": -0.0061, + "num_tokens": 16476727.0, + "reward": 0.875, + "reward_std": 0.13363061845302582, + "rewards/accuracy_reward/mean": 0.875, + "rewards/accuracy_reward/std": 0.33601075410842896, + "sampling/importance_sampling_ratio/max": 1.6581839323043823, + "sampling/importance_sampling_ratio/mean": 0.9997780323028564, + "sampling/importance_sampling_ratio/min": 0.5478554964065552, + "sampling/sampling_logp_difference/max": 0.6017436981201172, + "sampling/sampling_logp_difference/mean": 0.009161354042589664, + "step": 824 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0001178507627628278, + "clip_ratio/low_min": 0.0001178507627628278, + "clip_ratio/region_mean": 0.0001178507627628278, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 738.0, + "completions/mean_length": 515.1875, + "completions/mean_terminated_length": 468.370361328125, + "completions/min_length": 184.0, + "completions/min_terminated_length": 184.0, + "entropy": 0.3462813012301922, + "epoch": 0.4416488222698073, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.021326366811990738, + "learning_rate": 1e-05, + "loss": 0.0152, + "num_tokens": 16497197.0, + "reward": 0.6875, + "reward_std": 0.3924051821231842, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.4823440313339233, + "sampling/importance_sampling_ratio/mean": 1.000078797340393, + "sampling/importance_sampling_ratio/min": 0.6797965168952942, + "sampling/sampling_logp_difference/max": 0.3936246633529663, + "sampling/sampling_logp_difference/mean": 0.012647273950278759, + "step": 825 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00017289073730353266, + "clip_ratio/low_min": 0.00017289073730353266, + "clip_ratio/region_mean": 0.00017289073730353266, + "completions/clipped_ratio": 0.0, + "completions/max_length": 595.0, + "completions/max_terminated_length": 595.0, + "completions/mean_length": 362.25, + "completions/mean_terminated_length": 362.25, + "completions/min_length": 141.0, + "completions/min_terminated_length": 141.0, + "entropy": 0.2770177647471428, + "epoch": 0.4421841541755889, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.018211321905255318, + "learning_rate": 1e-05, + "loss": 0.0242, + "num_tokens": 16512157.0, + "reward": 0.75, + "reward_std": 0.2314550280570984, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.622521162033081, + "sampling/importance_sampling_ratio/mean": 1.0004990100860596, + "sampling/importance_sampling_ratio/min": 0.46850597858428955, + "sampling/sampling_logp_difference/max": 0.7582063674926758, + "sampling/sampling_logp_difference/mean": 0.011050463654100895, + "step": 826 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 7.649938925169408e-05, + "clip_ratio/low_min": 7.649938925169408e-05, + "clip_ratio/region_mean": 7.649938925169408e-05, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 638.0, + "completions/mean_length": 422.59375, + "completions/mean_terminated_length": 386.862060546875, + "completions/min_length": 193.0, + "completions/min_terminated_length": 193.0, + "entropy": 0.48511166125535965, + "epoch": 0.44271948608137046, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.019551947712898254, + "learning_rate": 1e-05, + "loss": 0.0532, + "num_tokens": 16529360.0, + "reward": 0.625, + "reward_std": 0.3104073107242584, + "rewards/accuracy_reward/mean": 0.625, + "rewards/accuracy_reward/std": 0.49186936020851135, + "sampling/importance_sampling_ratio/max": 1.5627206563949585, + "sampling/importance_sampling_ratio/mean": 1.0000622272491455, + "sampling/importance_sampling_ratio/min": 0.6936726570129395, + "sampling/sampling_logp_difference/max": 0.4464282989501953, + "sampling/sampling_logp_difference/mean": 0.014928963035345078, + "step": 827 + }, + { + "clip_ratio/high_max": 6.917543214512989e-05, + "clip_ratio/high_mean": 6.917543214512989e-05, + "clip_ratio/low_mean": 9.742790280142799e-05, + "clip_ratio/low_min": 9.742790280142799e-05, + "clip_ratio/region_mean": 0.00016660333494655788, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 639.0, + "completions/mean_length": 380.5, + "completions/mean_terminated_length": 354.66668701171875, + "completions/min_length": 192.0, + "completions/min_terminated_length": 192.0, + "entropy": 0.348480436950922, + "epoch": 0.44325481798715205, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.021738076582551003, + "learning_rate": 1e-05, + "loss": -0.0607, + "num_tokens": 16545160.0, + "reward": 0.71875, + "reward_std": 0.3471629321575165, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.485531210899353, + "sampling/importance_sampling_ratio/mean": 1.0001122951507568, + "sampling/importance_sampling_ratio/min": 0.6237407922744751, + "sampling/sampling_logp_difference/max": 0.47202038764953613, + "sampling/sampling_logp_difference/mean": 0.012043345719575882, + "step": 828 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 732.0, + "completions/mean_length": 491.90625, + "completions/mean_terminated_length": 463.3448181152344, + "completions/min_length": 247.0, + "completions/min_terminated_length": 247.0, + "entropy": 0.25553832203149796, + "epoch": 0.44379014989293364, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.004612386226654053, + "learning_rate": 1e-05, + "loss": 0.0455, + "num_tokens": 16564685.0, + "reward": 0.65625, + "reward_std": 0.2651650309562683, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 2.0, + "sampling/importance_sampling_ratio/mean": 1.000020146369934, + "sampling/importance_sampling_ratio/min": 0.6000391840934753, + "sampling/sampling_logp_difference/max": 0.7492682933807373, + "sampling/sampling_logp_difference/mean": 0.010166849941015244, + "step": 829 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 4.1295010305475444e-05, + "clip_ratio/low_min": 4.1295010305475444e-05, + "clip_ratio/region_mean": 4.1295010305475444e-05, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 724.0, + "completions/mean_length": 505.21875, + "completions/mean_terminated_length": 431.6399841308594, + "completions/min_length": 185.0, + "completions/min_terminated_length": 185.0, + "entropy": 0.48429663851857185, + "epoch": 0.4443254817987152, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.029873324558138847, + "learning_rate": 1e-05, + "loss": 0.0091, + "num_tokens": 16585540.0, + "reward": 0.59375, + "reward_std": 0.4628904461860657, + "rewards/accuracy_reward/mean": 0.59375, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.414668083190918, + "sampling/importance_sampling_ratio/mean": 0.9995958209037781, + "sampling/importance_sampling_ratio/min": 0.5749284625053406, + "sampling/sampling_logp_difference/max": 0.5535097122192383, + "sampling/sampling_logp_difference/mean": 0.015926599502563477, + "step": 830 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 6.466632476076484e-05, + "clip_ratio/low_min": 6.466632476076484e-05, + "clip_ratio/region_mean": 6.466632476076484e-05, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 719.0, + "completions/mean_length": 503.65625, + "completions/mean_terminated_length": 454.7037048339844, + "completions/min_length": 201.0, + "completions/min_terminated_length": 201.0, + "entropy": 0.3221813887357712, + "epoch": 0.44486081370449676, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.010256975889205933, + "learning_rate": 1e-05, + "loss": 0.0358, + "num_tokens": 16605745.0, + "reward": 0.71875, + "reward_std": 0.3471629321575165, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.736228585243225, + "sampling/importance_sampling_ratio/mean": 1.0005266666412354, + "sampling/importance_sampling_ratio/min": 0.6991448402404785, + "sampling/sampling_logp_difference/max": 0.5517152547836304, + "sampling/sampling_logp_difference/mean": 0.011762522161006927, + "step": 831 + }, + { + "clip_ratio/high_max": 0.00014737022866029292, + "clip_ratio/high_mean": 0.00014737022866029292, + "clip_ratio/low_mean": 0.0002193370382883586, + "clip_ratio/low_min": 0.0002193370382883586, + "clip_ratio/region_mean": 0.0003667072669486515, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 731.0, + "completions/mean_length": 431.375, + "completions/mean_terminated_length": 396.5517272949219, + "completions/min_length": 155.0, + "completions/min_terminated_length": 155.0, + "entropy": 0.3429530579596758, + "epoch": 0.44539614561027835, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.011722094379365444, + "learning_rate": 1e-05, + "loss": 0.0277, + "num_tokens": 16622845.0, + "reward": 0.78125, + "reward_std": 0.3608423173427582, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.451257348060608, + "sampling/importance_sampling_ratio/mean": 0.9998858571052551, + "sampling/importance_sampling_ratio/min": 0.3799667954444885, + "sampling/sampling_logp_difference/max": 0.9676713943481445, + "sampling/sampling_logp_difference/mean": 0.011598000302910805, + "step": 832 + }, + { + "clip_ratio/high_max": 6.811989442212507e-05, + "clip_ratio/high_mean": 6.811989442212507e-05, + "clip_ratio/low_mean": 6.380806735251099e-05, + "clip_ratio/low_min": 6.380806735251099e-05, + "clip_ratio/region_mean": 0.00013192796177463606, + "completions/clipped_ratio": 0.0, + "completions/max_length": 759.0, + "completions/max_terminated_length": 759.0, + "completions/mean_length": 451.6875, + "completions/mean_terminated_length": 451.6875, + "completions/min_length": 176.0, + "completions/min_terminated_length": 176.0, + "entropy": 0.24354325979948044, + "epoch": 0.44593147751605994, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.010895389132201672, + "learning_rate": 1e-05, + "loss": -0.0495, + "num_tokens": 16641283.0, + "reward": 0.875, + "reward_std": 0.2314550280570984, + "rewards/accuracy_reward/mean": 0.875, + "rewards/accuracy_reward/std": 0.33601075410842896, + "sampling/importance_sampling_ratio/max": 1.2997705936431885, + "sampling/importance_sampling_ratio/mean": 0.9999913573265076, + "sampling/importance_sampling_ratio/min": 0.308633416891098, + "sampling/sampling_logp_difference/max": 1.1756010055541992, + "sampling/sampling_logp_difference/mean": 0.00954093225300312, + "step": 833 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 7.327080675167963e-05, + "clip_ratio/low_min": 7.327080675167963e-05, + "clip_ratio/region_mean": 7.327080675167963e-05, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 743.0, + "completions/mean_length": 440.09375, + "completions/mean_terminated_length": 406.17242431640625, + "completions/min_length": 237.0, + "completions/min_terminated_length": 237.0, + "entropy": 0.4135780967772007, + "epoch": 0.4464668094218415, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.02072235755622387, + "learning_rate": 1e-05, + "loss": 0.0604, + "num_tokens": 16659102.0, + "reward": 0.5625, + "reward_std": 0.3514062464237213, + "rewards/accuracy_reward/mean": 0.5625, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 2.0, + "sampling/importance_sampling_ratio/mean": 1.000566840171814, + "sampling/importance_sampling_ratio/min": 0.5816425085067749, + "sampling/sampling_logp_difference/max": 0.7674446105957031, + "sampling/sampling_logp_difference/mean": 0.013563827611505985, + "step": 834 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 749.0, + "completions/mean_length": 479.90625, + "completions/mean_terminated_length": 413.423095703125, + "completions/min_length": 242.0, + "completions/min_terminated_length": 242.0, + "entropy": 0.2676452100276947, + "epoch": 0.4470021413276231, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.019833054393529892, + "learning_rate": 1e-05, + "loss": 0.026, + "num_tokens": 16678307.0, + "reward": 0.40625, + "reward_std": 0.378745436668396, + "rewards/accuracy_reward/mean": 0.40625, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 2.0, + "sampling/importance_sampling_ratio/mean": 1.0006767511367798, + "sampling/importance_sampling_ratio/min": 0.6516871452331543, + "sampling/sampling_logp_difference/max": 0.9411327838897705, + "sampling/sampling_logp_difference/mean": 0.009629964828491211, + "step": 835 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 530.0, + "completions/max_terminated_length": 530.0, + "completions/mean_length": 354.15625, + "completions/mean_terminated_length": 354.15625, + "completions/min_length": 208.0, + "completions/min_terminated_length": 208.0, + "entropy": 0.22570085898041725, + "epoch": 0.4475374732334047, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.03132038190960884, + "learning_rate": 1e-05, + "loss": 0.0139, + "num_tokens": 16692928.0, + "reward": 0.96875, + "reward_std": 0.0883883461356163, + "rewards/accuracy_reward/mean": 0.96875, + "rewards/accuracy_reward/std": 0.1767766922712326, + "sampling/importance_sampling_ratio/max": 1.7164597511291504, + "sampling/importance_sampling_ratio/mean": 1.0002472400665283, + "sampling/importance_sampling_ratio/min": 0.4757560193538666, + "sampling/sampling_logp_difference/max": 0.7428500652313232, + "sampling/sampling_logp_difference/mean": 0.00921986810863018, + "step": 836 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 716.0, + "completions/mean_length": 530.15625, + "completions/mean_terminated_length": 463.55999755859375, + "completions/min_length": 233.0, + "completions/min_terminated_length": 233.0, + "entropy": 0.3691186085343361, + "epoch": 0.4480728051391863, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.017191870138049126, + "learning_rate": 1e-05, + "loss": 0.0078, + "num_tokens": 16713805.0, + "reward": 0.5, + "reward_std": 0.3650856614112854, + "rewards/accuracy_reward/mean": 0.5, + "rewards/accuracy_reward/std": 0.5080004930496216, + "sampling/importance_sampling_ratio/max": 1.7796034812927246, + "sampling/importance_sampling_ratio/mean": 1.000518560409546, + "sampling/importance_sampling_ratio/min": 0.7095998525619507, + "sampling/sampling_logp_difference/max": 0.5763905048370361, + "sampling/sampling_logp_difference/mean": 0.012961843982338905, + "step": 837 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 6.380806735251099e-05, + "clip_ratio/low_min": 6.380806735251099e-05, + "clip_ratio/region_mean": 6.380806735251099e-05, + "completions/clipped_ratio": 0.0, + "completions/max_length": 724.0, + "completions/max_terminated_length": 724.0, + "completions/mean_length": 519.8125, + "completions/mean_terminated_length": 519.8125, + "completions/min_length": 295.0, + "completions/min_terminated_length": 295.0, + "entropy": 0.26343732699751854, + "epoch": 0.4486081370449679, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": 0.0167, + "num_tokens": 16734303.0, + "reward": 0.90625, + "reward_std": 0.1293872892856598, + "rewards/accuracy_reward/mean": 0.90625, + "rewards/accuracy_reward/std": 0.2961445748806, + "sampling/importance_sampling_ratio/max": 1.9722262620925903, + "sampling/importance_sampling_ratio/mean": 0.9998405575752258, + "sampling/importance_sampling_ratio/min": 0.5341039299964905, + "sampling/sampling_logp_difference/max": 0.6791629791259766, + "sampling/sampling_logp_difference/mean": 0.011166399344801903, + "step": 838 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00021222411305643618, + "clip_ratio/low_min": 0.00021222411305643618, + "clip_ratio/region_mean": 0.00021222411305643618, + "completions/clipped_ratio": 0.0, + "completions/max_length": 622.0, + "completions/max_terminated_length": 622.0, + "completions/mean_length": 388.5625, + "completions/mean_terminated_length": 388.5625, + "completions/min_length": 92.0, + "completions/min_terminated_length": 92.0, + "entropy": 0.34673846885561943, + "epoch": 0.44914346895074947, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.02089758776128292, + "learning_rate": 1e-05, + "loss": 0.0058, + "num_tokens": 16750809.0, + "reward": 0.8125, + "reward_std": 0.3335031569004059, + "rewards/accuracy_reward/mean": 0.8125, + "rewards/accuracy_reward/std": 0.3965577781200409, + "sampling/importance_sampling_ratio/max": 2.0, + "sampling/importance_sampling_ratio/mean": 1.0003050565719604, + "sampling/importance_sampling_ratio/min": 0.33226293325424194, + "sampling/sampling_logp_difference/max": 1.1018285751342773, + "sampling/sampling_logp_difference/mean": 0.012727145105600357, + "step": 839 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00020833333837799728, + "clip_ratio/low_min": 0.00020833333837799728, + "clip_ratio/region_mean": 0.00020833333837799728, + "completions/clipped_ratio": 0.0, + "completions/max_length": 716.0, + "completions/max_terminated_length": 716.0, + "completions/mean_length": 465.59375, + "completions/mean_terminated_length": 465.59375, + "completions/min_length": 134.0, + "completions/min_terminated_length": 134.0, + "entropy": 0.21557242050766945, + "epoch": 0.44967880085653106, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.01053936593234539, + "learning_rate": 1e-05, + "loss": 0.0065, + "num_tokens": 16769740.0, + "reward": 0.875, + "reward_std": 0.13363061845302582, + "rewards/accuracy_reward/mean": 0.875, + "rewards/accuracy_reward/std": 0.33601075410842896, + "sampling/importance_sampling_ratio/max": 1.4637336730957031, + "sampling/importance_sampling_ratio/mean": 1.000529408454895, + "sampling/importance_sampling_ratio/min": 0.6094999313354492, + "sampling/sampling_logp_difference/max": 0.4951164722442627, + "sampling/sampling_logp_difference/mean": 0.00881272740662098, + "step": 840 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.3125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 746.0, + "completions/mean_length": 579.09375, + "completions/mean_terminated_length": 493.227294921875, + "completions/min_length": 254.0, + "completions/min_terminated_length": 254.0, + "entropy": 0.3651367239654064, + "epoch": 0.45021413276231265, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.002016344340518117, + "learning_rate": 1e-05, + "loss": 0.0184, + "num_tokens": 16792871.0, + "reward": 0.4375, + "reward_std": 0.1767766922712326, + "rewards/accuracy_reward/mean": 0.4375, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 1.6037328243255615, + "sampling/importance_sampling_ratio/mean": 1.0000085830688477, + "sampling/importance_sampling_ratio/min": 0.6860070824623108, + "sampling/sampling_logp_difference/max": 0.4723339080810547, + "sampling/sampling_logp_difference/mean": 0.012109356932342052, + "step": 841 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 715.0, + "completions/mean_length": 440.125, + "completions/mean_terminated_length": 429.5483703613281, + "completions/min_length": 156.0, + "completions/min_terminated_length": 156.0, + "entropy": 0.31424909457564354, + "epoch": 0.45074946466809424, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.02117704227566719, + "learning_rate": 1e-05, + "loss": 0.013, + "num_tokens": 16810555.0, + "reward": 0.75, + "reward_std": 0.3514062464237213, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.5482828617095947, + "sampling/importance_sampling_ratio/mean": 0.9997996091842651, + "sampling/importance_sampling_ratio/min": 0.5919291973114014, + "sampling/sampling_logp_difference/max": 0.5243682861328125, + "sampling/sampling_logp_difference/mean": 0.011482161469757557, + "step": 842 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 630.0, + "completions/max_terminated_length": 630.0, + "completions/mean_length": 384.0, + "completions/mean_terminated_length": 384.0, + "completions/min_length": 224.0, + "completions/min_terminated_length": 224.0, + "entropy": 0.2951059080660343, + "epoch": 0.4512847965738758, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.014482242986559868, + "learning_rate": 1e-05, + "loss": 0.007, + "num_tokens": 16826579.0, + "reward": 0.84375, + "reward_std": 0.1293872892856598, + "rewards/accuracy_reward/mean": 0.84375, + "rewards/accuracy_reward/std": 0.3689020276069641, + "sampling/importance_sampling_ratio/max": 1.599572777748108, + "sampling/importance_sampling_ratio/mean": 0.9997570514678955, + "sampling/importance_sampling_ratio/min": 0.7156587839126587, + "sampling/sampling_logp_difference/max": 0.46973657608032227, + "sampling/sampling_logp_difference/mean": 0.010345741175115108, + "step": 843 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 7.431628910126165e-05, + "clip_ratio/low_min": 7.431628910126165e-05, + "clip_ratio/region_mean": 7.431628910126165e-05, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 573.0, + "completions/mean_length": 405.4375, + "completions/mean_terminated_length": 367.9310302734375, + "completions/min_length": 187.0, + "completions/min_terminated_length": 187.0, + "entropy": 0.45297108963131905, + "epoch": 0.4518201284796574, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.008480343036353588, + "learning_rate": 1e-05, + "loss": -0.013, + "num_tokens": 16843049.0, + "reward": 0.65625, + "reward_std": 0.1293872892856598, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.5063261985778809, + "sampling/importance_sampling_ratio/mean": 0.9999780058860779, + "sampling/importance_sampling_ratio/min": 0.5270180106163025, + "sampling/sampling_logp_difference/max": 0.6405205726623535, + "sampling/sampling_logp_difference/mean": 0.015170668251812458, + "step": 844 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 679.0, + "completions/mean_length": 519.03125, + "completions/mean_terminated_length": 483.46429443359375, + "completions/min_length": 230.0, + "completions/min_terminated_length": 230.0, + "entropy": 0.3445496968924999, + "epoch": 0.45235546038543895, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.004610029514878988, + "learning_rate": 1e-05, + "loss": -0.008, + "num_tokens": 16863986.0, + "reward": 0.75, + "reward_std": 0.1767766922712326, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.4119755029678345, + "sampling/importance_sampling_ratio/mean": 0.9999881386756897, + "sampling/importance_sampling_ratio/min": 0.6070384383201599, + "sampling/sampling_logp_difference/max": 0.4991631507873535, + "sampling/sampling_logp_difference/mean": 0.012054568156599998, + "step": 845 + }, + { + "clip_ratio/high_max": 7.720815483480692e-05, + "clip_ratio/high_mean": 7.720815483480692e-05, + "clip_ratio/low_mean": 0.00013008988025831059, + "clip_ratio/low_min": 0.00013008988025831059, + "clip_ratio/region_mean": 0.0002072980350931175, + "completions/clipped_ratio": 0.0, + "completions/max_length": 698.0, + "completions/max_terminated_length": 698.0, + "completions/mean_length": 464.65625, + "completions/mean_terminated_length": 464.65625, + "completions/min_length": 262.0, + "completions/min_terminated_length": 262.0, + "entropy": 0.2506127431988716, + "epoch": 0.45289079229122053, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.011979878880083561, + "learning_rate": 1e-05, + "loss": -0.0336, + "num_tokens": 16882423.0, + "reward": 0.6875, + "reward_std": 0.249358132481575, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.5981649160385132, + "sampling/importance_sampling_ratio/mean": 1.0001941919326782, + "sampling/importance_sampling_ratio/min": 0.6572299003601074, + "sampling/sampling_logp_difference/max": 0.4688560962677002, + "sampling/sampling_logp_difference/mean": 0.01008039154112339, + "step": 846 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 6.830601341789588e-05, + "clip_ratio/low_min": 6.830601341789588e-05, + "clip_ratio/region_mean": 6.830601341789588e-05, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 681.0, + "completions/mean_length": 472.125, + "completions/mean_terminated_length": 462.58062744140625, + "completions/min_length": 275.0, + "completions/min_terminated_length": 275.0, + "entropy": 0.3238905295729637, + "epoch": 0.4534261241970021, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.0057082343846559525, + "learning_rate": 1e-05, + "loss": 0.0592, + "num_tokens": 16901515.0, + "reward": 0.75, + "reward_std": 0.2314550280570984, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.5296075344085693, + "sampling/importance_sampling_ratio/mean": 0.9998763203620911, + "sampling/importance_sampling_ratio/min": 0.7057211995124817, + "sampling/sampling_logp_difference/max": 0.42501115798950195, + "sampling/sampling_logp_difference/mean": 0.011838079430162907, + "step": 847 + }, + { + "clip_ratio/high_max": 4.5620436139870435e-05, + "clip_ratio/high_mean": 4.5620436139870435e-05, + "clip_ratio/low_mean": 0.00015090232773218304, + "clip_ratio/low_min": 0.00015090232773218304, + "clip_ratio/region_mean": 0.00019652276387205347, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 728.0, + "completions/mean_length": 601.40625, + "completions/mean_terminated_length": 562.9615478515625, + "completions/min_length": 369.0, + "completions/min_terminated_length": 369.0, + "entropy": 0.32016063295304775, + "epoch": 0.4539614561027837, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.0071393828839063644, + "learning_rate": 1e-05, + "loss": -0.0026, + "num_tokens": 16925272.0, + "reward": 0.71875, + "reward_std": 0.24511480331420898, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.461082100868225, + "sampling/importance_sampling_ratio/mean": 0.9995551109313965, + "sampling/importance_sampling_ratio/min": 0.680505633354187, + "sampling/sampling_logp_difference/max": 0.3849191665649414, + "sampling/sampling_logp_difference/mean": 0.011756669729948044, + "step": 848 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 6.641870277235284e-05, + "clip_ratio/low_min": 6.641870277235284e-05, + "clip_ratio/region_mean": 6.641870277235284e-05, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 669.0, + "completions/mean_length": 453.71875, + "completions/mean_terminated_length": 443.58062744140625, + "completions/min_length": 241.0, + "completions/min_terminated_length": 241.0, + "entropy": 0.29842460341751575, + "epoch": 0.4544967880085653, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.01773042045533657, + "learning_rate": 1e-05, + "loss": 0.0629, + "num_tokens": 16943479.0, + "reward": 0.78125, + "reward_std": 0.4355708956718445, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.3509780168533325, + "sampling/importance_sampling_ratio/mean": 0.9998967051506042, + "sampling/importance_sampling_ratio/min": 0.5830528736114502, + "sampling/sampling_logp_difference/max": 0.5394773483276367, + "sampling/sampling_logp_difference/mean": 0.010512765496969223, + "step": 849 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 746.0, + "completions/mean_length": 468.65625, + "completions/mean_terminated_length": 459.0, + "completions/min_length": 242.0, + "completions/min_terminated_length": 242.0, + "entropy": 0.19418418407440186, + "epoch": 0.4550321199143469, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": 0.0081, + "num_tokens": 16962076.0, + "reward": 0.84375, + "reward_std": 0.1293872892856598, + "rewards/accuracy_reward/mean": 0.84375, + "rewards/accuracy_reward/std": 0.3689020276069641, + "sampling/importance_sampling_ratio/max": 1.3339976072311401, + "sampling/importance_sampling_ratio/mean": 0.9999716877937317, + "sampling/importance_sampling_ratio/min": 0.6263445615768433, + "sampling/sampling_logp_difference/max": 0.46785473823547363, + "sampling/sampling_logp_difference/mean": 0.00778750516474247, + "step": 850 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 746.0, + "completions/mean_length": 510.375, + "completions/mean_terminated_length": 493.20001220703125, + "completions/min_length": 251.0, + "completions/min_terminated_length": 251.0, + "entropy": 0.3192160576581955, + "epoch": 0.4555674518201285, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.041246507316827774, + "learning_rate": 1e-05, + "loss": 0.0158, + "num_tokens": 16982248.0, + "reward": 0.75, + "reward_std": 0.4492306709289551, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.451161503791809, + "sampling/importance_sampling_ratio/mean": 0.9997957348823547, + "sampling/importance_sampling_ratio/min": 0.311160147190094, + "sampling/sampling_logp_difference/max": 1.167447566986084, + "sampling/sampling_logp_difference/mean": 0.011297615244984627, + "step": 851 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00018371613987255841, + "clip_ratio/low_min": 0.00018371613987255841, + "clip_ratio/region_mean": 0.00018371613987255841, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 744.0, + "completions/mean_length": 465.21875, + "completions/mean_terminated_length": 421.96429443359375, + "completions/min_length": 241.0, + "completions/min_terminated_length": 241.0, + "entropy": 0.29270661622285843, + "epoch": 0.45610278372591007, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.017661161720752716, + "learning_rate": 1e-05, + "loss": 0.0628, + "num_tokens": 17000943.0, + "reward": 0.5, + "reward_std": 0.2587745785713196, + "rewards/accuracy_reward/mean": 0.5, + "rewards/accuracy_reward/std": 0.5080004930496216, + "sampling/importance_sampling_ratio/max": 1.434083104133606, + "sampling/importance_sampling_ratio/mean": 0.999801516532898, + "sampling/importance_sampling_ratio/min": 0.6706390976905823, + "sampling/sampling_logp_difference/max": 0.3995242118835449, + "sampling/sampling_logp_difference/mean": 0.010368138551712036, + "step": 852 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.404236799222417e-05, + "clip_ratio/low_min": 5.404236799222417e-05, + "clip_ratio/region_mean": 5.404236799222417e-05, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 736.0, + "completions/mean_length": 436.625, + "completions/mean_terminated_length": 389.2857360839844, + "completions/min_length": 243.0, + "completions/min_terminated_length": 243.0, + "entropy": 0.34419820457696915, + "epoch": 0.45663811563169165, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.0037945820949971676, + "learning_rate": 1e-05, + "loss": 0.0854, + "num_tokens": 17018275.0, + "reward": 0.65625, + "reward_std": 0.2041158676147461, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.6534979343414307, + "sampling/importance_sampling_ratio/mean": 1.0004469156265259, + "sampling/importance_sampling_ratio/min": 0.514949381351471, + "sampling/sampling_logp_difference/max": 0.6636867523193359, + "sampling/sampling_logp_difference/mean": 0.011727089993655682, + "step": 853 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 687.0, + "completions/mean_length": 428.0625, + "completions/mean_terminated_length": 417.0967712402344, + "completions/min_length": 203.0, + "completions/min_terminated_length": 203.0, + "entropy": 0.3591027185320854, + "epoch": 0.45717344753747324, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.0074991523288190365, + "learning_rate": 1e-05, + "loss": 0.0344, + "num_tokens": 17035901.0, + "reward": 0.6875, + "reward_std": 0.292504221200943, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.604923129081726, + "sampling/importance_sampling_ratio/mean": 0.9997456073760986, + "sampling/importance_sampling_ratio/min": 0.635753333568573, + "sampling/sampling_logp_difference/max": 0.47307586669921875, + "sampling/sampling_logp_difference/mean": 0.012660740874707699, + "step": 854 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 758.0, + "completions/mean_length": 458.46875, + "completions/mean_terminated_length": 401.1481628417969, + "completions/min_length": 250.0, + "completions/min_terminated_length": 250.0, + "entropy": 0.3138784375041723, + "epoch": 0.45770877944325483, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.005024927668273449, + "learning_rate": 1e-05, + "loss": 0.0949, + "num_tokens": 17054196.0, + "reward": 0.8125, + "reward_std": 0.2177756428718567, + "rewards/accuracy_reward/mean": 0.8125, + "rewards/accuracy_reward/std": 0.3965577781200409, + "sampling/importance_sampling_ratio/max": 1.7859833240509033, + "sampling/importance_sampling_ratio/mean": 0.9999326467514038, + "sampling/importance_sampling_ratio/min": 0.6756333112716675, + "sampling/sampling_logp_difference/max": 0.5799691677093506, + "sampling/sampling_logp_difference/mean": 0.011334572918713093, + "step": 855 + }, + { + "clip_ratio/high_max": 6.998880417086184e-05, + "clip_ratio/high_mean": 6.998880417086184e-05, + "clip_ratio/low_mean": 7.817385630914941e-05, + "clip_ratio/low_min": 7.817385630914941e-05, + "clip_ratio/region_mean": 0.00014816266048001125, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 745.0, + "completions/mean_length": 460.0, + "completions/mean_terminated_length": 450.06451416015625, + "completions/min_length": 291.0, + "completions/min_terminated_length": 291.0, + "entropy": 0.31339183636009693, + "epoch": 0.4582441113490364, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": 0.0246, + "num_tokens": 17072284.0, + "reward": 0.90625, + "reward_std": 0.2041158676147461, + "rewards/accuracy_reward/mean": 0.90625, + "rewards/accuracy_reward/std": 0.2961445748806, + "sampling/importance_sampling_ratio/max": 1.5173048973083496, + "sampling/importance_sampling_ratio/mean": 0.9996866583824158, + "sampling/importance_sampling_ratio/min": 0.716724157333374, + "sampling/sampling_logp_difference/max": 0.41693568229675293, + "sampling/sampling_logp_difference/mean": 0.010737800039350986, + "step": 856 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0002016609869315289, + "clip_ratio/low_min": 0.0002016609869315289, + "clip_ratio/region_mean": 0.0002016609869315289, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 725.0, + "completions/mean_length": 431.15625, + "completions/mean_terminated_length": 396.3103332519531, + "completions/min_length": 226.0, + "completions/min_terminated_length": 226.0, + "entropy": 0.30677182972431183, + "epoch": 0.458779443254818, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.02329942211508751, + "learning_rate": 1e-05, + "loss": 0.0093, + "num_tokens": 17089393.0, + "reward": 0.75, + "reward_std": 0.2587745785713196, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.6312874555587769, + "sampling/importance_sampling_ratio/mean": 1.000083088874817, + "sampling/importance_sampling_ratio/min": 0.655465841293335, + "sampling/sampling_logp_difference/max": 0.48936963081359863, + "sampling/sampling_logp_difference/mean": 0.010949400253593922, + "step": 857 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.72868921153713e-05, + "clip_ratio/low_min": 5.72868921153713e-05, + "clip_ratio/region_mean": 5.72868921153713e-05, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 728.0, + "completions/mean_length": 460.71875, + "completions/mean_terminated_length": 440.2333679199219, + "completions/min_length": 242.0, + "completions/min_terminated_length": 242.0, + "entropy": 0.28196910955011845, + "epoch": 0.4593147751605996, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.006725633516907692, + "learning_rate": 1e-05, + "loss": 0.03, + "num_tokens": 17108288.0, + "reward": 0.75, + "reward_std": 0.292504221200943, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 2.0, + "sampling/importance_sampling_ratio/mean": 0.9996670484542847, + "sampling/importance_sampling_ratio/min": 0.513340175151825, + "sampling/sampling_logp_difference/max": 0.9619584083557129, + "sampling/sampling_logp_difference/mean": 0.011167189106345177, + "step": 858 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.521201455849223e-05, + "clip_ratio/low_min": 5.521201455849223e-05, + "clip_ratio/region_mean": 5.521201455849223e-05, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 713.0, + "completions/mean_length": 456.15625, + "completions/mean_terminated_length": 423.89654541015625, + "completions/min_length": 116.0, + "completions/min_terminated_length": 116.0, + "entropy": 0.3315916433930397, + "epoch": 0.45985010706638113, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.026618383824825287, + "learning_rate": 1e-05, + "loss": 0.0142, + "num_tokens": 17126133.0, + "reward": 0.65625, + "reward_std": 0.3061639964580536, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.5123687982559204, + "sampling/importance_sampling_ratio/mean": 0.9998215436935425, + "sampling/importance_sampling_ratio/min": 0.6895114183425903, + "sampling/sampling_logp_difference/max": 0.4136772155761719, + "sampling/sampling_logp_difference/mean": 0.011936474591493607, + "step": 859 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 718.0, + "completions/mean_length": 422.4375, + "completions/mean_terminated_length": 386.6896667480469, + "completions/min_length": 222.0, + "completions/min_terminated_length": 222.0, + "entropy": 0.35348470509052277, + "epoch": 0.4603854389721627, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.021620135754346848, + "learning_rate": 1e-05, + "loss": 0.0112, + "num_tokens": 17143011.0, + "reward": 0.6875, + "reward_std": 0.44403791427612305, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.7950732707977295, + "sampling/importance_sampling_ratio/mean": 0.9996082782745361, + "sampling/importance_sampling_ratio/min": 0.586424708366394, + "sampling/sampling_logp_difference/max": 0.5850458145141602, + "sampling/sampling_logp_difference/mean": 0.012666827067732811, + "step": 860 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 6.364562432281673e-05, + "clip_ratio/low_min": 6.364562432281673e-05, + "clip_ratio/region_mean": 6.364562432281673e-05, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 741.0, + "completions/mean_length": 511.96875, + "completions/mean_terminated_length": 485.4827575683594, + "completions/min_length": 344.0, + "completions/min_terminated_length": 344.0, + "entropy": 0.22219637222588062, + "epoch": 0.4609207708779443, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.0042509338818490505, + "learning_rate": 1e-05, + "loss": 0.0104, + "num_tokens": 17162690.0, + "reward": 0.6875, + "reward_std": 0.292504221200943, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.33616042137146, + "sampling/importance_sampling_ratio/mean": 1.000166416168213, + "sampling/importance_sampling_ratio/min": 0.6979255676269531, + "sampling/sampling_logp_difference/max": 0.3596428632736206, + "sampling/sampling_logp_difference/mean": 0.007480310276150703, + "step": 861 + }, + { + "clip_ratio/high_max": 6.406971078831702e-05, + "clip_ratio/high_mean": 6.406971078831702e-05, + "clip_ratio/low_mean": 9.430403588339686e-05, + "clip_ratio/low_min": 9.430403588339686e-05, + "clip_ratio/region_mean": 0.0001583737466717139, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 658.0, + "completions/mean_length": 481.96875, + "completions/mean_terminated_length": 462.9000244140625, + "completions/min_length": 278.0, + "completions/min_terminated_length": 278.0, + "entropy": 0.24463251046836376, + "epoch": 0.4614561027837259, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.009309791028499603, + "learning_rate": 1e-05, + "loss": 0.0239, + "num_tokens": 17181721.0, + "reward": 0.6875, + "reward_std": 0.44403791427612305, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.488285779953003, + "sampling/importance_sampling_ratio/mean": 1.000335454940796, + "sampling/importance_sampling_ratio/min": 0.639365553855896, + "sampling/sampling_logp_difference/max": 0.4472789764404297, + "sampling/sampling_logp_difference/mean": 0.009572452865540981, + "step": 862 + }, + { + "clip_ratio/high_max": 6.42673549009487e-05, + "clip_ratio/high_mean": 6.42673549009487e-05, + "clip_ratio/low_mean": 5.681818220182322e-05, + "clip_ratio/low_min": 5.681818220182322e-05, + "clip_ratio/region_mean": 0.00012108553710277192, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 734.0, + "completions/mean_length": 522.75, + "completions/mean_terminated_length": 466.15386962890625, + "completions/min_length": 164.0, + "completions/min_terminated_length": 164.0, + "entropy": 0.3433074038475752, + "epoch": 0.4619914346895075, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.018548304215073586, + "learning_rate": 1e-05, + "loss": 0.0291, + "num_tokens": 17202217.0, + "reward": 0.6875, + "reward_std": 0.3745020925998688, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.8536497354507446, + "sampling/importance_sampling_ratio/mean": 0.9996280074119568, + "sampling/importance_sampling_ratio/min": 0.5494065880775452, + "sampling/sampling_logp_difference/max": 0.6171565055847168, + "sampling/sampling_logp_difference/mean": 0.012258768081665039, + "step": 863 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 9.018759010359645e-05, + "clip_ratio/low_min": 9.018759010359645e-05, + "clip_ratio/region_mean": 9.018759010359645e-05, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 550.0, + "completions/mean_length": 348.75, + "completions/mean_terminated_length": 335.2257995605469, + "completions/min_length": 208.0, + "completions/min_terminated_length": 208.0, + "entropy": 0.3397308196872473, + "epoch": 0.4625267665952891, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": 0.026, + "num_tokens": 17216633.0, + "reward": 0.8125, + "reward_std": 0.1157275140285492, + "rewards/accuracy_reward/mean": 0.8125, + "rewards/accuracy_reward/std": 0.3965577781200409, + "sampling/importance_sampling_ratio/max": 1.7661750316619873, + "sampling/importance_sampling_ratio/mean": 1.0003950595855713, + "sampling/importance_sampling_ratio/min": 0.5858866572380066, + "sampling/sampling_logp_difference/max": 0.5688161849975586, + "sampling/sampling_logp_difference/mean": 0.01183544285595417, + "step": 864 + }, + { + "clip_ratio/high_max": 6.262525130296126e-05, + "clip_ratio/high_mean": 6.262525130296126e-05, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 6.262525130296126e-05, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 682.0, + "completions/mean_length": 481.1875, + "completions/mean_terminated_length": 428.0740661621094, + "completions/min_length": 114.0, + "completions/min_terminated_length": 114.0, + "entropy": 0.3127823583781719, + "epoch": 0.46306209850107066, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.009447633288800716, + "learning_rate": 1e-05, + "loss": 0.0104, + "num_tokens": 17236047.0, + "reward": 0.59375, + "reward_std": 0.3987956643104553, + "rewards/accuracy_reward/mean": 0.59375, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.560917854309082, + "sampling/importance_sampling_ratio/mean": 1.000223159790039, + "sampling/importance_sampling_ratio/min": 0.6159913539886475, + "sampling/sampling_logp_difference/max": 0.48452234268188477, + "sampling/sampling_logp_difference/mean": 0.011035087518393993, + "step": 865 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 687.0, + "completions/max_terminated_length": 687.0, + "completions/mean_length": 407.5, + "completions/mean_terminated_length": 407.5, + "completions/min_length": 172.0, + "completions/min_terminated_length": 172.0, + "entropy": 0.23410633765161037, + "epoch": 0.46359743040685225, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.006699251011013985, + "learning_rate": 1e-05, + "loss": 0.0114, + "num_tokens": 17252615.0, + "reward": 0.9375, + "reward_std": 0.1767766922712326, + "rewards/accuracy_reward/mean": 0.9375, + "rewards/accuracy_reward/std": 0.24593468010425568, + "sampling/importance_sampling_ratio/max": 1.3216911554336548, + "sampling/importance_sampling_ratio/mean": 1.0000323057174683, + "sampling/importance_sampling_ratio/min": 0.7802764773368835, + "sampling/sampling_logp_difference/max": 0.2789120674133301, + "sampling/sampling_logp_difference/mean": 0.009173828177154064, + "step": 866 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 8.765778329689056e-05, + "clip_ratio/low_min": 8.765778329689056e-05, + "clip_ratio/region_mean": 8.765778329689056e-05, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 756.0, + "completions/mean_length": 521.8125, + "completions/mean_terminated_length": 496.3448181152344, + "completions/min_length": 205.0, + "completions/min_terminated_length": 205.0, + "entropy": 0.3517126627266407, + "epoch": 0.46413276231263384, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.01118876039981842, + "learning_rate": 1e-05, + "loss": 0.0676, + "num_tokens": 17273049.0, + "reward": 0.90625, + "reward_std": 0.1293872892856598, + "rewards/accuracy_reward/mean": 0.90625, + "rewards/accuracy_reward/std": 0.2961445748806, + "sampling/importance_sampling_ratio/max": 1.4212286472320557, + "sampling/importance_sampling_ratio/mean": 0.9997889399528503, + "sampling/importance_sampling_ratio/min": 0.6396047472953796, + "sampling/sampling_logp_difference/max": 0.44690489768981934, + "sampling/sampling_logp_difference/mean": 0.011829306371510029, + "step": 867 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 755.0, + "completions/mean_length": 431.21875, + "completions/mean_terminated_length": 420.3548278808594, + "completions/min_length": 180.0, + "completions/min_terminated_length": 180.0, + "entropy": 0.2854548469185829, + "epoch": 0.46466809421841543, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.020841553807258606, + "learning_rate": 1e-05, + "loss": 0.0479, + "num_tokens": 17290680.0, + "reward": 0.6875, + "reward_std": 0.1157275140285492, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.4288326501846313, + "sampling/importance_sampling_ratio/mean": 0.9994004368782043, + "sampling/importance_sampling_ratio/min": 0.7336400747299194, + "sampling/sampling_logp_difference/max": 0.3568577766418457, + "sampling/sampling_logp_difference/mean": 0.010053771547973156, + "step": 868 + }, + { + "clip_ratio/high_max": 5.540779966395348e-05, + "clip_ratio/high_mean": 5.540779966395348e-05, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 5.540779966395348e-05, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 745.0, + "completions/mean_length": 459.375, + "completions/mean_terminated_length": 438.8000183105469, + "completions/min_length": 232.0, + "completions/min_terminated_length": 232.0, + "entropy": 0.28289372473955154, + "epoch": 0.465203426124197, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.029098093509674072, + "learning_rate": 1e-05, + "loss": -0.0066, + "num_tokens": 17309308.0, + "reward": 0.75, + "reward_std": 0.1767766922712326, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.379487156867981, + "sampling/importance_sampling_ratio/mean": 0.9998158812522888, + "sampling/importance_sampling_ratio/min": 0.7158328294754028, + "sampling/sampling_logp_difference/max": 0.3343086242675781, + "sampling/sampling_logp_difference/mean": 0.010313146747648716, + "step": 869 + }, + { + "clip_ratio/high_max": 5.972288636257872e-05, + "clip_ratio/high_mean": 5.972288636257872e-05, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 5.972288636257872e-05, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 750.0, + "completions/mean_length": 569.125, + "completions/mean_terminated_length": 562.7096557617188, + "completions/min_length": 404.0, + "completions/min_terminated_length": 404.0, + "entropy": 0.27961366064846516, + "epoch": 0.4657387580299786, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.015332610346376896, + "learning_rate": 1e-05, + "loss": -0.0162, + "num_tokens": 17331784.0, + "reward": 0.875, + "reward_std": 0.2314550280570984, + "rewards/accuracy_reward/mean": 0.875, + "rewards/accuracy_reward/std": 0.33601075410842896, + "sampling/importance_sampling_ratio/max": 1.443011999130249, + "sampling/importance_sampling_ratio/mean": 1.000084400177002, + "sampling/importance_sampling_ratio/min": 0.6820155382156372, + "sampling/sampling_logp_difference/max": 0.3827028274536133, + "sampling/sampling_logp_difference/mean": 0.009687328711152077, + "step": 870 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00014738050958840176, + "clip_ratio/low_min": 0.00014738050958840176, + "clip_ratio/region_mean": 0.00014738050958840176, + "completions/clipped_ratio": 0.0, + "completions/max_length": 716.0, + "completions/max_terminated_length": 716.0, + "completions/mean_length": 462.0625, + "completions/mean_terminated_length": 462.0625, + "completions/min_length": 313.0, + "completions/min_terminated_length": 313.0, + "entropy": 0.2723985593765974, + "epoch": 0.4662740899357602, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.013609047047793865, + "learning_rate": 1e-05, + "loss": -0.026, + "num_tokens": 17350794.0, + "reward": 0.78125, + "reward_std": 0.3608423173427582, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.4104193449020386, + "sampling/importance_sampling_ratio/mean": 1.0001128911972046, + "sampling/importance_sampling_ratio/min": 0.7268655300140381, + "sampling/sampling_logp_difference/max": 0.3438870906829834, + "sampling/sampling_logp_difference/mean": 0.009979769587516785, + "step": 871 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.9780009905807674e-05, + "clip_ratio/low_min": 5.9780009905807674e-05, + "clip_ratio/region_mean": 5.9780009905807674e-05, + "completions/clipped_ratio": 0.0, + "completions/max_length": 716.0, + "completions/max_terminated_length": 716.0, + "completions/mean_length": 503.375, + "completions/mean_terminated_length": 503.375, + "completions/min_length": 335.0, + "completions/min_terminated_length": 335.0, + "entropy": 0.25760078616440296, + "epoch": 0.4668094218415418, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.015023523010313511, + "learning_rate": 1e-05, + "loss": 0.0011, + "num_tokens": 17370846.0, + "reward": 0.78125, + "reward_std": 0.2041158676147461, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.3430633544921875, + "sampling/importance_sampling_ratio/mean": 1.000095009803772, + "sampling/importance_sampling_ratio/min": 0.6748009324073792, + "sampling/sampling_logp_difference/max": 0.393337607383728, + "sampling/sampling_logp_difference/mean": 0.009723112918436527, + "step": 872 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00011422744137234986, + "clip_ratio/low_min": 0.00011422744137234986, + "clip_ratio/region_mean": 0.00011422744137234986, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 758.0, + "completions/mean_length": 494.75, + "completions/mean_terminated_length": 444.1481628417969, + "completions/min_length": 215.0, + "completions/min_terminated_length": 215.0, + "entropy": 0.44542980566620827, + "epoch": 0.4673447537473233, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.00849328562617302, + "learning_rate": 1e-05, + "loss": 0.0638, + "num_tokens": 17390166.0, + "reward": 0.71875, + "reward_std": 0.3198433816432953, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.4321115016937256, + "sampling/importance_sampling_ratio/mean": 1.000278353691101, + "sampling/importance_sampling_ratio/min": 0.7282505631446838, + "sampling/sampling_logp_difference/max": 0.3591499328613281, + "sampling/sampling_logp_difference/mean": 0.01396699994802475, + "step": 873 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 569.0, + "completions/max_terminated_length": 569.0, + "completions/mean_length": 403.6875, + "completions/mean_terminated_length": 403.6875, + "completions/min_length": 262.0, + "completions/min_terminated_length": 262.0, + "entropy": 0.24195192381739616, + "epoch": 0.4678800856531049, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.005018971394747496, + "learning_rate": 1e-05, + "loss": 0.0197, + "num_tokens": 17406484.0, + "reward": 0.96875, + "reward_std": 0.0883883461356163, + "rewards/accuracy_reward/mean": 0.96875, + "rewards/accuracy_reward/std": 0.1767766922712326, + "sampling/importance_sampling_ratio/max": 1.4348853826522827, + "sampling/importance_sampling_ratio/mean": 0.9999661445617676, + "sampling/importance_sampling_ratio/min": 0.603873074054718, + "sampling/sampling_logp_difference/max": 0.5043913125991821, + "sampling/sampling_logp_difference/mean": 0.009061934426426888, + "step": 874 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00021602207198156975, + "clip_ratio/low_min": 0.00021602207198156975, + "clip_ratio/region_mean": 0.00021602207198156975, + "completions/clipped_ratio": 0.34375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 730.0, + "completions/mean_length": 574.59375, + "completions/mean_terminated_length": 473.2857360839844, + "completions/min_length": 292.0, + "completions/min_terminated_length": 292.0, + "entropy": 0.37176984921097755, + "epoch": 0.4684154175588865, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.007806619629263878, + "learning_rate": 1e-05, + "loss": 0.0187, + "num_tokens": 17428535.0, + "reward": 0.34375, + "reward_std": 0.3808925747871399, + "rewards/accuracy_reward/mean": 0.34375, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.633134126663208, + "sampling/importance_sampling_ratio/mean": 1.0000697374343872, + "sampling/importance_sampling_ratio/min": 0.6994459629058838, + "sampling/sampling_logp_difference/max": 0.49050092697143555, + "sampling/sampling_logp_difference/mean": 0.012547642923891544, + "step": 875 + }, + { + "clip_ratio/high_max": 0.00012194439477752894, + "clip_ratio/high_mean": 0.00012194439477752894, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.00012194439477752894, + "completions/clipped_ratio": 0.0, + "completions/max_length": 701.0, + "completions/max_terminated_length": 701.0, + "completions/mean_length": 476.375, + "completions/mean_terminated_length": 476.375, + "completions/min_length": 226.0, + "completions/min_terminated_length": 226.0, + "entropy": 0.30392312444746494, + "epoch": 0.4689507494646681, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.005617944058030844, + "learning_rate": 1e-05, + "loss": -0.007, + "num_tokens": 17447283.0, + "reward": 0.6875, + "reward_std": 0.1767766922712326, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.538852334022522, + "sampling/importance_sampling_ratio/mean": 1.0001436471939087, + "sampling/importance_sampling_ratio/min": 0.6784803867340088, + "sampling/sampling_logp_difference/max": 0.43103694915771484, + "sampling/sampling_logp_difference/mean": 0.011121723800897598, + "step": 876 + }, + { + "clip_ratio/high_max": 6.278252112679183e-05, + "clip_ratio/high_mean": 6.278252112679183e-05, + "clip_ratio/low_mean": 6.278252112679183e-05, + "clip_ratio/low_min": 6.278252112679183e-05, + "clip_ratio/region_mean": 0.00012556504225358367, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 687.0, + "completions/mean_length": 563.75, + "completions/mean_terminated_length": 506.55999755859375, + "completions/min_length": 304.0, + "completions/min_terminated_length": 304.0, + "entropy": 0.28311794623732567, + "epoch": 0.46948608137044967, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.02786269783973694, + "learning_rate": 1e-05, + "loss": 0.0943, + "num_tokens": 17469267.0, + "reward": 0.6875, + "reward_std": 0.4082317352294922, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.8582525253295898, + "sampling/importance_sampling_ratio/mean": 0.9999716877937317, + "sampling/importance_sampling_ratio/min": 0.6807003617286682, + "sampling/sampling_logp_difference/max": 0.6196365356445312, + "sampling/sampling_logp_difference/mean": 0.009687327779829502, + "step": 877 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 6.374299118760973e-05, + "clip_ratio/low_min": 6.374299118760973e-05, + "clip_ratio/region_mean": 6.374299118760973e-05, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 686.0, + "completions/mean_length": 469.0625, + "completions/mean_terminated_length": 438.137939453125, + "completions/min_length": 279.0, + "completions/min_terminated_length": 279.0, + "entropy": 0.3062589019536972, + "epoch": 0.47002141327623126, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.029604416340589523, + "learning_rate": 1e-05, + "loss": 0.0823, + "num_tokens": 17487821.0, + "reward": 0.84375, + "reward_std": 0.24511480331420898, + "rewards/accuracy_reward/mean": 0.84375, + "rewards/accuracy_reward/std": 0.3689020276069641, + "sampling/importance_sampling_ratio/max": 1.6236634254455566, + "sampling/importance_sampling_ratio/mean": 1.0003081560134888, + "sampling/importance_sampling_ratio/min": 0.698088526725769, + "sampling/sampling_logp_difference/max": 0.48468494415283203, + "sampling/sampling_logp_difference/mean": 0.010873785242438316, + "step": 878 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 9.457403575652279e-05, + "clip_ratio/low_min": 9.457403575652279e-05, + "clip_ratio/region_mean": 9.457403575652279e-05, + "completions/clipped_ratio": 0.34375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 743.0, + "completions/mean_length": 562.375, + "completions/mean_terminated_length": 454.66668701171875, + "completions/min_length": 274.0, + "completions/min_terminated_length": 274.0, + "entropy": 0.4696640409529209, + "epoch": 0.47055674518201285, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.00855310633778572, + "learning_rate": 1e-05, + "loss": 0.0386, + "num_tokens": 17509793.0, + "reward": 0.53125, + "reward_std": 0.3198433816432953, + "rewards/accuracy_reward/mean": 0.53125, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.7555736303329468, + "sampling/importance_sampling_ratio/mean": 1.0000959634780884, + "sampling/importance_sampling_ratio/min": 0.7506774663925171, + "sampling/sampling_logp_difference/max": 0.5627956390380859, + "sampling/sampling_logp_difference/mean": 0.015245474874973297, + "step": 879 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00011901665493496694, + "clip_ratio/low_min": 0.00011901665493496694, + "clip_ratio/region_mean": 0.00011901665493496694, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 750.0, + "completions/mean_length": 515.03125, + "completions/mean_terminated_length": 468.1851806640625, + "completions/min_length": 278.0, + "completions/min_terminated_length": 278.0, + "entropy": 0.39100852981209755, + "epoch": 0.47109207708779444, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.008996756747364998, + "learning_rate": 1e-05, + "loss": 0.0605, + "num_tokens": 17530554.0, + "reward": 0.59375, + "reward_std": 0.3198433816432953, + "rewards/accuracy_reward/mean": 0.59375, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.374659776687622, + "sampling/importance_sampling_ratio/mean": 0.9998103976249695, + "sampling/importance_sampling_ratio/min": 0.6815146803855896, + "sampling/sampling_logp_difference/max": 0.38343748450279236, + "sampling/sampling_logp_difference/mean": 0.013063844293355942, + "step": 880 + }, + { + "clip_ratio/high_max": 9.064539335668087e-05, + "clip_ratio/high_mean": 9.064539335668087e-05, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 9.064539335668087e-05, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 658.0, + "completions/mean_length": 430.125, + "completions/mean_terminated_length": 419.2257995605469, + "completions/min_length": 235.0, + "completions/min_terminated_length": 235.0, + "entropy": 0.29076406359672546, + "epoch": 0.471627408993576, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.0038288207724690437, + "learning_rate": 1e-05, + "loss": 0.0471, + "num_tokens": 17547958.0, + "reward": 0.9375, + "reward_std": 0.1767766922712326, + "rewards/accuracy_reward/mean": 0.9375, + "rewards/accuracy_reward/std": 0.24593468010425568, + "sampling/importance_sampling_ratio/max": 1.3932408094406128, + "sampling/importance_sampling_ratio/mean": 1.000108242034912, + "sampling/importance_sampling_ratio/min": 0.7467346787452698, + "sampling/sampling_logp_difference/max": 0.3316326141357422, + "sampling/sampling_logp_difference/mean": 0.01060293149203062, + "step": 881 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00015880407954682596, + "clip_ratio/low_min": 0.00015880407954682596, + "clip_ratio/region_mean": 0.00015880407954682596, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 737.0, + "completions/mean_length": 514.0, + "completions/mean_terminated_length": 505.8064270019531, + "completions/min_length": 265.0, + "completions/min_terminated_length": 265.0, + "entropy": 0.29901344515383244, + "epoch": 0.4721627408993576, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.03653116151690483, + "learning_rate": 1e-05, + "loss": -0.0376, + "num_tokens": 17568518.0, + "reward": 0.8125, + "reward_std": 0.2177756428718567, + "rewards/accuracy_reward/mean": 0.8125, + "rewards/accuracy_reward/std": 0.3965577781200409, + "sampling/importance_sampling_ratio/max": 1.572670340538025, + "sampling/importance_sampling_ratio/mean": 0.9999311566352844, + "sampling/importance_sampling_ratio/min": 0.7423138618469238, + "sampling/sampling_logp_difference/max": 0.4527750015258789, + "sampling/sampling_logp_difference/mean": 0.01066594012081623, + "step": 882 + }, + { + "clip_ratio/high_max": 0.00016806877101771533, + "clip_ratio/high_mean": 0.00016806877101771533, + "clip_ratio/low_mean": 5.952380888629705e-05, + "clip_ratio/low_min": 5.952380888629705e-05, + "clip_ratio/region_mean": 0.00022759257990401238, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 672.0, + "completions/mean_length": 394.90625, + "completions/mean_terminated_length": 382.8709411621094, + "completions/min_length": 198.0, + "completions/min_terminated_length": 198.0, + "entropy": 0.40722164139151573, + "epoch": 0.4726980728051392, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.02149309031665325, + "learning_rate": 1e-05, + "loss": 0.0743, + "num_tokens": 17584787.0, + "reward": 0.65625, + "reward_std": 0.378745436668396, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.514536738395691, + "sampling/importance_sampling_ratio/mean": 1.0001946687698364, + "sampling/importance_sampling_ratio/min": 0.7147672772407532, + "sampling/sampling_logp_difference/max": 0.41510963439941406, + "sampling/sampling_logp_difference/mean": 0.011294819414615631, + "step": 883 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 550.0, + "completions/max_terminated_length": 550.0, + "completions/mean_length": 358.65625, + "completions/mean_terminated_length": 358.65625, + "completions/min_length": 224.0, + "completions/min_terminated_length": 224.0, + "entropy": 0.3161104694008827, + "epoch": 0.4732334047109208, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": 0.0, + "num_tokens": 17599520.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/accuracy_reward/mean": 1.0, + "rewards/accuracy_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.604664921760559, + "sampling/importance_sampling_ratio/mean": 1.0000346899032593, + "sampling/importance_sampling_ratio/min": 0.6983143091201782, + "sampling/sampling_logp_difference/max": 0.4729149341583252, + "sampling/sampling_logp_difference/mean": 0.012083711102604866, + "step": 884 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 714.0, + "completions/mean_length": 477.96875, + "completions/mean_terminated_length": 424.25927734375, + "completions/min_length": 261.0, + "completions/min_terminated_length": 261.0, + "entropy": 0.36180451698601246, + "epoch": 0.4737687366167024, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.006163413170725107, + "learning_rate": 1e-05, + "loss": 0.0661, + "num_tokens": 17618311.0, + "reward": 0.71875, + "reward_std": 0.2651650309562683, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.5656629800796509, + "sampling/importance_sampling_ratio/mean": 1.000460147857666, + "sampling/importance_sampling_ratio/min": 0.6155635714530945, + "sampling/sampling_logp_difference/max": 0.4852170944213867, + "sampling/sampling_logp_difference/mean": 0.012288326397538185, + "step": 885 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 689.0, + "completions/mean_length": 505.59375, + "completions/mean_terminated_length": 478.4482727050781, + "completions/min_length": 313.0, + "completions/min_terminated_length": 313.0, + "entropy": 0.36343562975525856, + "epoch": 0.4743040685224839, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.010548599064350128, + "learning_rate": 1e-05, + "loss": 0.0216, + "num_tokens": 17638890.0, + "reward": 0.75, + "reward_std": 0.3514062464237213, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.4335078001022339, + "sampling/importance_sampling_ratio/mean": 0.9997308254241943, + "sampling/importance_sampling_ratio/min": 0.7642113566398621, + "sampling/sampling_logp_difference/max": 0.3601243495941162, + "sampling/sampling_logp_difference/mean": 0.01170903630554676, + "step": 886 + }, + { + "clip_ratio/high_max": 0.00017748800019035116, + "clip_ratio/high_mean": 0.00017748800019035116, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.00017748800019035116, + "completions/clipped_ratio": 0.0, + "completions/max_length": 638.0, + "completions/max_terminated_length": 638.0, + "completions/mean_length": 415.375, + "completions/mean_terminated_length": 415.375, + "completions/min_length": 230.0, + "completions/min_terminated_length": 230.0, + "entropy": 0.284851860255003, + "epoch": 0.4748394004282655, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.010278324596583843, + "learning_rate": 1e-05, + "loss": -0.0258, + "num_tokens": 17655742.0, + "reward": 0.8125, + "reward_std": 0.3945523500442505, + "rewards/accuracy_reward/mean": 0.8125, + "rewards/accuracy_reward/std": 0.3965577781200409, + "sampling/importance_sampling_ratio/max": 1.4035282135009766, + "sampling/importance_sampling_ratio/mean": 1.0001215934753418, + "sampling/importance_sampling_ratio/min": 0.5479459166526794, + "sampling/sampling_logp_difference/max": 0.6015787124633789, + "sampling/sampling_logp_difference/mean": 0.011346030980348587, + "step": 887 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 6.22200095676817e-05, + "clip_ratio/low_min": 6.22200095676817e-05, + "clip_ratio/region_mean": 6.22200095676817e-05, + "completions/clipped_ratio": 0.0, + "completions/max_length": 709.0, + "completions/max_terminated_length": 709.0, + "completions/mean_length": 414.5625, + "completions/mean_terminated_length": 414.5625, + "completions/min_length": 228.0, + "completions/min_terminated_length": 228.0, + "entropy": 0.23697533085942268, + "epoch": 0.4753747323340471, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": -0.0055, + "num_tokens": 17672960.0, + "reward": 0.78125, + "reward_std": 0.0883883461356163, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.4025033712387085, + "sampling/importance_sampling_ratio/mean": 1.0002896785736084, + "sampling/importance_sampling_ratio/min": 0.6271695494651794, + "sampling/sampling_logp_difference/max": 0.4665384292602539, + "sampling/sampling_logp_difference/mean": 0.008857685141265392, + "step": 888 + }, + { + "clip_ratio/high_max": 4.8206711653620005e-05, + "clip_ratio/high_mean": 4.8206711653620005e-05, + "clip_ratio/low_mean": 5.339598283171654e-05, + "clip_ratio/low_min": 5.339598283171654e-05, + "clip_ratio/region_mean": 0.00010160269448533654, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 721.0, + "completions/mean_length": 548.59375, + "completions/mean_terminated_length": 533.9666748046875, + "completions/min_length": 227.0, + "completions/min_terminated_length": 227.0, + "entropy": 0.31761903315782547, + "epoch": 0.4759100642398287, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.022894147783517838, + "learning_rate": 1e-05, + "loss": -0.0267, + "num_tokens": 17694611.0, + "reward": 0.6875, + "reward_std": 0.3335031569004059, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.5240819454193115, + "sampling/importance_sampling_ratio/mean": 0.9996128082275391, + "sampling/importance_sampling_ratio/min": 0.6478152275085449, + "sampling/sampling_logp_difference/max": 0.43414974212646484, + "sampling/sampling_logp_difference/mean": 0.012083141133189201, + "step": 889 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 717.0, + "completions/mean_length": 515.78125, + "completions/mean_terminated_length": 507.6451416015625, + "completions/min_length": 324.0, + "completions/min_terminated_length": 324.0, + "entropy": 0.31561365723609924, + "epoch": 0.47644539614561027, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.008195150643587112, + "learning_rate": 1e-05, + "loss": 0.0728, + "num_tokens": 17715268.0, + "reward": 0.875, + "reward_std": 0.292504221200943, + "rewards/accuracy_reward/mean": 0.875, + "rewards/accuracy_reward/std": 0.33601075410842896, + "sampling/importance_sampling_ratio/max": 1.435183048248291, + "sampling/importance_sampling_ratio/mean": 1.00010085105896, + "sampling/importance_sampling_ratio/min": 0.7544096112251282, + "sampling/sampling_logp_difference/max": 0.36129236221313477, + "sampling/sampling_logp_difference/mean": 0.010514057241380215, + "step": 890 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 758.0, + "completions/mean_length": 398.84375, + "completions/mean_terminated_length": 386.93548583984375, + "completions/min_length": 157.0, + "completions/min_terminated_length": 157.0, + "entropy": 0.29939622804522514, + "epoch": 0.47698072805139186, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.0051595428958535194, + "learning_rate": 1e-05, + "loss": 0.0278, + "num_tokens": 17731695.0, + "reward": 0.90625, + "reward_std": 0.2651650309562683, + "rewards/accuracy_reward/mean": 0.90625, + "rewards/accuracy_reward/std": 0.2961445748806, + "sampling/importance_sampling_ratio/max": 1.3975964784622192, + "sampling/importance_sampling_ratio/mean": 1.000022053718567, + "sampling/importance_sampling_ratio/min": 0.7142203450202942, + "sampling/sampling_logp_difference/max": 0.3365638256072998, + "sampling/sampling_logp_difference/mean": 0.010699898935854435, + "step": 891 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 681.0, + "completions/max_terminated_length": 681.0, + "completions/mean_length": 456.4375, + "completions/mean_terminated_length": 456.4375, + "completions/min_length": 260.0, + "completions/min_terminated_length": 260.0, + "entropy": 0.2786124423146248, + "epoch": 0.47751605995717344, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.018611766397953033, + "learning_rate": 1e-05, + "loss": -0.0686, + "num_tokens": 17750061.0, + "reward": 0.75, + "reward_std": 0.4355512857437134, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.5237611532211304, + "sampling/importance_sampling_ratio/mean": 1.0000518560409546, + "sampling/importance_sampling_ratio/min": 0.6943444013595581, + "sampling/sampling_logp_difference/max": 0.42118167877197266, + "sampling/sampling_logp_difference/mean": 0.010647882707417011, + "step": 892 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.374032843974419e-05, + "clip_ratio/low_min": 5.374032843974419e-05, + "clip_ratio/region_mean": 5.374032843974419e-05, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 767.0, + "completions/mean_length": 502.75, + "completions/mean_terminated_length": 485.0666809082031, + "completions/min_length": 239.0, + "completions/min_terminated_length": 239.0, + "entropy": 0.2564187031239271, + "epoch": 0.47805139186295503, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.007111367769539356, + "learning_rate": 1e-05, + "loss": 0.0248, + "num_tokens": 17770229.0, + "reward": 0.9375, + "reward_std": 0.1157275140285492, + "rewards/accuracy_reward/mean": 0.9375, + "rewards/accuracy_reward/std": 0.24593468010425568, + "sampling/importance_sampling_ratio/max": 1.3762917518615723, + "sampling/importance_sampling_ratio/mean": 1.0004146099090576, + "sampling/importance_sampling_ratio/min": 0.655583381652832, + "sampling/sampling_logp_difference/max": 0.4222297668457031, + "sampling/sampling_logp_difference/mean": 0.009604319930076599, + "step": 893 + }, + { + "clip_ratio/high_max": 8.579272252973169e-05, + "clip_ratio/high_mean": 8.579272252973169e-05, + "clip_ratio/low_mean": 5.771006544819102e-05, + "clip_ratio/low_min": 5.771006544819102e-05, + "clip_ratio/region_mean": 0.0001435027879779227, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 685.0, + "completions/mean_length": 462.34375, + "completions/mean_terminated_length": 452.4838562011719, + "completions/min_length": 182.0, + "completions/min_terminated_length": 182.0, + "entropy": 0.32170755974948406, + "epoch": 0.4785867237687366, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.01108202338218689, + "learning_rate": 1e-05, + "loss": 0.0361, + "num_tokens": 17788640.0, + "reward": 0.71875, + "reward_std": 0.3608423173427582, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.4440582990646362, + "sampling/importance_sampling_ratio/mean": 1.000043511390686, + "sampling/importance_sampling_ratio/min": 0.7355419397354126, + "sampling/sampling_logp_difference/max": 0.36745738983154297, + "sampling/sampling_logp_difference/mean": 0.01109524816274643, + "step": 894 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.312367284204811e-05, + "clip_ratio/low_min": 5.312367284204811e-05, + "clip_ratio/region_mean": 5.312367284204811e-05, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 521.0, + "completions/mean_length": 419.21875, + "completions/mean_terminated_length": 369.39288330078125, + "completions/min_length": 185.0, + "completions/min_terminated_length": 185.0, + "entropy": 0.37245791032910347, + "epoch": 0.4791220556745182, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.026289524510502815, + "learning_rate": 1e-05, + "loss": 0.0392, + "num_tokens": 17805663.0, + "reward": 0.625, + "reward_std": 0.3335031569004059, + "rewards/accuracy_reward/mean": 0.625, + "rewards/accuracy_reward/std": 0.49186936020851135, + "sampling/importance_sampling_ratio/max": 1.4213987588882446, + "sampling/importance_sampling_ratio/mean": 1.0001524686813354, + "sampling/importance_sampling_ratio/min": 0.7705093026161194, + "sampling/sampling_logp_difference/max": 0.3516414165496826, + "sampling/sampling_logp_difference/mean": 0.012494618073105812, + "step": 895 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 7.339988223975524e-05, + "clip_ratio/low_min": 7.339988223975524e-05, + "clip_ratio/region_mean": 7.339988223975524e-05, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 746.0, + "completions/mean_length": 436.0625, + "completions/mean_terminated_length": 413.933349609375, + "completions/min_length": 166.0, + "completions/min_terminated_length": 166.0, + "entropy": 0.3051671665161848, + "epoch": 0.4796573875802998, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.020245268940925598, + "learning_rate": 1e-05, + "loss": 0.0485, + "num_tokens": 17823529.0, + "reward": 0.875, + "reward_std": 0.2177756428718567, + "rewards/accuracy_reward/mean": 0.875, + "rewards/accuracy_reward/std": 0.33601075410842896, + "sampling/importance_sampling_ratio/max": 1.3547430038452148, + "sampling/importance_sampling_ratio/mean": 0.9999259114265442, + "sampling/importance_sampling_ratio/min": 0.6948326230049133, + "sampling/sampling_logp_difference/max": 0.36408424377441406, + "sampling/sampling_logp_difference/mean": 0.011529726907610893, + "step": 896 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 590.0, + "completions/mean_length": 434.625, + "completions/mean_terminated_length": 423.8709411621094, + "completions/min_length": 307.0, + "completions/min_terminated_length": 307.0, + "entropy": 0.30700914561748505, + "epoch": 0.4801927194860814, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.004601879511028528, + "learning_rate": 1e-05, + "loss": 0.0446, + "num_tokens": 17841141.0, + "reward": 0.96875, + "reward_std": 0.0883883461356163, + "rewards/accuracy_reward/mean": 0.96875, + "rewards/accuracy_reward/std": 0.1767766922712326, + "sampling/importance_sampling_ratio/max": 1.3869141340255737, + "sampling/importance_sampling_ratio/mean": 0.9993304014205933, + "sampling/importance_sampling_ratio/min": 0.6631650924682617, + "sampling/sampling_logp_difference/max": 0.41073131561279297, + "sampling/sampling_logp_difference/mean": 0.01103257853537798, + "step": 897 + }, + { + "clip_ratio/high_max": 0.00015404890291392803, + "clip_ratio/high_mean": 0.00015404890291392803, + "clip_ratio/low_mean": 6.423432932933792e-05, + "clip_ratio/low_min": 6.423432932933792e-05, + "clip_ratio/region_mean": 0.00021828323224326596, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 760.0, + "completions/mean_length": 399.5625, + "completions/mean_terminated_length": 387.6773986816406, + "completions/min_length": 242.0, + "completions/min_terminated_length": 242.0, + "entropy": 0.39390813559293747, + "epoch": 0.480728051391863, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.01933855004608631, + "learning_rate": 1e-05, + "loss": 0.0304, + "num_tokens": 17857607.0, + "reward": 0.75, + "reward_std": 0.3514062464237213, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.4763736724853516, + "sampling/importance_sampling_ratio/mean": 0.9998878836631775, + "sampling/importance_sampling_ratio/min": 0.5628733038902283, + "sampling/sampling_logp_difference/max": 0.5747007131576538, + "sampling/sampling_logp_difference/mean": 0.013597422279417515, + "step": 898 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 680.0, + "completions/mean_length": 491.21875, + "completions/mean_terminated_length": 482.2903137207031, + "completions/min_length": 282.0, + "completions/min_terminated_length": 282.0, + "entropy": 0.2863363791257143, + "epoch": 0.48126338329764456, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": 0.0, + "num_tokens": 17877254.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/accuracy_reward/mean": 1.0, + "rewards/accuracy_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.6229543685913086, + "sampling/importance_sampling_ratio/mean": 1.0002821683883667, + "sampling/importance_sampling_ratio/min": 0.6835976243019104, + "sampling/sampling_logp_difference/max": 0.48424816131591797, + "sampling/sampling_logp_difference/mean": 0.010139341466128826, + "step": 899 + }, + { + "clip_ratio/high_max": 7.413997809635475e-05, + "clip_ratio/high_mean": 7.413997809635475e-05, + "clip_ratio/low_mean": 0.00021566540090134367, + "clip_ratio/low_min": 0.00021566540090134367, + "clip_ratio/region_mean": 0.0002898053717217408, + "completions/clipped_ratio": 0.0, + "completions/max_length": 592.0, + "completions/max_terminated_length": 592.0, + "completions/mean_length": 398.625, + "completions/mean_terminated_length": 398.625, + "completions/min_length": 247.0, + "completions/min_terminated_length": 247.0, + "entropy": 0.31713657453656197, + "epoch": 0.4817987152034261, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.01953887566924095, + "learning_rate": 1e-05, + "loss": -0.0333, + "num_tokens": 17893962.0, + "reward": 0.78125, + "reward_std": 0.3608423173427582, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.3457244634628296, + "sampling/importance_sampling_ratio/mean": 1.0002864599227905, + "sampling/importance_sampling_ratio/min": 0.660288393497467, + "sampling/sampling_logp_difference/max": 0.41507863998413086, + "sampling/sampling_logp_difference/mean": 0.011422082781791687, + "step": 900 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 673.0, + "completions/mean_length": 487.59375, + "completions/mean_terminated_length": 478.5483703613281, + "completions/min_length": 253.0, + "completions/min_terminated_length": 253.0, + "entropy": 0.2983247824013233, + "epoch": 0.4823340471092077, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.03428788483142853, + "learning_rate": 1e-05, + "loss": 0.002, + "num_tokens": 17913477.0, + "reward": 0.9375, + "reward_std": 0.1767766922712326, + "rewards/accuracy_reward/mean": 0.9375, + "rewards/accuracy_reward/std": 0.24593468010425568, + "sampling/importance_sampling_ratio/max": 1.430167317390442, + "sampling/importance_sampling_ratio/mean": 0.9998111724853516, + "sampling/importance_sampling_ratio/min": 0.5547361373901367, + "sampling/sampling_logp_difference/max": 0.5892627239227295, + "sampling/sampling_logp_difference/mean": 0.011623206548392773, + "step": 901 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 703.0, + "completions/mean_length": 518.90625, + "completions/mean_terminated_length": 461.423095703125, + "completions/min_length": 283.0, + "completions/min_terminated_length": 283.0, + "entropy": 0.40202517434954643, + "epoch": 0.4828693790149893, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.006073471624404192, + "learning_rate": 1e-05, + "loss": 0.0523, + "num_tokens": 17933850.0, + "reward": 0.4375, + "reward_std": 0.2925041913986206, + "rewards/accuracy_reward/mean": 0.4375, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 2.0, + "sampling/importance_sampling_ratio/mean": 1.000144600868225, + "sampling/importance_sampling_ratio/min": 0.6256729960441589, + "sampling/sampling_logp_difference/max": 0.8568992614746094, + "sampling/sampling_logp_difference/mean": 0.014280413277447224, + "step": 902 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 742.0, + "completions/mean_length": 506.3125, + "completions/mean_terminated_length": 445.923095703125, + "completions/min_length": 202.0, + "completions/min_terminated_length": 202.0, + "entropy": 0.26994348876178265, + "epoch": 0.48340471092077086, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.007904134690761566, + "learning_rate": 1e-05, + "loss": 0.0765, + "num_tokens": 17954708.0, + "reward": 0.8125, + "reward_std": 0.3335031569004059, + "rewards/accuracy_reward/mean": 0.8125, + "rewards/accuracy_reward/std": 0.3965577781200409, + "sampling/importance_sampling_ratio/max": 1.4643410444259644, + "sampling/importance_sampling_ratio/mean": 0.9998607635498047, + "sampling/importance_sampling_ratio/min": 0.49734869599342346, + "sampling/sampling_logp_difference/max": 0.6984639167785645, + "sampling/sampling_logp_difference/mean": 0.010100830346345901, + "step": 903 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 7.122506940504536e-05, + "clip_ratio/low_min": 7.122506940504536e-05, + "clip_ratio/region_mean": 7.122506940504536e-05, + "completions/clipped_ratio": 0.0, + "completions/max_length": 733.0, + "completions/max_terminated_length": 733.0, + "completions/mean_length": 439.53125, + "completions/mean_terminated_length": 439.53125, + "completions/min_length": 191.0, + "completions/min_terminated_length": 191.0, + "entropy": 0.220725879073143, + "epoch": 0.48394004282655245, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.0029265962075442076, + "learning_rate": 1e-05, + "loss": -0.0214, + "num_tokens": 17972157.0, + "reward": 0.9375, + "reward_std": 0.1767766922712326, + "rewards/accuracy_reward/mean": 0.9375, + "rewards/accuracy_reward/std": 0.24593468010425568, + "sampling/importance_sampling_ratio/max": 1.5479936599731445, + "sampling/importance_sampling_ratio/mean": 0.9997026324272156, + "sampling/importance_sampling_ratio/min": 0.6082855463027954, + "sampling/sampling_logp_difference/max": 0.49711084365844727, + "sampling/sampling_logp_difference/mean": 0.009222800843417645, + "step": 904 + }, + { + "clip_ratio/high_max": 0.00014070072211325169, + "clip_ratio/high_mean": 0.00014070072211325169, + "clip_ratio/low_mean": 6.42013328615576e-05, + "clip_ratio/low_min": 6.42013328615576e-05, + "clip_ratio/region_mean": 0.0002049020549748093, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 598.0, + "completions/mean_length": 457.4375, + "completions/mean_terminated_length": 425.3103332519531, + "completions/min_length": 253.0, + "completions/min_terminated_length": 253.0, + "entropy": 0.41206288896501064, + "epoch": 0.48447537473233404, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.008017003536224365, + "learning_rate": 1e-05, + "loss": 0.1193, + "num_tokens": 17990931.0, + "reward": 0.75, + "reward_std": 0.3335031569004059, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.3692039251327515, + "sampling/importance_sampling_ratio/mean": 0.9999173879623413, + "sampling/importance_sampling_ratio/min": 0.5028315782546997, + "sampling/sampling_logp_difference/max": 0.6875, + "sampling/sampling_logp_difference/mean": 0.013152733445167542, + "step": 905 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00016133778262883425, + "clip_ratio/low_min": 0.00016133778262883425, + "clip_ratio/region_mean": 0.00016133778262883425, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 692.0, + "completions/mean_length": 458.5625, + "completions/mean_terminated_length": 426.5517272949219, + "completions/min_length": 173.0, + "completions/min_terminated_length": 173.0, + "entropy": 0.3623971212655306, + "epoch": 0.48501070663811563, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.023558853194117546, + "learning_rate": 1e-05, + "loss": 0.0153, + "num_tokens": 18009421.0, + "reward": 0.71875, + "reward_std": 0.24511480331420898, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.4311026334762573, + "sampling/importance_sampling_ratio/mean": 0.9998683929443359, + "sampling/importance_sampling_ratio/min": 0.6248417496681213, + "sampling/sampling_logp_difference/max": 0.4702568054199219, + "sampling/sampling_logp_difference/mean": 0.012686527334153652, + "step": 906 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.623032120638527e-05, + "clip_ratio/low_min": 5.623032120638527e-05, + "clip_ratio/region_mean": 5.623032120638527e-05, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 674.0, + "completions/mean_length": 474.46875, + "completions/mean_terminated_length": 465.0, + "completions/min_length": 302.0, + "completions/min_terminated_length": 302.0, + "entropy": 0.21107923239469528, + "epoch": 0.4855460385438972, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.014978027902543545, + "learning_rate": 1e-05, + "loss": 0.0238, + "num_tokens": 18027932.0, + "reward": 0.84375, + "reward_std": 0.1293872892856598, + "rewards/accuracy_reward/mean": 0.84375, + "rewards/accuracy_reward/std": 0.3689020276069641, + "sampling/importance_sampling_ratio/max": 1.41343355178833, + "sampling/importance_sampling_ratio/mean": 1.0001240968704224, + "sampling/importance_sampling_ratio/min": 0.7033680081367493, + "sampling/sampling_logp_difference/max": 0.35187506675720215, + "sampling/sampling_logp_difference/mean": 0.008225500583648682, + "step": 907 + }, + { + "clip_ratio/high_max": 0.0001752337957441341, + "clip_ratio/high_mean": 0.0001752337957441341, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0001752337957441341, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 751.0, + "completions/mean_length": 509.46875, + "completions/mean_terminated_length": 492.2333679199219, + "completions/min_length": 133.0, + "completions/min_terminated_length": 133.0, + "entropy": 0.3040708899497986, + "epoch": 0.4860813704496788, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.021471887826919556, + "learning_rate": 1e-05, + "loss": 0.0064, + "num_tokens": 18047883.0, + "reward": 0.46875, + "reward_std": 0.47137707471847534, + "rewards/accuracy_reward/mean": 0.46875, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.4772247076034546, + "sampling/importance_sampling_ratio/mean": 0.9998332858085632, + "sampling/importance_sampling_ratio/min": 0.595400869846344, + "sampling/sampling_logp_difference/max": 0.5185203552246094, + "sampling/sampling_logp_difference/mean": 0.011188496835529804, + "step": 908 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 6.652474985457957e-05, + "clip_ratio/low_min": 6.652474985457957e-05, + "clip_ratio/region_mean": 6.652474985457957e-05, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 764.0, + "completions/mean_length": 489.40625, + "completions/mean_terminated_length": 470.8333435058594, + "completions/min_length": 242.0, + "completions/min_terminated_length": 242.0, + "entropy": 0.39791784808039665, + "epoch": 0.4866167023554604, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.03481146693229675, + "learning_rate": 1e-05, + "loss": -0.0179, + "num_tokens": 18068384.0, + "reward": 0.59375, + "reward_std": 0.4628904461860657, + "rewards/accuracy_reward/mean": 0.59375, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.351954698562622, + "sampling/importance_sampling_ratio/mean": 0.9997958540916443, + "sampling/importance_sampling_ratio/min": 0.6212679743766785, + "sampling/sampling_logp_difference/max": 0.4759927988052368, + "sampling/sampling_logp_difference/mean": 0.013370419852435589, + "step": 909 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 726.0, + "completions/max_terminated_length": 726.0, + "completions/mean_length": 468.90625, + "completions/mean_terminated_length": 468.90625, + "completions/min_length": 280.0, + "completions/min_terminated_length": 280.0, + "entropy": 0.26747251860797405, + "epoch": 0.487152034261242, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": 0.0, + "num_tokens": 18087261.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/accuracy_reward/mean": 1.0, + "rewards/accuracy_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.4127734899520874, + "sampling/importance_sampling_ratio/mean": 1.0001856088638306, + "sampling/importance_sampling_ratio/min": 0.6977643966674805, + "sampling/sampling_logp_difference/max": 0.35987377166748047, + "sampling/sampling_logp_difference/mean": 0.009961742907762527, + "step": 910 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 4.76917193736881e-05, + "clip_ratio/low_min": 4.76917193736881e-05, + "clip_ratio/region_mean": 4.76917193736881e-05, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 699.0, + "completions/mean_length": 516.3125, + "completions/mean_terminated_length": 469.7037048339844, + "completions/min_length": 286.0, + "completions/min_terminated_length": 286.0, + "entropy": 0.28392020240426064, + "epoch": 0.4876873661670236, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": 0.0226, + "num_tokens": 18107415.0, + "reward": 0.84375, + "reward_std": 0.1293872892856598, + "rewards/accuracy_reward/mean": 0.84375, + "rewards/accuracy_reward/std": 0.3689020276069641, + "sampling/importance_sampling_ratio/max": 1.8648970127105713, + "sampling/importance_sampling_ratio/mean": 1.000344157218933, + "sampling/importance_sampling_ratio/min": 0.6863139867782593, + "sampling/sampling_logp_difference/max": 0.6232057809829712, + "sampling/sampling_logp_difference/mean": 0.01108316145837307, + "step": 911 + }, + { + "clip_ratio/high_max": 5.1845705456798896e-05, + "clip_ratio/high_mean": 5.1845705456798896e-05, + "clip_ratio/low_mean": 0.0001192373783851508, + "clip_ratio/low_min": 0.0001192373783851508, + "clip_ratio/region_mean": 0.0001710830838419497, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 745.0, + "completions/mean_length": 534.21875, + "completions/mean_terminated_length": 500.8214416503906, + "completions/min_length": 340.0, + "completions/min_terminated_length": 340.0, + "entropy": 0.3648163303732872, + "epoch": 0.48822269807280516, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.010235407389700413, + "learning_rate": 1e-05, + "loss": 0.0583, + "num_tokens": 18128574.0, + "reward": 0.40625, + "reward_std": 0.4534739851951599, + "rewards/accuracy_reward/mean": 0.40625, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.448590636253357, + "sampling/importance_sampling_ratio/mean": 1.0002288818359375, + "sampling/importance_sampling_ratio/min": 0.7280926704406738, + "sampling/sampling_logp_difference/max": 0.3705911636352539, + "sampling/sampling_logp_difference/mean": 0.012508274056017399, + "step": 912 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.8934463595505804e-05, + "clip_ratio/low_min": 5.8934463595505804e-05, + "clip_ratio/region_mean": 5.8934463595505804e-05, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 758.0, + "completions/mean_length": 477.8125, + "completions/mean_terminated_length": 458.4667053222656, + "completions/min_length": 171.0, + "completions/min_terminated_length": 171.0, + "entropy": 0.3433845341205597, + "epoch": 0.48875802997858675, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.021932439878582954, + "learning_rate": 1e-05, + "loss": 0.0348, + "num_tokens": 18148096.0, + "reward": 0.78125, + "reward_std": 0.3471629321575165, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.3927536010742188, + "sampling/importance_sampling_ratio/mean": 0.9999805688858032, + "sampling/importance_sampling_ratio/min": 0.5484291911125183, + "sampling/sampling_logp_difference/max": 0.6006970405578613, + "sampling/sampling_logp_difference/mean": 0.011226137168705463, + "step": 913 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 6.856829713797197e-05, + "clip_ratio/low_min": 6.856829713797197e-05, + "clip_ratio/region_mean": 6.856829713797197e-05, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 742.0, + "completions/mean_length": 476.875, + "completions/mean_terminated_length": 457.4667053222656, + "completions/min_length": 262.0, + "completions/min_terminated_length": 262.0, + "entropy": 0.3077407404780388, + "epoch": 0.4892933618843683, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.005473727826029062, + "learning_rate": 1e-05, + "loss": -0.0092, + "num_tokens": 18167020.0, + "reward": 0.78125, + "reward_std": 0.24511480331420898, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.439332365989685, + "sampling/importance_sampling_ratio/mean": 0.9999160766601562, + "sampling/importance_sampling_ratio/min": 0.08567306399345398, + "sampling/sampling_logp_difference/max": 2.457216739654541, + "sampling/sampling_logp_difference/mean": 0.011770566925406456, + "step": 914 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00011579434794839472, + "clip_ratio/low_min": 0.00011579434794839472, + "clip_ratio/region_mean": 0.00011579434794839472, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 658.0, + "completions/mean_length": 466.4375, + "completions/mean_terminated_length": 423.3571472167969, + "completions/min_length": 279.0, + "completions/min_terminated_length": 279.0, + "entropy": 0.35018456168472767, + "epoch": 0.48982869379014987, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.029982883483171463, + "learning_rate": 1e-05, + "loss": 0.0987, + "num_tokens": 18186130.0, + "reward": 0.53125, + "reward_std": 0.24511480331420898, + "rewards/accuracy_reward/mean": 0.53125, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.6562178134918213, + "sampling/importance_sampling_ratio/mean": 1.00028395652771, + "sampling/importance_sampling_ratio/min": 0.6066259145736694, + "sampling/sampling_logp_difference/max": 0.5045366287231445, + "sampling/sampling_logp_difference/mean": 0.01201210543513298, + "step": 915 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 6.994963769102469e-05, + "clip_ratio/low_min": 6.994963769102469e-05, + "clip_ratio/region_mean": 6.994963769102469e-05, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 631.0, + "completions/mean_length": 430.71875, + "completions/mean_terminated_length": 408.2333679199219, + "completions/min_length": 217.0, + "completions/min_terminated_length": 217.0, + "entropy": 0.29074944369494915, + "epoch": 0.49036402569593146, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.03246884047985077, + "learning_rate": 1e-05, + "loss": 0.0344, + "num_tokens": 18203801.0, + "reward": 0.9375, + "reward_std": 0.1767766922712326, + "rewards/accuracy_reward/mean": 0.9375, + "rewards/accuracy_reward/std": 0.24593468010425568, + "sampling/importance_sampling_ratio/max": 1.4905964136123657, + "sampling/importance_sampling_ratio/mean": 1.0003291368484497, + "sampling/importance_sampling_ratio/min": 0.6721236109733582, + "sampling/sampling_logp_difference/max": 0.39917635917663574, + "sampling/sampling_logp_difference/mean": 0.011309084482491016, + "step": 916 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00011041528341593221, + "clip_ratio/low_min": 0.00011041528341593221, + "clip_ratio/region_mean": 0.00011041528341593221, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 737.0, + "completions/mean_length": 548.53125, + "completions/mean_terminated_length": 497.8846435546875, + "completions/min_length": 202.0, + "completions/min_terminated_length": 202.0, + "entropy": 0.28678014129400253, + "epoch": 0.49089935760171305, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.01799173094332218, + "learning_rate": 1e-05, + "loss": 0.0943, + "num_tokens": 18225154.0, + "reward": 0.6875, + "reward_std": 0.3745020925998688, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.6590856313705444, + "sampling/importance_sampling_ratio/mean": 1.0002859830856323, + "sampling/importance_sampling_ratio/min": 0.6085002422332764, + "sampling/sampling_logp_difference/max": 0.5062665939331055, + "sampling/sampling_logp_difference/mean": 0.010599044151604176, + "step": 917 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00014400921645574272, + "clip_ratio/low_min": 0.00014400921645574272, + "clip_ratio/region_mean": 0.00014400921645574272, + "completions/clipped_ratio": 0.0, + "completions/max_length": 553.0, + "completions/max_terminated_length": 553.0, + "completions/mean_length": 398.28125, + "completions/mean_terminated_length": 398.28125, + "completions/min_length": 241.0, + "completions/min_terminated_length": 241.0, + "entropy": 0.30088853277266026, + "epoch": 0.49143468950749464, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.007649065461009741, + "learning_rate": 1e-05, + "loss": 0.0712, + "num_tokens": 18241643.0, + "reward": 0.84375, + "reward_std": 0.3198433816432953, + "rewards/accuracy_reward/mean": 0.84375, + "rewards/accuracy_reward/std": 0.3689020276069641, + "sampling/importance_sampling_ratio/max": 1.2989885807037354, + "sampling/importance_sampling_ratio/mean": 1.0002886056900024, + "sampling/importance_sampling_ratio/min": 0.66785728931427, + "sampling/sampling_logp_difference/max": 0.40368080139160156, + "sampling/sampling_logp_difference/mean": 0.011044279672205448, + "step": 918 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 738.0, + "completions/mean_length": 487.4375, + "completions/mean_terminated_length": 458.4137878417969, + "completions/min_length": 223.0, + "completions/min_terminated_length": 223.0, + "entropy": 0.3637304864823818, + "epoch": 0.4919700214132762, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.007687463890761137, + "learning_rate": 1e-05, + "loss": 0.0604, + "num_tokens": 18261177.0, + "reward": 0.5625, + "reward_std": 0.2177756428718567, + "rewards/accuracy_reward/mean": 0.5625, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 1.3802224397659302, + "sampling/importance_sampling_ratio/mean": 0.9997232556343079, + "sampling/importance_sampling_ratio/min": 0.6623309254646301, + "sampling/sampling_logp_difference/max": 0.4119899272918701, + "sampling/sampling_logp_difference/mean": 0.01202123798429966, + "step": 919 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00010454619769006968, + "clip_ratio/low_min": 0.00010454619769006968, + "clip_ratio/region_mean": 0.00010454619769006968, + "completions/clipped_ratio": 0.40625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 738.0, + "completions/mean_length": 611.59375, + "completions/mean_terminated_length": 504.5789489746094, + "completions/min_length": 335.0, + "completions/min_terminated_length": 335.0, + "entropy": 0.5391234718263149, + "epoch": 0.4925053533190578, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.006431222427636385, + "learning_rate": 1e-05, + "loss": 0.0252, + "num_tokens": 18285100.0, + "reward": 0.09375, + "reward_std": 0.2041158676147461, + "rewards/accuracy_reward/mean": 0.09375, + "rewards/accuracy_reward/std": 0.2961445748806, + "sampling/importance_sampling_ratio/max": 1.4450308084487915, + "sampling/importance_sampling_ratio/mean": 0.9998129606246948, + "sampling/importance_sampling_ratio/min": 0.6925428509712219, + "sampling/sampling_logp_difference/max": 0.3681306838989258, + "sampling/sampling_logp_difference/mean": 0.01693020947277546, + "step": 920 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 736.0, + "completions/max_terminated_length": 736.0, + "completions/mean_length": 471.21875, + "completions/mean_terminated_length": 471.21875, + "completions/min_length": 263.0, + "completions/min_terminated_length": 263.0, + "entropy": 0.26552114076912403, + "epoch": 0.4930406852248394, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.02040373533964157, + "learning_rate": 1e-05, + "loss": 0.0075, + "num_tokens": 18304203.0, + "reward": 0.84375, + "reward_std": 0.24511480331420898, + "rewards/accuracy_reward/mean": 0.84375, + "rewards/accuracy_reward/std": 0.3689020276069641, + "sampling/importance_sampling_ratio/max": 1.4626009464263916, + "sampling/importance_sampling_ratio/mean": 0.9998367428779602, + "sampling/importance_sampling_ratio/min": 0.7014557123184204, + "sampling/sampling_logp_difference/max": 0.3802163600921631, + "sampling/sampling_logp_difference/mean": 0.009553835727274418, + "step": 921 + }, + { + "clip_ratio/high_max": 5.9045818488812074e-05, + "clip_ratio/high_mean": 5.9045818488812074e-05, + "clip_ratio/low_mean": 0.00010313531674910337, + "clip_ratio/low_min": 0.00010313531674910337, + "clip_ratio/region_mean": 0.00016218113523791544, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 764.0, + "completions/mean_length": 515.28125, + "completions/mean_terminated_length": 489.137939453125, + "completions/min_length": 236.0, + "completions/min_terminated_length": 236.0, + "entropy": 0.35010990500450134, + "epoch": 0.493576017130621, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.01850818283855915, + "learning_rate": 1e-05, + "loss": 0.1123, + "num_tokens": 18324716.0, + "reward": 0.71875, + "reward_std": 0.4534739851951599, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.5360453128814697, + "sampling/importance_sampling_ratio/mean": 0.9998300671577454, + "sampling/importance_sampling_ratio/min": 0.6004226207733154, + "sampling/sampling_logp_difference/max": 0.5101215839385986, + "sampling/sampling_logp_difference/mean": 0.012675362639129162, + "step": 922 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 7.17565999366343e-05, + "clip_ratio/low_min": 7.17565999366343e-05, + "clip_ratio/region_mean": 7.17565999366343e-05, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 762.0, + "completions/mean_length": 459.1875, + "completions/mean_terminated_length": 427.2413635253906, + "completions/min_length": 186.0, + "completions/min_terminated_length": 186.0, + "entropy": 0.3355816677212715, + "epoch": 0.4941113490364026, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.018069904297590256, + "learning_rate": 1e-05, + "loss": 0.0188, + "num_tokens": 18343210.0, + "reward": 0.78125, + "reward_std": 0.24511480331420898, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.4299403429031372, + "sampling/importance_sampling_ratio/mean": 1.000341773033142, + "sampling/importance_sampling_ratio/min": 0.6935073137283325, + "sampling/sampling_logp_difference/max": 0.3659934997558594, + "sampling/sampling_logp_difference/mean": 0.011436311528086662, + "step": 923 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.7129800552502275e-05, + "clip_ratio/low_min": 5.7129800552502275e-05, + "clip_ratio/region_mean": 5.7129800552502275e-05, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 745.0, + "completions/mean_length": 533.4375, + "completions/mean_terminated_length": 517.800048828125, + "completions/min_length": 245.0, + "completions/min_terminated_length": 245.0, + "entropy": 0.32349422574043274, + "epoch": 0.49464668094218417, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.013042080216109753, + "learning_rate": 1e-05, + "loss": 0.0563, + "num_tokens": 18364112.0, + "reward": 0.625, + "reward_std": 0.3335031569004059, + "rewards/accuracy_reward/mean": 0.625, + "rewards/accuracy_reward/std": 0.49186936020851135, + "sampling/importance_sampling_ratio/max": 1.4326272010803223, + "sampling/importance_sampling_ratio/mean": 0.9996811151504517, + "sampling/importance_sampling_ratio/min": 0.7073561549186707, + "sampling/sampling_logp_difference/max": 0.3595099449157715, + "sampling/sampling_logp_difference/mean": 0.011251432821154594, + "step": 924 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 645.0, + "completions/mean_length": 508.25, + "completions/mean_terminated_length": 448.3077087402344, + "completions/min_length": 273.0, + "completions/min_terminated_length": 273.0, + "entropy": 0.3550127577036619, + "epoch": 0.49518201284796576, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": 0.008, + "num_tokens": 18384400.0, + "reward": 0.65625, + "reward_std": 0.1293872892856598, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.2908086776733398, + "sampling/importance_sampling_ratio/mean": 0.999809741973877, + "sampling/importance_sampling_ratio/min": 0.7054306268692017, + "sampling/sampling_logp_difference/max": 0.34894680976867676, + "sampling/sampling_logp_difference/mean": 0.012393838725984097, + "step": 925 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 6.782419950468466e-05, + "clip_ratio/low_min": 6.782419950468466e-05, + "clip_ratio/region_mean": 6.782419950468466e-05, + "completions/clipped_ratio": 0.0, + "completions/max_length": 763.0, + "completions/max_terminated_length": 763.0, + "completions/mean_length": 483.71875, + "completions/mean_terminated_length": 483.71875, + "completions/min_length": 269.0, + "completions/min_terminated_length": 269.0, + "entropy": 0.31992656737565994, + "epoch": 0.49571734475374735, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.01657484471797943, + "learning_rate": 1e-05, + "loss": 0.0573, + "num_tokens": 18403991.0, + "reward": 0.78125, + "reward_std": 0.3377464711666107, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.4087289571762085, + "sampling/importance_sampling_ratio/mean": 0.9999836087226868, + "sampling/importance_sampling_ratio/min": 0.7234011888504028, + "sampling/sampling_logp_difference/max": 0.34268784523010254, + "sampling/sampling_logp_difference/mean": 0.011354964226484299, + "step": 926 + }, + { + "clip_ratio/high_max": 6.659563223365694e-05, + "clip_ratio/high_mean": 6.659563223365694e-05, + "clip_ratio/low_mean": 0.0001500398138887249, + "clip_ratio/low_min": 0.0001500398138887249, + "clip_ratio/region_mean": 0.00021663544612238184, + "completions/clipped_ratio": 0.0, + "completions/max_length": 682.0, + "completions/max_terminated_length": 682.0, + "completions/mean_length": 423.59375, + "completions/mean_terminated_length": 423.59375, + "completions/min_length": 176.0, + "completions/min_terminated_length": 176.0, + "entropy": 0.35049179941415787, + "epoch": 0.49625267665952894, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.026263821870088577, + "learning_rate": 1e-05, + "loss": -0.0016, + "num_tokens": 18420602.0, + "reward": 0.6875, + "reward_std": 0.3471825420856476, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.4508293867111206, + "sampling/importance_sampling_ratio/mean": 1.0002559423446655, + "sampling/importance_sampling_ratio/min": 0.6940593123435974, + "sampling/sampling_logp_difference/max": 0.3721354007720947, + "sampling/sampling_logp_difference/mean": 0.011414631269872189, + "step": 927 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.34375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 766.0, + "completions/mean_length": 527.8125, + "completions/mean_terminated_length": 402.0, + "completions/min_length": 282.0, + "completions/min_terminated_length": 282.0, + "entropy": 0.4881604462862015, + "epoch": 0.49678800856531047, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.008296244777739048, + "learning_rate": 1e-05, + "loss": 0.0004, + "num_tokens": 18441876.0, + "reward": 0.5625, + "reward_std": 0.1157275140285492, + "rewards/accuracy_reward/mean": 0.5625, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 1.5937150716781616, + "sampling/importance_sampling_ratio/mean": 0.9997748732566833, + "sampling/importance_sampling_ratio/min": 0.6914979815483093, + "sampling/sampling_logp_difference/max": 0.4660677909851074, + "sampling/sampling_logp_difference/mean": 0.015581191517412663, + "step": 928 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 726.0, + "completions/mean_length": 533.34375, + "completions/mean_terminated_length": 489.8888854980469, + "completions/min_length": 235.0, + "completions/min_terminated_length": 235.0, + "entropy": 0.3309678826481104, + "epoch": 0.49732334047109206, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.008973688818514347, + "learning_rate": 1e-05, + "loss": 0.057, + "num_tokens": 18462743.0, + "reward": 0.78125, + "reward_std": 0.4218915104866028, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.398266315460205, + "sampling/importance_sampling_ratio/mean": 0.9999146461486816, + "sampling/importance_sampling_ratio/min": 0.6988242864608765, + "sampling/sampling_logp_difference/max": 0.3583559989929199, + "sampling/sampling_logp_difference/mean": 0.01103132776916027, + "step": 929 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 633.0, + "completions/max_terminated_length": 633.0, + "completions/mean_length": 430.40625, + "completions/mean_terminated_length": 430.40625, + "completions/min_length": 236.0, + "completions/min_terminated_length": 236.0, + "entropy": 0.22674109786748886, + "epoch": 0.49785867237687365, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.032213736325502396, + "learning_rate": 1e-05, + "loss": -0.012, + "num_tokens": 18479812.0, + "reward": 0.59375, + "reward_std": 0.3377464711666107, + "rewards/accuracy_reward/mean": 0.59375, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.3437047004699707, + "sampling/importance_sampling_ratio/mean": 1.0004483461380005, + "sampling/importance_sampling_ratio/min": 0.7815067768096924, + "sampling/sampling_logp_difference/max": 0.29543042182922363, + "sampling/sampling_logp_difference/mean": 0.00871048029512167, + "step": 930 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 751.0, + "completions/mean_length": 484.125, + "completions/mean_terminated_length": 454.75860595703125, + "completions/min_length": 219.0, + "completions/min_terminated_length": 219.0, + "entropy": 0.35440368205308914, + "epoch": 0.49839400428265523, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.01829420030117035, + "learning_rate": 1e-05, + "loss": 0.0341, + "num_tokens": 18499112.0, + "reward": 0.3125, + "reward_std": 0.3335031569004059, + "rewards/accuracy_reward/mean": 0.3125, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.365425705909729, + "sampling/importance_sampling_ratio/mean": 0.9999644160270691, + "sampling/importance_sampling_ratio/min": 0.1889118105173111, + "sampling/sampling_logp_difference/max": 1.6664749383926392, + "sampling/sampling_logp_difference/mean": 0.012364272028207779, + "step": 931 + }, + { + "clip_ratio/high_max": 6.124448555056006e-05, + "clip_ratio/high_mean": 6.124448555056006e-05, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 6.124448555056006e-05, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 575.0, + "completions/mean_length": 439.5625, + "completions/mean_terminated_length": 405.5862121582031, + "completions/min_length": 190.0, + "completions/min_terminated_length": 190.0, + "entropy": 0.40438541769981384, + "epoch": 0.4989293361884368, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.014041540212929249, + "learning_rate": 1e-05, + "loss": -0.025, + "num_tokens": 18517426.0, + "reward": 0.71875, + "reward_std": 0.3787454068660736, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.5562971830368042, + "sampling/importance_sampling_ratio/mean": 1.0000966787338257, + "sampling/importance_sampling_ratio/min": 0.6646143198013306, + "sampling/sampling_logp_difference/max": 0.4423093795776367, + "sampling/sampling_logp_difference/mean": 0.01447139959782362, + "step": 932 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 6.191183638293296e-05, + "clip_ratio/low_min": 6.191183638293296e-05, + "clip_ratio/region_mean": 6.191183638293296e-05, + "completions/clipped_ratio": 0.0, + "completions/max_length": 612.0, + "completions/max_terminated_length": 612.0, + "completions/mean_length": 467.59375, + "completions/mean_terminated_length": 467.59375, + "completions/min_length": 267.0, + "completions/min_terminated_length": 267.0, + "entropy": 0.2528857309371233, + "epoch": 0.4994646680942184, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.023867884650826454, + "learning_rate": 1e-05, + "loss": -0.0065, + "num_tokens": 18535997.0, + "reward": 0.6875, + "reward_std": 0.3335031569004059, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.3603086471557617, + "sampling/importance_sampling_ratio/mean": 0.9998392462730408, + "sampling/importance_sampling_ratio/min": 0.681882917881012, + "sampling/sampling_logp_difference/max": 0.38289737701416016, + "sampling/sampling_logp_difference/mean": 0.009321975521743298, + "step": 933 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 9.619084448786452e-05, + "clip_ratio/low_min": 9.619084448786452e-05, + "clip_ratio/region_mean": 9.619084448786452e-05, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 718.0, + "completions/mean_length": 463.0625, + "completions/mean_terminated_length": 442.7333679199219, + "completions/min_length": 249.0, + "completions/min_terminated_length": 249.0, + "entropy": 0.3133617974817753, + "epoch": 0.5, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.011186545714735985, + "learning_rate": 1e-05, + "loss": 0.0664, + "num_tokens": 18554479.0, + "reward": 0.65625, + "reward_std": 0.3198433816432953, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.3334717750549316, + "sampling/importance_sampling_ratio/mean": 1.0001076459884644, + "sampling/importance_sampling_ratio/min": 0.7463027238845825, + "sampling/sampling_logp_difference/max": 0.29262399673461914, + "sampling/sampling_logp_difference/mean": 0.011498593725264072, + "step": 934 + }, + { + "clip_ratio/high_max": 7.11845132173039e-05, + "clip_ratio/high_mean": 7.11845132173039e-05, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 7.11845132173039e-05, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 687.0, + "completions/mean_length": 445.0625, + "completions/mean_terminated_length": 411.6551818847656, + "completions/min_length": 211.0, + "completions/min_terminated_length": 211.0, + "entropy": 0.2541938256472349, + "epoch": 0.5005353319057816, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.011226793751120567, + "learning_rate": 1e-05, + "loss": 0.0045, + "num_tokens": 18572641.0, + "reward": 0.78125, + "reward_std": 0.3061639666557312, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.9631344079971313, + "sampling/importance_sampling_ratio/mean": 1.000319242477417, + "sampling/importance_sampling_ratio/min": 0.7012016773223877, + "sampling/sampling_logp_difference/max": 0.6745424270629883, + "sampling/sampling_logp_difference/mean": 0.009384569711983204, + "step": 935 + }, + { + "clip_ratio/high_max": 0.00012135922588640824, + "clip_ratio/high_mean": 0.00012135922588640824, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.00012135922588640824, + "completions/clipped_ratio": 0.0, + "completions/max_length": 715.0, + "completions/max_terminated_length": 715.0, + "completions/mean_length": 403.5625, + "completions/mean_terminated_length": 403.5625, + "completions/min_length": 169.0, + "completions/min_terminated_length": 169.0, + "entropy": 0.2944677174091339, + "epoch": 0.5010706638115632, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.032058071345090866, + "learning_rate": 1e-05, + "loss": -0.0307, + "num_tokens": 18588859.0, + "reward": 0.5625, + "reward_std": 0.3471825420856476, + "rewards/accuracy_reward/mean": 0.5625, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 1.7584322690963745, + "sampling/importance_sampling_ratio/mean": 0.9996654391288757, + "sampling/importance_sampling_ratio/min": 0.7382290363311768, + "sampling/sampling_logp_difference/max": 0.564422607421875, + "sampling/sampling_logp_difference/mean": 0.01127475406974554, + "step": 936 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.76834318053443e-05, + "clip_ratio/low_min": 5.76834318053443e-05, + "clip_ratio/region_mean": 5.76834318053443e-05, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 660.0, + "completions/mean_length": 502.21875, + "completions/mean_terminated_length": 484.5000305175781, + "completions/min_length": 316.0, + "completions/min_terminated_length": 316.0, + "entropy": 0.3367830812931061, + "epoch": 0.5016059957173448, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.015154167078435421, + "learning_rate": 1e-05, + "loss": -0.0086, + "num_tokens": 18608762.0, + "reward": 0.4375, + "reward_std": 0.3335031569004059, + "rewards/accuracy_reward/mean": 0.4375, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 1.4711087942123413, + "sampling/importance_sampling_ratio/mean": 1.0000927448272705, + "sampling/importance_sampling_ratio/min": 0.6639459729194641, + "sampling/sampling_logp_difference/max": 0.40955448150634766, + "sampling/sampling_logp_difference/mean": 0.011766834184527397, + "step": 937 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 7.575757626909763e-05, + "clip_ratio/low_min": 7.575757626909763e-05, + "clip_ratio/region_mean": 7.575757626909763e-05, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 493.0, + "completions/mean_length": 353.65625, + "completions/mean_terminated_length": 340.2903137207031, + "completions/min_length": 215.0, + "completions/min_terminated_length": 215.0, + "entropy": 0.3739020489156246, + "epoch": 0.5021413276231264, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.03872900456190109, + "learning_rate": 1e-05, + "loss": 0.1192, + "num_tokens": 18624191.0, + "reward": 0.9375, + "reward_std": 0.1767766922712326, + "rewards/accuracy_reward/mean": 0.9375, + "rewards/accuracy_reward/std": 0.24593468010425568, + "sampling/importance_sampling_ratio/max": 1.3281844854354858, + "sampling/importance_sampling_ratio/mean": 0.9997400045394897, + "sampling/importance_sampling_ratio/min": 0.5952030420303345, + "sampling/sampling_logp_difference/max": 0.518852710723877, + "sampling/sampling_logp_difference/mean": 0.01244304608553648, + "step": 938 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 613.0, + "completions/mean_length": 435.8125, + "completions/mean_terminated_length": 425.0967712402344, + "completions/min_length": 245.0, + "completions/min_terminated_length": 245.0, + "entropy": 0.2987457551062107, + "epoch": 0.5026766595289079, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.007831594906747341, + "learning_rate": 1e-05, + "loss": 0.0256, + "num_tokens": 18641537.0, + "reward": 0.75, + "reward_std": 0.2314550280570984, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.4131138324737549, + "sampling/importance_sampling_ratio/mean": 1.0005991458892822, + "sampling/importance_sampling_ratio/min": 0.6951550841331482, + "sampling/sampling_logp_difference/max": 0.3636202812194824, + "sampling/sampling_logp_difference/mean": 0.010999714024364948, + "step": 939 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.6306307669728994e-05, + "clip_ratio/low_min": 5.6306307669728994e-05, + "clip_ratio/region_mean": 5.6306307669728994e-05, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 717.0, + "completions/mean_length": 521.25, + "completions/mean_terminated_length": 513.290283203125, + "completions/min_length": 332.0, + "completions/min_terminated_length": 332.0, + "entropy": 0.31170865148305893, + "epoch": 0.5032119914346895, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": 0.0044, + "num_tokens": 18661945.0, + "reward": 0.71875, + "reward_std": 0.0883883461356163, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.5880764722824097, + "sampling/importance_sampling_ratio/mean": 1.000365138053894, + "sampling/importance_sampling_ratio/min": 0.585188627243042, + "sampling/sampling_logp_difference/max": 0.5358209609985352, + "sampling/sampling_logp_difference/mean": 0.011812165379524231, + "step": 940 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00012867815530626103, + "clip_ratio/low_min": 0.00012867815530626103, + "clip_ratio/region_mean": 0.00012867815530626103, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 563.0, + "completions/mean_length": 446.3125, + "completions/mean_terminated_length": 435.93548583984375, + "completions/min_length": 265.0, + "completions/min_terminated_length": 265.0, + "entropy": 0.2936486452817917, + "epoch": 0.5037473233404711, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.02061215043067932, + "learning_rate": 1e-05, + "loss": 0.0116, + "num_tokens": 18680363.0, + "reward": 0.71875, + "reward_std": 0.2630178928375244, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.485531210899353, + "sampling/importance_sampling_ratio/mean": 1.0001529455184937, + "sampling/importance_sampling_ratio/min": 0.6224051713943481, + "sampling/sampling_logp_difference/max": 0.4741640090942383, + "sampling/sampling_logp_difference/mean": 0.010788899846374989, + "step": 941 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 6.003842281643301e-05, + "clip_ratio/low_min": 6.003842281643301e-05, + "clip_ratio/region_mean": 6.003842281643301e-05, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 747.0, + "completions/mean_length": 473.40625, + "completions/mean_terminated_length": 442.9310302734375, + "completions/min_length": 249.0, + "completions/min_terminated_length": 249.0, + "entropy": 0.2970592677593231, + "epoch": 0.5042826552462527, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.02228197641670704, + "learning_rate": 1e-05, + "loss": 0.0424, + "num_tokens": 18699296.0, + "reward": 0.6875, + "reward_std": 0.3335031569004059, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.9042567014694214, + "sampling/importance_sampling_ratio/mean": 0.9999818205833435, + "sampling/importance_sampling_ratio/min": 0.6825937032699585, + "sampling/sampling_logp_difference/max": 0.6440917253494263, + "sampling/sampling_logp_difference/mean": 0.01097430195659399, + "step": 942 + }, + { + "clip_ratio/high_max": 5.857544601894915e-05, + "clip_ratio/high_mean": 5.857544601894915e-05, + "clip_ratio/low_mean": 0.00013716623652726412, + "clip_ratio/low_min": 0.00013716623652726412, + "clip_ratio/region_mean": 0.00019574168254621327, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 681.0, + "completions/mean_length": 463.3125, + "completions/mean_terminated_length": 453.4838562011719, + "completions/min_length": 230.0, + "completions/min_terminated_length": 230.0, + "entropy": 0.32913353852927685, + "epoch": 0.5048179871520343, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.010207954794168472, + "learning_rate": 1e-05, + "loss": 0.0461, + "num_tokens": 18717842.0, + "reward": 0.8125, + "reward_std": 0.3335031569004059, + "rewards/accuracy_reward/mean": 0.8125, + "rewards/accuracy_reward/std": 0.3965577781200409, + "sampling/importance_sampling_ratio/max": 1.385231375694275, + "sampling/importance_sampling_ratio/mean": 1.0003337860107422, + "sampling/importance_sampling_ratio/min": 0.6972249150276184, + "sampling/sampling_logp_difference/max": 0.36064720153808594, + "sampling/sampling_logp_difference/mean": 0.011917158961296082, + "step": 943 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 6.163708167150617e-05, + "clip_ratio/low_min": 6.163708167150617e-05, + "clip_ratio/region_mean": 6.163708167150617e-05, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 613.0, + "completions/mean_length": 428.03125, + "completions/mean_terminated_length": 417.06451416015625, + "completions/min_length": 197.0, + "completions/min_terminated_length": 197.0, + "entropy": 0.2715218998491764, + "epoch": 0.5053533190578159, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.007468083873391151, + "learning_rate": 1e-05, + "loss": 0.0921, + "num_tokens": 18734731.0, + "reward": 0.65625, + "reward_std": 0.2041158676147461, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.309759497642517, + "sampling/importance_sampling_ratio/mean": 1.0000685453414917, + "sampling/importance_sampling_ratio/min": 0.6160060167312622, + "sampling/sampling_logp_difference/max": 0.4844985008239746, + "sampling/sampling_logp_difference/mean": 0.010679789818823338, + "step": 944 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 714.0, + "completions/mean_length": 488.09375, + "completions/mean_terminated_length": 448.107177734375, + "completions/min_length": 225.0, + "completions/min_terminated_length": 225.0, + "entropy": 0.2994806170463562, + "epoch": 0.5058886509635975, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.024860825389623642, + "learning_rate": 1e-05, + "loss": 0.0204, + "num_tokens": 18753590.0, + "reward": 0.75, + "reward_std": 0.3335031569004059, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.4279488325119019, + "sampling/importance_sampling_ratio/mean": 0.9999743700027466, + "sampling/importance_sampling_ratio/min": 0.7017055749893188, + "sampling/sampling_logp_difference/max": 0.35623908042907715, + "sampling/sampling_logp_difference/mean": 0.009921894408762455, + "step": 945 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 4.56537636637222e-05, + "clip_ratio/low_min": 4.56537636637222e-05, + "clip_ratio/region_mean": 4.56537636637222e-05, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 758.0, + "completions/mean_length": 539.40625, + "completions/mean_terminated_length": 497.0740661621094, + "completions/min_length": 188.0, + "completions/min_terminated_length": 188.0, + "entropy": 0.48707421869039536, + "epoch": 0.5064239828693791, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.01256539486348629, + "learning_rate": 1e-05, + "loss": 0.0928, + "num_tokens": 18774515.0, + "reward": 0.75, + "reward_std": 0.3745020925998688, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.5281684398651123, + "sampling/importance_sampling_ratio/mean": 1.0002484321594238, + "sampling/importance_sampling_ratio/min": 0.6692571640014648, + "sampling/sampling_logp_difference/max": 0.424069881439209, + "sampling/sampling_logp_difference/mean": 0.015065666288137436, + "step": 946 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 6.887052586534992e-05, + "clip_ratio/low_min": 6.887052586534992e-05, + "clip_ratio/region_mean": 6.887052586534992e-05, + "completions/clipped_ratio": 0.0, + "completions/max_length": 590.0, + "completions/max_terminated_length": 590.0, + "completions/mean_length": 388.75, + "completions/mean_terminated_length": 388.75, + "completions/min_length": 138.0, + "completions/min_terminated_length": 138.0, + "entropy": 0.28587933629751205, + "epoch": 0.5069593147751607, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.035891104489564896, + "learning_rate": 1e-05, + "loss": -0.0064, + "num_tokens": 18790155.0, + "reward": 0.8125, + "reward_std": 0.1157275140285492, + "rewards/accuracy_reward/mean": 0.8125, + "rewards/accuracy_reward/std": 0.3965577781200409, + "sampling/importance_sampling_ratio/max": 1.4828362464904785, + "sampling/importance_sampling_ratio/mean": 1.0002168416976929, + "sampling/importance_sampling_ratio/min": 0.6562126278877258, + "sampling/sampling_logp_difference/max": 0.42127037048339844, + "sampling/sampling_logp_difference/mean": 0.010166658088564873, + "step": 947 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00014253899280447513, + "clip_ratio/low_min": 0.00014253899280447513, + "clip_ratio/region_mean": 0.00014253899280447513, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 614.0, + "completions/mean_length": 452.1875, + "completions/mean_terminated_length": 442.0, + "completions/min_length": 337.0, + "completions/min_terminated_length": 337.0, + "entropy": 0.2540544904768467, + "epoch": 0.5074946466809421, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.00658551836386323, + "learning_rate": 1e-05, + "loss": 0.0424, + "num_tokens": 18808257.0, + "reward": 0.875, + "reward_std": 0.2314550280570984, + "rewards/accuracy_reward/mean": 0.875, + "rewards/accuracy_reward/std": 0.33601075410842896, + "sampling/importance_sampling_ratio/max": 2.0, + "sampling/importance_sampling_ratio/mean": 0.9995406866073608, + "sampling/importance_sampling_ratio/min": 0.6581525802612305, + "sampling/sampling_logp_difference/max": 1.0281407833099365, + "sampling/sampling_logp_difference/mean": 0.010223720222711563, + "step": 948 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00023598983898409642, + "clip_ratio/low_min": 0.00023598983898409642, + "clip_ratio/region_mean": 0.00023598983898409642, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 744.0, + "completions/mean_length": 504.0, + "completions/mean_terminated_length": 466.2857360839844, + "completions/min_length": 285.0, + "completions/min_terminated_length": 285.0, + "entropy": 0.3444122467190027, + "epoch": 0.5080299785867237, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.01840297132730484, + "learning_rate": 1e-05, + "loss": 0.0339, + "num_tokens": 18828345.0, + "reward": 0.625, + "reward_std": 0.3335031569004059, + "rewards/accuracy_reward/mean": 0.625, + "rewards/accuracy_reward/std": 0.49186936020851135, + "sampling/importance_sampling_ratio/max": 1.636265754699707, + "sampling/importance_sampling_ratio/mean": 1.0002254247665405, + "sampling/importance_sampling_ratio/min": 0.6133067011833191, + "sampling/sampling_logp_difference/max": 0.4924166202545166, + "sampling/sampling_logp_difference/mean": 0.012093781493604183, + "step": 949 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 756.0, + "completions/max_terminated_length": 756.0, + "completions/mean_length": 494.3125, + "completions/mean_terminated_length": 494.3125, + "completions/min_length": 249.0, + "completions/min_terminated_length": 249.0, + "entropy": 0.27808588184416294, + "epoch": 0.5085653104925053, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.013713176362216473, + "learning_rate": 1e-05, + "loss": -0.0167, + "num_tokens": 18847931.0, + "reward": 0.8125, + "reward_std": 0.2587745785713196, + "rewards/accuracy_reward/mean": 0.8125, + "rewards/accuracy_reward/std": 0.3965577781200409, + "sampling/importance_sampling_ratio/max": 1.5939327478408813, + "sampling/importance_sampling_ratio/mean": 0.9996863007545471, + "sampling/importance_sampling_ratio/min": 0.6843679547309875, + "sampling/sampling_logp_difference/max": 0.4662044048309326, + "sampling/sampling_logp_difference/mean": 0.010790374130010605, + "step": 950 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 716.0, + "completions/max_terminated_length": 716.0, + "completions/mean_length": 405.1875, + "completions/mean_terminated_length": 405.1875, + "completions/min_length": 207.0, + "completions/min_terminated_length": 207.0, + "entropy": 0.34625665098428726, + "epoch": 0.5091006423982869, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.019684698432683945, + "learning_rate": 1e-05, + "loss": -0.0817, + "num_tokens": 18864417.0, + "reward": 0.875, + "reward_std": 0.2925041913986206, + "rewards/accuracy_reward/mean": 0.875, + "rewards/accuracy_reward/std": 0.33601075410842896, + "sampling/importance_sampling_ratio/max": 1.4535462856292725, + "sampling/importance_sampling_ratio/mean": 0.9996865391731262, + "sampling/importance_sampling_ratio/min": 0.7225186824798584, + "sampling/sampling_logp_difference/max": 0.3740062713623047, + "sampling/sampling_logp_difference/mean": 0.01191488653421402, + "step": 951 + }, + { + "clip_ratio/high_max": 4.736642586067319e-05, + "clip_ratio/high_mean": 4.736642586067319e-05, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 4.736642586067319e-05, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 764.0, + "completions/mean_length": 487.3125, + "completions/mean_terminated_length": 478.258056640625, + "completions/min_length": 212.0, + "completions/min_terminated_length": 212.0, + "entropy": 0.2598949037492275, + "epoch": 0.5096359743040685, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.028428837656974792, + "learning_rate": 1e-05, + "loss": 0.0199, + "num_tokens": 18883467.0, + "reward": 0.9375, + "reward_std": 0.1767766922712326, + "rewards/accuracy_reward/mean": 0.9375, + "rewards/accuracy_reward/std": 0.24593468010425568, + "sampling/importance_sampling_ratio/max": 1.8784284591674805, + "sampling/importance_sampling_ratio/mean": 1.0001908540725708, + "sampling/importance_sampling_ratio/min": 0.640715479850769, + "sampling/sampling_logp_difference/max": 0.6304354667663574, + "sampling/sampling_logp_difference/mean": 0.009616902098059654, + "step": 952 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00011550290582817979, + "clip_ratio/low_min": 0.00011550290582817979, + "clip_ratio/region_mean": 0.00011550290582817979, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 718.0, + "completions/mean_length": 461.84375, + "completions/mean_terminated_length": 441.433349609375, + "completions/min_length": 251.0, + "completions/min_terminated_length": 251.0, + "entropy": 0.3623839858919382, + "epoch": 0.5101713062098501, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.009525010362267494, + "learning_rate": 1e-05, + "loss": 0.0311, + "num_tokens": 18902030.0, + "reward": 0.625, + "reward_std": 0.2177756428718567, + "rewards/accuracy_reward/mean": 0.625, + "rewards/accuracy_reward/std": 0.49186936020851135, + "sampling/importance_sampling_ratio/max": 1.32841157913208, + "sampling/importance_sampling_ratio/mean": 1.0002055168151855, + "sampling/importance_sampling_ratio/min": 0.4049910306930542, + "sampling/sampling_logp_difference/max": 0.9038903713226318, + "sampling/sampling_logp_difference/mean": 0.011523718945682049, + "step": 953 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.225752465776168e-05, + "clip_ratio/low_min": 5.225752465776168e-05, + "clip_ratio/region_mean": 5.225752465776168e-05, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 694.0, + "completions/mean_length": 500.5, + "completions/mean_terminated_length": 462.2857360839844, + "completions/min_length": 301.0, + "completions/min_terminated_length": 301.0, + "entropy": 0.35181020572781563, + "epoch": 0.5107066381156317, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.013160860165953636, + "learning_rate": 1e-05, + "loss": -0.0162, + "num_tokens": 18922510.0, + "reward": 0.71875, + "reward_std": 0.2041158676147461, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.6601618528366089, + "sampling/importance_sampling_ratio/mean": 1.0003364086151123, + "sampling/importance_sampling_ratio/min": 0.7017132639884949, + "sampling/sampling_logp_difference/max": 0.5069150924682617, + "sampling/sampling_logp_difference/mean": 0.011754768900573254, + "step": 954 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 754.0, + "completions/max_terminated_length": 754.0, + "completions/mean_length": 443.6875, + "completions/mean_terminated_length": 443.6875, + "completions/min_length": 154.0, + "completions/min_terminated_length": 154.0, + "entropy": 0.2924421951174736, + "epoch": 0.5112419700214133, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": 0.0, + "num_tokens": 18940556.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/accuracy_reward/mean": 1.0, + "rewards/accuracy_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.4143191576004028, + "sampling/importance_sampling_ratio/mean": 0.9999143481254578, + "sampling/importance_sampling_ratio/min": 0.7306100130081177, + "sampling/sampling_logp_difference/max": 0.3466482162475586, + "sampling/sampling_logp_difference/mean": 0.01069424208253622, + "step": 955 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.955216693109833e-05, + "clip_ratio/low_min": 5.955216693109833e-05, + "clip_ratio/region_mean": 5.955216693109833e-05, + "completions/clipped_ratio": 0.0, + "completions/max_length": 699.0, + "completions/max_terminated_length": 699.0, + "completions/mean_length": 442.28125, + "completions/mean_terminated_length": 442.28125, + "completions/min_length": 166.0, + "completions/min_terminated_length": 166.0, + "entropy": 0.34365570545196533, + "epoch": 0.5117773019271948, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.013136308640241623, + "learning_rate": 1e-05, + "loss": 0.0678, + "num_tokens": 18958221.0, + "reward": 0.84375, + "reward_std": 0.22201895713806152, + "rewards/accuracy_reward/mean": 0.84375, + "rewards/accuracy_reward/std": 0.3689020276069641, + "sampling/importance_sampling_ratio/max": 1.5441542863845825, + "sampling/importance_sampling_ratio/mean": 0.9999651908874512, + "sampling/importance_sampling_ratio/min": 0.7165300250053406, + "sampling/sampling_logp_difference/max": 0.434476375579834, + "sampling/sampling_logp_difference/mean": 0.011885290965437889, + "step": 956 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00012048192729707807, + "clip_ratio/low_min": 0.00012048192729707807, + "clip_ratio/region_mean": 0.00012048192729707807, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 736.0, + "completions/mean_length": 458.625, + "completions/mean_terminated_length": 448.6451416015625, + "completions/min_length": 280.0, + "completions/min_terminated_length": 280.0, + "entropy": 0.24939272366464138, + "epoch": 0.5123126338329764, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.021936535835266113, + "learning_rate": 1e-05, + "loss": 0.0042, + "num_tokens": 18976265.0, + "reward": 0.90625, + "reward_std": 0.2041158676147461, + "rewards/accuracy_reward/mean": 0.90625, + "rewards/accuracy_reward/std": 0.2961445748806, + "sampling/importance_sampling_ratio/max": 1.4704183340072632, + "sampling/importance_sampling_ratio/mean": 0.9997578859329224, + "sampling/importance_sampling_ratio/min": 0.6308900117874146, + "sampling/sampling_logp_difference/max": 0.46062374114990234, + "sampling/sampling_logp_difference/mean": 0.009267176501452923, + "step": 957 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 698.0, + "completions/mean_length": 489.09375, + "completions/mean_terminated_length": 470.5000305175781, + "completions/min_length": 265.0, + "completions/min_terminated_length": 265.0, + "entropy": 0.29304561018943787, + "epoch": 0.512847965738758, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.003565209684893489, + "learning_rate": 1e-05, + "loss": 0.0305, + "num_tokens": 18995868.0, + "reward": 0.96875, + "reward_std": 0.0883883461356163, + "rewards/accuracy_reward/mean": 0.96875, + "rewards/accuracy_reward/std": 0.1767766922712326, + "sampling/importance_sampling_ratio/max": 1.336312174797058, + "sampling/importance_sampling_ratio/mean": 0.9999875426292419, + "sampling/importance_sampling_ratio/min": 0.6728830933570862, + "sampling/sampling_logp_difference/max": 0.39618372917175293, + "sampling/sampling_logp_difference/mean": 0.010725563392043114, + "step": 958 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 6.364562432281673e-05, + "clip_ratio/low_min": 6.364562432281673e-05, + "clip_ratio/region_mean": 6.364562432281673e-05, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 701.0, + "completions/mean_length": 473.90625, + "completions/mean_terminated_length": 464.4193420410156, + "completions/min_length": 88.0, + "completions/min_terminated_length": 88.0, + "entropy": 0.24543077498674393, + "epoch": 0.5133832976445396, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.015022432431578636, + "learning_rate": 1e-05, + "loss": 0.0147, + "num_tokens": 19014689.0, + "reward": 0.875, + "reward_std": 0.2177756428718567, + "rewards/accuracy_reward/mean": 0.875, + "rewards/accuracy_reward/std": 0.33601075410842896, + "sampling/importance_sampling_ratio/max": 1.8724751472473145, + "sampling/importance_sampling_ratio/mean": 1.000110387802124, + "sampling/importance_sampling_ratio/min": 0.7357137799263, + "sampling/sampling_logp_difference/max": 0.6272611618041992, + "sampling/sampling_logp_difference/mean": 0.00897144339978695, + "step": 959 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 6.49688154226169e-05, + "clip_ratio/low_min": 6.49688154226169e-05, + "clip_ratio/region_mean": 6.49688154226169e-05, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 696.0, + "completions/mean_length": 582.5625, + "completions/mean_terminated_length": 510.0, + "completions/min_length": 216.0, + "completions/min_terminated_length": 216.0, + "entropy": 0.4166363328695297, + "epoch": 0.5139186295503212, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.016490228474140167, + "learning_rate": 1e-05, + "loss": -0.014, + "num_tokens": 19037267.0, + "reward": 0.125, + "reward_std": 0.2314550280570984, + "rewards/accuracy_reward/mean": 0.125, + "rewards/accuracy_reward/std": 0.33601075410842896, + "sampling/importance_sampling_ratio/max": 1.3851897716522217, + "sampling/importance_sampling_ratio/mean": 1.0000791549682617, + "sampling/importance_sampling_ratio/min": 0.7044484615325928, + "sampling/sampling_logp_difference/max": 0.3503401279449463, + "sampling/sampling_logp_difference/mean": 0.013296709395945072, + "step": 960 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 726.0, + "completions/max_terminated_length": 726.0, + "completions/mean_length": 448.21875, + "completions/mean_terminated_length": 448.21875, + "completions/min_length": 258.0, + "completions/min_terminated_length": 258.0, + "entropy": 0.2712840549647808, + "epoch": 0.5144539614561028, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.0034663125406950712, + "learning_rate": 1e-05, + "loss": 0.061, + "num_tokens": 19055258.0, + "reward": 0.6875, + "reward_std": 0.2177756428718567, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.6785802841186523, + "sampling/importance_sampling_ratio/mean": 0.9998134970664978, + "sampling/importance_sampling_ratio/min": 0.6850326657295227, + "sampling/sampling_logp_difference/max": 0.5179483890533447, + "sampling/sampling_logp_difference/mean": 0.010201629251241684, + "step": 961 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00017628124260227196, + "clip_ratio/low_min": 0.00017628124260227196, + "clip_ratio/region_mean": 0.00017628124260227196, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 660.0, + "completions/mean_length": 519.25, + "completions/mean_terminated_length": 461.8461608886719, + "completions/min_length": 298.0, + "completions/min_terminated_length": 298.0, + "entropy": 0.38563186675310135, + "epoch": 0.5149892933618844, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.008774523623287678, + "learning_rate": 1e-05, + "loss": -0.0085, + "num_tokens": 19075802.0, + "reward": 0.625, + "reward_std": 0.2314550280570984, + "rewards/accuracy_reward/mean": 0.625, + "rewards/accuracy_reward/std": 0.49186936020851135, + "sampling/importance_sampling_ratio/max": 1.4578242301940918, + "sampling/importance_sampling_ratio/mean": 0.999571681022644, + "sampling/importance_sampling_ratio/min": 0.626001238822937, + "sampling/sampling_logp_difference/max": 0.4684029817581177, + "sampling/sampling_logp_difference/mean": 0.012711354531347752, + "step": 962 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 657.0, + "completions/mean_length": 441.875, + "completions/mean_terminated_length": 431.3548278808594, + "completions/min_length": 229.0, + "completions/min_terminated_length": 229.0, + "entropy": 0.30633309856057167, + "epoch": 0.515524625267666, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.02011452428996563, + "learning_rate": 1e-05, + "loss": 0.006, + "num_tokens": 19093990.0, + "reward": 0.90625, + "reward_std": 0.2041158676147461, + "rewards/accuracy_reward/mean": 0.90625, + "rewards/accuracy_reward/std": 0.2961445748806, + "sampling/importance_sampling_ratio/max": 1.4361798763275146, + "sampling/importance_sampling_ratio/mean": 0.999936580657959, + "sampling/importance_sampling_ratio/min": 0.6223956346511841, + "sampling/sampling_logp_difference/max": 0.4741792678833008, + "sampling/sampling_logp_difference/mean": 0.011413052678108215, + "step": 963 + }, + { + "clip_ratio/high_max": 6.513809057651088e-05, + "clip_ratio/high_mean": 6.513809057651088e-05, + "clip_ratio/low_mean": 6.094588025007397e-05, + "clip_ratio/low_min": 6.094588025007397e-05, + "clip_ratio/region_mean": 0.00012608397082658485, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 636.0, + "completions/mean_length": 460.90625, + "completions/mean_terminated_length": 440.433349609375, + "completions/min_length": 306.0, + "completions/min_terminated_length": 306.0, + "entropy": 0.2541094161570072, + "epoch": 0.5160599571734475, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.015052350237965584, + "learning_rate": 1e-05, + "loss": 0.0081, + "num_tokens": 19112347.0, + "reward": 0.71875, + "reward_std": 0.35564959049224854, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.4809685945510864, + "sampling/importance_sampling_ratio/mean": 1.000511646270752, + "sampling/importance_sampling_ratio/min": 0.6855025291442871, + "sampling/sampling_logp_difference/max": 0.3926963806152344, + "sampling/sampling_logp_difference/mean": 0.009388880804181099, + "step": 964 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 765.0, + "completions/mean_length": 492.84375, + "completions/mean_terminated_length": 464.3793029785156, + "completions/min_length": 279.0, + "completions/min_terminated_length": 279.0, + "entropy": 0.31386849097907543, + "epoch": 0.5165952890792291, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.006239220499992371, + "learning_rate": 1e-05, + "loss": 0.0961, + "num_tokens": 19131846.0, + "reward": 0.78125, + "reward_std": 0.3061639666557312, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.675916314125061, + "sampling/importance_sampling_ratio/mean": 1.0001627206802368, + "sampling/importance_sampling_ratio/min": 0.7222133278846741, + "sampling/sampling_logp_difference/max": 0.5163600444793701, + "sampling/sampling_logp_difference/mean": 0.010937148705124855, + "step": 965 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 6.329113966785371e-05, + "clip_ratio/low_min": 6.329113966785371e-05, + "clip_ratio/region_mean": 6.329113966785371e-05, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 737.0, + "completions/mean_length": 478.78125, + "completions/mean_terminated_length": 469.45159912109375, + "completions/min_length": 247.0, + "completions/min_terminated_length": 247.0, + "entropy": 0.277352349832654, + "epoch": 0.5171306209850107, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.013667028397321701, + "learning_rate": 1e-05, + "loss": -0.0055, + "num_tokens": 19151223.0, + "reward": 0.78125, + "reward_std": 0.24511480331420898, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.4211808443069458, + "sampling/importance_sampling_ratio/mean": 0.9998354911804199, + "sampling/importance_sampling_ratio/min": 0.7232437133789062, + "sampling/sampling_logp_difference/max": 0.3514881134033203, + "sampling/sampling_logp_difference/mean": 0.010061788372695446, + "step": 966 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 716.0, + "completions/mean_length": 484.71875, + "completions/mean_terminated_length": 475.58062744140625, + "completions/min_length": 124.0, + "completions/min_terminated_length": 124.0, + "entropy": 0.24185454845428467, + "epoch": 0.5176659528907923, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.006015704944729805, + "learning_rate": 1e-05, + "loss": -0.0032, + "num_tokens": 19170782.0, + "reward": 0.78125, + "reward_std": 0.0883883461356163, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.3794660568237305, + "sampling/importance_sampling_ratio/mean": 0.9993918538093567, + "sampling/importance_sampling_ratio/min": 0.6304227113723755, + "sampling/sampling_logp_difference/max": 0.46136474609375, + "sampling/sampling_logp_difference/mean": 0.009323207661509514, + "step": 967 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0001298701245104894, + "clip_ratio/low_min": 0.0001298701245104894, + "clip_ratio/region_mean": 0.0001298701245104894, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 682.0, + "completions/mean_length": 503.15625, + "completions/mean_terminated_length": 485.5000305175781, + "completions/min_length": 304.0, + "completions/min_terminated_length": 304.0, + "entropy": 0.3035869039595127, + "epoch": 0.5182012847965739, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.01092250645160675, + "learning_rate": 1e-05, + "loss": -0.0309, + "num_tokens": 19191171.0, + "reward": 0.59375, + "reward_std": 0.378745436668396, + "rewards/accuracy_reward/mean": 0.59375, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.4265621900558472, + "sampling/importance_sampling_ratio/mean": 0.9999486804008484, + "sampling/importance_sampling_ratio/min": 0.6853776574134827, + "sampling/sampling_logp_difference/max": 0.37778520584106445, + "sampling/sampling_logp_difference/mean": 0.01078245509415865, + "step": 968 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 760.0, + "completions/mean_length": 619.96875, + "completions/mean_terminated_length": 531.1500244140625, + "completions/min_length": 290.0, + "completions/min_terminated_length": 290.0, + "entropy": 0.43357282504439354, + "epoch": 0.5187366167023555, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.013624757528305054, + "learning_rate": 1e-05, + "loss": 0.0109, + "num_tokens": 19215442.0, + "reward": 0.46875, + "reward_std": 0.2630178928375244, + "rewards/accuracy_reward/mean": 0.46875, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.3855798244476318, + "sampling/importance_sampling_ratio/mean": 1.0001927614212036, + "sampling/importance_sampling_ratio/min": 0.34400755167007446, + "sampling/sampling_logp_difference/max": 1.067091703414917, + "sampling/sampling_logp_difference/mean": 0.012979980558156967, + "step": 969 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 567.0, + "completions/max_terminated_length": 567.0, + "completions/mean_length": 344.6875, + "completions/mean_terminated_length": 344.6875, + "completions/min_length": 218.0, + "completions/min_terminated_length": 218.0, + "entropy": 0.2605263330042362, + "epoch": 0.5192719486081371, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": 0.0, + "num_tokens": 19230104.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/accuracy_reward/mean": 1.0, + "rewards/accuracy_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.431227445602417, + "sampling/importance_sampling_ratio/mean": 1.0000169277191162, + "sampling/importance_sampling_ratio/min": 0.6947574019432068, + "sampling/sampling_logp_difference/max": 0.36419248580932617, + "sampling/sampling_logp_difference/mean": 0.00965052843093872, + "step": 970 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00011839962462545373, + "clip_ratio/low_min": 0.00011839962462545373, + "clip_ratio/region_mean": 0.00011839962462545373, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 765.0, + "completions/mean_length": 551.09375, + "completions/mean_terminated_length": 528.6551513671875, + "completions/min_length": 270.0, + "completions/min_terminated_length": 270.0, + "entropy": 0.3060551155358553, + "epoch": 0.5198072805139187, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.016152273863554, + "learning_rate": 1e-05, + "loss": 0.0353, + "num_tokens": 19251691.0, + "reward": 0.6875, + "reward_std": 0.3514062464237213, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.365064024925232, + "sampling/importance_sampling_ratio/mean": 0.999962568283081, + "sampling/importance_sampling_ratio/min": 0.5747963190078735, + "sampling/sampling_logp_difference/max": 0.5537395477294922, + "sampling/sampling_logp_difference/mean": 0.010949786752462387, + "step": 971 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00021914325043326244, + "clip_ratio/low_min": 0.00021914325043326244, + "clip_ratio/region_mean": 0.00021914325043326244, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 745.0, + "completions/mean_length": 506.53125, + "completions/mean_terminated_length": 446.19232177734375, + "completions/min_length": 250.0, + "completions/min_terminated_length": 250.0, + "entropy": 0.3677872810512781, + "epoch": 0.5203426124197003, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.012668418698012829, + "learning_rate": 1e-05, + "loss": 0.1049, + "num_tokens": 19271316.0, + "reward": 0.625, + "reward_std": 0.3924052119255066, + "rewards/accuracy_reward/mean": 0.625, + "rewards/accuracy_reward/std": 0.49186936020851135, + "sampling/importance_sampling_ratio/max": 1.3050062656402588, + "sampling/importance_sampling_ratio/mean": 0.9999018907546997, + "sampling/importance_sampling_ratio/min": 0.6798360347747803, + "sampling/sampling_logp_difference/max": 0.38590359687805176, + "sampling/sampling_logp_difference/mean": 0.012253649532794952, + "step": 972 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 4.791107858181931e-05, + "clip_ratio/low_min": 4.791107858181931e-05, + "clip_ratio/region_mean": 4.791107858181931e-05, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 657.0, + "completions/mean_length": 490.28125, + "completions/mean_terminated_length": 438.85186767578125, + "completions/min_length": 181.0, + "completions/min_terminated_length": 181.0, + "entropy": 0.3771228492259979, + "epoch": 0.5208779443254818, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.008566985838115215, + "learning_rate": 1e-05, + "loss": 0.0809, + "num_tokens": 19291005.0, + "reward": 0.8125, + "reward_std": 0.2587745785713196, + "rewards/accuracy_reward/mean": 0.8125, + "rewards/accuracy_reward/std": 0.3965577781200409, + "sampling/importance_sampling_ratio/max": 1.5105116367340088, + "sampling/importance_sampling_ratio/mean": 1.0001238584518433, + "sampling/importance_sampling_ratio/min": 0.6620843410491943, + "sampling/sampling_logp_difference/max": 0.4124484062194824, + "sampling/sampling_logp_difference/mean": 0.01264272816479206, + "step": 973 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 534.0, + "completions/max_terminated_length": 534.0, + "completions/mean_length": 364.59375, + "completions/mean_terminated_length": 364.59375, + "completions/min_length": 217.0, + "completions/min_terminated_length": 217.0, + "entropy": 0.2749151811003685, + "epoch": 0.5214132762312634, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": 0.0, + "num_tokens": 19306368.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/accuracy_reward/mean": 1.0, + "rewards/accuracy_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.333916425704956, + "sampling/importance_sampling_ratio/mean": 1.0003809928894043, + "sampling/importance_sampling_ratio/min": 0.6982260346412659, + "sampling/sampling_logp_difference/max": 0.35921239852905273, + "sampling/sampling_logp_difference/mean": 0.010014554485678673, + "step": 974 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.3717230912297964e-05, + "clip_ratio/low_min": 5.3717230912297964e-05, + "clip_ratio/region_mean": 5.3717230912297964e-05, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 748.0, + "completions/mean_length": 506.9375, + "completions/mean_terminated_length": 489.5333557128906, + "completions/min_length": 215.0, + "completions/min_terminated_length": 215.0, + "entropy": 0.2950927261263132, + "epoch": 0.521948608137045, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.009787298738956451, + "learning_rate": 1e-05, + "loss": -0.0156, + "num_tokens": 19326622.0, + "reward": 0.3125, + "reward_std": 0.4082317352294922, + "rewards/accuracy_reward/mean": 0.3125, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.5575547218322754, + "sampling/importance_sampling_ratio/mean": 0.9995861649513245, + "sampling/importance_sampling_ratio/min": 0.6320212483406067, + "sampling/sampling_logp_difference/max": 0.4588322639465332, + "sampling/sampling_logp_difference/mean": 0.011625087819993496, + "step": 975 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 637.0, + "completions/max_terminated_length": 637.0, + "completions/mean_length": 432.1875, + "completions/mean_terminated_length": 432.1875, + "completions/min_length": 192.0, + "completions/min_terminated_length": 192.0, + "entropy": 0.31722994707524776, + "epoch": 0.5224839400428265, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.019412700086832047, + "learning_rate": 1e-05, + "loss": -0.0215, + "num_tokens": 19344028.0, + "reward": 0.75, + "reward_std": 0.3514062464237213, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.5474977493286133, + "sampling/importance_sampling_ratio/mean": 0.9997346997261047, + "sampling/importance_sampling_ratio/min": 0.6909081935882568, + "sampling/sampling_logp_difference/max": 0.43663930892944336, + "sampling/sampling_logp_difference/mean": 0.011539718136191368, + "step": 976 + }, + { + "clip_ratio/high_max": 5.064829747425392e-05, + "clip_ratio/high_mean": 5.064829747425392e-05, + "clip_ratio/low_mean": 9.850141213973984e-05, + "clip_ratio/low_min": 9.850141213973984e-05, + "clip_ratio/region_mean": 0.00014914970961399376, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 744.0, + "completions/mean_length": 647.6875, + "completions/mean_terminated_length": 619.923095703125, + "completions/min_length": 440.0, + "completions/min_terminated_length": 440.0, + "entropy": 0.2991927396506071, + "epoch": 0.5230192719486081, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.011010593734681606, + "learning_rate": 1e-05, + "loss": 0.0353, + "num_tokens": 19369434.0, + "reward": 0.625, + "reward_std": 0.3745020925998688, + "rewards/accuracy_reward/mean": 0.625, + "rewards/accuracy_reward/std": 0.49186936020851135, + "sampling/importance_sampling_ratio/max": 1.7090359926223755, + "sampling/importance_sampling_ratio/mean": 0.9997743368148804, + "sampling/importance_sampling_ratio/min": 0.6227453947067261, + "sampling/sampling_logp_difference/max": 0.5359294414520264, + "sampling/sampling_logp_difference/mean": 0.010291263461112976, + "step": 977 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 703.0, + "completions/mean_length": 476.96875, + "completions/mean_terminated_length": 467.58062744140625, + "completions/min_length": 265.0, + "completions/min_terminated_length": 265.0, + "entropy": 0.27222805842757225, + "epoch": 0.5235546038543897, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.0072904909029603004, + "learning_rate": 1e-05, + "loss": 0.0522, + "num_tokens": 19387937.0, + "reward": 0.90625, + "reward_std": 0.2651650309562683, + "rewards/accuracy_reward/mean": 0.90625, + "rewards/accuracy_reward/std": 0.2961445748806, + "sampling/importance_sampling_ratio/max": 1.6329829692840576, + "sampling/importance_sampling_ratio/mean": 1.000044345855713, + "sampling/importance_sampling_ratio/min": 0.6657766103744507, + "sampling/sampling_logp_difference/max": 0.49040842056274414, + "sampling/sampling_logp_difference/mean": 0.009205597452819347, + "step": 978 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 710.0, + "completions/mean_length": 535.53125, + "completions/mean_terminated_length": 502.3214416503906, + "completions/min_length": 206.0, + "completions/min_terminated_length": 206.0, + "entropy": 0.38416952081024647, + "epoch": 0.5240899357601713, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.010845184326171875, + "learning_rate": 1e-05, + "loss": 0.0512, + "num_tokens": 19409034.0, + "reward": 0.75, + "reward_std": 0.4261348247528076, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.4432660341262817, + "sampling/importance_sampling_ratio/mean": 1.000100016593933, + "sampling/importance_sampling_ratio/min": 0.5611334443092346, + "sampling/sampling_logp_difference/max": 0.577796459197998, + "sampling/sampling_logp_difference/mean": 0.012798493728041649, + "step": 979 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 8.074935612967238e-05, + "clip_ratio/low_min": 8.074935612967238e-05, + "clip_ratio/region_mean": 8.074935612967238e-05, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 685.0, + "completions/mean_length": 523.0625, + "completions/mean_terminated_length": 488.0714416503906, + "completions/min_length": 327.0, + "completions/min_terminated_length": 327.0, + "entropy": 0.30425986647605896, + "epoch": 0.5246252676659529, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.018338380381464958, + "learning_rate": 1e-05, + "loss": 0.0305, + "num_tokens": 19429740.0, + "reward": 0.75, + "reward_std": 0.26726123690605164, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.2874058485031128, + "sampling/importance_sampling_ratio/mean": 1.0000114440917969, + "sampling/importance_sampling_ratio/min": 0.6871376037597656, + "sampling/sampling_logp_difference/max": 0.37522077560424805, + "sampling/sampling_logp_difference/mean": 0.01055125892162323, + "step": 980 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 6.624271190958098e-05, + "clip_ratio/low_min": 6.624271190958098e-05, + "clip_ratio/region_mean": 6.624271190958098e-05, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 705.0, + "completions/mean_length": 512.28125, + "completions/mean_terminated_length": 495.2333679199219, + "completions/min_length": 339.0, + "completions/min_terminated_length": 339.0, + "entropy": 0.28203972429037094, + "epoch": 0.5251605995717344, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.00662532402202487, + "learning_rate": 1e-05, + "loss": 0.016, + "num_tokens": 19449669.0, + "reward": 0.78125, + "reward_std": 0.2041158676147461, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.6777180433273315, + "sampling/importance_sampling_ratio/mean": 0.9999814033508301, + "sampling/importance_sampling_ratio/min": 0.7238746881484985, + "sampling/sampling_logp_difference/max": 0.5174345970153809, + "sampling/sampling_logp_difference/mean": 0.011040785349905491, + "step": 981 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 751.0, + "completions/max_terminated_length": 751.0, + "completions/mean_length": 429.375, + "completions/mean_terminated_length": 429.375, + "completions/min_length": 186.0, + "completions/min_terminated_length": 186.0, + "entropy": 0.33899323269724846, + "epoch": 0.525695931477516, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.016133377328515053, + "learning_rate": 1e-05, + "loss": -0.025, + "num_tokens": 19466689.0, + "reward": 0.84375, + "reward_std": 0.1293872892856598, + "rewards/accuracy_reward/mean": 0.84375, + "rewards/accuracy_reward/std": 0.3689020276069641, + "sampling/importance_sampling_ratio/max": 1.5335289239883423, + "sampling/importance_sampling_ratio/mean": 1.0002915859222412, + "sampling/importance_sampling_ratio/min": 0.4264197051525116, + "sampling/sampling_logp_difference/max": 0.8523311614990234, + "sampling/sampling_logp_difference/mean": 0.011671149171888828, + "step": 982 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00024546719578211196, + "clip_ratio/low_min": 0.00024546719578211196, + "clip_ratio/region_mean": 0.00024546719578211196, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 733.0, + "completions/mean_length": 565.09375, + "completions/mean_terminated_length": 508.2799987792969, + "completions/min_length": 308.0, + "completions/min_terminated_length": 308.0, + "entropy": 0.34883488342165947, + "epoch": 0.5262312633832976, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.01370038092136383, + "learning_rate": 1e-05, + "loss": 0.063, + "num_tokens": 19488956.0, + "reward": 0.5625, + "reward_std": 0.3514062762260437, + "rewards/accuracy_reward/mean": 0.5625, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 1.4622024297714233, + "sampling/importance_sampling_ratio/mean": 1.0004162788391113, + "sampling/importance_sampling_ratio/min": 0.6158801913261414, + "sampling/sampling_logp_difference/max": 0.48470282554626465, + "sampling/sampling_logp_difference/mean": 0.01206190139055252, + "step": 983 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00010218562238151208, + "clip_ratio/low_min": 0.00010218562238151208, + "clip_ratio/region_mean": 0.00010218562238151208, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 716.0, + "completions/mean_length": 546.6875, + "completions/mean_terminated_length": 495.61541748046875, + "completions/min_length": 340.0, + "completions/min_terminated_length": 340.0, + "entropy": 0.38961392268538475, + "epoch": 0.5267665952890792, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.019659826532006264, + "learning_rate": 1e-05, + "loss": 0.1077, + "num_tokens": 19510434.0, + "reward": 0.625, + "reward_std": 0.5081326961517334, + "rewards/accuracy_reward/mean": 0.625, + "rewards/accuracy_reward/std": 0.49186936020851135, + "sampling/importance_sampling_ratio/max": 1.4071341753005981, + "sampling/importance_sampling_ratio/mean": 0.9999825954437256, + "sampling/importance_sampling_ratio/min": 0.5802328586578369, + "sampling/sampling_logp_difference/max": 0.5443258285522461, + "sampling/sampling_logp_difference/mean": 0.012630732730031013, + "step": 984 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 7.701786671532318e-05, + "clip_ratio/low_min": 7.701786671532318e-05, + "clip_ratio/region_mean": 7.701786671532318e-05, + "completions/clipped_ratio": 0.0, + "completions/max_length": 722.0, + "completions/max_terminated_length": 722.0, + "completions/mean_length": 429.28125, + "completions/mean_terminated_length": 429.28125, + "completions/min_length": 218.0, + "completions/min_terminated_length": 218.0, + "entropy": 0.2875707168132067, + "epoch": 0.5273019271948608, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.00487469881772995, + "learning_rate": 1e-05, + "loss": -0.0164, + "num_tokens": 19527747.0, + "reward": 0.78125, + "reward_std": 0.0883883461356163, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.2798290252685547, + "sampling/importance_sampling_ratio/mean": 0.9999646544456482, + "sampling/importance_sampling_ratio/min": 0.6921064257621765, + "sampling/sampling_logp_difference/max": 0.3680155277252197, + "sampling/sampling_logp_difference/mean": 0.010059557855129242, + "step": 985 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 660.0, + "completions/mean_length": 396.40625, + "completions/mean_terminated_length": 384.4193420410156, + "completions/min_length": 218.0, + "completions/min_terminated_length": 218.0, + "entropy": 0.2835698686540127, + "epoch": 0.5278372591006424, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.019992968067526817, + "learning_rate": 1e-05, + "loss": 0.0428, + "num_tokens": 19544768.0, + "reward": 0.875, + "reward_std": 0.2177756428718567, + "rewards/accuracy_reward/mean": 0.875, + "rewards/accuracy_reward/std": 0.33601075410842896, + "sampling/importance_sampling_ratio/max": 1.42948579788208, + "sampling/importance_sampling_ratio/mean": 0.9999502301216125, + "sampling/importance_sampling_ratio/min": 0.6881143450737, + "sampling/sampling_logp_difference/max": 0.37380027770996094, + "sampling/sampling_logp_difference/mean": 0.01113945059478283, + "step": 986 + }, + { + "clip_ratio/high_max": 0.0001092864986276254, + "clip_ratio/high_mean": 0.0001092864986276254, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0001092864986276254, + "completions/clipped_ratio": 0.25, + "completions/max_length": 768.0, + "completions/max_terminated_length": 761.0, + "completions/mean_length": 560.78125, + "completions/mean_terminated_length": 491.7083435058594, + "completions/min_length": 249.0, + "completions/min_terminated_length": 249.0, + "entropy": 0.36560990288853645, + "epoch": 0.528372591006424, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.015714678913354874, + "learning_rate": 1e-05, + "loss": 0.0451, + "num_tokens": 19566945.0, + "reward": 0.40625, + "reward_std": 0.4534739851951599, + "rewards/accuracy_reward/mean": 0.40625, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.39082670211792, + "sampling/importance_sampling_ratio/mean": 0.9996116757392883, + "sampling/importance_sampling_ratio/min": 0.6515651941299438, + "sampling/sampling_logp_difference/max": 0.4283778667449951, + "sampling/sampling_logp_difference/mean": 0.012630755081772804, + "step": 987 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 695.0, + "completions/mean_length": 454.21875, + "completions/mean_terminated_length": 444.0967712402344, + "completions/min_length": 252.0, + "completions/min_terminated_length": 252.0, + "entropy": 0.2823651097714901, + "epoch": 0.5289079229122056, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.013048914261162281, + "learning_rate": 1e-05, + "loss": 0.0101, + "num_tokens": 19585160.0, + "reward": 0.875, + "reward_std": 0.2177756428718567, + "rewards/accuracy_reward/mean": 0.875, + "rewards/accuracy_reward/std": 0.33601075410842896, + "sampling/importance_sampling_ratio/max": 1.5002124309539795, + "sampling/importance_sampling_ratio/mean": 0.9995115399360657, + "sampling/importance_sampling_ratio/min": 0.5749893188476562, + "sampling/sampling_logp_difference/max": 0.5534038543701172, + "sampling/sampling_logp_difference/mean": 0.010700213722884655, + "step": 988 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 560.0, + "completions/max_terminated_length": 560.0, + "completions/mean_length": 392.9375, + "completions/mean_terminated_length": 392.9375, + "completions/min_length": 237.0, + "completions/min_terminated_length": 237.0, + "entropy": 0.2659153249114752, + "epoch": 0.5294432548179872, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": 0.0201, + "num_tokens": 19601806.0, + "reward": 0.96875, + "reward_std": 0.0883883461356163, + "rewards/accuracy_reward/mean": 0.96875, + "rewards/accuracy_reward/std": 0.1767766922712326, + "sampling/importance_sampling_ratio/max": 1.5709412097930908, + "sampling/importance_sampling_ratio/mean": 1.0001845359802246, + "sampling/importance_sampling_ratio/min": 0.5926231145858765, + "sampling/sampling_logp_difference/max": 0.5231966972351074, + "sampling/sampling_logp_difference/mean": 0.010107295587658882, + "step": 989 + }, + { + "clip_ratio/high_max": 5.955216693109833e-05, + "clip_ratio/high_mean": 5.955216693109833e-05, + "clip_ratio/low_mean": 7.318500865949318e-05, + "clip_ratio/low_min": 7.318500865949318e-05, + "clip_ratio/region_mean": 0.0001327371755905915, + "completions/clipped_ratio": 0.0, + "completions/max_length": 674.0, + "completions/max_terminated_length": 674.0, + "completions/mean_length": 461.78125, + "completions/mean_terminated_length": 461.78125, + "completions/min_length": 159.0, + "completions/min_terminated_length": 159.0, + "entropy": 0.3579945210367441, + "epoch": 0.5299785867237687, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.014617021195590496, + "learning_rate": 1e-05, + "loss": -0.0012, + "num_tokens": 19620383.0, + "reward": 0.71875, + "reward_std": 0.35564959049224854, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.4609228372573853, + "sampling/importance_sampling_ratio/mean": 0.9997321963310242, + "sampling/importance_sampling_ratio/min": 0.7005718946456909, + "sampling/sampling_logp_difference/max": 0.37906837463378906, + "sampling/sampling_logp_difference/mean": 0.01290670782327652, + "step": 990 + }, + { + "clip_ratio/high_max": 5.526083259610459e-05, + "clip_ratio/high_mean": 5.526083259610459e-05, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 5.526083259610459e-05, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 757.0, + "completions/mean_length": 539.75, + "completions/mean_terminated_length": 532.3870849609375, + "completions/min_length": 311.0, + "completions/min_terminated_length": 311.0, + "entropy": 0.2842806428670883, + "epoch": 0.5305139186295503, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.006144804880023003, + "learning_rate": 1e-05, + "loss": 0.0508, + "num_tokens": 19641103.0, + "reward": 0.90625, + "reward_std": 0.2651650309562683, + "rewards/accuracy_reward/mean": 0.90625, + "rewards/accuracy_reward/std": 0.2961445748806, + "sampling/importance_sampling_ratio/max": 1.5386121273040771, + "sampling/importance_sampling_ratio/mean": 1.0001035928726196, + "sampling/importance_sampling_ratio/min": 0.6881845593452454, + "sampling/sampling_logp_difference/max": 0.4308807849884033, + "sampling/sampling_logp_difference/mean": 0.010615945793688297, + "step": 991 + }, + { + "clip_ratio/high_max": 8.411843737121671e-05, + "clip_ratio/high_mean": 8.411843737121671e-05, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 8.411843737121671e-05, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 525.0, + "completions/mean_length": 390.0625, + "completions/mean_terminated_length": 364.86669921875, + "completions/min_length": 166.0, + "completions/min_terminated_length": 166.0, + "entropy": 0.31818179227411747, + "epoch": 0.5310492505353319, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.007501174695789814, + "learning_rate": 1e-05, + "loss": 0.0988, + "num_tokens": 19657033.0, + "reward": 0.75, + "reward_std": 0.2314550280570984, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.7083754539489746, + "sampling/importance_sampling_ratio/mean": 1.0004456043243408, + "sampling/importance_sampling_ratio/min": 0.669009804725647, + "sampling/sampling_logp_difference/max": 0.5355429649353027, + "sampling/sampling_logp_difference/mean": 0.011256661266088486, + "step": 992 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 614.0, + "completions/max_terminated_length": 614.0, + "completions/mean_length": 426.125, + "completions/mean_terminated_length": 426.125, + "completions/min_length": 244.0, + "completions/min_terminated_length": 244.0, + "entropy": 0.24906635843217373, + "epoch": 0.5315845824411135, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.003510852111503482, + "learning_rate": 1e-05, + "loss": -0.0206, + "num_tokens": 19674205.0, + "reward": 0.96875, + "reward_std": 0.0883883461356163, + "rewards/accuracy_reward/mean": 0.96875, + "rewards/accuracy_reward/std": 0.1767766922712326, + "sampling/importance_sampling_ratio/max": 1.5225409269332886, + "sampling/importance_sampling_ratio/mean": 0.9998700022697449, + "sampling/importance_sampling_ratio/min": 0.5216495990753174, + "sampling/sampling_logp_difference/max": 0.650759220123291, + "sampling/sampling_logp_difference/mean": 0.010037382133305073, + "step": 993 + }, + { + "clip_ratio/high_max": 6.860592839075252e-05, + "clip_ratio/high_mean": 6.860592839075252e-05, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 6.860592839075252e-05, + "completions/clipped_ratio": 0.0, + "completions/max_length": 732.0, + "completions/max_terminated_length": 732.0, + "completions/mean_length": 395.40625, + "completions/mean_terminated_length": 395.40625, + "completions/min_length": 255.0, + "completions/min_terminated_length": 255.0, + "entropy": 0.24751000106334686, + "epoch": 0.5321199143468951, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.019045062363147736, + "learning_rate": 1e-05, + "loss": -0.0267, + "num_tokens": 19690306.0, + "reward": 0.9375, + "reward_std": 0.1157275140285492, + "rewards/accuracy_reward/mean": 0.9375, + "rewards/accuracy_reward/std": 0.24593468010425568, + "sampling/importance_sampling_ratio/max": 1.572339653968811, + "sampling/importance_sampling_ratio/mean": 1.0002657175064087, + "sampling/importance_sampling_ratio/min": 0.6859140396118164, + "sampling/sampling_logp_difference/max": 0.45256471633911133, + "sampling/sampling_logp_difference/mean": 0.010092799551784992, + "step": 994 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.148270065546967e-05, + "clip_ratio/low_min": 5.148270065546967e-05, + "clip_ratio/region_mean": 5.148270065546967e-05, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 738.0, + "completions/mean_length": 482.0625, + "completions/mean_terminated_length": 463.0000305175781, + "completions/min_length": 262.0, + "completions/min_terminated_length": 262.0, + "entropy": 0.2893268521875143, + "epoch": 0.5326552462526767, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.02204366959631443, + "learning_rate": 1e-05, + "loss": 0.0335, + "num_tokens": 19709484.0, + "reward": 0.9375, + "reward_std": 0.1157275140285492, + "rewards/accuracy_reward/mean": 0.9375, + "rewards/accuracy_reward/std": 0.24593468010425568, + "sampling/importance_sampling_ratio/max": 1.4051787853240967, + "sampling/importance_sampling_ratio/mean": 1.0002083778381348, + "sampling/importance_sampling_ratio/min": 0.7070884108543396, + "sampling/sampling_logp_difference/max": 0.3465995788574219, + "sampling/sampling_logp_difference/mean": 0.010966657660901546, + "step": 995 + }, + { + "clip_ratio/high_max": 6.551362457685173e-05, + "clip_ratio/high_mean": 6.551362457685173e-05, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 6.551362457685173e-05, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 710.0, + "completions/mean_length": 462.21875, + "completions/mean_terminated_length": 376.6000061035156, + "completions/min_length": 205.0, + "completions/min_terminated_length": 205.0, + "entropy": 0.5366212762892246, + "epoch": 0.5331905781584583, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.007105550728738308, + "learning_rate": 1e-05, + "loss": 0.0335, + "num_tokens": 19728091.0, + "reward": 0.625, + "reward_std": 0.3335031569004059, + "rewards/accuracy_reward/mean": 0.625, + "rewards/accuracy_reward/std": 0.49186936020851135, + "sampling/importance_sampling_ratio/max": 1.8256914615631104, + "sampling/importance_sampling_ratio/mean": 0.9998644590377808, + "sampling/importance_sampling_ratio/min": 0.6105164289474487, + "sampling/sampling_logp_difference/max": 0.6019587516784668, + "sampling/sampling_logp_difference/mean": 0.013786759227514267, + "step": 996 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 732.0, + "completions/mean_length": 568.53125, + "completions/mean_terminated_length": 562.0967407226562, + "completions/min_length": 304.0, + "completions/min_terminated_length": 304.0, + "entropy": 0.24387717805802822, + "epoch": 0.5337259100642399, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.004168031737208366, + "learning_rate": 1e-05, + "loss": 0.0362, + "num_tokens": 19750332.0, + "reward": 0.6875, + "reward_std": 0.1767766922712326, + "rewards/accuracy_reward/mean": 0.6875, + "rewards/accuracy_reward/std": 0.4709290862083435, + "sampling/importance_sampling_ratio/max": 1.4168020486831665, + "sampling/importance_sampling_ratio/mean": 1.0003665685653687, + "sampling/importance_sampling_ratio/min": 0.6397484540939331, + "sampling/sampling_logp_difference/max": 0.44668030738830566, + "sampling/sampling_logp_difference/mean": 0.008794558234512806, + "step": 997 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 693.0, + "completions/max_terminated_length": 693.0, + "completions/mean_length": 480.1875, + "completions/mean_terminated_length": 480.1875, + "completions/min_length": 289.0, + "completions/min_terminated_length": 289.0, + "entropy": 0.23125405795872211, + "epoch": 0.5342612419700214, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.01607550121843815, + "learning_rate": 1e-05, + "loss": 0.0423, + "num_tokens": 19769498.0, + "reward": 0.9375, + "reward_std": 0.1157275140285492, + "rewards/accuracy_reward/mean": 0.9375, + "rewards/accuracy_reward/std": 0.24593468010425568, + "sampling/importance_sampling_ratio/max": 1.3502311706542969, + "sampling/importance_sampling_ratio/mean": 0.999817430973053, + "sampling/importance_sampling_ratio/min": 0.5263004899024963, + "sampling/sampling_logp_difference/max": 0.6418828964233398, + "sampling/sampling_logp_difference/mean": 0.009172629565000534, + "step": 998 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 508.0, + "completions/max_terminated_length": 508.0, + "completions/mean_length": 359.0625, + "completions/mean_terminated_length": 359.0625, + "completions/min_length": 259.0, + "completions/min_terminated_length": 259.0, + "entropy": 0.2785221226513386, + "epoch": 0.534796573875803, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": 0.0, + "num_tokens": 19784412.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/accuracy_reward/mean": 1.0, + "rewards/accuracy_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.5911743640899658, + "sampling/importance_sampling_ratio/mean": 1.0001888275146484, + "sampling/importance_sampling_ratio/min": 0.6769352555274963, + "sampling/sampling_logp_difference/max": 0.46447229385375977, + "sampling/sampling_logp_difference/mean": 0.010836697183549404, + "step": 999 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.385609620134346e-05, + "clip_ratio/low_min": 5.385609620134346e-05, + "clip_ratio/region_mean": 5.385609620134346e-05, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 746.0, + "completions/mean_length": 545.75, + "completions/mean_terminated_length": 483.5199890136719, + "completions/min_length": 264.0, + "completions/min_terminated_length": 264.0, + "entropy": 0.38301945105195045, + "epoch": 0.5353319057815846, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.01854352094233036, + "learning_rate": 1e-05, + "loss": 0.1301, + "num_tokens": 19805500.0, + "reward": 0.5, + "reward_std": 0.5081326961517334, + "rewards/accuracy_reward/mean": 0.5, + "rewards/accuracy_reward/std": 0.5080004930496216, + "sampling/importance_sampling_ratio/max": 1.3488013744354248, + "sampling/importance_sampling_ratio/mean": 0.9999552965164185, + "sampling/importance_sampling_ratio/min": 0.7033720016479492, + "sampling/sampling_logp_difference/max": 0.3518693447113037, + "sampling/sampling_logp_difference/mean": 0.012176279909908772, + "step": 1000 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 628.0, + "completions/mean_length": 449.1875, + "completions/mean_terminated_length": 438.9031982421875, + "completions/min_length": 312.0, + "completions/min_terminated_length": 312.0, + "entropy": 0.2753011118620634, + "epoch": 0.5358672376873662, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.02747787907719612, + "learning_rate": 1e-05, + "loss": -0.0111, + "num_tokens": 19824154.0, + "reward": 0.90625, + "reward_std": 0.2651650309562683, + "rewards/accuracy_reward/mean": 0.90625, + "rewards/accuracy_reward/std": 0.2961445748806, + "sampling/importance_sampling_ratio/max": 1.5519434213638306, + "sampling/importance_sampling_ratio/mean": 1.000260353088379, + "sampling/importance_sampling_ratio/min": 0.7156000733375549, + "sampling/sampling_logp_difference/max": 0.43950796127319336, + "sampling/sampling_logp_difference/mean": 0.010242156684398651, + "step": 1001 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 716.0, + "completions/mean_length": 482.125, + "completions/mean_terminated_length": 463.0666809082031, + "completions/min_length": 266.0, + "completions/min_terminated_length": 266.0, + "entropy": 0.29971857368946075, + "epoch": 0.5364025695931478, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.009986001998186111, + "learning_rate": 1e-05, + "loss": 0.0325, + "num_tokens": 19843542.0, + "reward": 0.8125, + "reward_std": 0.3471825420856476, + "rewards/accuracy_reward/mean": 0.8125, + "rewards/accuracy_reward/std": 0.3965577781200409, + "sampling/importance_sampling_ratio/max": 1.4225226640701294, + "sampling/importance_sampling_ratio/mean": 1.0002081394195557, + "sampling/importance_sampling_ratio/min": 0.7357439398765564, + "sampling/sampling_logp_difference/max": 0.3524317741394043, + "sampling/sampling_logp_difference/mean": 0.010664070025086403, + "step": 1002 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 642.0, + "completions/max_terminated_length": 642.0, + "completions/mean_length": 411.09375, + "completions/mean_terminated_length": 411.09375, + "completions/min_length": 261.0, + "completions/min_terminated_length": 261.0, + "entropy": 0.30399791710078716, + "epoch": 0.5369379014989293, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.007462628651410341, + "learning_rate": 1e-05, + "loss": 0.0056, + "num_tokens": 19860617.0, + "reward": 0.90625, + "reward_std": 0.1293872892856598, + "rewards/accuracy_reward/mean": 0.90625, + "rewards/accuracy_reward/std": 0.2961445748806, + "sampling/importance_sampling_ratio/max": 1.4144448041915894, + "sampling/importance_sampling_ratio/mean": 0.9999246001243591, + "sampling/importance_sampling_ratio/min": 0.5888384580612183, + "sampling/sampling_logp_difference/max": 0.5296034812927246, + "sampling/sampling_logp_difference/mean": 0.01047993078827858, + "step": 1003 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00014293882122728974, + "clip_ratio/low_min": 0.00014293882122728974, + "clip_ratio/region_mean": 0.00014293882122728974, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 698.0, + "completions/mean_length": 525.46875, + "completions/mean_terminated_length": 490.8214416503906, + "completions/min_length": 241.0, + "completions/min_terminated_length": 241.0, + "entropy": 0.4285593256354332, + "epoch": 0.5374732334047109, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.02118309959769249, + "learning_rate": 1e-05, + "loss": 0.0742, + "num_tokens": 19881176.0, + "reward": 0.59375, + "reward_std": 0.3787454068660736, + "rewards/accuracy_reward/mean": 0.59375, + "rewards/accuracy_reward/std": 0.49899089336395264, + "sampling/importance_sampling_ratio/max": 1.6302584409713745, + "sampling/importance_sampling_ratio/mean": 1.0003572702407837, + "sampling/importance_sampling_ratio/min": 0.4600124657154083, + "sampling/sampling_logp_difference/max": 0.7765016555786133, + "sampling/sampling_logp_difference/mean": 0.014100081287324429, + "step": 1004 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 638.0, + "completions/mean_length": 381.25, + "completions/mean_terminated_length": 368.774169921875, + "completions/min_length": 149.0, + "completions/min_terminated_length": 149.0, + "entropy": 0.28941107355058193, + "epoch": 0.5380085653104925, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.005712924525141716, + "learning_rate": 1e-05, + "loss": 0.0168, + "num_tokens": 19896384.0, + "reward": 0.9375, + "reward_std": 0.1157275140285492, + "rewards/accuracy_reward/mean": 0.9375, + "rewards/accuracy_reward/std": 0.24593468010425568, + "sampling/importance_sampling_ratio/max": 1.4629381895065308, + "sampling/importance_sampling_ratio/mean": 1.0005462169647217, + "sampling/importance_sampling_ratio/min": 0.7008655667304993, + "sampling/sampling_logp_difference/max": 0.3804469108581543, + "sampling/sampling_logp_difference/mean": 0.01073462050408125, + "step": 1005 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 5.056634472566657e-05, + "clip_ratio/low_min": 5.056634472566657e-05, + "clip_ratio/region_mean": 5.056634472566657e-05, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 677.0, + "completions/mean_length": 457.28125, + "completions/mean_terminated_length": 436.5666809082031, + "completions/min_length": 274.0, + "completions/min_terminated_length": 274.0, + "entropy": 0.33258638344705105, + "epoch": 0.538543897216274, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.01634703204035759, + "learning_rate": 1e-05, + "loss": 0.0405, + "num_tokens": 19914609.0, + "reward": 0.78125, + "reward_std": 0.3377464711666107, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.3282477855682373, + "sampling/importance_sampling_ratio/mean": 0.9999423623085022, + "sampling/importance_sampling_ratio/min": 0.6146914958953857, + "sampling/sampling_logp_difference/max": 0.4866347312927246, + "sampling/sampling_logp_difference/mean": 0.011314889416098595, + "step": 1006 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00018642631766851991, + "clip_ratio/low_min": 0.00018642631766851991, + "clip_ratio/region_mean": 0.00018642631766851991, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 762.0, + "completions/mean_length": 489.1875, + "completions/mean_terminated_length": 480.19354248046875, + "completions/min_length": 256.0, + "completions/min_terminated_length": 256.0, + "entropy": 0.2814666237682104, + "epoch": 0.5390792291220556, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.014355567283928394, + "learning_rate": 1e-05, + "loss": 0.0065, + "num_tokens": 19934447.0, + "reward": 0.65625, + "reward_std": 0.3966485261917114, + "rewards/accuracy_reward/mean": 0.65625, + "rewards/accuracy_reward/std": 0.4825586974620819, + "sampling/importance_sampling_ratio/max": 1.4836410284042358, + "sampling/importance_sampling_ratio/mean": 0.9998093843460083, + "sampling/importance_sampling_ratio/min": 0.6395634412765503, + "sampling/sampling_logp_difference/max": 0.44696950912475586, + "sampling/sampling_logp_difference/mean": 0.010976734571158886, + "step": 1007 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 755.0, + "completions/max_terminated_length": 755.0, + "completions/mean_length": 490.125, + "completions/mean_terminated_length": 490.125, + "completions/min_length": 270.0, + "completions/min_terminated_length": 270.0, + "entropy": 0.3004448153078556, + "epoch": 0.5396145610278372, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": 0.0, + "num_tokens": 19953699.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/accuracy_reward/mean": 1.0, + "rewards/accuracy_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.6653494834899902, + "sampling/importance_sampling_ratio/mean": 1.0000314712524414, + "sampling/importance_sampling_ratio/min": 0.6230331659317017, + "sampling/sampling_logp_difference/max": 0.5100350379943848, + "sampling/sampling_logp_difference/mean": 0.01125288661569357, + "step": 1008 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 4.8449612222611904e-05, + "clip_ratio/low_min": 4.8449612222611904e-05, + "clip_ratio/region_mean": 4.8449612222611904e-05, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 733.0, + "completions/mean_length": 503.6875, + "completions/mean_terminated_length": 429.67999267578125, + "completions/min_length": 217.0, + "completions/min_terminated_length": 217.0, + "entropy": 0.3737986646592617, + "epoch": 0.5401498929336188, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.016752269119024277, + "learning_rate": 1e-05, + "loss": 0.0322, + "num_tokens": 19973993.0, + "reward": 0.71875, + "reward_std": 0.2630178928375244, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.435368537902832, + "sampling/importance_sampling_ratio/mean": 0.9996938705444336, + "sampling/importance_sampling_ratio/min": 0.6979666948318481, + "sampling/sampling_logp_difference/max": 0.3614215850830078, + "sampling/sampling_logp_difference/mean": 0.012252227403223515, + "step": 1009 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00029390008421614766, + "clip_ratio/low_min": 0.00029390008421614766, + "clip_ratio/region_mean": 0.00029390008421614766, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 649.0, + "completions/mean_length": 497.3125, + "completions/mean_terminated_length": 434.8461608886719, + "completions/min_length": 238.0, + "completions/min_terminated_length": 238.0, + "entropy": 0.40621403232216835, + "epoch": 0.5406852248394004, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.008822666481137276, + "learning_rate": 1e-05, + "loss": 0.0064, + "num_tokens": 19993595.0, + "reward": 0.375, + "reward_std": 0.292504221200943, + "rewards/accuracy_reward/mean": 0.375, + "rewards/accuracy_reward/std": 0.49186936020851135, + "sampling/importance_sampling_ratio/max": 1.4496333599090576, + "sampling/importance_sampling_ratio/mean": 0.9998958706855774, + "sampling/importance_sampling_ratio/min": 0.6977450847625732, + "sampling/sampling_logp_difference/max": 0.3713107109069824, + "sampling/sampling_logp_difference/mean": 0.013254624791443348, + "step": 1010 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 595.0, + "completions/max_terminated_length": 595.0, + "completions/mean_length": 378.71875, + "completions/mean_terminated_length": 378.71875, + "completions/min_length": 168.0, + "completions/min_terminated_length": 168.0, + "entropy": 0.256598187610507, + "epoch": 0.541220556745182, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": -0.0133, + "num_tokens": 20009186.0, + "reward": 0.96875, + "reward_std": 0.0883883461356163, + "rewards/accuracy_reward/mean": 0.96875, + "rewards/accuracy_reward/std": 0.1767766922712326, + "sampling/importance_sampling_ratio/max": 1.4466054439544678, + "sampling/importance_sampling_ratio/mean": 1.0001771450042725, + "sampling/importance_sampling_ratio/min": 0.7326815128326416, + "sampling/sampling_logp_difference/max": 0.3692197799682617, + "sampling/sampling_logp_difference/mean": 0.009641504846513271, + "step": 1011 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00011813741366495378, + "clip_ratio/low_min": 0.00011813741366495378, + "clip_ratio/region_mean": 0.00011813741366495378, + "completions/clipped_ratio": 0.375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 768.0, + "completions/mean_length": 563.40625, + "completions/mean_terminated_length": 440.6499938964844, + "completions/min_length": 278.0, + "completions/min_terminated_length": 278.0, + "entropy": 0.41633135825395584, + "epoch": 0.5417558886509636, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.007999341003596783, + "learning_rate": 1e-05, + "loss": 0.0028, + "num_tokens": 20031607.0, + "reward": 0.5625, + "reward_std": 0.1767766922712326, + "rewards/accuracy_reward/mean": 0.5625, + "rewards/accuracy_reward/std": 0.504016101360321, + "sampling/importance_sampling_ratio/max": 1.4557040929794312, + "sampling/importance_sampling_ratio/mean": 1.000044584274292, + "sampling/importance_sampling_ratio/min": 0.7197877168655396, + "sampling/sampling_logp_difference/max": 0.3754897117614746, + "sampling/sampling_logp_difference/mean": 0.012992034666240215, + "step": 1012 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 683.0, + "completions/max_terminated_length": 683.0, + "completions/mean_length": 381.0625, + "completions/mean_terminated_length": 381.0625, + "completions/min_length": 234.0, + "completions/min_terminated_length": 234.0, + "entropy": 0.2639304231852293, + "epoch": 0.5422912205567452, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": 0.0097, + "num_tokens": 20047625.0, + "reward": 0.96875, + "reward_std": 0.0883883461356163, + "rewards/accuracy_reward/mean": 0.96875, + "rewards/accuracy_reward/std": 0.1767766922712326, + "sampling/importance_sampling_ratio/max": 1.6141644716262817, + "sampling/importance_sampling_ratio/mean": 0.9996117949485779, + "sampling/importance_sampling_ratio/min": 0.6979535818099976, + "sampling/sampling_logp_difference/max": 0.4788174629211426, + "sampling/sampling_logp_difference/mean": 0.009938123635947704, + "step": 1013 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 6.416838004952297e-05, + "clip_ratio/low_min": 6.416838004952297e-05, + "clip_ratio/region_mean": 6.416838004952297e-05, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 531.0, + "completions/mean_length": 422.71875, + "completions/mean_terminated_length": 411.58062744140625, + "completions/min_length": 240.0, + "completions/min_terminated_length": 240.0, + "entropy": 0.22711737640202045, + "epoch": 0.5428265524625268, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.005366690922528505, + "learning_rate": 1e-05, + "loss": 0.0511, + "num_tokens": 20065200.0, + "reward": 0.96875, + "reward_std": 0.0883883461356163, + "rewards/accuracy_reward/mean": 0.96875, + "rewards/accuracy_reward/std": 0.1767766922712326, + "sampling/importance_sampling_ratio/max": 1.4140678644180298, + "sampling/importance_sampling_ratio/mean": 0.9999990463256836, + "sampling/importance_sampling_ratio/min": 0.692293107509613, + "sampling/sampling_logp_difference/max": 0.36774587631225586, + "sampling/sampling_logp_difference/mean": 0.009445932693779469, + "step": 1014 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 716.0, + "completions/max_terminated_length": 716.0, + "completions/mean_length": 428.0625, + "completions/mean_terminated_length": 428.0625, + "completions/min_length": 240.0, + "completions/min_terminated_length": 240.0, + "entropy": 0.2568762693554163, + "epoch": 0.5433618843683083, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.01963024213910103, + "learning_rate": 1e-05, + "loss": -0.0212, + "num_tokens": 20082386.0, + "reward": 0.90625, + "reward_std": 0.2041158676147461, + "rewards/accuracy_reward/mean": 0.90625, + "rewards/accuracy_reward/std": 0.2961445748806, + "sampling/importance_sampling_ratio/max": 1.4306024312973022, + "sampling/importance_sampling_ratio/mean": 0.9998763799667358, + "sampling/importance_sampling_ratio/min": 0.697256863117218, + "sampling/sampling_logp_difference/max": 0.36060142517089844, + "sampling/sampling_logp_difference/mean": 0.009530318900942802, + "step": 1015 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 642.0, + "completions/mean_length": 448.71875, + "completions/mean_terminated_length": 403.1071472167969, + "completions/min_length": 223.0, + "completions/min_terminated_length": 223.0, + "entropy": 0.36373442970216274, + "epoch": 0.5438972162740899, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": 0.0, + "num_tokens": 20100249.0, + "reward": 0.75, + "reward_std": 0.0, + "rewards/accuracy_reward/mean": 0.75, + "rewards/accuracy_reward/std": 0.4399413466453552, + "sampling/importance_sampling_ratio/max": 1.4357532262802124, + "sampling/importance_sampling_ratio/mean": 0.9998073577880859, + "sampling/importance_sampling_ratio/min": 0.7113254070281982, + "sampling/sampling_logp_difference/max": 0.36168956756591797, + "sampling/sampling_logp_difference/mean": 0.011635943315923214, + "step": 1016 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 641.0, + "completions/mean_length": 428.0625, + "completions/mean_terminated_length": 417.0967712402344, + "completions/min_length": 190.0, + "completions/min_terminated_length": 190.0, + "entropy": 0.25626081973314285, + "epoch": 0.5444325481798715, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.011524244211614132, + "learning_rate": 1e-05, + "loss": 0.0695, + "num_tokens": 20117715.0, + "reward": 0.8125, + "reward_std": 0.2177756428718567, + "rewards/accuracy_reward/mean": 0.8125, + "rewards/accuracy_reward/std": 0.3965577781200409, + "sampling/importance_sampling_ratio/max": 1.5319937467575073, + "sampling/importance_sampling_ratio/mean": 1.0003708600997925, + "sampling/importance_sampling_ratio/min": 0.7119307518005371, + "sampling/sampling_logp_difference/max": 0.42656993865966797, + "sampling/sampling_logp_difference/mean": 0.009544669650495052, + "step": 1017 + }, + { + "clip_ratio/high_max": 6.231306178960949e-05, + "clip_ratio/high_mean": 6.231306178960949e-05, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 6.231306178960949e-05, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 768.0, + "completions/max_terminated_length": 744.0, + "completions/mean_length": 538.875, + "completions/mean_terminated_length": 515.1724243164062, + "completions/min_length": 223.0, + "completions/min_terminated_length": 223.0, + "entropy": 0.2870079483836889, + "epoch": 0.5449678800856531, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.027886562049388885, + "learning_rate": 1e-05, + "loss": 0.0037, + "num_tokens": 20138975.0, + "reward": 0.53125, + "reward_std": 0.4534739851951599, + "rewards/accuracy_reward/mean": 0.53125, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.624444842338562, + "sampling/importance_sampling_ratio/mean": 1.0002611875534058, + "sampling/importance_sampling_ratio/min": 0.5770533084869385, + "sampling/sampling_logp_difference/max": 0.5498206615447998, + "sampling/sampling_logp_difference/mean": 0.010416710749268532, + "step": 1018 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0001102035676012747, + "clip_ratio/low_min": 0.0001102035676012747, + "clip_ratio/region_mean": 0.0001102035676012747, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 768.0, + "completions/max_terminated_length": 610.0, + "completions/mean_length": 492.03125, + "completions/mean_terminated_length": 428.3461608886719, + "completions/min_length": 265.0, + "completions/min_terminated_length": 265.0, + "entropy": 0.39040610007941723, + "epoch": 0.5455032119914347, + "frac_reward_zero_std": 0.25, + "grad_norm": 0.018447870388627052, + "learning_rate": 1e-05, + "loss": 0.0321, + "num_tokens": 20158560.0, + "reward": 0.53125, + "reward_std": 0.3471629321575165, + "rewards/accuracy_reward/mean": 0.53125, + "rewards/accuracy_reward/std": 0.507007360458374, + "sampling/importance_sampling_ratio/max": 1.409449577331543, + "sampling/importance_sampling_ratio/mean": 1.0000241994857788, + "sampling/importance_sampling_ratio/min": 0.5752276182174683, + "sampling/sampling_logp_difference/max": 0.5529894828796387, + "sampling/sampling_logp_difference/mean": 0.011754137463867664, + "step": 1019 + }, + { + "clip_ratio/high_max": 8.327781688421965e-05, + "clip_ratio/high_mean": 8.327781688421965e-05, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 8.327781688421965e-05, + "completions/clipped_ratio": 0.0, + "completions/max_length": 639.0, + "completions/max_terminated_length": 639.0, + "completions/mean_length": 420.25, + "completions/mean_terminated_length": 420.25, + "completions/min_length": 211.0, + "completions/min_terminated_length": 211.0, + "entropy": 0.25564243644475937, + "epoch": 0.5460385438972163, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.007572890259325504, + "learning_rate": 1e-05, + "loss": 0.035, + "num_tokens": 20175368.0, + "reward": 0.9375, + "reward_std": 0.1767766922712326, + "rewards/accuracy_reward/mean": 0.9375, + "rewards/accuracy_reward/std": 0.24593468010425568, + "sampling/importance_sampling_ratio/max": 1.5367435216903687, + "sampling/importance_sampling_ratio/mean": 0.9999706149101257, + "sampling/importance_sampling_ratio/min": 0.7095188498497009, + "sampling/sampling_logp_difference/max": 0.42966556549072266, + "sampling/sampling_logp_difference/mean": 0.010025544092059135, + "step": 1020 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 719.0, + "completions/mean_length": 547.5625, + "completions/mean_terminated_length": 506.7407531738281, + "completions/min_length": 275.0, + "completions/min_terminated_length": 275.0, + "entropy": 0.40882474556565285, + "epoch": 0.5465738758029979, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": -0.0241, + "num_tokens": 20196762.0, + "reward": 0.71875, + "reward_std": 0.0883883461356163, + "rewards/accuracy_reward/mean": 0.71875, + "rewards/accuracy_reward/std": 0.45680341124534607, + "sampling/importance_sampling_ratio/max": 1.4408026933670044, + "sampling/importance_sampling_ratio/mean": 1.0000535249710083, + "sampling/importance_sampling_ratio/min": 0.6971439719200134, + "sampling/sampling_logp_difference/max": 0.3652002811431885, + "sampling/sampling_logp_difference/mean": 0.013723382726311684, + "step": 1021 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.00014731880219187587, + "clip_ratio/low_min": 0.00014731880219187587, + "clip_ratio/region_mean": 0.00014731880219187587, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 768.0, + "completions/max_terminated_length": 763.0, + "completions/mean_length": 427.78125, + "completions/mean_terminated_length": 416.8064270019531, + "completions/min_length": 172.0, + "completions/min_terminated_length": 172.0, + "entropy": 0.3384027648717165, + "epoch": 0.5471092077087795, + "frac_reward_zero_std": 0.5, + "grad_norm": 0.026471413671970367, + "learning_rate": 1e-05, + "loss": 0.051, + "num_tokens": 20214123.0, + "reward": 0.78125, + "reward_std": 0.24511480331420898, + "rewards/accuracy_reward/mean": 0.78125, + "rewards/accuracy_reward/std": 0.420013427734375, + "sampling/importance_sampling_ratio/max": 1.3845716714859009, + "sampling/importance_sampling_ratio/mean": 1.0003368854522705, + "sampling/importance_sampling_ratio/min": 0.37910571694374084, + "sampling/sampling_logp_difference/max": 0.969940185546875, + "sampling/sampling_logp_difference/mean": 0.01268053986132145, + "step": 1022 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 653.0, + "completions/max_terminated_length": 653.0, + "completions/mean_length": 431.59375, + "completions/mean_terminated_length": 431.59375, + "completions/min_length": 316.0, + "completions/min_terminated_length": 316.0, + "entropy": 0.24964939430356026, + "epoch": 0.547644539614561, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1e-05, + "loss": 0.0, + "num_tokens": 20232214.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/accuracy_reward/mean": 1.0, + "rewards/accuracy_reward/std": 0.0, + "sampling/importance_sampling_ratio/max": 1.4343326091766357, + "sampling/importance_sampling_ratio/mean": 0.9999062418937683, + "sampling/importance_sampling_ratio/min": 0.6914226412773132, + "sampling/sampling_logp_difference/max": 0.3690040111541748, + "sampling/sampling_logp_difference/mean": 0.01000040490180254, + "step": 1023 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 7.301401637960225e-05, + "clip_ratio/low_min": 7.301401637960225e-05, + "clip_ratio/region_mean": 7.301401637960225e-05, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 768.0, + "completions/max_terminated_length": 656.0, + "completions/mean_length": 522.53125, + "completions/mean_terminated_length": 506.16668701171875, + "completions/min_length": 326.0, + "completions/min_terminated_length": 326.0, + "entropy": 0.2866067998111248, + "epoch": 0.5481798715203426, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.011995550245046616, + "learning_rate": 1e-05, + "loss": -0.0003, + "num_tokens": 20252823.0, + "reward": 0.5, + "reward_std": 0.4671337604522705, + "rewards/accuracy_reward/mean": 0.5, + "rewards/accuracy_reward/std": 0.5080004930496216, + "sampling/importance_sampling_ratio/max": 1.8201465606689453, + "sampling/importance_sampling_ratio/mean": 1.000035285949707, + "sampling/importance_sampling_ratio/min": 0.6696924567222595, + "sampling/sampling_logp_difference/max": 0.5989170074462891, + "sampling/sampling_logp_difference/mean": 0.010623245500028133, + "step": 1024 + } + ], + "logging_steps": 1, + "max_steps": 1024, + "num_input_tokens_seen": 20252823, + "num_train_epochs": 1, + "save_steps": 64, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 0.0, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +}