{ "best_global_step": null, "best_metric": null, "best_model_checkpoint": null, "epoch": 0.5481798715203426, "eval_steps": 500, "global_step": 1024, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.5, "completions/max_length": 768.0, "completions/max_terminated_length": 753.0, "completions/mean_length": 664.84375, "completions/mean_terminated_length": 561.6875, "completions/min_length": 322.0, "completions/min_terminated_length": 322.0, "entropy": 0.48598330840468407, "epoch": 0.0005353319057815846, "frac_reward_zero_std": 0.25, "grad_norm": 0.003221343969926238, "learning_rate": 1e-05, "loss": 0.0473, "num_tokens": 25147.0, "reward": 0.46875, "reward_std": 0.2651650309562683, "rewards/accuracy_reward/mean": 0.46875, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.2930827140808105, "sampling/importance_sampling_ratio/mean": 0.9998305439949036, "sampling/importance_sampling_ratio/min": 0.6979114413261414, "sampling/sampling_logp_difference/max": 0.3596630096435547, "sampling/sampling_logp_difference/mean": 0.012516415677964687, "step": 1 }, { "clip_ratio/high_max": 5.68698815186508e-05, "clip_ratio/high_mean": 5.68698815186508e-05, "clip_ratio/low_mean": 4.831851401831955e-05, "clip_ratio/low_min": 4.831851401831955e-05, "clip_ratio/region_mean": 0.00010518839553697035, "completions/clipped_ratio": 0.28125, "completions/max_length": 768.0, "completions/max_terminated_length": 760.0, "completions/mean_length": 643.875, "completions/mean_terminated_length": 595.3043823242188, "completions/min_length": 274.0, "completions/min_terminated_length": 274.0, "entropy": 0.42878107354044914, "epoch": 0.0010706638115631692, "frac_reward_zero_std": 0.25, "grad_norm": 0.011494417674839497, "learning_rate": 1e-05, "loss": 0.0406, "num_tokens": 49599.0, "reward": 0.5625, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.2916598320007324, "sampling/importance_sampling_ratio/mean": 1.0002554655075073, "sampling/importance_sampling_ratio/min": 0.7152712345123291, "sampling/sampling_logp_difference/max": 0.33509349822998047, "sampling/sampling_logp_difference/mean": 0.01164340227842331, "step": 2 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.718761920230463e-05, "clip_ratio/low_min": 4.718761920230463e-05, "clip_ratio/region_mean": 4.718761920230463e-05, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 756.0, "completions/mean_length": 580.40625, "completions/mean_terminated_length": 517.875, "completions/min_length": 224.0, "completions/min_terminated_length": 224.0, "entropy": 0.4754200950264931, "epoch": 0.0016059957173447537, "frac_reward_zero_std": 0.25, "grad_norm": 0.007718953769654036, "learning_rate": 1e-05, "loss": 0.0792, "num_tokens": 71644.0, "reward": 0.78125, "reward_std": 0.3377464711666107, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.000353217124939, "sampling/importance_sampling_ratio/min": 0.74234539270401, "sampling/sampling_logp_difference/max": 0.9044589996337891, "sampling/sampling_logp_difference/mean": 0.013075978495180607, "step": 3 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.7258978156605735e-05, "clip_ratio/low_min": 4.7258978156605735e-05, "clip_ratio/region_mean": 4.7258978156605735e-05, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 752.0, "completions/mean_length": 619.8125, "completions/mean_terminated_length": 570.4166870117188, "completions/min_length": 404.0, "completions/min_terminated_length": 404.0, "entropy": 0.5283972099423409, "epoch": 0.0021413276231263384, "frac_reward_zero_std": 0.5, "grad_norm": 0.014179565943777561, "learning_rate": 1e-05, "loss": 0.0032, "num_tokens": 95790.0, "reward": 0.5, "reward_std": 0.2314550280570984, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5080004930496216, "sampling/importance_sampling_ratio/max": 1.427490472793579, "sampling/importance_sampling_ratio/mean": 0.9997374415397644, "sampling/importance_sampling_ratio/min": 0.5011419057846069, "sampling/sampling_logp_difference/max": 0.6908659934997559, "sampling/sampling_logp_difference/mean": 0.01302722655236721, "step": 4 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 8.389262075070292e-05, "clip_ratio/low_min": 8.389262075070292e-05, "clip_ratio/region_mean": 8.389262075070292e-05, "completions/clipped_ratio": 0.5, "completions/max_length": 768.0, "completions/max_terminated_length": 754.0, "completions/mean_length": 677.09375, "completions/mean_terminated_length": 586.1875, "completions/min_length": 428.0, "completions/min_terminated_length": 428.0, "entropy": 0.5897238254547119, "epoch": 0.0026766595289079227, "frac_reward_zero_std": 0.0, "grad_norm": 0.009040264412760735, "learning_rate": 1e-05, "loss": 0.0631, "num_tokens": 121249.0, "reward": 0.46875, "reward_std": 0.5123760104179382, "rewards/accuracy_reward/mean": 0.46875, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.669755458831787, "sampling/importance_sampling_ratio/mean": 1.000028133392334, "sampling/importance_sampling_ratio/min": 0.6303314566612244, "sampling/sampling_logp_difference/max": 0.5126771926879883, "sampling/sampling_logp_difference/mean": 0.013944849371910095, "step": 5 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00015668613195884973, "clip_ratio/low_min": 0.00015668613195884973, "clip_ratio/region_mean": 0.00015668613195884973, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 768.0, "completions/mean_length": 531.5625, "completions/mean_terminated_length": 515.800048828125, "completions/min_length": 310.0, "completions/min_terminated_length": 310.0, "entropy": 0.44371388852596283, "epoch": 0.0032119914346895075, "frac_reward_zero_std": 0.75, "grad_norm": 0.01289108395576477, "learning_rate": 1e-05, "loss": 0.0031, "num_tokens": 142163.0, "reward": 0.8125, "reward_std": 0.1157275140285492, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.436777114868164, "sampling/importance_sampling_ratio/mean": 1.0001883506774902, "sampling/importance_sampling_ratio/min": 0.6979279518127441, "sampling/sampling_logp_difference/max": 0.36240243911743164, "sampling/sampling_logp_difference/mean": 0.012074148282408714, "step": 6 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 687.0, "completions/mean_length": 553.5, "completions/mean_terminated_length": 493.44000244140625, "completions/min_length": 212.0, "completions/min_terminated_length": 212.0, "entropy": 0.4113178588449955, "epoch": 0.003747323340471092, "frac_reward_zero_std": 0.0, "grad_norm": 0.017510822042822838, "learning_rate": 1e-05, "loss": 0.0667, "num_tokens": 163499.0, "reward": 0.59375, "reward_std": 0.4944729208946228, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.2691634893417358, "sampling/importance_sampling_ratio/mean": 1.000402569770813, "sampling/importance_sampling_ratio/min": 0.7248293161392212, "sampling/sampling_logp_difference/max": 0.3218190670013428, "sampling/sampling_logp_difference/mean": 0.011392496526241302, "step": 7 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 8.816482295515016e-05, "clip_ratio/low_min": 8.816482295515016e-05, "clip_ratio/region_mean": 8.816482295515016e-05, "completions/clipped_ratio": 0.5, "completions/max_length": 768.0, "completions/max_terminated_length": 728.0, "completions/mean_length": 693.1875, "completions/mean_terminated_length": 618.375, "completions/min_length": 442.0, "completions/min_terminated_length": 442.0, "entropy": 0.3966786302626133, "epoch": 0.004282655246252677, "frac_reward_zero_std": 0.0, "grad_norm": 0.006211712956428528, "learning_rate": 1e-05, "loss": 0.0605, "num_tokens": 189201.0, "reward": 0.40625, "reward_std": 0.4534739851951599, "rewards/accuracy_reward/mean": 0.40625, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.5011963844299316, "sampling/importance_sampling_ratio/mean": 1.0001108646392822, "sampling/importance_sampling_ratio/min": 0.6621246337890625, "sampling/sampling_logp_difference/max": 0.41230154037475586, "sampling/sampling_logp_difference/mean": 0.01060819998383522, "step": 8 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.758279465022497e-05, "clip_ratio/low_min": 4.758279465022497e-05, "clip_ratio/region_mean": 4.758279465022497e-05, "completions/clipped_ratio": 0.4375, "completions/max_length": 768.0, "completions/max_terminated_length": 768.0, "completions/mean_length": 634.46875, "completions/mean_terminated_length": 530.6111450195312, "completions/min_length": 326.0, "completions/min_terminated_length": 326.0, "entropy": 0.399143572896719, "epoch": 0.004817987152034261, "frac_reward_zero_std": 0.25, "grad_norm": 0.002772281877696514, "learning_rate": 1e-05, "loss": 0.0326, "num_tokens": 213440.0, "reward": 0.59375, "reward_std": 0.3061639964580536, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.3725049495697021, "sampling/importance_sampling_ratio/mean": 1.0003511905670166, "sampling/importance_sampling_ratio/min": 0.7243841290473938, "sampling/sampling_logp_difference/max": 0.3224334716796875, "sampling/sampling_logp_difference/mean": 0.010564294643700123, "step": 9 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 702.0, "completions/mean_length": 539.03125, "completions/mean_terminated_length": 506.3214416503906, "completions/min_length": 305.0, "completions/min_terminated_length": 305.0, "entropy": 0.3731744661927223, "epoch": 0.0053533190578158455, "frac_reward_zero_std": 0.5, "grad_norm": 0.004717267584055662, "learning_rate": 1e-05, "loss": 0.0598, "num_tokens": 233721.0, "reward": 0.90625, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.2861584424972534, "sampling/importance_sampling_ratio/mean": 0.9997180700302124, "sampling/importance_sampling_ratio/min": 0.7788320183753967, "sampling/sampling_logp_difference/max": 0.2516598701477051, "sampling/sampling_logp_difference/mean": 0.010257482528686523, "step": 10 }, { "clip_ratio/high_max": 4.15144459111616e-05, "clip_ratio/high_mean": 4.15144459111616e-05, "clip_ratio/low_mean": 0.00014118030958343297, "clip_ratio/low_min": 0.00014118030958343297, "clip_ratio/region_mean": 0.00018269475549459457, "completions/clipped_ratio": 0.46875, "completions/max_length": 768.0, "completions/max_terminated_length": 728.0, "completions/mean_length": 672.21875, "completions/mean_terminated_length": 587.7058715820312, "completions/min_length": 429.0, "completions/min_terminated_length": 429.0, "entropy": 0.644787147641182, "epoch": 0.005888650963597431, "frac_reward_zero_std": 0.25, "grad_norm": 0.0076923375017941, "learning_rate": 1e-05, "loss": 0.0328, "num_tokens": 259200.0, "reward": 0.34375, "reward_std": 0.3608423173427582, "rewards/accuracy_reward/mean": 0.34375, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.3399858474731445, "sampling/importance_sampling_ratio/mean": 0.999772846698761, "sampling/importance_sampling_ratio/min": 0.5169666409492493, "sampling/sampling_logp_difference/max": 0.6597769260406494, "sampling/sampling_logp_difference/mean": 0.015453845262527466, "step": 11 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00010247058162349276, "clip_ratio/low_min": 0.00010247058162349276, "clip_ratio/region_mean": 0.00010247058162349276, "completions/clipped_ratio": 0.4375, "completions/max_length": 768.0, "completions/max_terminated_length": 729.0, "completions/mean_length": 657.34375, "completions/mean_terminated_length": 571.2777709960938, "completions/min_length": 356.0, "completions/min_terminated_length": 356.0, "entropy": 0.4300087243318558, "epoch": 0.006423982869379015, "frac_reward_zero_std": 0.0, "grad_norm": 0.008051280863583088, "learning_rate": 1e-05, "loss": 0.0759, "num_tokens": 283739.0, "reward": 0.625, "reward_std": 0.4492306709289551, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.2970905303955078, "sampling/importance_sampling_ratio/mean": 0.9996793866157532, "sampling/importance_sampling_ratio/min": 0.7377063035964966, "sampling/sampling_logp_difference/max": 0.3042095899581909, "sampling/sampling_logp_difference/mean": 0.011997226625680923, "step": 12 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00018348376761423424, "clip_ratio/low_min": 0.00018348376761423424, "clip_ratio/region_mean": 0.00018348376761423424, "completions/clipped_ratio": 0.34375, "completions/max_length": 768.0, "completions/max_terminated_length": 641.0, "completions/mean_length": 560.34375, "completions/mean_terminated_length": 451.5714416503906, "completions/min_length": 291.0, "completions/min_terminated_length": 291.0, "entropy": 0.4340112954378128, "epoch": 0.006959314775160599, "frac_reward_zero_std": 0.0, "grad_norm": 0.012295019812881947, "learning_rate": 1e-05, "loss": 0.0408, "num_tokens": 305094.0, "reward": 0.40625, "reward_std": 0.4218915104866028, "rewards/accuracy_reward/mean": 0.40625, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.7559185028076172, "sampling/importance_sampling_ratio/mean": 0.9999406337738037, "sampling/importance_sampling_ratio/min": 0.6984894275665283, "sampling/sampling_logp_difference/max": 0.5629920363426208, "sampling/sampling_logp_difference/mean": 0.011640074662864208, "step": 13 }, { "clip_ratio/high_max": 4.9860391300171614e-05, "clip_ratio/high_mean": 4.9860391300171614e-05, "clip_ratio/low_mean": 0.00013638269956572913, "clip_ratio/low_min": 0.00013638269956572913, "clip_ratio/region_mean": 0.00018624309086590074, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 768.0, "completions/mean_length": 589.46875, "completions/mean_terminated_length": 529.9583740234375, "completions/min_length": 278.0, "completions/min_terminated_length": 278.0, "entropy": 0.37963151931762695, "epoch": 0.007494646680942184, "frac_reward_zero_std": 0.25, "grad_norm": 0.013525142334401608, "learning_rate": 1e-05, "loss": 0.0342, "num_tokens": 327509.0, "reward": 0.65625, "reward_std": 0.3608423173427582, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.4621731042861938, "sampling/importance_sampling_ratio/mean": 1.0000073909759521, "sampling/importance_sampling_ratio/min": 0.7251969575881958, "sampling/sampling_logp_difference/max": 0.3799237012863159, "sampling/sampling_logp_difference/mean": 0.011182216927409172, "step": 14 }, { "clip_ratio/high_max": 5.367110497900285e-05, "clip_ratio/high_mean": 5.367110497900285e-05, "clip_ratio/low_mean": 4.475474270293489e-05, "clip_ratio/low_min": 4.475474270293489e-05, "clip_ratio/region_mean": 9.842584768193774e-05, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 760.0, "completions/mean_length": 595.40625, "completions/mean_terminated_length": 563.4444580078125, "completions/min_length": 261.0, "completions/min_terminated_length": 261.0, "entropy": 0.37569622322916985, "epoch": 0.008029978586723769, "frac_reward_zero_std": 0.25, "grad_norm": 0.01099634449928999, "learning_rate": 1e-05, "loss": 0.0269, "num_tokens": 350010.0, "reward": 0.6875, "reward_std": 0.38298875093460083, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.4191175699234009, "sampling/importance_sampling_ratio/mean": 1.0001291036605835, "sampling/importance_sampling_ratio/min": 0.7041683197021484, "sampling/sampling_logp_difference/max": 0.3507378101348877, "sampling/sampling_logp_difference/mean": 0.010376600548624992, "step": 15 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 9.394958397024311e-05, "clip_ratio/low_min": 9.394958397024311e-05, "clip_ratio/region_mean": 9.394958397024311e-05, "completions/clipped_ratio": 0.3125, "completions/max_length": 768.0, "completions/max_terminated_length": 733.0, "completions/mean_length": 618.21875, "completions/mean_terminated_length": 550.1363525390625, "completions/min_length": 287.0, "completions/min_terminated_length": 287.0, "entropy": 0.8762477859854698, "epoch": 0.008565310492505354, "frac_reward_zero_std": 0.25, "grad_norm": 0.009937820956110954, "learning_rate": 1e-05, "loss": 0.0738, "num_tokens": 374545.0, "reward": 0.34375, "reward_std": 0.378745436668396, "rewards/accuracy_reward/mean": 0.34375, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.36752450466156, "sampling/importance_sampling_ratio/mean": 0.9995363354682922, "sampling/importance_sampling_ratio/min": 0.6208454966545105, "sampling/sampling_logp_difference/max": 0.4766731262207031, "sampling/sampling_logp_difference/mean": 0.017068270593881607, "step": 16 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.6159526391420513e-05, "clip_ratio/low_min": 4.6159526391420513e-05, "clip_ratio/region_mean": 4.6159526391420513e-05, "completions/clipped_ratio": 0.46875, "completions/max_length": 768.0, "completions/max_terminated_length": 740.0, "completions/mean_length": 658.78125, "completions/mean_terminated_length": 562.4117431640625, "completions/min_length": 368.0, "completions/min_terminated_length": 368.0, "entropy": 0.5209148675203323, "epoch": 0.009100642398286937, "frac_reward_zero_std": 0.25, "grad_norm": 0.008431400172412395, "learning_rate": 1e-05, "loss": 0.0081, "num_tokens": 399434.0, "reward": 0.125, "reward_std": 0.2925041913986206, "rewards/accuracy_reward/mean": 0.125, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 1.504237413406372, "sampling/importance_sampling_ratio/mean": 0.999762237071991, "sampling/importance_sampling_ratio/min": 0.13839177787303925, "sampling/sampling_logp_difference/max": 1.9776666164398193, "sampling/sampling_logp_difference/mean": 0.013602105900645256, "step": 17 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.2793562897713855e-05, "clip_ratio/low_min": 4.2793562897713855e-05, "clip_ratio/region_mean": 4.2793562897713855e-05, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 756.0, "completions/mean_length": 614.53125, "completions/mean_terminated_length": 571.5599975585938, "completions/min_length": 304.0, "completions/min_terminated_length": 304.0, "entropy": 0.4716529883444309, "epoch": 0.009635974304068522, "frac_reward_zero_std": 0.5, "grad_norm": 0.01435131672769785, "learning_rate": 1e-05, "loss": 0.0442, "num_tokens": 422979.0, "reward": 0.75, "reward_std": 0.2314550280570984, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.665273666381836, "sampling/importance_sampling_ratio/mean": 1.0000128746032715, "sampling/importance_sampling_ratio/min": 0.6897562146186829, "sampling/sampling_logp_difference/max": 0.5099895000457764, "sampling/sampling_logp_difference/mean": 0.011879773810505867, "step": 18 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.813246050616726e-05, "clip_ratio/low_min": 4.813246050616726e-05, "clip_ratio/region_mean": 4.813246050616726e-05, "completions/clipped_ratio": 0.53125, "completions/max_length": 768.0, "completions/max_terminated_length": 767.0, "completions/mean_length": 664.21875, "completions/mean_terminated_length": 546.6000366210938, "completions/min_length": 293.0, "completions/min_terminated_length": 293.0, "entropy": 0.4797363206744194, "epoch": 0.010171306209850108, "frac_reward_zero_std": 0.5, "grad_norm": 0.008629962801933289, "learning_rate": 1e-05, "loss": 0.0297, "num_tokens": 448402.0, "reward": 0.46875, "reward_std": 0.2630178928375244, "rewards/accuracy_reward/mean": 0.46875, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.3417432308197021, "sampling/importance_sampling_ratio/mean": 0.9999527931213379, "sampling/importance_sampling_ratio/min": 0.6983755230903625, "sampling/sampling_logp_difference/max": 0.35899829864501953, "sampling/sampling_logp_difference/mean": 0.013303980231285095, "step": 19 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0001342818941338919, "clip_ratio/low_min": 0.0001342818941338919, "clip_ratio/region_mean": 0.0001342818941338919, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 741.0, "completions/mean_length": 467.0625, "completions/mean_terminated_length": 411.3333435058594, "completions/min_length": 253.0, "completions/min_terminated_length": 253.0, "entropy": 0.45434001460671425, "epoch": 0.010706638115631691, "frac_reward_zero_std": 0.0, "grad_norm": 0.016064638271927834, "learning_rate": 1e-05, "loss": 0.0832, "num_tokens": 467028.0, "reward": 0.5, "reward_std": 0.49871626496315, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5080004930496216, "sampling/importance_sampling_ratio/max": 1.4572550058364868, "sampling/importance_sampling_ratio/mean": 0.9995190501213074, "sampling/importance_sampling_ratio/min": 0.7872229814529419, "sampling/sampling_logp_difference/max": 0.3765544891357422, "sampling/sampling_logp_difference/mean": 0.012009660713374615, "step": 20 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 765.0, "completions/mean_length": 575.25, "completions/mean_terminated_length": 539.5555419921875, "completions/min_length": 276.0, "completions/min_terminated_length": 276.0, "entropy": 0.3936432972550392, "epoch": 0.011241970021413276, "frac_reward_zero_std": 0.5, "grad_norm": 0.008663182146847248, "learning_rate": 1e-05, "loss": 0.0233, "num_tokens": 489468.0, "reward": 0.75, "reward_std": 0.2587745785713196, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.2776802778244019, "sampling/importance_sampling_ratio/mean": 0.9997345209121704, "sampling/importance_sampling_ratio/min": 0.7000576257705688, "sampling/sampling_logp_difference/max": 0.35659265518188477, "sampling/sampling_logp_difference/mean": 0.011136982589960098, "step": 21 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00015927586355246603, "clip_ratio/low_min": 0.00015927586355246603, "clip_ratio/region_mean": 0.00015927586355246603, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 744.0, "completions/mean_length": 597.59375, "completions/mean_terminated_length": 549.8800048828125, "completions/min_length": 337.0, "completions/min_terminated_length": 337.0, "entropy": 0.42605889588594437, "epoch": 0.011777301927194861, "frac_reward_zero_std": 0.0, "grad_norm": 0.014341483823955059, "learning_rate": 1e-05, "loss": 0.088, "num_tokens": 511967.0, "reward": 0.5625, "reward_std": 0.4492306709289551, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.3279242515563965, "sampling/importance_sampling_ratio/mean": 1.0003553628921509, "sampling/importance_sampling_ratio/min": 0.6799124479293823, "sampling/sampling_logp_difference/max": 0.38579118251800537, "sampling/sampling_logp_difference/mean": 0.012305875308811665, "step": 22 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.28125, "completions/max_length": 768.0, "completions/max_terminated_length": 700.0, "completions/mean_length": 601.4375, "completions/mean_terminated_length": 536.2608642578125, "completions/min_length": 317.0, "completions/min_terminated_length": 317.0, "entropy": 0.4323686696588993, "epoch": 0.012312633832976445, "frac_reward_zero_std": 0.25, "grad_norm": 0.012643562629818916, "learning_rate": 1e-05, "loss": 0.0275, "num_tokens": 535325.0, "reward": 0.71875, "reward_std": 0.378745436668396, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.4192399978637695, "sampling/importance_sampling_ratio/mean": 1.0001556873321533, "sampling/importance_sampling_ratio/min": 0.7160170078277588, "sampling/sampling_logp_difference/max": 0.35012149810791016, "sampling/sampling_logp_difference/mean": 0.012007133103907108, "step": 23 }, { "clip_ratio/high_max": 5.628095459542237e-05, "clip_ratio/high_mean": 5.628095459542237e-05, "clip_ratio/low_mean": 0.00011048113810829818, "clip_ratio/low_min": 0.00011048113810829818, "clip_ratio/region_mean": 0.00016676209270372055, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 704.0, "completions/mean_length": 597.40625, "completions/mean_terminated_length": 549.6400146484375, "completions/min_length": 375.0, "completions/min_terminated_length": 375.0, "entropy": 0.5417136512696743, "epoch": 0.01284796573875803, "frac_reward_zero_std": 0.5, "grad_norm": 0.0033429788891226053, "learning_rate": 1e-05, "loss": 0.0307, "num_tokens": 558514.0, "reward": 0.59375, "reward_std": 0.22201895713806152, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.4942647218704224, "sampling/importance_sampling_ratio/mean": 0.9999322891235352, "sampling/importance_sampling_ratio/min": 0.684075653553009, "sampling/sampling_logp_difference/max": 0.40163421630859375, "sampling/sampling_logp_difference/mean": 0.01343031320720911, "step": 24 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.251700738677755e-05, "clip_ratio/low_min": 4.251700738677755e-05, "clip_ratio/region_mean": 4.251700738677755e-05, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 768.0, "completions/mean_length": 604.6875, "completions/mean_terminated_length": 558.9599609375, "completions/min_length": 282.0, "completions/min_terminated_length": 282.0, "entropy": 0.6011920757591724, "epoch": 0.013383297644539615, "frac_reward_zero_std": 0.25, "grad_norm": 0.002957536606118083, "learning_rate": 1e-05, "loss": 0.0463, "num_tokens": 581360.0, "reward": 0.625, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.4236646890640259, "sampling/importance_sampling_ratio/mean": 1.000066876411438, "sampling/importance_sampling_ratio/min": 0.7464213967323303, "sampling/sampling_logp_difference/max": 0.35323429107666016, "sampling/sampling_logp_difference/mean": 0.01519707590341568, "step": 25 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00013296937322593294, "clip_ratio/low_min": 0.00013296937322593294, "clip_ratio/region_mean": 0.00013296937322593294, "completions/clipped_ratio": 0.59375, "completions/max_length": 768.0, "completions/max_terminated_length": 751.0, "completions/mean_length": 717.75, "completions/mean_terminated_length": 644.3077392578125, "completions/min_length": 504.0, "completions/min_terminated_length": 504.0, "entropy": 0.4574657008051872, "epoch": 0.013918629550321198, "frac_reward_zero_std": 0.0, "grad_norm": 0.012310518883168697, "learning_rate": 1e-05, "loss": 0.0334, "num_tokens": 608368.0, "reward": 0.3125, "reward_std": 0.49022960662841797, "rewards/accuracy_reward/mean": 0.3125, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.3321303129196167, "sampling/importance_sampling_ratio/mean": 0.9998629093170166, "sampling/importance_sampling_ratio/min": 0.7263608574867249, "sampling/sampling_logp_difference/max": 0.31970834732055664, "sampling/sampling_logp_difference/mean": 0.012832907028496265, "step": 26 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0001234677511092741, "clip_ratio/low_min": 0.0001234677511092741, "clip_ratio/region_mean": 0.0001234677511092741, "completions/clipped_ratio": 0.625, "completions/max_length": 768.0, "completions/max_terminated_length": 765.0, "completions/mean_length": 715.46875, "completions/mean_terminated_length": 627.9166870117188, "completions/min_length": 482.0, "completions/min_terminated_length": 482.0, "entropy": 0.4889042265713215, "epoch": 0.014453961456102784, "frac_reward_zero_std": 0.0, "grad_norm": 0.011511260643601418, "learning_rate": 1e-05, "loss": 0.0532, "num_tokens": 635119.0, "reward": 0.40625, "reward_std": 0.4765698313713074, "rewards/accuracy_reward/mean": 0.40625, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.3562421798706055, "sampling/importance_sampling_ratio/mean": 0.9999900460243225, "sampling/importance_sampling_ratio/min": 0.7082851529121399, "sampling/sampling_logp_difference/max": 0.3449084758758545, "sampling/sampling_logp_difference/mean": 0.012470146641135216, "step": 27 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 9.630745262256823e-05, "clip_ratio/low_min": 9.630745262256823e-05, "clip_ratio/region_mean": 9.630745262256823e-05, "completions/clipped_ratio": 0.4375, "completions/max_length": 768.0, "completions/max_terminated_length": 747.0, "completions/mean_length": 668.59375, "completions/mean_terminated_length": 591.2777709960938, "completions/min_length": 371.0, "completions/min_terminated_length": 371.0, "entropy": 0.4034435786306858, "epoch": 0.014989293361884369, "frac_reward_zero_std": 0.0, "grad_norm": 0.009466876275837421, "learning_rate": 1e-05, "loss": -0.0038, "num_tokens": 660794.0, "reward": 0.34375, "reward_std": 0.47137710452079773, "rewards/accuracy_reward/mean": 0.34375, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.4180912971496582, "sampling/importance_sampling_ratio/mean": 1.000315546989441, "sampling/importance_sampling_ratio/min": 0.7504450678825378, "sampling/sampling_logp_difference/max": 0.34931182861328125, "sampling/sampling_logp_difference/mean": 0.011455441825091839, "step": 28 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.201830936130136e-05, "clip_ratio/low_min": 5.201830936130136e-05, "clip_ratio/region_mean": 5.201830936130136e-05, "completions/clipped_ratio": 0.28125, "completions/max_length": 768.0, "completions/max_terminated_length": 734.0, "completions/mean_length": 632.65625, "completions/mean_terminated_length": 579.6956787109375, "completions/min_length": 376.0, "completions/min_terminated_length": 376.0, "entropy": 0.3653472289443016, "epoch": 0.015524625267665952, "frac_reward_zero_std": 0.0, "grad_norm": 0.013039426878094673, "learning_rate": 1e-05, "loss": 0.05, "num_tokens": 685279.0, "reward": 0.5625, "reward_std": 0.47655022144317627, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.330334186553955, "sampling/importance_sampling_ratio/mean": 0.9999987483024597, "sampling/importance_sampling_ratio/min": 0.7631338834762573, "sampling/sampling_logp_difference/max": 0.2854301929473877, "sampling/sampling_logp_difference/mean": 0.009900057688355446, "step": 29 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00010230386760667898, "clip_ratio/low_min": 0.00010230386760667898, "clip_ratio/region_mean": 0.00010230386760667898, "completions/clipped_ratio": 0.5625, "completions/max_length": 768.0, "completions/max_terminated_length": 767.0, "completions/mean_length": 688.59375, "completions/mean_terminated_length": 586.5, "completions/min_length": 406.0, "completions/min_terminated_length": 406.0, "entropy": 0.7968601994216442, "epoch": 0.016059957173447537, "frac_reward_zero_std": 0.0, "grad_norm": 0.018240302801132202, "learning_rate": 1e-05, "loss": -0.0144, "num_tokens": 711610.0, "reward": 0.375, "reward_std": 0.4261348247528076, "rewards/accuracy_reward/mean": 0.375, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.3993512392044067, "sampling/importance_sampling_ratio/mean": 1.0000684261322021, "sampling/importance_sampling_ratio/min": 0.7310611605644226, "sampling/sampling_logp_difference/max": 0.33600878715515137, "sampling/sampling_logp_difference/mean": 0.01700461097061634, "step": 30 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0001818625896703452, "clip_ratio/low_min": 0.0001818625896703452, "clip_ratio/region_mean": 0.0001818625896703452, "completions/clipped_ratio": 0.4375, "completions/max_length": 768.0, "completions/max_terminated_length": 740.0, "completions/mean_length": 651.28125, "completions/mean_terminated_length": 560.5, "completions/min_length": 306.0, "completions/min_terminated_length": 306.0, "entropy": 0.3586757071316242, "epoch": 0.016595289079229122, "frac_reward_zero_std": 0.0, "grad_norm": 0.021731872111558914, "learning_rate": 1e-05, "loss": 0.0656, "num_tokens": 736131.0, "reward": 0.53125, "reward_std": 0.4218915104866028, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.2882115840911865, "sampling/importance_sampling_ratio/mean": 0.9996837973594666, "sampling/importance_sampling_ratio/min": 0.6830993890762329, "sampling/sampling_logp_difference/max": 0.3811149597167969, "sampling/sampling_logp_difference/mean": 0.010377843864262104, "step": 31 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 9.170946577796713e-05, "clip_ratio/low_min": 9.170946577796713e-05, "clip_ratio/region_mean": 9.170946577796713e-05, "completions/clipped_ratio": 0.40625, "completions/max_length": 768.0, "completions/max_terminated_length": 768.0, "completions/mean_length": 615.9375, "completions/mean_terminated_length": 511.8947448730469, "completions/min_length": 320.0, "completions/min_terminated_length": 320.0, "entropy": 0.42582567408680916, "epoch": 0.017130620985010708, "frac_reward_zero_std": 0.25, "grad_norm": 0.0080404132604599, "learning_rate": 1e-05, "loss": 0.0426, "num_tokens": 759585.0, "reward": 0.40625, "reward_std": 0.3061639666557312, "rewards/accuracy_reward/mean": 0.40625, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.3272674083709717, "sampling/importance_sampling_ratio/mean": 0.9995907545089722, "sampling/importance_sampling_ratio/min": 0.7206646203994751, "sampling/sampling_logp_difference/max": 0.32758140563964844, "sampling/sampling_logp_difference/mean": 0.01153803151100874, "step": 32 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.849315104773268e-05, "clip_ratio/low_min": 6.849315104773268e-05, "clip_ratio/region_mean": 6.849315104773268e-05, "completions/clipped_ratio": 0.34375, "completions/max_length": 768.0, "completions/max_terminated_length": 768.0, "completions/mean_length": 596.71875, "completions/mean_terminated_length": 507.0, "completions/min_length": 259.0, "completions/min_terminated_length": 259.0, "entropy": 0.5254503786563873, "epoch": 0.017665952890792293, "frac_reward_zero_std": 0.25, "grad_norm": 0.009133022278547287, "learning_rate": 1e-05, "loss": 0.0647, "num_tokens": 782304.0, "reward": 0.5625, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.6737715005874634, "sampling/importance_sampling_ratio/mean": 1.000001072883606, "sampling/importance_sampling_ratio/min": 0.7468359470367432, "sampling/sampling_logp_difference/max": 0.5150794982910156, "sampling/sampling_logp_difference/mean": 0.013609294779598713, "step": 33 }, { "clip_ratio/high_max": 4.824392090085894e-05, "clip_ratio/high_mean": 4.824392090085894e-05, "clip_ratio/low_mean": 8.361593063455075e-05, "clip_ratio/low_min": 8.361593063455075e-05, "clip_ratio/region_mean": 0.0001318598515354097, "completions/clipped_ratio": 0.53125, "completions/max_length": 768.0, "completions/max_terminated_length": 743.0, "completions/mean_length": 683.53125, "completions/mean_terminated_length": 587.800048828125, "completions/min_length": 295.0, "completions/min_terminated_length": 295.0, "entropy": 0.46616850793361664, "epoch": 0.018201284796573874, "frac_reward_zero_std": 0.0, "grad_norm": 0.00606547063216567, "learning_rate": 1e-05, "loss": 0.0291, "num_tokens": 808065.0, "reward": 0.40625, "reward_std": 0.3987956643104553, "rewards/accuracy_reward/mean": 0.40625, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.4248172044754028, "sampling/importance_sampling_ratio/mean": 1.000334620475769, "sampling/importance_sampling_ratio/min": 0.6216268539428711, "sampling/sampling_logp_difference/max": 0.4754152297973633, "sampling/sampling_logp_difference/mean": 0.012083113193511963, "step": 34 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.375, "completions/max_length": 768.0, "completions/max_terminated_length": 730.0, "completions/mean_length": 635.96875, "completions/mean_terminated_length": 556.75, "completions/min_length": 339.0, "completions/min_terminated_length": 339.0, "entropy": 0.5142815373837948, "epoch": 0.01873661670235546, "frac_reward_zero_std": 0.25, "grad_norm": 0.005932202562689781, "learning_rate": 1e-05, "loss": 0.0578, "num_tokens": 832312.0, "reward": 0.59375, "reward_std": 0.378745436668396, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.7484441995620728, "sampling/importance_sampling_ratio/mean": 1.0000585317611694, "sampling/importance_sampling_ratio/min": 0.7307000756263733, "sampling/sampling_logp_difference/max": 0.5587263107299805, "sampling/sampling_logp_difference/mean": 0.013376685790717602, "step": 35 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 9.928691360983066e-05, "clip_ratio/low_min": 9.928691360983066e-05, "clip_ratio/region_mean": 9.928691360983066e-05, "completions/clipped_ratio": 0.28125, "completions/max_length": 768.0, "completions/max_terminated_length": 739.0, "completions/mean_length": 582.71875, "completions/mean_terminated_length": 510.2174072265625, "completions/min_length": 251.0, "completions/min_terminated_length": 251.0, "entropy": 0.401416290551424, "epoch": 0.019271948608137045, "frac_reward_zero_std": 0.5, "grad_norm": 0.008968272246420383, "learning_rate": 1e-05, "loss": 0.0544, "num_tokens": 854615.0, "reward": 0.75, "reward_std": 0.26726123690605164, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.2956496477127075, "sampling/importance_sampling_ratio/mean": 1.000141978263855, "sampling/importance_sampling_ratio/min": 0.3398108184337616, "sampling/sampling_logp_difference/max": 1.0793662071228027, "sampling/sampling_logp_difference/mean": 0.01159136462956667, "step": 36 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 728.0, "completions/mean_length": 561.1875, "completions/mean_terminated_length": 531.6428833007812, "completions/min_length": 338.0, "completions/min_terminated_length": 338.0, "entropy": 0.42634254321455956, "epoch": 0.01980728051391863, "frac_reward_zero_std": 0.75, "grad_norm": 0.0023323469795286655, "learning_rate": 1e-05, "loss": 0.036, "num_tokens": 876533.0, "reward": 0.71875, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.723872423171997, "sampling/importance_sampling_ratio/mean": 0.999769389629364, "sampling/importance_sampling_ratio/min": 0.5008617043495178, "sampling/sampling_logp_difference/max": 0.6914253234863281, "sampling/sampling_logp_difference/mean": 0.011795828118920326, "step": 37 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 9.189795673592016e-05, "clip_ratio/low_min": 9.189795673592016e-05, "clip_ratio/region_mean": 9.189795673592016e-05, "completions/clipped_ratio": 0.34375, "completions/max_length": 768.0, "completions/max_terminated_length": 761.0, "completions/mean_length": 632.59375, "completions/mean_terminated_length": 561.6666870117188, "completions/min_length": 266.0, "completions/min_terminated_length": 266.0, "entropy": 0.4238196052610874, "epoch": 0.020342612419700215, "frac_reward_zero_std": 0.25, "grad_norm": 0.008609730750322342, "learning_rate": 1e-05, "loss": 0.0429, "num_tokens": 900656.0, "reward": 0.65625, "reward_std": 0.3787454068660736, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.305167317390442, "sampling/importance_sampling_ratio/mean": 0.9998759627342224, "sampling/importance_sampling_ratio/min": 0.7219241857528687, "sampling/sampling_logp_difference/max": 0.3258352279663086, "sampling/sampling_logp_difference/mean": 0.011658228933811188, "step": 38 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.34375, "completions/max_length": 768.0, "completions/max_terminated_length": 758.0, "completions/mean_length": 602.0625, "completions/mean_terminated_length": 515.1428833007812, "completions/min_length": 266.0, "completions/min_terminated_length": 266.0, "entropy": 0.3751674108207226, "epoch": 0.0208779443254818, "frac_reward_zero_std": 0.5, "grad_norm": 0.006625959184020758, "learning_rate": 1e-05, "loss": -0.0008, "num_tokens": 923482.0, "reward": 0.53125, "reward_std": 0.2630178928375244, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.3292808532714844, "sampling/importance_sampling_ratio/mean": 1.000037670135498, "sampling/importance_sampling_ratio/min": 0.7790080308914185, "sampling/sampling_logp_difference/max": 0.2846381664276123, "sampling/sampling_logp_difference/mean": 0.010517522692680359, "step": 39 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.7129800552502275e-05, "clip_ratio/low_min": 5.7129800552502275e-05, "clip_ratio/region_mean": 5.7129800552502275e-05, "completions/clipped_ratio": 0.40625, "completions/max_length": 768.0, "completions/max_terminated_length": 751.0, "completions/mean_length": 592.71875, "completions/mean_terminated_length": 472.78948974609375, "completions/min_length": 258.0, "completions/min_terminated_length": 258.0, "entropy": 0.6938692703843117, "epoch": 0.021413276231263382, "frac_reward_zero_std": 0.5, "grad_norm": 0.005904671736061573, "learning_rate": 1e-05, "loss": 0.0258, "num_tokens": 946313.0, "reward": 0.375, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.375, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.2952286005020142, "sampling/importance_sampling_ratio/mean": 1.000117540359497, "sampling/importance_sampling_ratio/min": 0.6890854835510254, "sampling/sampling_logp_difference/max": 0.37238991260528564, "sampling/sampling_logp_difference/mean": 0.016663314774632454, "step": 40 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0002457740847603418, "clip_ratio/low_min": 0.0002457740847603418, "clip_ratio/region_mean": 0.0002457740847603418, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 755.0, "completions/mean_length": 508.65625, "completions/mean_terminated_length": 471.607177734375, "completions/min_length": 180.0, "completions/min_terminated_length": 180.0, "entropy": 0.42964523658156395, "epoch": 0.021948608137044967, "frac_reward_zero_std": 0.0, "grad_norm": 0.006222760304808617, "learning_rate": 1e-05, "loss": 0.1182, "num_tokens": 966102.0, "reward": 0.5, "reward_std": 0.4082317352294922, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5080004930496216, "sampling/importance_sampling_ratio/max": 1.3748246431350708, "sampling/importance_sampling_ratio/mean": 0.999871015548706, "sampling/importance_sampling_ratio/min": 0.782130777835846, "sampling/sampling_logp_difference/max": 0.3183262348175049, "sampling/sampling_logp_difference/mean": 0.011689409613609314, "step": 41 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 752.0, "completions/mean_length": 562.3125, "completions/mean_terminated_length": 493.75, "completions/min_length": 248.0, "completions/min_terminated_length": 248.0, "entropy": 0.4230865277349949, "epoch": 0.022483940042826552, "frac_reward_zero_std": 0.25, "grad_norm": 0.008011308498680592, "learning_rate": 1e-05, "loss": 0.0662, "num_tokens": 987520.0, "reward": 0.625, "reward_std": 0.3745020925998688, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.3456498384475708, "sampling/importance_sampling_ratio/mean": 0.9999926090240479, "sampling/importance_sampling_ratio/min": 0.5969445109367371, "sampling/sampling_logp_difference/max": 0.5159311294555664, "sampling/sampling_logp_difference/mean": 0.012312295846641064, "step": 42 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0002048586648015771, "clip_ratio/low_min": 0.0002048586648015771, "clip_ratio/region_mean": 0.0002048586648015771, "completions/clipped_ratio": 0.4375, "completions/max_length": 768.0, "completions/max_terminated_length": 696.0, "completions/mean_length": 600.84375, "completions/mean_terminated_length": 470.8333435058594, "completions/min_length": 264.0, "completions/min_terminated_length": 264.0, "entropy": 0.3819310888648033, "epoch": 0.023019271948608137, "frac_reward_zero_std": 0.5, "grad_norm": 0.002085540210828185, "learning_rate": 1e-05, "loss": 0.0691, "num_tokens": 1010627.0, "reward": 0.4375, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.4375, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0002131462097168, "sampling/importance_sampling_ratio/min": 0.6660265922546387, "sampling/sampling_logp_difference/max": 0.9354488849639893, "sampling/sampling_logp_difference/mean": 0.01111356820911169, "step": 43 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.3125, "completions/max_length": 768.0, "completions/max_terminated_length": 742.0, "completions/mean_length": 608.40625, "completions/mean_terminated_length": 535.8636474609375, "completions/min_length": 283.0, "completions/min_terminated_length": 283.0, "entropy": 0.33856771513819695, "epoch": 0.023554603854389723, "frac_reward_zero_std": 0.25, "grad_norm": 0.010461578145623207, "learning_rate": 1e-05, "loss": 0.0148, "num_tokens": 1033824.0, "reward": 0.6875, "reward_std": 0.2925041913986206, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.3511464595794678, "sampling/importance_sampling_ratio/mean": 0.99992835521698, "sampling/importance_sampling_ratio/min": 0.69541335105896, "sampling/sampling_logp_difference/max": 0.3632488250732422, "sampling/sampling_logp_difference/mean": 0.009910862892866135, "step": 44 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.8412082833237946e-05, "clip_ratio/low_min": 4.8412082833237946e-05, "clip_ratio/region_mean": 4.8412082833237946e-05, "completions/clipped_ratio": 0.4375, "completions/max_length": 768.0, "completions/max_terminated_length": 761.0, "completions/mean_length": 660.71875, "completions/mean_terminated_length": 577.2777709960938, "completions/min_length": 376.0, "completions/min_terminated_length": 376.0, "entropy": 0.4713536947965622, "epoch": 0.024089935760171308, "frac_reward_zero_std": 0.25, "grad_norm": 0.009117750450968742, "learning_rate": 1e-05, "loss": 0.0326, "num_tokens": 1059095.0, "reward": 0.46875, "reward_std": 0.3198433816432953, "rewards/accuracy_reward/mean": 0.46875, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.7383469343185425, "sampling/importance_sampling_ratio/mean": 1.0000401735305786, "sampling/importance_sampling_ratio/min": 0.7212705016136169, "sampling/sampling_logp_difference/max": 0.5529346466064453, "sampling/sampling_logp_difference/mean": 0.012183919548988342, "step": 45 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.699248165707104e-05, "clip_ratio/low_min": 4.699248165707104e-05, "clip_ratio/region_mean": 4.699248165707104e-05, "completions/clipped_ratio": 0.40625, "completions/max_length": 768.0, "completions/max_terminated_length": 740.0, "completions/mean_length": 632.34375, "completions/mean_terminated_length": 539.5263061523438, "completions/min_length": 351.0, "completions/min_terminated_length": 351.0, "entropy": 0.4380064718425274, "epoch": 0.02462526766595289, "frac_reward_zero_std": 0.0, "grad_norm": 0.006470283027738333, "learning_rate": 1e-05, "loss": 0.0495, "num_tokens": 1083346.0, "reward": 0.5625, "reward_std": 0.4492306709289551, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.304788589477539, "sampling/importance_sampling_ratio/mean": 0.9999873638153076, "sampling/importance_sampling_ratio/min": 0.7482465505599976, "sampling/sampling_logp_difference/max": 0.29002273082733154, "sampling/sampling_logp_difference/mean": 0.012076225131750107, "step": 46 }, { "clip_ratio/high_max": 4.736642586067319e-05, "clip_ratio/high_mean": 4.736642586067319e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 4.736642586067319e-05, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 697.0, "completions/mean_length": 562.90625, "completions/mean_terminated_length": 533.607177734375, "completions/min_length": 320.0, "completions/min_terminated_length": 320.0, "entropy": 0.5093584284186363, "epoch": 0.025160599571734475, "frac_reward_zero_std": 0.25, "grad_norm": 0.009556936100125313, "learning_rate": 1e-05, "loss": 0.0088, "num_tokens": 1105119.0, "reward": 0.65625, "reward_std": 0.378745436668396, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.388994812965393, "sampling/importance_sampling_ratio/mean": 0.99997878074646, "sampling/importance_sampling_ratio/min": 0.7357785701751709, "sampling/sampling_logp_difference/max": 0.328580379486084, "sampling/sampling_logp_difference/mean": 0.013824340887367725, "step": 47 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 706.0, "completions/mean_length": 554.0625, "completions/mean_terminated_length": 531.9310302734375, "completions/min_length": 338.0, "completions/min_terminated_length": 338.0, "entropy": 0.34858502447605133, "epoch": 0.02569593147751606, "frac_reward_zero_std": 0.5, "grad_norm": 0.0031233886256814003, "learning_rate": 1e-05, "loss": 0.0324, "num_tokens": 1125985.0, "reward": 0.90625, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.2635139226913452, "sampling/importance_sampling_ratio/mean": 0.9999485611915588, "sampling/importance_sampling_ratio/min": 0.7291891574859619, "sampling/sampling_logp_difference/max": 0.31582218408584595, "sampling/sampling_logp_difference/mean": 0.00996592827141285, "step": 48 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 723.0, "completions/mean_length": 516.25, "completions/mean_terminated_length": 445.7599792480469, "completions/min_length": 181.0, "completions/min_terminated_length": 181.0, "entropy": 0.6114879846572876, "epoch": 0.026231263383297645, "frac_reward_zero_std": 0.25, "grad_norm": 0.004022506065666676, "learning_rate": 1e-05, "loss": 0.0689, "num_tokens": 1146441.0, "reward": 0.46875, "reward_std": 0.2651650309562683, "rewards/accuracy_reward/mean": 0.46875, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.3954023122787476, "sampling/importance_sampling_ratio/mean": 0.9996545314788818, "sampling/importance_sampling_ratio/min": 0.6914423704147339, "sampling/sampling_logp_difference/max": 0.36897552013397217, "sampling/sampling_logp_difference/mean": 0.015522178262472153, "step": 49 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.451373726828024e-05, "clip_ratio/low_min": 5.451373726828024e-05, "clip_ratio/region_mean": 5.451373726828024e-05, "completions/clipped_ratio": 0.28125, "completions/max_length": 768.0, "completions/max_terminated_length": 734.0, "completions/mean_length": 577.34375, "completions/mean_terminated_length": 502.7391357421875, "completions/min_length": 363.0, "completions/min_terminated_length": 363.0, "entropy": 0.4173782952129841, "epoch": 0.02676659528907923, "frac_reward_zero_std": 0.75, "grad_norm": 0.009276395663619041, "learning_rate": 1e-05, "loss": 0.0151, "num_tokens": 1168468.0, "reward": 0.625, "reward_std": 0.13363061845302582, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.3346651792526245, "sampling/importance_sampling_ratio/mean": 1.0000964403152466, "sampling/importance_sampling_ratio/min": 0.5289000868797302, "sampling/sampling_logp_difference/max": 0.636955738067627, "sampling/sampling_logp_difference/mean": 0.011444348841905594, "step": 50 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.6484408560208976e-05, "clip_ratio/low_min": 5.6484408560208976e-05, "clip_ratio/region_mean": 5.6484408560208976e-05, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 759.0, "completions/mean_length": 605.5, "completions/mean_terminated_length": 594.6666870117188, "completions/min_length": 365.0, "completions/min_terminated_length": 365.0, "entropy": 0.4438452161848545, "epoch": 0.027301927194860815, "frac_reward_zero_std": 0.0, "grad_norm": 0.013731824234127998, "learning_rate": 1e-05, "loss": 0.0206, "num_tokens": 1191740.0, "reward": 0.65625, "reward_std": 0.4355708956718445, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.3785372972488403, "sampling/importance_sampling_ratio/mean": 0.999667763710022, "sampling/importance_sampling_ratio/min": 0.6162854433059692, "sampling/sampling_logp_difference/max": 0.48404502868652344, "sampling/sampling_logp_difference/mean": 0.011669471859931946, "step": 51 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 8.839568545226939e-05, "clip_ratio/low_min": 8.839568545226939e-05, "clip_ratio/region_mean": 8.839568545226939e-05, "completions/clipped_ratio": 0.34375, "completions/max_length": 768.0, "completions/max_terminated_length": 726.0, "completions/mean_length": 572.5625, "completions/mean_terminated_length": 470.19049072265625, "completions/min_length": 219.0, "completions/min_terminated_length": 219.0, "entropy": 0.43201351165771484, "epoch": 0.027837259100642397, "frac_reward_zero_std": 0.5, "grad_norm": 0.008128860965371132, "learning_rate": 1e-05, "loss": 0.0628, "num_tokens": 1213718.0, "reward": 0.4375, "reward_std": 0.249358132481575, "rewards/accuracy_reward/mean": 0.4375, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.2701990604400635, "sampling/importance_sampling_ratio/mean": 0.9999561905860901, "sampling/importance_sampling_ratio/min": 0.7129706740379333, "sampling/sampling_logp_difference/max": 0.3383150100708008, "sampling/sampling_logp_difference/mean": 0.011831110343337059, "step": 52 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 766.0, "completions/mean_length": 614.03125, "completions/mean_terminated_length": 578.5, "completions/min_length": 404.0, "completions/min_terminated_length": 404.0, "entropy": 0.41860589012503624, "epoch": 0.028372591006423982, "frac_reward_zero_std": 0.25, "grad_norm": 0.006652893498539925, "learning_rate": 1e-05, "loss": 0.0424, "num_tokens": 1236959.0, "reward": 0.78125, "reward_std": 0.3608423173427582, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.467631459236145, "sampling/importance_sampling_ratio/mean": 1.0000083446502686, "sampling/importance_sampling_ratio/min": 0.7228109836578369, "sampling/sampling_logp_difference/max": 0.3836498260498047, "sampling/sampling_logp_difference/mean": 0.011651166714727879, "step": 53 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0001522653219581116, "clip_ratio/low_min": 0.0001522653219581116, "clip_ratio/region_mean": 0.0001522653219581116, "completions/clipped_ratio": 0.28125, "completions/max_length": 768.0, "completions/max_terminated_length": 765.0, "completions/mean_length": 595.78125, "completions/mean_terminated_length": 528.3912963867188, "completions/min_length": 303.0, "completions/min_terminated_length": 303.0, "entropy": 0.57270622625947, "epoch": 0.028907922912205567, "frac_reward_zero_std": 0.25, "grad_norm": 0.01190225500613451, "learning_rate": 1e-05, "loss": 0.0416, "num_tokens": 1259808.0, "reward": 0.65625, "reward_std": 0.3966485261917114, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.332876205444336, "sampling/importance_sampling_ratio/mean": 0.9999961256980896, "sampling/importance_sampling_ratio/min": 0.6416661143302917, "sampling/sampling_logp_difference/max": 0.44368720054626465, "sampling/sampling_logp_difference/mean": 0.01362132839858532, "step": 54 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.4375, "completions/max_length": 768.0, "completions/max_terminated_length": 758.0, "completions/mean_length": 680.28125, "completions/mean_terminated_length": 612.0555419921875, "completions/min_length": 301.0, "completions/min_terminated_length": 301.0, "entropy": 0.4660458490252495, "epoch": 0.029443254817987152, "frac_reward_zero_std": 0.0, "grad_norm": 0.020519772544503212, "learning_rate": 1e-05, "loss": 0.0619, "num_tokens": 1285321.0, "reward": 0.625, "reward_std": 0.4355512857437134, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.3958441019058228, "sampling/importance_sampling_ratio/mean": 0.9997105598449707, "sampling/importance_sampling_ratio/min": 0.6905045509338379, "sampling/sampling_logp_difference/max": 0.3703327178955078, "sampling/sampling_logp_difference/mean": 0.012635939754545689, "step": 55 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00020559043332468718, "clip_ratio/low_min": 0.00020559043332468718, "clip_ratio/region_mean": 0.00020559043332468718, "completions/clipped_ratio": 0.40625, "completions/max_length": 768.0, "completions/max_terminated_length": 728.0, "completions/mean_length": 598.6875, "completions/mean_terminated_length": 482.84210205078125, "completions/min_length": 316.0, "completions/min_terminated_length": 316.0, "entropy": 0.5831898897886276, "epoch": 0.029978586723768737, "frac_reward_zero_std": 0.25, "grad_norm": 0.004278325010091066, "learning_rate": 1e-05, "loss": 0.0603, "num_tokens": 1308239.0, "reward": 0.40625, "reward_std": 0.3061639666557312, "rewards/accuracy_reward/mean": 0.40625, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.280590295791626, "sampling/importance_sampling_ratio/mean": 0.9995114803314209, "sampling/importance_sampling_ratio/min": 0.5621932744979858, "sampling/sampling_logp_difference/max": 0.5759096145629883, "sampling/sampling_logp_difference/mean": 0.013754518702626228, "step": 56 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 748.0, "completions/mean_length": 458.1875, "completions/mean_terminated_length": 448.19354248046875, "completions/min_length": 194.0, "completions/min_terminated_length": 194.0, "entropy": 0.4043896570801735, "epoch": 0.030513918629550323, "frac_reward_zero_std": 0.5, "grad_norm": 0.012731232680380344, "learning_rate": 1e-05, "loss": -0.0191, "num_tokens": 1326141.0, "reward": 0.84375, "reward_std": 0.22201895713806152, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.3083082437515259, "sampling/importance_sampling_ratio/mean": 0.9999406337738037, "sampling/importance_sampling_ratio/min": 0.6367043852806091, "sampling/sampling_logp_difference/max": 0.4514498710632324, "sampling/sampling_logp_difference/mean": 0.011479364708065987, "step": 57 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.857544601894915e-05, "clip_ratio/low_min": 5.857544601894915e-05, "clip_ratio/region_mean": 5.857544601894915e-05, "completions/clipped_ratio": 0.3125, "completions/max_length": 768.0, "completions/max_terminated_length": 764.0, "completions/mean_length": 643.375, "completions/mean_terminated_length": 586.727294921875, "completions/min_length": 408.0, "completions/min_terminated_length": 408.0, "entropy": 0.5183435417711735, "epoch": 0.031049250535331904, "frac_reward_zero_std": 0.5, "grad_norm": 0.009470314718782902, "learning_rate": 1e-05, "loss": 0.0102, "num_tokens": 1350793.0, "reward": 0.5, "reward_std": 0.26726123690605164, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5080004930496216, "sampling/importance_sampling_ratio/max": 1.2867701053619385, "sampling/importance_sampling_ratio/mean": 0.9996908903121948, "sampling/importance_sampling_ratio/min": 0.7265493869781494, "sampling/sampling_logp_difference/max": 0.3194488286972046, "sampling/sampling_logp_difference/mean": 0.013498436659574509, "step": 58 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00011429791265982203, "clip_ratio/low_min": 0.00011429791265982203, "clip_ratio/region_mean": 0.00011429791265982203, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 744.0, "completions/mean_length": 479.5, "completions/mean_terminated_length": 426.0740661621094, "completions/min_length": 61.0, "completions/min_terminated_length": 61.0, "entropy": 0.48458342254161835, "epoch": 0.03158458244111349, "frac_reward_zero_std": 0.25, "grad_norm": 0.019218454137444496, "learning_rate": 1e-05, "loss": 0.0819, "num_tokens": 1370001.0, "reward": 0.53125, "reward_std": 0.3787454068660736, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.2893465757369995, "sampling/importance_sampling_ratio/mean": 1.0000818967819214, "sampling/importance_sampling_ratio/min": 0.623902440071106, "sampling/sampling_logp_difference/max": 0.47176122665405273, "sampling/sampling_logp_difference/mean": 0.013390498235821724, "step": 59 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00014285331053542905, "clip_ratio/low_min": 0.00014285331053542905, "clip_ratio/region_mean": 0.00014285331053542905, "completions/clipped_ratio": 0.3125, "completions/max_length": 768.0, "completions/max_terminated_length": 737.0, "completions/mean_length": 580.46875, "completions/mean_terminated_length": 495.227294921875, "completions/min_length": 291.0, "completions/min_terminated_length": 291.0, "entropy": 0.585710447281599, "epoch": 0.032119914346895075, "frac_reward_zero_std": 0.5, "grad_norm": 0.008108831010758877, "learning_rate": 1e-05, "loss": 0.055, "num_tokens": 1392952.0, "reward": 0.5625, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.562903881072998, "sampling/importance_sampling_ratio/mean": 1.0000708103179932, "sampling/importance_sampling_ratio/min": 0.6652374863624573, "sampling/sampling_logp_difference/max": 0.4465456008911133, "sampling/sampling_logp_difference/mean": 0.015322019346058369, "step": 60 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.34375, "completions/max_length": 768.0, "completions/max_terminated_length": 754.0, "completions/mean_length": 587.625, "completions/mean_terminated_length": 493.1428527832031, "completions/min_length": 265.0, "completions/min_terminated_length": 265.0, "entropy": 0.4231221675872803, "epoch": 0.032655246252676656, "frac_reward_zero_std": 0.0, "grad_norm": 0.00792720541357994, "learning_rate": 1e-05, "loss": 0.0198, "num_tokens": 1415596.0, "reward": 0.5, "reward_std": 0.4492306709289551, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5080004930496216, "sampling/importance_sampling_ratio/max": 1.386669635772705, "sampling/importance_sampling_ratio/mean": 1.0000114440917969, "sampling/importance_sampling_ratio/min": 0.4110261797904968, "sampling/sampling_logp_difference/max": 0.8890984058380127, "sampling/sampling_logp_difference/mean": 0.01243380643427372, "step": 61 }, { "clip_ratio/high_max": 7.427213131450117e-05, "clip_ratio/high_mean": 7.427213131450117e-05, "clip_ratio/low_mean": 6.01250612817239e-05, "clip_ratio/low_min": 6.01250612817239e-05, "clip_ratio/region_mean": 0.00013439719259622507, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 657.0, "completions/mean_length": 470.53125, "completions/mean_terminated_length": 428.0357360839844, "completions/min_length": 270.0, "completions/min_terminated_length": 270.0, "entropy": 0.4739554189145565, "epoch": 0.033190578158458245, "frac_reward_zero_std": 0.25, "grad_norm": 0.0063326735980808735, "learning_rate": 1e-05, "loss": 0.0683, "num_tokens": 1434101.0, "reward": 0.71875, "reward_std": 0.3198433816432953, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.4615764617919922, "sampling/importance_sampling_ratio/mean": 1.000329852104187, "sampling/importance_sampling_ratio/min": 0.3510209619998932, "sampling/sampling_logp_difference/max": 1.0469093322753906, "sampling/sampling_logp_difference/mean": 0.012787309475243092, "step": 62 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.875195008935407e-05, "clip_ratio/low_min": 4.875195008935407e-05, "clip_ratio/region_mean": 4.875195008935407e-05, "completions/clipped_ratio": 0.375, "completions/max_length": 768.0, "completions/max_terminated_length": 734.0, "completions/mean_length": 585.9375, "completions/mean_terminated_length": 476.70001220703125, "completions/min_length": 327.0, "completions/min_terminated_length": 327.0, "entropy": 0.5160165019333363, "epoch": 0.03372591006423983, "frac_reward_zero_std": 0.25, "grad_norm": 0.009666070342063904, "learning_rate": 1e-05, "loss": 0.0492, "num_tokens": 1456643.0, "reward": 0.5, "reward_std": 0.3745020925998688, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5080004930496216, "sampling/importance_sampling_ratio/max": 1.3192894458770752, "sampling/importance_sampling_ratio/mean": 0.9999880194664001, "sampling/importance_sampling_ratio/min": 0.4595102071762085, "sampling/sampling_logp_difference/max": 0.7775940895080566, "sampling/sampling_logp_difference/mean": 0.014054049737751484, "step": 63 }, { "clip_ratio/high_max": 0.00010205427679466084, "clip_ratio/high_mean": 0.00010205427679466084, "clip_ratio/low_mean": 0.00016632045299047604, "clip_ratio/low_min": 0.00016632045299047604, "clip_ratio/region_mean": 0.0002683747297851369, "completions/clipped_ratio": 0.3125, "completions/max_length": 768.0, "completions/max_terminated_length": 763.0, "completions/mean_length": 585.4375, "completions/mean_terminated_length": 502.4545593261719, "completions/min_length": 152.0, "completions/min_terminated_length": 152.0, "entropy": 0.5863043442368507, "epoch": 0.034261241970021415, "frac_reward_zero_std": 0.0, "grad_norm": 0.013950761407613754, "learning_rate": 1e-05, "loss": -0.0454, "num_tokens": 1479281.0, "reward": 0.40625, "reward_std": 0.4218915104866028, "rewards/accuracy_reward/mean": 0.40625, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.351951241493225, "sampling/importance_sampling_ratio/mean": 0.999655544757843, "sampling/importance_sampling_ratio/min": 0.6168337464332581, "sampling/sampling_logp_difference/max": 0.4831557273864746, "sampling/sampling_logp_difference/mean": 0.014881279319524765, "step": 64 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.307855462888256e-05, "clip_ratio/low_min": 5.307855462888256e-05, "clip_ratio/region_mean": 5.307855462888256e-05, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 724.0, "completions/mean_length": 587.71875, "completions/mean_terminated_length": 527.625, "completions/min_length": 276.0, "completions/min_terminated_length": 276.0, "entropy": 0.387448538094759, "epoch": 0.034796573875803, "frac_reward_zero_std": 0.25, "grad_norm": 0.010778703726828098, "learning_rate": 1e-05, "loss": 0.0369, "num_tokens": 1501840.0, "reward": 0.75, "reward_std": 0.3745020925998688, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.3139044046401978, "sampling/importance_sampling_ratio/mean": 1.00022292137146, "sampling/importance_sampling_ratio/min": 0.6147822141647339, "sampling/sampling_logp_difference/max": 0.48648715019226074, "sampling/sampling_logp_difference/mean": 0.010643831454217434, "step": 65 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.375, "completions/max_length": 768.0, "completions/max_terminated_length": 747.0, "completions/mean_length": 628.03125, "completions/mean_terminated_length": 544.0499877929688, "completions/min_length": 205.0, "completions/min_terminated_length": 205.0, "entropy": 0.4458521343767643, "epoch": 0.035331905781584586, "frac_reward_zero_std": 0.5, "grad_norm": 0.02286791242659092, "learning_rate": 1e-05, "loss": 0.0486, "num_tokens": 1525729.0, "reward": 0.65625, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.3667349815368652, "sampling/importance_sampling_ratio/mean": 0.9998641014099121, "sampling/importance_sampling_ratio/min": 0.5646668076515198, "sampling/sampling_logp_difference/max": 0.5715193748474121, "sampling/sampling_logp_difference/mean": 0.012170457281172276, "step": 66 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 9.674482498667203e-05, "clip_ratio/low_min": 9.674482498667203e-05, "clip_ratio/region_mean": 9.674482498667203e-05, "completions/clipped_ratio": 0.34375, "completions/max_length": 768.0, "completions/max_terminated_length": 760.0, "completions/mean_length": 631.5625, "completions/mean_terminated_length": 560.0952758789062, "completions/min_length": 280.0, "completions/min_terminated_length": 280.0, "entropy": 0.5247922241687775, "epoch": 0.03586723768736617, "frac_reward_zero_std": 0.25, "grad_norm": 0.005835197400301695, "learning_rate": 1e-05, "loss": 0.0164, "num_tokens": 1549971.0, "reward": 0.3125, "reward_std": 0.292504221200943, "rewards/accuracy_reward/mean": 0.3125, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.5354112386703491, "sampling/importance_sampling_ratio/mean": 1.0004278421401978, "sampling/importance_sampling_ratio/min": 0.5730288624763489, "sampling/sampling_logp_difference/max": 0.5568192005157471, "sampling/sampling_logp_difference/mean": 0.013546698726713657, "step": 67 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.887894440093078e-05, "clip_ratio/low_min": 5.887894440093078e-05, "clip_ratio/region_mean": 5.887894440093078e-05, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 721.0, "completions/mean_length": 525.28125, "completions/mean_terminated_length": 500.17242431640625, "completions/min_length": 348.0, "completions/min_terminated_length": 348.0, "entropy": 0.4689428135752678, "epoch": 0.03640256959314775, "frac_reward_zero_std": 0.5, "grad_norm": 0.0051872618496418, "learning_rate": 1e-05, "loss": 0.0514, "num_tokens": 1570332.0, "reward": 0.8125, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.46169912815094, "sampling/importance_sampling_ratio/mean": 1.000031590461731, "sampling/importance_sampling_ratio/min": 0.5865122079849243, "sampling/sampling_logp_difference/max": 0.5335617065429688, "sampling/sampling_logp_difference/mean": 0.012794294394552708, "step": 68 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00016885084187379107, "clip_ratio/low_min": 0.00016885084187379107, "clip_ratio/region_mean": 0.00016885084187379107, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 731.0, "completions/mean_length": 541.71875, "completions/mean_terminated_length": 509.39288330078125, "completions/min_length": 286.0, "completions/min_terminated_length": 286.0, "entropy": 0.3781624920666218, "epoch": 0.03693790149892934, "frac_reward_zero_std": 0.0, "grad_norm": 0.02200511284172535, "learning_rate": 1e-05, "loss": 0.0951, "num_tokens": 1591267.0, "reward": 0.84375, "reward_std": 0.3808925747871399, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.2878000736236572, "sampling/importance_sampling_ratio/mean": 1.0002068281173706, "sampling/importance_sampling_ratio/min": 0.77150559425354, "sampling/sampling_logp_difference/max": 0.2594113349914551, "sampling/sampling_logp_difference/mean": 0.010493318550288677, "step": 69 }, { "clip_ratio/high_max": 6.742178811691701e-05, "clip_ratio/high_mean": 6.742178811691701e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 6.742178811691701e-05, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 730.0, "completions/mean_length": 568.09375, "completions/mean_terminated_length": 531.0740966796875, "completions/min_length": 343.0, "completions/min_terminated_length": 343.0, "entropy": 0.3990517556667328, "epoch": 0.03747323340471092, "frac_reward_zero_std": 0.25, "grad_norm": 0.005453715566545725, "learning_rate": 1e-05, "loss": 0.098, "num_tokens": 1612814.0, "reward": 0.875, "reward_std": 0.292504221200943, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 1.412319302558899, "sampling/importance_sampling_ratio/mean": 0.9995604753494263, "sampling/importance_sampling_ratio/min": 0.6802285313606262, "sampling/sampling_logp_difference/max": 0.3853263854980469, "sampling/sampling_logp_difference/mean": 0.01026479434221983, "step": 70 }, { "clip_ratio/high_max": 5.0689373892964795e-05, "clip_ratio/high_mean": 5.0689373892964795e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 5.0689373892964795e-05, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 742.0, "completions/mean_length": 511.96875, "completions/mean_terminated_length": 494.9000244140625, "completions/min_length": 162.0, "completions/min_terminated_length": 162.0, "entropy": 0.45358023419976234, "epoch": 0.03800856531049251, "frac_reward_zero_std": 0.25, "grad_norm": 0.012236704118549824, "learning_rate": 1e-05, "loss": -0.0299, "num_tokens": 1632669.0, "reward": 0.75, "reward_std": 0.3514062762260437, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.8347247838974, "sampling/importance_sampling_ratio/mean": 0.9996175169944763, "sampling/importance_sampling_ratio/min": 0.5587140321731567, "sampling/sampling_logp_difference/max": 0.6068944931030273, "sampling/sampling_logp_difference/mean": 0.012087250128388405, "step": 71 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 702.0, "completions/mean_length": 514.125, "completions/mean_terminated_length": 455.5384826660156, "completions/min_length": 208.0, "completions/min_terminated_length": 208.0, "entropy": 0.5026115663349628, "epoch": 0.03854389721627409, "frac_reward_zero_std": 0.75, "grad_norm": 0.009692998602986336, "learning_rate": 1e-05, "loss": -0.0253, "num_tokens": 1652609.0, "reward": 0.40625, "reward_std": 0.1293872892856598, "rewards/accuracy_reward/mean": 0.40625, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.3599499464035034, "sampling/importance_sampling_ratio/mean": 1.0003825426101685, "sampling/importance_sampling_ratio/min": 0.710231363773346, "sampling/sampling_logp_difference/max": 0.3421645164489746, "sampling/sampling_logp_difference/mean": 0.013382923789322376, "step": 72 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.28125, "completions/max_length": 768.0, "completions/max_terminated_length": 720.0, "completions/mean_length": 596.40625, "completions/mean_terminated_length": 529.2608642578125, "completions/min_length": 306.0, "completions/min_terminated_length": 306.0, "entropy": 0.447014432400465, "epoch": 0.03907922912205567, "frac_reward_zero_std": 0.25, "grad_norm": 0.008403382264077663, "learning_rate": 1e-05, "loss": 0.0633, "num_tokens": 1674982.0, "reward": 0.53125, "reward_std": 0.3061639666557312, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.6530752182006836, "sampling/importance_sampling_ratio/mean": 1.0001921653747559, "sampling/importance_sampling_ratio/min": 0.5703743100166321, "sampling/sampling_logp_difference/max": 0.56146240234375, "sampling/sampling_logp_difference/mean": 0.012438956648111343, "step": 73 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 751.0, "completions/mean_length": 532.25, "completions/mean_terminated_length": 488.59259033203125, "completions/min_length": 314.0, "completions/min_terminated_length": 314.0, "entropy": 0.3938888870179653, "epoch": 0.03961456102783726, "frac_reward_zero_std": 0.0, "grad_norm": 0.020746279507875443, "learning_rate": 1e-05, "loss": 0.0652, "num_tokens": 1695878.0, "reward": 0.78125, "reward_std": 0.4355708956718445, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.8193347454071045, "sampling/importance_sampling_ratio/mean": 1.000119686126709, "sampling/importance_sampling_ratio/min": 0.6978682279586792, "sampling/sampling_logp_difference/max": 0.59847092628479, "sampling/sampling_logp_difference/mean": 0.010908914729952812, "step": 74 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.28125, "completions/max_length": 768.0, "completions/max_terminated_length": 722.0, "completions/mean_length": 567.90625, "completions/mean_terminated_length": 489.60870361328125, "completions/min_length": 241.0, "completions/min_terminated_length": 241.0, "entropy": 0.3957683704793453, "epoch": 0.04014989293361884, "frac_reward_zero_std": 0.5, "grad_norm": 0.004737218841910362, "learning_rate": 1e-05, "loss": 0.0433, "num_tokens": 1717419.0, "reward": 0.75, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.3065564632415771, "sampling/importance_sampling_ratio/mean": 0.9997105002403259, "sampling/importance_sampling_ratio/min": 0.6393411755561829, "sampling/sampling_logp_difference/max": 0.44731712341308594, "sampling/sampling_logp_difference/mean": 0.01032248791307211, "step": 75 }, { "clip_ratio/high_max": 6.003842281643301e-05, "clip_ratio/high_mean": 6.003842281643301e-05, "clip_ratio/low_mean": 0.00011532022472238168, "clip_ratio/low_min": 0.00011532022472238168, "clip_ratio/region_mean": 0.0001753586475388147, "completions/clipped_ratio": 0.28125, "completions/max_length": 768.0, "completions/max_terminated_length": 745.0, "completions/mean_length": 587.53125, "completions/mean_terminated_length": 516.9130249023438, "completions/min_length": 218.0, "completions/min_terminated_length": 218.0, "entropy": 0.6047898083925247, "epoch": 0.04068522483940043, "frac_reward_zero_std": 0.0, "grad_norm": 0.014375505037605762, "learning_rate": 1e-05, "loss": 0.0525, "num_tokens": 1740620.0, "reward": 0.5625, "reward_std": 0.5081326961517334, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.515614628791809, "sampling/importance_sampling_ratio/mean": 0.9999227523803711, "sampling/importance_sampling_ratio/min": 0.49422574043273926, "sampling/sampling_logp_difference/max": 0.7047629356384277, "sampling/sampling_logp_difference/mean": 0.015101158060133457, "step": 76 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 8.747709580347873e-05, "clip_ratio/low_min": 8.747709580347873e-05, "clip_ratio/region_mean": 8.747709580347873e-05, "completions/clipped_ratio": 0.375, "completions/max_length": 768.0, "completions/max_terminated_length": 761.0, "completions/mean_length": 662.9375, "completions/mean_terminated_length": 599.9000244140625, "completions/min_length": 227.0, "completions/min_terminated_length": 227.0, "entropy": 0.5256207138299942, "epoch": 0.04122055674518201, "frac_reward_zero_std": 0.25, "grad_norm": 0.009198879823088646, "learning_rate": 1e-05, "loss": 0.0462, "num_tokens": 1765802.0, "reward": 0.65625, "reward_std": 0.3608423173427582, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.3540449142456055, "sampling/importance_sampling_ratio/mean": 1.0003117322921753, "sampling/importance_sampling_ratio/min": 0.6656081080436707, "sampling/sampling_logp_difference/max": 0.40705418586730957, "sampling/sampling_logp_difference/mean": 0.013424842618405819, "step": 77 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.617657759808935e-05, "clip_ratio/low_min": 4.617657759808935e-05, "clip_ratio/region_mean": 4.617657759808935e-05, "completions/clipped_ratio": 0.28125, "completions/max_length": 768.0, "completions/max_terminated_length": 721.0, "completions/mean_length": 577.34375, "completions/mean_terminated_length": 502.7391357421875, "completions/min_length": 228.0, "completions/min_terminated_length": 228.0, "entropy": 0.5271884277462959, "epoch": 0.0417558886509636, "frac_reward_zero_std": 0.5, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0212, "num_tokens": 1788429.0, "reward": 0.5, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5080004930496216, "sampling/importance_sampling_ratio/max": 1.3982810974121094, "sampling/importance_sampling_ratio/mean": 1.0000624656677246, "sampling/importance_sampling_ratio/min": 0.7151228785514832, "sampling/sampling_logp_difference/max": 0.33530092239379883, "sampling/sampling_logp_difference/mean": 0.014294518157839775, "step": 78 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00015163622447289526, "clip_ratio/low_min": 0.00015163622447289526, "clip_ratio/region_mean": 0.00015163622447289526, "completions/clipped_ratio": 0.28125, "completions/max_length": 768.0, "completions/max_terminated_length": 747.0, "completions/mean_length": 646.84375, "completions/mean_terminated_length": 599.434814453125, "completions/min_length": 412.0, "completions/min_terminated_length": 412.0, "entropy": 0.3845251575112343, "epoch": 0.04229122055674518, "frac_reward_zero_std": 0.0, "grad_norm": 0.0057140556164085865, "learning_rate": 1e-05, "loss": 0.0779, "num_tokens": 1812520.0, "reward": 0.5625, "reward_std": 0.4082317352294922, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.460033655166626, "sampling/importance_sampling_ratio/mean": 0.9999608993530273, "sampling/importance_sampling_ratio/min": 0.7029695510864258, "sampling/sampling_logp_difference/max": 0.37845945358276367, "sampling/sampling_logp_difference/mean": 0.01091760490089655, "step": 79 }, { "clip_ratio/high_max": 5.110384154249914e-05, "clip_ratio/high_mean": 5.110384154249914e-05, "clip_ratio/low_mean": 9.743091504788026e-05, "clip_ratio/low_min": 9.743091504788026e-05, "clip_ratio/region_mean": 0.0001485347565903794, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 732.0, "completions/mean_length": 580.0, "completions/mean_terminated_length": 517.3333740234375, "completions/min_length": 298.0, "completions/min_terminated_length": 298.0, "entropy": 0.5739789605140686, "epoch": 0.042826552462526764, "frac_reward_zero_std": 0.25, "grad_norm": 0.006165775004774332, "learning_rate": 1e-05, "loss": 0.0771, "num_tokens": 1835352.0, "reward": 0.71875, "reward_std": 0.3198433816432953, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.4025832414627075, "sampling/importance_sampling_ratio/mean": 0.9997737407684326, "sampling/importance_sampling_ratio/min": 0.2956174612045288, "sampling/sampling_logp_difference/max": 1.21868896484375, "sampling/sampling_logp_difference/mean": 0.01467198971658945, "step": 80 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 714.0, "completions/mean_length": 558.5625, "completions/mean_terminated_length": 499.91998291015625, "completions/min_length": 256.0, "completions/min_terminated_length": 256.0, "entropy": 0.40372762829065323, "epoch": 0.04336188436830835, "frac_reward_zero_std": 0.5, "grad_norm": 0.012004398740828037, "learning_rate": 1e-05, "loss": 0.0537, "num_tokens": 1857202.0, "reward": 0.75, "reward_std": 0.26726123690605164, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.5625461339950562, "sampling/importance_sampling_ratio/mean": 0.99977707862854, "sampling/importance_sampling_ratio/min": 0.578526496887207, "sampling/sampling_logp_difference/max": 0.5472710132598877, "sampling/sampling_logp_difference/mean": 0.011829355731606483, "step": 81 }, { "clip_ratio/high_max": 4.5687134843319654e-05, "clip_ratio/high_mean": 4.5687134843319654e-05, "clip_ratio/low_mean": 0.00013247920287540182, "clip_ratio/low_min": 0.00013247920287540182, "clip_ratio/region_mean": 0.00017816633771872148, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 754.0, "completions/mean_length": 552.75, "completions/mean_terminated_length": 481.0, "completions/min_length": 10.0, "completions/min_terminated_length": 10.0, "entropy": 0.39825017005205154, "epoch": 0.043897216274089934, "frac_reward_zero_std": 0.0, "grad_norm": 0.008354890160262585, "learning_rate": 1e-05, "loss": 0.0355, "num_tokens": 1878490.0, "reward": 0.5, "reward_std": 0.4355512857437134, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5080004930496216, "sampling/importance_sampling_ratio/max": 1.3465768098831177, "sampling/importance_sampling_ratio/mean": 0.999875545501709, "sampling/importance_sampling_ratio/min": 0.7088252902030945, "sampling/sampling_logp_difference/max": 0.3441462516784668, "sampling/sampling_logp_difference/mean": 0.01083304826170206, "step": 82 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 9.244907414540648e-05, "clip_ratio/low_min": 9.244907414540648e-05, "clip_ratio/region_mean": 9.244907414540648e-05, "completions/clipped_ratio": 0.34375, "completions/max_length": 768.0, "completions/max_terminated_length": 721.0, "completions/mean_length": 655.71875, "completions/mean_terminated_length": 596.90478515625, "completions/min_length": 447.0, "completions/min_terminated_length": 447.0, "entropy": 0.41648585721850395, "epoch": 0.04443254817987152, "frac_reward_zero_std": 0.0, "grad_norm": 0.008299059234559536, "learning_rate": 1e-05, "loss": 0.0414, "num_tokens": 1903329.0, "reward": 0.5, "reward_std": 0.4671337604522705, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5080004930496216, "sampling/importance_sampling_ratio/max": 1.3997514247894287, "sampling/importance_sampling_ratio/mean": 1.00001060962677, "sampling/importance_sampling_ratio/min": 0.7458000779151917, "sampling/sampling_logp_difference/max": 0.33629465103149414, "sampling/sampling_logp_difference/mean": 0.011712967418134212, "step": 83 }, { "clip_ratio/high_max": 4.448398613021709e-05, "clip_ratio/high_mean": 4.448398613021709e-05, "clip_ratio/low_mean": 0.0001298681745538488, "clip_ratio/low_min": 0.0001298681745538488, "clip_ratio/region_mean": 0.0001743521606840659, "completions/clipped_ratio": 0.5, "completions/max_length": 768.0, "completions/max_terminated_length": 756.0, "completions/mean_length": 712.0, "completions/mean_terminated_length": 656.0, "completions/min_length": 479.0, "completions/min_terminated_length": 479.0, "entropy": 0.5676172859966755, "epoch": 0.044967880085653104, "frac_reward_zero_std": 0.0, "grad_norm": 0.014507370069622993, "learning_rate": 1e-05, "loss": 0.0415, "num_tokens": 1930081.0, "reward": 0.46875, "reward_std": 0.4807935357093811, "rewards/accuracy_reward/mean": 0.46875, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.4132459163665771, "sampling/importance_sampling_ratio/mean": 0.9998955130577087, "sampling/importance_sampling_ratio/min": 0.4077630639076233, "sampling/sampling_logp_difference/max": 0.897068977355957, "sampling/sampling_logp_difference/mean": 0.013884996995329857, "step": 84 }, { "clip_ratio/high_max": 5.3418803872773424e-05, "clip_ratio/high_mean": 5.3418803872773424e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 5.3418803872773424e-05, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 731.0, "completions/mean_length": 545.8125, "completions/mean_terminated_length": 504.6666564941406, "completions/min_length": 297.0, "completions/min_terminated_length": 297.0, "entropy": 0.365762073546648, "epoch": 0.045503211991434686, "frac_reward_zero_std": 0.5, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0693, "num_tokens": 1951363.0, "reward": 0.59375, "reward_std": 0.22201895713806152, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.3155086040496826, "sampling/importance_sampling_ratio/mean": 0.9999384880065918, "sampling/importance_sampling_ratio/min": 0.3215399980545044, "sampling/sampling_logp_difference/max": 1.1346333026885986, "sampling/sampling_logp_difference/mean": 0.00987795926630497, "step": 85 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.444250928121619e-05, "clip_ratio/low_min": 5.444250928121619e-05, "clip_ratio/region_mean": 5.444250928121619e-05, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 750.0, "completions/mean_length": 522.90625, "completions/mean_terminated_length": 497.5517272949219, "completions/min_length": 264.0, "completions/min_terminated_length": 264.0, "entropy": 0.4572664350271225, "epoch": 0.046038543897216275, "frac_reward_zero_std": 0.25, "grad_norm": 0.0327676385641098, "learning_rate": 1e-05, "loss": 0.0754, "num_tokens": 1971336.0, "reward": 0.4375, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.4375, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.2798858880996704, "sampling/importance_sampling_ratio/mean": 0.9997351765632629, "sampling/importance_sampling_ratio/min": 0.5278854370117188, "sampling/sampling_logp_difference/max": 0.6388759613037109, "sampling/sampling_logp_difference/mean": 0.012657161802053452, "step": 86 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 725.0, "completions/mean_length": 570.84375, "completions/mean_terminated_length": 525.34619140625, "completions/min_length": 289.0, "completions/min_terminated_length": 289.0, "entropy": 0.347834974527359, "epoch": 0.046573875802997856, "frac_reward_zero_std": 0.75, "grad_norm": 0.01057675201445818, "learning_rate": 1e-05, "loss": 0.0045, "num_tokens": 1993627.0, "reward": 0.65625, "reward_std": 0.1293872892856598, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.4069409370422363, "sampling/importance_sampling_ratio/mean": 1.0003336668014526, "sampling/importance_sampling_ratio/min": 0.6120913028717041, "sampling/sampling_logp_difference/max": 0.4908738136291504, "sampling/sampling_logp_difference/mean": 0.010178979486227036, "step": 87 }, { "clip_ratio/high_max": 4.858142347075045e-05, "clip_ratio/high_mean": 4.858142347075045e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 4.858142347075045e-05, "completions/clipped_ratio": 0.375, "completions/max_length": 768.0, "completions/max_terminated_length": 749.0, "completions/mean_length": 669.65625, "completions/mean_terminated_length": 610.6500244140625, "completions/min_length": 409.0, "completions/min_terminated_length": 409.0, "entropy": 0.4982682652771473, "epoch": 0.047109207708779445, "frac_reward_zero_std": 0.25, "grad_norm": 0.007830352522432804, "learning_rate": 1e-05, "loss": 0.0613, "num_tokens": 2019040.0, "reward": 0.53125, "reward_std": 0.3061639964580536, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.4326367378234863, "sampling/importance_sampling_ratio/mean": 1.0002639293670654, "sampling/importance_sampling_ratio/min": 0.6901931166648865, "sampling/sampling_logp_difference/max": 0.37078389525413513, "sampling/sampling_logp_difference/mean": 0.01206160057336092, "step": 88 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.415401053847745e-05, "clip_ratio/low_min": 4.415401053847745e-05, "clip_ratio/region_mean": 4.415401053847745e-05, "completions/clipped_ratio": 0.28125, "completions/max_length": 768.0, "completions/max_terminated_length": 764.0, "completions/mean_length": 618.5, "completions/mean_terminated_length": 560.0, "completions/min_length": 246.0, "completions/min_terminated_length": 246.0, "entropy": 0.4850558340549469, "epoch": 0.04764453961456103, "frac_reward_zero_std": 0.0, "grad_norm": 0.011177493259310722, "learning_rate": 1e-05, "loss": 0.0306, "num_tokens": 2042840.0, "reward": 0.5625, "reward_std": 0.48503684997558594, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.3503026962280273, "sampling/importance_sampling_ratio/mean": 0.9996246099472046, "sampling/importance_sampling_ratio/min": 0.5023510456085205, "sampling/sampling_logp_difference/max": 0.6884560585021973, "sampling/sampling_logp_difference/mean": 0.013991152867674828, "step": 89 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00010283642041031271, "clip_ratio/low_min": 0.00010283642041031271, "clip_ratio/region_mean": 0.00010283642041031271, "completions/clipped_ratio": 0.40625, "completions/max_length": 768.0, "completions/max_terminated_length": 749.0, "completions/mean_length": 662.71875, "completions/mean_terminated_length": 590.6842041015625, "completions/min_length": 421.0, "completions/min_terminated_length": 421.0, "entropy": 0.6079593561589718, "epoch": 0.048179871520342615, "frac_reward_zero_std": 0.5, "grad_norm": 0.006331273354589939, "learning_rate": 1e-05, "loss": 0.0282, "num_tokens": 2068375.0, "reward": 0.40625, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.40625, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.347251534461975, "sampling/importance_sampling_ratio/mean": 0.9998496770858765, "sampling/importance_sampling_ratio/min": 0.6717338562011719, "sampling/sampling_logp_difference/max": 0.3978930711746216, "sampling/sampling_logp_difference/mean": 0.014920149929821491, "step": 90 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00016003364726202562, "clip_ratio/low_min": 0.00016003364726202562, "clip_ratio/region_mean": 0.00016003364726202562, "completions/clipped_ratio": 0.28125, "completions/max_length": 768.0, "completions/max_terminated_length": 757.0, "completions/mean_length": 593.0, "completions/mean_terminated_length": 524.521728515625, "completions/min_length": 257.0, "completions/min_terminated_length": 257.0, "entropy": 0.3775136098265648, "epoch": 0.0487152034261242, "frac_reward_zero_std": 0.25, "grad_norm": 0.0033323902171105146, "learning_rate": 1e-05, "loss": 0.0177, "num_tokens": 2091079.0, "reward": 0.46875, "reward_std": 0.3061639666557312, "rewards/accuracy_reward/mean": 0.46875, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.3280028104782104, "sampling/importance_sampling_ratio/mean": 0.9996941089630127, "sampling/importance_sampling_ratio/min": 0.6664149165153503, "sampling/sampling_logp_difference/max": 0.40584278106689453, "sampling/sampling_logp_difference/mean": 0.010856150649487972, "step": 91 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 766.0, "completions/mean_length": 544.375, "completions/mean_terminated_length": 502.96295166015625, "completions/min_length": 301.0, "completions/min_terminated_length": 301.0, "entropy": 0.5462964624166489, "epoch": 0.04925053533190578, "frac_reward_zero_std": 0.5, "grad_norm": 0.019380314275622368, "learning_rate": 1e-05, "loss": 0.0169, "num_tokens": 2111907.0, "reward": 0.8125, "reward_std": 0.249358132481575, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.3186160326004028, "sampling/importance_sampling_ratio/mean": 1.0001270771026611, "sampling/importance_sampling_ratio/min": 0.529840886592865, "sampling/sampling_logp_difference/max": 0.6351785659790039, "sampling/sampling_logp_difference/mean": 0.013746769167482853, "step": 92 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 8.876702850102447e-05, "clip_ratio/low_min": 8.876702850102447e-05, "clip_ratio/region_mean": 8.876702850102447e-05, "completions/clipped_ratio": 0.4375, "completions/max_length": 768.0, "completions/max_terminated_length": 727.0, "completions/mean_length": 666.03125, "completions/mean_terminated_length": 586.7222290039062, "completions/min_length": 442.0, "completions/min_terminated_length": 442.0, "entropy": 0.614947035908699, "epoch": 0.04978586723768737, "frac_reward_zero_std": 0.0, "grad_norm": 0.004191740415990353, "learning_rate": 1e-05, "loss": 0.0558, "num_tokens": 2137044.0, "reward": 0.53125, "reward_std": 0.47137707471847534, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.4060710668563843, "sampling/importance_sampling_ratio/mean": 1.0000110864639282, "sampling/importance_sampling_ratio/min": 0.5382021069526672, "sampling/sampling_logp_difference/max": 0.6195211410522461, "sampling/sampling_logp_difference/mean": 0.01403165701776743, "step": 93 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.40625, "completions/max_length": 768.0, "completions/max_terminated_length": 761.0, "completions/mean_length": 661.40625, "completions/mean_terminated_length": 588.4736938476562, "completions/min_length": 422.0, "completions/min_terminated_length": 422.0, "entropy": 0.4756098948419094, "epoch": 0.05032119914346895, "frac_reward_zero_std": 0.5, "grad_norm": 0.00242839427664876, "learning_rate": 1e-05, "loss": 0.0402, "num_tokens": 2162049.0, "reward": 0.5625, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.3409123420715332, "sampling/importance_sampling_ratio/mean": 1.0000770092010498, "sampling/importance_sampling_ratio/min": 0.6455109119415283, "sampling/sampling_logp_difference/max": 0.4377131462097168, "sampling/sampling_logp_difference/mean": 0.011750009842216969, "step": 94 }, { "clip_ratio/high_max": 5.975143358227797e-05, "clip_ratio/high_mean": 5.975143358227797e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 5.975143358227797e-05, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 767.0, "completions/mean_length": 559.40625, "completions/mean_terminated_length": 520.7777709960938, "completions/min_length": 278.0, "completions/min_terminated_length": 278.0, "entropy": 0.47756603360176086, "epoch": 0.05085653104925054, "frac_reward_zero_std": 0.0, "grad_norm": 0.02502770908176899, "learning_rate": 1e-05, "loss": 0.1244, "num_tokens": 2183142.0, "reward": 0.625, "reward_std": 0.3945523500442505, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.4260393381118774, "sampling/importance_sampling_ratio/mean": 1.0001617670059204, "sampling/importance_sampling_ratio/min": 0.39494454860687256, "sampling/sampling_logp_difference/max": 0.9290099143981934, "sampling/sampling_logp_difference/mean": 0.011839261278510094, "step": 95 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00011316847667330876, "clip_ratio/low_min": 0.00011316847667330876, "clip_ratio/region_mean": 0.00011316847667330876, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 738.0, "completions/mean_length": 514.6875, "completions/mean_terminated_length": 497.8000183105469, "completions/min_length": 167.0, "completions/min_terminated_length": 167.0, "entropy": 0.548037227243185, "epoch": 0.05139186295503212, "frac_reward_zero_std": 0.5, "grad_norm": 0.009991215541958809, "learning_rate": 1e-05, "loss": 0.0379, "num_tokens": 2203132.0, "reward": 0.6875, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.52627432346344, "sampling/importance_sampling_ratio/mean": 0.9997628331184387, "sampling/importance_sampling_ratio/min": 0.6441085934638977, "sampling/sampling_logp_difference/max": 0.43988800048828125, "sampling/sampling_logp_difference/mean": 0.013994378969073296, "step": 96 }, { "clip_ratio/high_max": 5.941064591752365e-05, "clip_ratio/high_mean": 5.941064591752365e-05, "clip_ratio/low_mean": 0.0001188212918350473, "clip_ratio/low_min": 0.0001188212918350473, "clip_ratio/region_mean": 0.00017823194502852857, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 747.0, "completions/mean_length": 586.21875, "completions/mean_terminated_length": 544.2692260742188, "completions/min_length": 334.0, "completions/min_terminated_length": 334.0, "entropy": 0.3432692065834999, "epoch": 0.05192719486081371, "frac_reward_zero_std": 0.25, "grad_norm": 0.008097965270280838, "learning_rate": 1e-05, "loss": 0.0368, "num_tokens": 2225595.0, "reward": 0.75, "reward_std": 0.3745020925998688, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.350540280342102, "sampling/importance_sampling_ratio/mean": 0.9999731779098511, "sampling/importance_sampling_ratio/min": 0.4406386911869049, "sampling/sampling_logp_difference/max": 0.8195300102233887, "sampling/sampling_logp_difference/mean": 0.009654526598751545, "step": 97 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 719.0, "completions/mean_length": 544.0, "completions/mean_terminated_length": 529.0667114257812, "completions/min_length": 313.0, "completions/min_terminated_length": 313.0, "entropy": 0.3262908738106489, "epoch": 0.05246252676659529, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0427, "num_tokens": 2246771.0, "reward": 0.71875, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.410939335823059, "sampling/importance_sampling_ratio/mean": 0.9998470544815063, "sampling/importance_sampling_ratio/min": 0.542165219783783, "sampling/sampling_logp_difference/max": 0.6121845245361328, "sampling/sampling_logp_difference/mean": 0.009420693852007389, "step": 98 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.798464578925632e-05, "clip_ratio/low_min": 4.798464578925632e-05, "clip_ratio/region_mean": 4.798464578925632e-05, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 747.0, "completions/mean_length": 554.28125, "completions/mean_terminated_length": 494.44000244140625, "completions/min_length": 148.0, "completions/min_terminated_length": 148.0, "entropy": 0.41109946370124817, "epoch": 0.05299785867237687, "frac_reward_zero_std": 0.0, "grad_norm": 0.008418666198849678, "learning_rate": 1e-05, "loss": 0.0199, "num_tokens": 2268068.0, "reward": 0.78125, "reward_std": 0.4218915104866028, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.3368817567825317, "sampling/importance_sampling_ratio/mean": 1.0000663995742798, "sampling/importance_sampling_ratio/min": 0.2998821437358856, "sampling/sampling_logp_difference/max": 1.2043657302856445, "sampling/sampling_logp_difference/mean": 0.01116845104843378, "step": 99 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.53125, "completions/max_length": 768.0, "completions/max_terminated_length": 711.0, "completions/mean_length": 690.09375, "completions/mean_terminated_length": 601.800048828125, "completions/min_length": 447.0, "completions/min_terminated_length": 447.0, "entropy": 0.5567548722028732, "epoch": 0.05353319057815846, "frac_reward_zero_std": 0.25, "grad_norm": 0.009668181650340557, "learning_rate": 1e-05, "loss": 0.0671, "num_tokens": 2294319.0, "reward": 0.40625, "reward_std": 0.3377465009689331, "rewards/accuracy_reward/mean": 0.40625, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.5256686210632324, "sampling/importance_sampling_ratio/mean": 1.0002436637878418, "sampling/importance_sampling_ratio/min": 0.5617189407348633, "sampling/sampling_logp_difference/max": 0.5767536163330078, "sampling/sampling_logp_difference/mean": 0.013522452674806118, "step": 100 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00021020192798459902, "clip_ratio/low_min": 0.00021020192798459902, "clip_ratio/region_mean": 0.00021020192798459902, "completions/clipped_ratio": 0.5, "completions/max_length": 768.0, "completions/max_terminated_length": 745.0, "completions/mean_length": 666.375, "completions/mean_terminated_length": 564.75, "completions/min_length": 319.0, "completions/min_terminated_length": 319.0, "entropy": 0.6232212483882904, "epoch": 0.05406852248394004, "frac_reward_zero_std": 0.25, "grad_norm": 0.013746596872806549, "learning_rate": 1e-05, "loss": 0.0382, "num_tokens": 2319683.0, "reward": 0.53125, "reward_std": 0.378745436668396, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.5801082849502563, "sampling/importance_sampling_ratio/mean": 1.0001741647720337, "sampling/importance_sampling_ratio/min": 0.5974046587944031, "sampling/sampling_logp_difference/max": 0.5151605606079102, "sampling/sampling_logp_difference/mean": 0.014388276264071465, "step": 101 }, { "clip_ratio/high_max": 5.3648069297196344e-05, "clip_ratio/high_mean": 5.3648069297196344e-05, "clip_ratio/low_mean": 5.4159445426194e-05, "clip_ratio/low_min": 5.4159445426194e-05, "clip_ratio/region_mean": 0.00010780751472339034, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 754.0, "completions/mean_length": 547.71875, "completions/mean_terminated_length": 496.8846435546875, "completions/min_length": 326.0, "completions/min_terminated_length": 326.0, "entropy": 0.43036941811442375, "epoch": 0.05460385438972163, "frac_reward_zero_std": 0.25, "grad_norm": 0.0024584766943007708, "learning_rate": 1e-05, "loss": 0.0534, "num_tokens": 2340722.0, "reward": 0.5625, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.3089535236358643, "sampling/importance_sampling_ratio/mean": 1.0002161264419556, "sampling/importance_sampling_ratio/min": 0.3775835335254669, "sampling/sampling_logp_difference/max": 0.9739634990692139, "sampling/sampling_logp_difference/mean": 0.01143811084330082, "step": 102 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00010505907630431466, "clip_ratio/low_min": 0.00010505907630431466, "clip_ratio/region_mean": 0.00010505907630431466, "completions/clipped_ratio": 0.34375, "completions/max_length": 768.0, "completions/max_terminated_length": 759.0, "completions/mean_length": 630.9375, "completions/mean_terminated_length": 559.1428833007812, "completions/min_length": 182.0, "completions/min_terminated_length": 182.0, "entropy": 0.38904211297631264, "epoch": 0.05513918629550321, "frac_reward_zero_std": 0.0, "grad_norm": 0.008734066039323807, "learning_rate": 1e-05, "loss": 0.04, "num_tokens": 2364568.0, "reward": 0.5625, "reward_std": 0.4261348247528076, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.4303452968597412, "sampling/importance_sampling_ratio/mean": 1.0002168416976929, "sampling/importance_sampling_ratio/min": 0.5523419380187988, "sampling/sampling_logp_difference/max": 0.5935878753662109, "sampling/sampling_logp_difference/mean": 0.011160231195390224, "step": 103 }, { "clip_ratio/high_max": 5.048465391155332e-05, "clip_ratio/high_mean": 5.048465391155332e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 5.048465391155332e-05, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 735.0, "completions/mean_length": 508.125, "completions/mean_terminated_length": 481.2413635253906, "completions/min_length": 152.0, "completions/min_terminated_length": 152.0, "entropy": 0.4318654015660286, "epoch": 0.055674518201284794, "frac_reward_zero_std": 0.75, "grad_norm": 0.007156253792345524, "learning_rate": 1e-05, "loss": 0.0781, "num_tokens": 2384188.0, "reward": 0.90625, "reward_std": 0.1293872892856598, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.3144526481628418, "sampling/importance_sampling_ratio/mean": 0.999993085861206, "sampling/importance_sampling_ratio/min": 0.5605980753898621, "sampling/sampling_logp_difference/max": 0.5787510871887207, "sampling/sampling_logp_difference/mean": 0.011459408327937126, "step": 104 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.4996399083174765e-05, "clip_ratio/low_min": 4.4996399083174765e-05, "clip_ratio/region_mean": 4.4996399083174765e-05, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 746.0, "completions/mean_length": 568.21875, "completions/mean_terminated_length": 539.6785888671875, "completions/min_length": 282.0, "completions/min_terminated_length": 282.0, "entropy": 0.5981403402984142, "epoch": 0.05620985010706638, "frac_reward_zero_std": 0.0, "grad_norm": 0.02444583736360073, "learning_rate": 1e-05, "loss": 0.0736, "num_tokens": 2405899.0, "reward": 0.71875, "reward_std": 0.4534739851951599, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.2787832021713257, "sampling/importance_sampling_ratio/mean": 1.000063180923462, "sampling/importance_sampling_ratio/min": 0.6764567494392395, "sampling/sampling_logp_difference/max": 0.3908867835998535, "sampling/sampling_logp_difference/mean": 0.013214386068284512, "step": 105 }, { "clip_ratio/high_max": 5.425347262644209e-05, "clip_ratio/high_mean": 5.425347262644209e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 5.425347262644209e-05, "completions/clipped_ratio": 0.34375, "completions/max_length": 768.0, "completions/max_terminated_length": 642.0, "completions/mean_length": 607.65625, "completions/mean_terminated_length": 523.6666870117188, "completions/min_length": 255.0, "completions/min_terminated_length": 255.0, "entropy": 0.42543483152985573, "epoch": 0.056745182012847964, "frac_reward_zero_std": 0.0, "grad_norm": 0.005100829526782036, "learning_rate": 1e-05, "loss": 0.032, "num_tokens": 2428992.0, "reward": 0.65625, "reward_std": 0.3808925747871399, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.2988433837890625, "sampling/importance_sampling_ratio/mean": 1.0001482963562012, "sampling/importance_sampling_ratio/min": 0.5767703652381897, "sampling/sampling_logp_difference/max": 0.5503110885620117, "sampling/sampling_logp_difference/mean": 0.011606302112340927, "step": 106 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0001682632318988908, "clip_ratio/low_min": 0.0001682632318988908, "clip_ratio/region_mean": 0.0001682632318988908, "completions/clipped_ratio": 0.375, "completions/max_length": 768.0, "completions/max_terminated_length": 753.0, "completions/mean_length": 618.84375, "completions/mean_terminated_length": 529.3500366210938, "completions/min_length": 138.0, "completions/min_terminated_length": 138.0, "entropy": 0.46930770948529243, "epoch": 0.05728051391862955, "frac_reward_zero_std": 0.0, "grad_norm": 0.016056913882493973, "learning_rate": 1e-05, "loss": -0.0106, "num_tokens": 2452507.0, "reward": 0.53125, "reward_std": 0.4765698313713074, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.355771780014038, "sampling/importance_sampling_ratio/mean": 1.0001250505447388, "sampling/importance_sampling_ratio/min": 0.6253914833068848, "sampling/sampling_logp_difference/max": 0.4693775177001953, "sampling/sampling_logp_difference/mean": 0.012566701509058475, "step": 107 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 9.01831554074306e-05, "clip_ratio/low_min": 9.01831554074306e-05, "clip_ratio/region_mean": 9.01831554074306e-05, "completions/clipped_ratio": 0.46875, "completions/max_length": 768.0, "completions/max_terminated_length": 732.0, "completions/mean_length": 657.34375, "completions/mean_terminated_length": 559.7058715820312, "completions/min_length": 409.0, "completions/min_terminated_length": 409.0, "entropy": 0.42219090834259987, "epoch": 0.057815845824411134, "frac_reward_zero_std": 0.0, "grad_norm": 0.004926656372845173, "learning_rate": 1e-05, "loss": 0.0041, "num_tokens": 2477494.0, "reward": 0.4375, "reward_std": 0.4261348247528076, "rewards/accuracy_reward/mean": 0.4375, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.4135397672653198, "sampling/importance_sampling_ratio/mean": 1.0002245903015137, "sampling/importance_sampling_ratio/min": 0.6830143928527832, "sampling/sampling_logp_difference/max": 0.3812394142150879, "sampling/sampling_logp_difference/mean": 0.011800967156887054, "step": 108 }, { "clip_ratio/high_max": 4.8506015446037054e-05, "clip_ratio/high_mean": 4.8506015446037054e-05, "clip_ratio/low_mean": 9.067827340913936e-05, "clip_ratio/low_min": 9.067827340913936e-05, "clip_ratio/region_mean": 0.00013918428885517642, "completions/clipped_ratio": 0.4375, "completions/max_length": 768.0, "completions/max_terminated_length": 749.0, "completions/mean_length": 684.1875, "completions/mean_terminated_length": 619.0, "completions/min_length": 480.0, "completions/min_terminated_length": 480.0, "entropy": 0.42762457206845284, "epoch": 0.05835117773019272, "frac_reward_zero_std": 0.0, "grad_norm": 0.01747298799455166, "learning_rate": 1e-05, "loss": 0.0294, "num_tokens": 2503860.0, "reward": 0.59375, "reward_std": 0.3987956643104553, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.4077019691467285, "sampling/importance_sampling_ratio/mean": 0.9999071359634399, "sampling/importance_sampling_ratio/min": 0.6927469372749329, "sampling/sampling_logp_difference/max": 0.36709046363830566, "sampling/sampling_logp_difference/mean": 0.011081259697675705, "step": 109 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 704.0, "completions/mean_length": 538.90625, "completions/mean_terminated_length": 486.0384826660156, "completions/min_length": 251.0, "completions/min_terminated_length": 251.0, "entropy": 0.46098020672798157, "epoch": 0.058886509635974305, "frac_reward_zero_std": 0.25, "grad_norm": 0.011608164757490158, "learning_rate": 1e-05, "loss": 0.0437, "num_tokens": 2525065.0, "reward": 0.59375, "reward_std": 0.3061639964580536, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.4105576276779175, "sampling/importance_sampling_ratio/mean": 1.0002728700637817, "sampling/importance_sampling_ratio/min": 0.7486305832862854, "sampling/sampling_logp_difference/max": 0.34398508071899414, "sampling/sampling_logp_difference/mean": 0.012847142294049263, "step": 110 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0001269035565201193, "clip_ratio/low_min": 0.0001269035565201193, "clip_ratio/region_mean": 0.0001269035565201193, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 759.0, "completions/mean_length": 587.4375, "completions/mean_terminated_length": 545.7692260742188, "completions/min_length": 295.0, "completions/min_terminated_length": 295.0, "entropy": 0.438092615455389, "epoch": 0.059421841541755886, "frac_reward_zero_std": 0.25, "grad_norm": 0.0035028019919991493, "learning_rate": 1e-05, "loss": 0.0354, "num_tokens": 2547567.0, "reward": 0.21875, "reward_std": 0.3061639964580536, "rewards/accuracy_reward/mean": 0.21875, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.314680814743042, "sampling/importance_sampling_ratio/mean": 0.9998528957366943, "sampling/importance_sampling_ratio/min": 0.623349130153656, "sampling/sampling_logp_difference/max": 0.472648561000824, "sampling/sampling_logp_difference/mean": 0.012160422280430794, "step": 111 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 9.173203943646513e-05, "clip_ratio/low_min": 9.173203943646513e-05, "clip_ratio/region_mean": 9.173203943646513e-05, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 722.0, "completions/mean_length": 568.59375, "completions/mean_terminated_length": 522.5769653320312, "completions/min_length": 332.0, "completions/min_terminated_length": 332.0, "entropy": 0.49586763232946396, "epoch": 0.059957173447537475, "frac_reward_zero_std": 0.25, "grad_norm": 0.010358037427067757, "learning_rate": 1e-05, "loss": 0.0575, "num_tokens": 2569322.0, "reward": 0.625, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.504044532775879, "sampling/importance_sampling_ratio/mean": 0.9998618364334106, "sampling/importance_sampling_ratio/min": 0.7376149892807007, "sampling/sampling_logp_difference/max": 0.4081578254699707, "sampling/sampling_logp_difference/mean": 0.013150558806955814, "step": 112 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0001805224601412192, "clip_ratio/low_min": 0.0001805224601412192, "clip_ratio/region_mean": 0.0001805224601412192, "completions/clipped_ratio": 0.3125, "completions/max_length": 768.0, "completions/max_terminated_length": 741.0, "completions/mean_length": 641.84375, "completions/mean_terminated_length": 584.5, "completions/min_length": 357.0, "completions/min_terminated_length": 357.0, "entropy": 0.3489786498248577, "epoch": 0.06049250535331906, "frac_reward_zero_std": 0.0, "grad_norm": 0.011260202154517174, "learning_rate": 1e-05, "loss": 0.0681, "num_tokens": 2593485.0, "reward": 0.375, "reward_std": 0.4082317352294922, "rewards/accuracy_reward/mean": 0.375, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.6591311693191528, "sampling/importance_sampling_ratio/mean": 1.000071406364441, "sampling/importance_sampling_ratio/min": 0.6210263967514038, "sampling/sampling_logp_difference/max": 0.5062940120697021, "sampling/sampling_logp_difference/mean": 0.009856711141765118, "step": 113 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.40625, "completions/max_length": 768.0, "completions/max_terminated_length": 714.0, "completions/mean_length": 647.90625, "completions/mean_terminated_length": 565.73681640625, "completions/min_length": 362.0, "completions/min_terminated_length": 362.0, "entropy": 0.3779727593064308, "epoch": 0.061027837259100645, "frac_reward_zero_std": 0.25, "grad_norm": 0.007471651770174503, "learning_rate": 1e-05, "loss": 0.0598, "num_tokens": 2617866.0, "reward": 0.53125, "reward_std": 0.3608423173427582, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.8370476961135864, "sampling/importance_sampling_ratio/mean": 1.0000334978103638, "sampling/importance_sampling_ratio/min": 0.7170367240905762, "sampling/sampling_logp_difference/max": 0.6081597805023193, "sampling/sampling_logp_difference/mean": 0.01039117481559515, "step": 114 }, { "clip_ratio/high_max": 5.392579987528734e-05, "clip_ratio/high_mean": 5.392579987528734e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 5.392579987528734e-05, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 742.0, "completions/mean_length": 587.96875, "completions/mean_terminated_length": 554.629638671875, "completions/min_length": 397.0, "completions/min_terminated_length": 397.0, "entropy": 0.38369783386588097, "epoch": 0.06156316916488223, "frac_reward_zero_std": 0.25, "grad_norm": 0.003959751687943935, "learning_rate": 1e-05, "loss": 0.0847, "num_tokens": 2640433.0, "reward": 0.4375, "reward_std": 0.292504221200943, "rewards/accuracy_reward/mean": 0.4375, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.390690803527832, "sampling/importance_sampling_ratio/mean": 0.9999404549598694, "sampling/importance_sampling_ratio/min": 0.6957584619522095, "sampling/sampling_logp_difference/max": 0.36275267601013184, "sampling/sampling_logp_difference/mean": 0.01046355627477169, "step": 115 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.461099160835147e-05, "clip_ratio/low_min": 4.461099160835147e-05, "clip_ratio/region_mean": 4.461099160835147e-05, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 760.0, "completions/mean_length": 609.125, "completions/mean_terminated_length": 586.4285888671875, "completions/min_length": 338.0, "completions/min_terminated_length": 338.0, "entropy": 0.35069551318883896, "epoch": 0.06209850107066381, "frac_reward_zero_std": 0.25, "grad_norm": 0.01873995177447796, "learning_rate": 1e-05, "loss": 0.045, "num_tokens": 2664013.0, "reward": 0.875, "reward_std": 0.292504221200943, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 1.2887252569198608, "sampling/importance_sampling_ratio/mean": 1.0001789331436157, "sampling/importance_sampling_ratio/min": 0.5800834894180298, "sampling/sampling_logp_difference/max": 0.5445833206176758, "sampling/sampling_logp_difference/mean": 0.010990909300744534, "step": 116 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 758.0, "completions/mean_length": 554.0, "completions/mean_terminated_length": 531.862060546875, "completions/min_length": 309.0, "completions/min_terminated_length": 309.0, "entropy": 0.38614571839571, "epoch": 0.0626338329764454, "frac_reward_zero_std": 0.25, "grad_norm": 0.01285243034362793, "learning_rate": 1e-05, "loss": 0.0039, "num_tokens": 2685421.0, "reward": 0.75, "reward_std": 0.3514062762260437, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.292870044708252, "sampling/importance_sampling_ratio/mean": 0.9997349381446838, "sampling/importance_sampling_ratio/min": 0.6336048245429993, "sampling/sampling_logp_difference/max": 0.4563298225402832, "sampling/sampling_logp_difference/mean": 0.010782275348901749, "step": 117 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 9.733352271723561e-05, "clip_ratio/low_min": 9.733352271723561e-05, "clip_ratio/region_mean": 9.733352271723561e-05, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 761.0, "completions/mean_length": 557.8125, "completions/mean_terminated_length": 498.9599914550781, "completions/min_length": 231.0, "completions/min_terminated_length": 231.0, "entropy": 0.3148543946444988, "epoch": 0.06316916488222699, "frac_reward_zero_std": 0.0, "grad_norm": 0.005136107560247183, "learning_rate": 1e-05, "loss": 0.0698, "num_tokens": 2706519.0, "reward": 0.53125, "reward_std": 0.3808925747871399, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.3862152099609375, "sampling/importance_sampling_ratio/mean": 0.9998864531517029, "sampling/importance_sampling_ratio/min": 0.6390144228935242, "sampling/sampling_logp_difference/max": 0.4478282928466797, "sampling/sampling_logp_difference/mean": 0.009354082867503166, "step": 118 }, { "clip_ratio/high_max": 4.069010537932627e-05, "clip_ratio/high_mean": 4.069010537932627e-05, "clip_ratio/low_mean": 0.00013032825518166646, "clip_ratio/low_min": 0.00013032825518166646, "clip_ratio/region_mean": 0.00017101836056099273, "completions/clipped_ratio": 0.53125, "completions/max_length": 768.0, "completions/max_terminated_length": 706.0, "completions/mean_length": 693.0, "completions/mean_terminated_length": 608.0000610351562, "completions/min_length": 440.0, "completions/min_terminated_length": 440.0, "entropy": 0.4280709773302078, "epoch": 0.06370449678800856, "frac_reward_zero_std": 0.25, "grad_norm": 0.016133075580000877, "learning_rate": 1e-05, "loss": 0.0394, "num_tokens": 2733135.0, "reward": 0.34375, "reward_std": 0.3061639964580536, "rewards/accuracy_reward/mean": 0.34375, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.4363564252853394, "sampling/importance_sampling_ratio/mean": 1.0001070499420166, "sampling/importance_sampling_ratio/min": 0.706879198551178, "sampling/sampling_logp_difference/max": 0.3621096611022949, "sampling/sampling_logp_difference/mean": 0.012061288580298424, "step": 119 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00011622501187957823, "clip_ratio/low_min": 0.00011622501187957823, "clip_ratio/region_mean": 0.00011622501187957823, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 742.0, "completions/mean_length": 556.4375, "completions/mean_terminated_length": 542.3333740234375, "completions/min_length": 366.0, "completions/min_terminated_length": 366.0, "entropy": 0.4840794987976551, "epoch": 0.06423982869379015, "frac_reward_zero_std": 0.75, "grad_norm": 0.005025084130465984, "learning_rate": 1e-05, "loss": 0.012, "num_tokens": 2754437.0, "reward": 0.65625, "reward_std": 0.1293872892856598, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.2942429780960083, "sampling/importance_sampling_ratio/mean": 0.9997637271881104, "sampling/importance_sampling_ratio/min": 0.6982724666595459, "sampling/sampling_logp_difference/max": 0.3591458797454834, "sampling/sampling_logp_difference/mean": 0.012346608564257622, "step": 120 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 755.0, "completions/mean_length": 584.5, "completions/mean_terminated_length": 542.1538696289062, "completions/min_length": 272.0, "completions/min_terminated_length": 272.0, "entropy": 0.41059280931949615, "epoch": 0.06477516059957174, "frac_reward_zero_std": 0.5, "grad_norm": 0.004495520610362291, "learning_rate": 1e-05, "loss": 0.0013, "num_tokens": 2776909.0, "reward": 0.625, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.5147361755371094, "sampling/importance_sampling_ratio/mean": 0.9997055530548096, "sampling/importance_sampling_ratio/min": 0.6869720816612244, "sampling/sampling_logp_difference/max": 0.4152412414550781, "sampling/sampling_logp_difference/mean": 0.011804929934442043, "step": 121 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 765.0, "completions/mean_length": 559.0, "completions/mean_terminated_length": 545.0667114257812, "completions/min_length": 336.0, "completions/min_terminated_length": 336.0, "entropy": 0.47115417942404747, "epoch": 0.06531049250535331, "frac_reward_zero_std": 0.5, "grad_norm": 0.010263456963002682, "learning_rate": 1e-05, "loss": -0.0138, "num_tokens": 2799293.0, "reward": 0.8125, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.8492910861968994, "sampling/importance_sampling_ratio/mean": 1.0000624656677246, "sampling/importance_sampling_ratio/min": 0.7267021536827087, "sampling/sampling_logp_difference/max": 0.614802360534668, "sampling/sampling_logp_difference/mean": 0.013139222748577595, "step": 122 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 8.574192543164827e-05, "clip_ratio/low_min": 8.574192543164827e-05, "clip_ratio/region_mean": 8.574192543164827e-05, "completions/clipped_ratio": 0.53125, "completions/max_length": 768.0, "completions/max_terminated_length": 749.0, "completions/mean_length": 680.4375, "completions/mean_terminated_length": 581.2000122070312, "completions/min_length": 363.0, "completions/min_terminated_length": 363.0, "entropy": 0.3957010358572006, "epoch": 0.0658458244111349, "frac_reward_zero_std": 0.25, "grad_norm": 0.008676748722791672, "learning_rate": 1e-05, "loss": 0.0544, "num_tokens": 2825003.0, "reward": 0.53125, "reward_std": 0.3471629321575165, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.4573827981948853, "sampling/importance_sampling_ratio/mean": 0.9998784065246582, "sampling/importance_sampling_ratio/min": 0.701416552066803, "sampling/sampling_logp_difference/max": 0.37664222717285156, "sampling/sampling_logp_difference/mean": 0.010493094101548195, "step": 123 }, { "clip_ratio/high_max": 8.090614574030042e-05, "clip_ratio/high_mean": 8.090614574030042e-05, "clip_ratio/low_mean": 5.509034963324666e-05, "clip_ratio/low_min": 5.509034963324666e-05, "clip_ratio/region_mean": 0.00013599649537354708, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 768.0, "completions/mean_length": 534.65625, "completions/mean_terminated_length": 501.3214416503906, "completions/min_length": 200.0, "completions/min_terminated_length": 200.0, "entropy": 0.4937793128192425, "epoch": 0.06638115631691649, "frac_reward_zero_std": 0.25, "grad_norm": 0.02326061576604843, "learning_rate": 1e-05, "loss": 0.119, "num_tokens": 2845544.0, "reward": 0.75, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.4219590425491333, "sampling/importance_sampling_ratio/mean": 0.9997606873512268, "sampling/importance_sampling_ratio/min": 0.6375591158866882, "sampling/sampling_logp_difference/max": 0.45010828971862793, "sampling/sampling_logp_difference/mean": 0.012804565951228142, "step": 124 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00010036129970103502, "clip_ratio/low_min": 0.00010036129970103502, "clip_ratio/region_mean": 0.00010036129970103502, "completions/clipped_ratio": 0.3125, "completions/max_length": 768.0, "completions/max_terminated_length": 739.0, "completions/mean_length": 632.0, "completions/mean_terminated_length": 570.1818237304688, "completions/min_length": 407.0, "completions/min_terminated_length": 407.0, "entropy": 0.46971315145492554, "epoch": 0.06691648822269808, "frac_reward_zero_std": 0.0, "grad_norm": 0.02554064616560936, "learning_rate": 1e-05, "loss": 0.0524, "num_tokens": 2869736.0, "reward": 0.5625, "reward_std": 0.4671337604522705, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.2999053001403809, "sampling/importance_sampling_ratio/mean": 0.999940812587738, "sampling/importance_sampling_ratio/min": 0.5058492422103882, "sampling/sampling_logp_difference/max": 0.6815166473388672, "sampling/sampling_logp_difference/mean": 0.012605911120772362, "step": 125 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.34375, "completions/max_length": 768.0, "completions/max_terminated_length": 688.0, "completions/mean_length": 619.6875, "completions/mean_terminated_length": 542.0, "completions/min_length": 282.0, "completions/min_terminated_length": 282.0, "entropy": 0.4839254133403301, "epoch": 0.06745182012847965, "frac_reward_zero_std": 0.0, "grad_norm": 0.009828461334109306, "learning_rate": 1e-05, "loss": 0.0114, "num_tokens": 2893078.0, "reward": 0.46875, "reward_std": 0.5038893818855286, "rewards/accuracy_reward/mean": 0.46875, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.3906389474868774, "sampling/importance_sampling_ratio/mean": 1.000062108039856, "sampling/importance_sampling_ratio/min": 0.664653480052948, "sampling/sampling_logp_difference/max": 0.408489465713501, "sampling/sampling_logp_difference/mean": 0.012790380045771599, "step": 126 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00016165318083949387, "clip_ratio/low_min": 0.00016165318083949387, "clip_ratio/region_mean": 0.00016165318083949387, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 709.0, "completions/mean_length": 536.90625, "completions/mean_terminated_length": 459.875, "completions/min_length": 156.0, "completions/min_terminated_length": 156.0, "entropy": 0.6607670187950134, "epoch": 0.06798715203426124, "frac_reward_zero_std": 0.25, "grad_norm": 0.007993953302502632, "learning_rate": 1e-05, "loss": 0.0435, "num_tokens": 2914587.0, "reward": 0.40625, "reward_std": 0.3608423173427582, "rewards/accuracy_reward/mean": 0.40625, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.4073468446731567, "sampling/importance_sampling_ratio/mean": 0.9998863339424133, "sampling/importance_sampling_ratio/min": 0.6154264807701111, "sampling/sampling_logp_difference/max": 0.4854397773742676, "sampling/sampling_logp_difference/mean": 0.01635979302227497, "step": 127 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00015176042506936938, "clip_ratio/low_min": 0.00015176042506936938, "clip_ratio/region_mean": 0.00015176042506936938, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 755.0, "completions/mean_length": 568.03125, "completions/mean_terminated_length": 501.375, "completions/min_length": 150.0, "completions/min_terminated_length": 150.0, "entropy": 0.5782148689031601, "epoch": 0.06852248394004283, "frac_reward_zero_std": 0.5, "grad_norm": 0.004083705134689808, "learning_rate": 1e-05, "loss": -0.0123, "num_tokens": 2936700.0, "reward": 0.34375, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.34375, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.4441615343093872, "sampling/importance_sampling_ratio/mean": 1.0001335144042969, "sampling/importance_sampling_ratio/min": 0.6603602766990662, "sampling/sampling_logp_difference/max": 0.414969801902771, "sampling/sampling_logp_difference/mean": 0.01503153145313263, "step": 128 }, { "clip_ratio/high_max": 5.0120288506150246e-05, "clip_ratio/high_mean": 5.0120288506150246e-05, "clip_ratio/low_mean": 9.586511077941395e-05, "clip_ratio/low_min": 9.586511077941395e-05, "clip_ratio/region_mean": 0.0001459853992855642, "completions/clipped_ratio": 0.40625, "completions/max_length": 768.0, "completions/max_terminated_length": 762.0, "completions/mean_length": 671.3125, "completions/mean_terminated_length": 605.1578979492188, "completions/min_length": 442.0, "completions/min_terminated_length": 442.0, "entropy": 0.5217020772397518, "epoch": 0.0690578158458244, "frac_reward_zero_std": 0.25, "grad_norm": 0.007857496850192547, "learning_rate": 1e-05, "loss": 0.0425, "num_tokens": 2962934.0, "reward": 0.4375, "reward_std": 0.3924051821231842, "rewards/accuracy_reward/mean": 0.4375, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.6288236379623413, "sampling/importance_sampling_ratio/mean": 0.9999582767486572, "sampling/importance_sampling_ratio/min": 0.7605227828025818, "sampling/sampling_logp_difference/max": 0.4878580570220947, "sampling/sampling_logp_difference/mean": 0.013301405124366283, "step": 129 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.089576370664872e-05, "clip_ratio/low_min": 5.089576370664872e-05, "clip_ratio/region_mean": 5.089576370664872e-05, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 713.0, "completions/mean_length": 524.40625, "completions/mean_terminated_length": 499.2069091796875, "completions/min_length": 220.0, "completions/min_terminated_length": 220.0, "entropy": 0.430951289832592, "epoch": 0.069593147751606, "frac_reward_zero_std": 0.5, "grad_norm": 0.0051418147049844265, "learning_rate": 1e-05, "loss": 0.0922, "num_tokens": 2984051.0, "reward": 0.75, "reward_std": 0.2314550280570984, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0001728534698486, "sampling/importance_sampling_ratio/min": 0.6581515073776245, "sampling/sampling_logp_difference/max": 1.094374656677246, "sampling/sampling_logp_difference/mean": 0.011661775410175323, "step": 130 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.843852341058664e-05, "clip_ratio/low_min": 5.843852341058664e-05, "clip_ratio/region_mean": 5.843852341058664e-05, "completions/clipped_ratio": 0.3125, "completions/max_length": 768.0, "completions/max_terminated_length": 766.0, "completions/mean_length": 606.03125, "completions/mean_terminated_length": 532.4091186523438, "completions/min_length": 329.0, "completions/min_terminated_length": 329.0, "entropy": 0.4265390373766422, "epoch": 0.07012847965738758, "frac_reward_zero_std": 0.0, "grad_norm": 0.011738032102584839, "learning_rate": 1e-05, "loss": 0.0178, "num_tokens": 3006892.0, "reward": 0.5, "reward_std": 0.4671337604522705, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5080004930496216, "sampling/importance_sampling_ratio/max": 1.2919646501541138, "sampling/importance_sampling_ratio/mean": 1.0000965595245361, "sampling/importance_sampling_ratio/min": 0.6226043701171875, "sampling/sampling_logp_difference/max": 0.473844051361084, "sampling/sampling_logp_difference/mean": 0.01166878454387188, "step": 131 }, { "clip_ratio/high_max": 4.8113932280102745e-05, "clip_ratio/high_mean": 4.8113932280102745e-05, "clip_ratio/low_mean": 4.638218888430856e-05, "clip_ratio/low_min": 4.638218888430856e-05, "clip_ratio/region_mean": 9.44961211644113e-05, "completions/clipped_ratio": 0.28125, "completions/max_length": 768.0, "completions/max_terminated_length": 761.0, "completions/mean_length": 635.15625, "completions/mean_terminated_length": 583.1739501953125, "completions/min_length": 327.0, "completions/min_terminated_length": 327.0, "entropy": 0.3812892846763134, "epoch": 0.07066381156316917, "frac_reward_zero_std": 0.25, "grad_norm": 0.009988192468881607, "learning_rate": 1e-05, "loss": 0.0355, "num_tokens": 3031249.0, "reward": 0.625, "reward_std": 0.3924051821231842, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.3657511472702026, "sampling/importance_sampling_ratio/mean": 0.9999386668205261, "sampling/importance_sampling_ratio/min": 0.5940070748329163, "sampling/sampling_logp_difference/max": 0.5208640098571777, "sampling/sampling_logp_difference/mean": 0.010986251756548882, "step": 132 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 766.0, "completions/mean_length": 528.625, "completions/mean_terminated_length": 484.2962951660156, "completions/min_length": 231.0, "completions/min_terminated_length": 231.0, "entropy": 0.37385377660393715, "epoch": 0.07119914346895075, "frac_reward_zero_std": 0.25, "grad_norm": 0.009472616948187351, "learning_rate": 1e-05, "loss": 0.0301, "num_tokens": 3051933.0, "reward": 0.75, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.3858510255813599, "sampling/importance_sampling_ratio/mean": 1.0000290870666504, "sampling/importance_sampling_ratio/min": 0.7769193053245544, "sampling/sampling_logp_difference/max": 0.32631444931030273, "sampling/sampling_logp_difference/mean": 0.010760965757071972, "step": 133 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 8.594018436269835e-05, "clip_ratio/low_min": 8.594018436269835e-05, "clip_ratio/region_mean": 8.594018436269835e-05, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 759.0, "completions/mean_length": 616.5, "completions/mean_terminated_length": 566.0, "completions/min_length": 347.0, "completions/min_terminated_length": 347.0, "entropy": 0.4294635057449341, "epoch": 0.07173447537473233, "frac_reward_zero_std": 0.0, "grad_norm": 0.028067462146282196, "learning_rate": 1e-05, "loss": 0.0484, "num_tokens": 3075669.0, "reward": 0.6875, "reward_std": 0.4082317352294922, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.436657190322876, "sampling/importance_sampling_ratio/mean": 1.0004854202270508, "sampling/importance_sampling_ratio/min": 0.6535131931304932, "sampling/sampling_logp_difference/max": 0.42539262771606445, "sampling/sampling_logp_difference/mean": 0.012337197549641132, "step": 134 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 714.0, "completions/mean_length": 472.28125, "completions/mean_terminated_length": 462.7419128417969, "completions/min_length": 285.0, "completions/min_terminated_length": 285.0, "entropy": 0.45794105529785156, "epoch": 0.07226980728051392, "frac_reward_zero_std": 0.5, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0344, "num_tokens": 3094454.0, "reward": 0.78125, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.2952197790145874, "sampling/importance_sampling_ratio/mean": 0.9998743534088135, "sampling/importance_sampling_ratio/min": 0.7145240306854248, "sampling/sampling_logp_difference/max": 0.3361387252807617, "sampling/sampling_logp_difference/mean": 0.01316301990300417, "step": 135 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 713.0, "completions/max_terminated_length": 713.0, "completions/mean_length": 516.84375, "completions/mean_terminated_length": 516.84375, "completions/min_length": 236.0, "completions/min_terminated_length": 236.0, "entropy": 0.3649085760116577, "epoch": 0.0728051391862955, "frac_reward_zero_std": 0.75, "grad_norm": 0.0048465123400092125, "learning_rate": 1e-05, "loss": -0.0117, "num_tokens": 3115017.0, "reward": 0.875, "reward_std": 0.13363061845302582, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 1.3063361644744873, "sampling/importance_sampling_ratio/mean": 1.0002888441085815, "sampling/importance_sampling_ratio/min": 0.7597522735595703, "sampling/sampling_logp_difference/max": 0.2747628688812256, "sampling/sampling_logp_difference/mean": 0.009677169844508171, "step": 136 }, { "clip_ratio/high_max": 6.32271112408489e-05, "clip_ratio/high_mean": 6.32271112408489e-05, "clip_ratio/low_mean": 4.901960710412823e-05, "clip_ratio/low_min": 4.901960710412823e-05, "clip_ratio/region_mean": 0.00011224671834497713, "completions/clipped_ratio": 0.28125, "completions/max_length": 768.0, "completions/max_terminated_length": 746.0, "completions/mean_length": 594.0, "completions/mean_terminated_length": 525.9130249023438, "completions/min_length": 282.0, "completions/min_terminated_length": 282.0, "entropy": 0.41735487431287766, "epoch": 0.07334047109207709, "frac_reward_zero_std": 0.0, "grad_norm": 0.018654856830835342, "learning_rate": 1e-05, "loss": 0.1186, "num_tokens": 3137457.0, "reward": 0.6875, "reward_std": 0.4671337604522705, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.5576105117797852, "sampling/importance_sampling_ratio/mean": 1.0002013444900513, "sampling/importance_sampling_ratio/min": 0.7714291214942932, "sampling/sampling_logp_difference/max": 0.44315290451049805, "sampling/sampling_logp_difference/mean": 0.011493084952235222, "step": 137 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.3599582568276674e-05, "clip_ratio/low_min": 4.3599582568276674e-05, "clip_ratio/region_mean": 4.3599582568276674e-05, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 758.0, "completions/mean_length": 556.5625, "completions/mean_terminated_length": 526.357177734375, "completions/min_length": 312.0, "completions/min_terminated_length": 312.0, "entropy": 0.39944466203451157, "epoch": 0.07387580299785867, "frac_reward_zero_std": 0.0, "grad_norm": 0.011650525964796543, "learning_rate": 1e-05, "loss": 0.0601, "num_tokens": 3158987.0, "reward": 0.75, "reward_std": 0.4261348247528076, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.3022949695587158, "sampling/importance_sampling_ratio/mean": 1.0001643896102905, "sampling/importance_sampling_ratio/min": 0.7634257674217224, "sampling/sampling_logp_difference/max": 0.2699394226074219, "sampling/sampling_logp_difference/mean": 0.011838635429739952, "step": 138 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00012147716188337654, "clip_ratio/low_min": 0.00012147716188337654, "clip_ratio/region_mean": 0.00012147716188337654, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 735.0, "completions/mean_length": 578.0625, "completions/mean_terminated_length": 542.888916015625, "completions/min_length": 206.0, "completions/min_terminated_length": 206.0, "entropy": 0.4281155802309513, "epoch": 0.07441113490364026, "frac_reward_zero_std": 0.25, "grad_norm": 0.011990956030786037, "learning_rate": 1e-05, "loss": 0.0493, "num_tokens": 3180685.0, "reward": 0.71875, "reward_std": 0.3198433816432953, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.282145619392395, "sampling/importance_sampling_ratio/mean": 1.000073790550232, "sampling/importance_sampling_ratio/min": 0.7347516417503357, "sampling/sampling_logp_difference/max": 0.30822277069091797, "sampling/sampling_logp_difference/mean": 0.011692803353071213, "step": 139 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.950494985678233e-05, "clip_ratio/low_min": 4.950494985678233e-05, "clip_ratio/region_mean": 4.950494985678233e-05, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 744.0, "completions/mean_length": 613.84375, "completions/mean_terminated_length": 570.6799926757812, "completions/min_length": 416.0, "completions/min_terminated_length": 416.0, "entropy": 0.379335206001997, "epoch": 0.07494646680942184, "frac_reward_zero_std": 0.5, "grad_norm": 0.003859455231577158, "learning_rate": 1e-05, "loss": -0.0015, "num_tokens": 3203648.0, "reward": 0.75, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.4280729293823242, "sampling/importance_sampling_ratio/mean": 1.0001428127288818, "sampling/importance_sampling_ratio/min": 0.7387887239456177, "sampling/sampling_logp_difference/max": 0.3563258647918701, "sampling/sampling_logp_difference/mean": 0.01078173890709877, "step": 140 }, { "clip_ratio/high_max": 4.288164564059116e-05, "clip_ratio/high_mean": 4.288164564059116e-05, "clip_ratio/low_mean": 8.576329128118232e-05, "clip_ratio/low_min": 8.576329128118232e-05, "clip_ratio/region_mean": 0.00012864494055975229, "completions/clipped_ratio": 0.59375, "completions/max_length": 768.0, "completions/max_terminated_length": 738.0, "completions/mean_length": 691.25, "completions/mean_terminated_length": 579.0769653320312, "completions/min_length": 372.0, "completions/min_terminated_length": 372.0, "entropy": 0.43631627783179283, "epoch": 0.07548179871520343, "frac_reward_zero_std": 0.0, "grad_norm": 0.010278386063873768, "learning_rate": 1e-05, "loss": 0.0765, "num_tokens": 3229656.0, "reward": 0.375, "reward_std": 0.4492306709289551, "rewards/accuracy_reward/mean": 0.375, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0001037120819092, "sampling/importance_sampling_ratio/min": 0.6426151990890503, "sampling/sampling_logp_difference/max": 0.7930843830108643, "sampling/sampling_logp_difference/mean": 0.012555928900837898, "step": 141 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 674.0, "completions/mean_length": 491.09375, "completions/mean_terminated_length": 482.1612854003906, "completions/min_length": 272.0, "completions/min_terminated_length": 272.0, "entropy": 0.3479072377085686, "epoch": 0.07601713062098502, "frac_reward_zero_std": 0.75, "grad_norm": 0.014172155410051346, "learning_rate": 1e-05, "loss": 0.016, "num_tokens": 3248723.0, "reward": 0.875, "reward_std": 0.13363061845302582, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 1.2840180397033691, "sampling/importance_sampling_ratio/mean": 0.9999534487724304, "sampling/importance_sampling_ratio/min": 0.733255922794342, "sampling/sampling_logp_difference/max": 0.310260534286499, "sampling/sampling_logp_difference/mean": 0.010322680696845055, "step": 142 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 9.306521678809077e-05, "clip_ratio/low_min": 9.306521678809077e-05, "clip_ratio/region_mean": 9.306521678809077e-05, "completions/clipped_ratio": 0.34375, "completions/max_length": 768.0, "completions/max_terminated_length": 696.0, "completions/mean_length": 593.46875, "completions/mean_terminated_length": 502.0476379394531, "completions/min_length": 304.0, "completions/min_terminated_length": 304.0, "entropy": 0.445149801671505, "epoch": 0.07655246252676659, "frac_reward_zero_std": 0.0, "grad_norm": 0.01311558485031128, "learning_rate": 1e-05, "loss": 0.1091, "num_tokens": 3271378.0, "reward": 0.5625, "reward_std": 0.4261348247528076, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.5771468877792358, "sampling/importance_sampling_ratio/mean": 1.0002362728118896, "sampling/importance_sampling_ratio/min": 0.7307872176170349, "sampling/sampling_logp_difference/max": 0.45561742782592773, "sampling/sampling_logp_difference/mean": 0.012233340181410313, "step": 143 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00011185682524228469, "clip_ratio/low_min": 0.00011185682524228469, "clip_ratio/region_mean": 0.00011185682524228469, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 717.0, "completions/mean_length": 523.96875, "completions/mean_terminated_length": 498.72412109375, "completions/min_length": 282.0, "completions/min_terminated_length": 282.0, "entropy": 0.3835696242749691, "epoch": 0.07708779443254818, "frac_reward_zero_std": 0.25, "grad_norm": 0.00548822246491909, "learning_rate": 1e-05, "loss": 0.0289, "num_tokens": 3291761.0, "reward": 0.75, "reward_std": 0.2925041913986206, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.2991029024124146, "sampling/importance_sampling_ratio/mean": 0.9997556209564209, "sampling/importance_sampling_ratio/min": 0.7319806814193726, "sampling/sampling_logp_difference/max": 0.31200122833251953, "sampling/sampling_logp_difference/mean": 0.011083073914051056, "step": 144 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.02008042531088e-05, "clip_ratio/low_min": 5.02008042531088e-05, "clip_ratio/region_mean": 5.02008042531088e-05, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 758.0, "completions/mean_length": 572.75, "completions/mean_terminated_length": 518.0800170898438, "completions/min_length": 356.0, "completions/min_terminated_length": 356.0, "entropy": 0.4650462605059147, "epoch": 0.07762312633832977, "frac_reward_zero_std": 0.25, "grad_norm": 0.01172321755439043, "learning_rate": 1e-05, "loss": 0.0935, "num_tokens": 3313745.0, "reward": 0.6875, "reward_std": 0.3745020925998688, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.4786471128463745, "sampling/importance_sampling_ratio/mean": 1.000137448310852, "sampling/importance_sampling_ratio/min": 0.6587621569633484, "sampling/sampling_logp_difference/max": 0.4173927307128906, "sampling/sampling_logp_difference/mean": 0.012362263165414333, "step": 145 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.263157800072804e-05, "clip_ratio/low_min": 5.263157800072804e-05, "clip_ratio/region_mean": 5.263157800072804e-05, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 762.0, "completions/mean_length": 522.1875, "completions/mean_terminated_length": 465.4615478515625, "completions/min_length": 279.0, "completions/min_terminated_length": 279.0, "entropy": 0.49817724898457527, "epoch": 0.07815845824411134, "frac_reward_zero_std": 0.25, "grad_norm": 0.0048125144094228745, "learning_rate": 1e-05, "loss": 0.0513, "num_tokens": 3333943.0, "reward": 0.71875, "reward_std": 0.3198433816432953, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.3833361864089966, "sampling/importance_sampling_ratio/mean": 0.9996218085289001, "sampling/importance_sampling_ratio/min": 0.7237681746482849, "sampling/sampling_logp_difference/max": 0.32449817657470703, "sampling/sampling_logp_difference/mean": 0.012647133320569992, "step": 146 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.083367068436928e-05, "clip_ratio/low_min": 5.083367068436928e-05, "clip_ratio/region_mean": 5.083367068436928e-05, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 739.0, "completions/mean_length": 554.15625, "completions/mean_terminated_length": 504.8077087402344, "completions/min_length": 303.0, "completions/min_terminated_length": 303.0, "entropy": 0.42663123086094856, "epoch": 0.07869379014989293, "frac_reward_zero_std": 0.0, "grad_norm": 0.004106262698769569, "learning_rate": 1e-05, "loss": 0.1323, "num_tokens": 3355332.0, "reward": 0.75, "reward_std": 0.4492306709289551, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.3002183437347412, "sampling/importance_sampling_ratio/mean": 0.9997296929359436, "sampling/importance_sampling_ratio/min": 0.6375466585159302, "sampling/sampling_logp_difference/max": 0.45012784004211426, "sampling/sampling_logp_difference/mean": 0.012056585401296616, "step": 147 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00011313626237097196, "clip_ratio/low_min": 0.00011313626237097196, "clip_ratio/region_mean": 0.00011313626237097196, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 768.0, "completions/mean_length": 544.46875, "completions/mean_terminated_length": 537.258056640625, "completions/min_length": 339.0, "completions/min_terminated_length": 339.0, "entropy": 0.3727027215063572, "epoch": 0.07922912205567452, "frac_reward_zero_std": 0.25, "grad_norm": 0.006529542617499828, "learning_rate": 1e-05, "loss": -0.029, "num_tokens": 3376403.0, "reward": 0.59375, "reward_std": 0.3061639666557312, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.4008318185806274, "sampling/importance_sampling_ratio/mean": 0.9996922016143799, "sampling/importance_sampling_ratio/min": 0.6501274704933167, "sampling/sampling_logp_difference/max": 0.4305868148803711, "sampling/sampling_logp_difference/mean": 0.011418145149946213, "step": 148 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 739.0, "completions/mean_length": 552.78125, "completions/mean_terminated_length": 512.9259033203125, "completions/min_length": 316.0, "completions/min_terminated_length": 316.0, "entropy": 0.3545210584998131, "epoch": 0.07976445396145611, "frac_reward_zero_std": 0.0, "grad_norm": 0.009868360124528408, "learning_rate": 1e-05, "loss": 0.1144, "num_tokens": 3397556.0, "reward": 0.75, "reward_std": 0.4492306709289551, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.459777593612671, "sampling/importance_sampling_ratio/mean": 0.9996419548988342, "sampling/importance_sampling_ratio/min": 0.591137707233429, "sampling/sampling_logp_difference/max": 0.5257062911987305, "sampling/sampling_logp_difference/mean": 0.010681376792490482, "step": 149 }, { "clip_ratio/high_max": 6.929046503501013e-05, "clip_ratio/high_mean": 6.929046503501013e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 6.929046503501013e-05, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 657.0, "completions/mean_length": 519.8125, "completions/mean_terminated_length": 494.137939453125, "completions/min_length": 197.0, "completions/min_terminated_length": 197.0, "entropy": 0.3497144766151905, "epoch": 0.08029978586723768, "frac_reward_zero_std": 0.25, "grad_norm": 0.007856633514165878, "learning_rate": 1e-05, "loss": 0.0728, "num_tokens": 3417814.0, "reward": 0.75, "reward_std": 0.3514062762260437, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.4564754962921143, "sampling/importance_sampling_ratio/mean": 1.0000619888305664, "sampling/importance_sampling_ratio/min": 0.5521512031555176, "sampling/sampling_logp_difference/max": 0.5939333438873291, "sampling/sampling_logp_difference/mean": 0.011367524974048138, "step": 150 }, { "clip_ratio/high_max": 4.9270791350863874e-05, "clip_ratio/high_mean": 4.9270791350863874e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 4.9270791350863874e-05, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 763.0, "completions/mean_length": 595.0625, "completions/mean_terminated_length": 555.1538696289062, "completions/min_length": 318.0, "completions/min_terminated_length": 318.0, "entropy": 0.4125949963927269, "epoch": 0.08083511777301927, "frac_reward_zero_std": 0.0, "grad_norm": 0.01485457643866539, "learning_rate": 1e-05, "loss": 0.0465, "num_tokens": 3440848.0, "reward": 0.5625, "reward_std": 0.4671337604522705, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.3517029285430908, "sampling/importance_sampling_ratio/mean": 0.9998703598976135, "sampling/importance_sampling_ratio/min": 0.7250288128852844, "sampling/sampling_logp_difference/max": 0.32154393196105957, "sampling/sampling_logp_difference/mean": 0.012134257704019547, "step": 151 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.886630267719738e-05, "clip_ratio/low_min": 4.886630267719738e-05, "clip_ratio/region_mean": 4.886630267719738e-05, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 751.0, "completions/mean_length": 610.75, "completions/mean_terminated_length": 558.3333740234375, "completions/min_length": 260.0, "completions/min_terminated_length": 260.0, "entropy": 0.45361290499567986, "epoch": 0.08137044967880086, "frac_reward_zero_std": 0.25, "grad_norm": 0.010131021030247211, "learning_rate": 1e-05, "loss": 0.0221, "num_tokens": 3464232.0, "reward": 0.71875, "reward_std": 0.3198433816432953, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.4945924282073975, "sampling/importance_sampling_ratio/mean": 1.000422716140747, "sampling/importance_sampling_ratio/min": 0.6482064127922058, "sampling/sampling_logp_difference/max": 0.4335460662841797, "sampling/sampling_logp_difference/mean": 0.012475905008614063, "step": 152 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 763.0, "completions/mean_length": 600.53125, "completions/mean_terminated_length": 569.5184936523438, "completions/min_length": 319.0, "completions/min_terminated_length": 319.0, "entropy": 0.4938753917813301, "epoch": 0.08190578158458243, "frac_reward_zero_std": 0.5, "grad_norm": 0.022933492437005043, "learning_rate": 1e-05, "loss": 0.0236, "num_tokens": 3487073.0, "reward": 0.71875, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.2920268774032593, "sampling/importance_sampling_ratio/mean": 0.9997101426124573, "sampling/importance_sampling_ratio/min": 0.7264668345451355, "sampling/sampling_logp_difference/max": 0.3195624351501465, "sampling/sampling_logp_difference/mean": 0.012675675563514233, "step": 153 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00010532044325373136, "clip_ratio/low_min": 0.00010532044325373136, "clip_ratio/region_mean": 0.00010532044325373136, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 734.0, "completions/mean_length": 560.78125, "completions/mean_terminated_length": 546.9666748046875, "completions/min_length": 302.0, "completions/min_terminated_length": 302.0, "entropy": 0.4342854470014572, "epoch": 0.08244111349036402, "frac_reward_zero_std": 0.5, "grad_norm": 0.003199859755113721, "learning_rate": 1e-05, "loss": 0.0053, "num_tokens": 3509114.0, "reward": 0.84375, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.3869831562042236, "sampling/importance_sampling_ratio/mean": 0.9997544288635254, "sampling/importance_sampling_ratio/min": 0.7236307263374329, "sampling/sampling_logp_difference/max": 0.3271310329437256, "sampling/sampling_logp_difference/mean": 0.0121637973934412, "step": 154 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 714.0, "completions/mean_length": 546.96875, "completions/mean_terminated_length": 473.29168701171875, "completions/min_length": 210.0, "completions/min_terminated_length": 210.0, "entropy": 0.3199138306081295, "epoch": 0.08297644539614561, "frac_reward_zero_std": 0.75, "grad_norm": 0.003953386563807726, "learning_rate": 1e-05, "loss": -0.006, "num_tokens": 3530473.0, "reward": 0.6875, "reward_std": 0.1157275140285492, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.3936272859573364, "sampling/importance_sampling_ratio/mean": 0.9998939037322998, "sampling/importance_sampling_ratio/min": 0.7508476972579956, "sampling/sampling_logp_difference/max": 0.3319098949432373, "sampling/sampling_logp_difference/mean": 0.009719339199364185, "step": 155 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.1082959544146433e-05, "clip_ratio/low_min": 5.1082959544146433e-05, "clip_ratio/region_mean": 5.1082959544146433e-05, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 692.0, "completions/mean_length": 549.0, "completions/mean_terminated_length": 508.4444580078125, "completions/min_length": 303.0, "completions/min_terminated_length": 303.0, "entropy": 0.36035246029496193, "epoch": 0.0835117773019272, "frac_reward_zero_std": 0.25, "grad_norm": 0.009710676968097687, "learning_rate": 1e-05, "loss": -0.0117, "num_tokens": 3551937.0, "reward": 0.75, "reward_std": 0.3745020925998688, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.2875422239303589, "sampling/importance_sampling_ratio/mean": 0.9994460940361023, "sampling/importance_sampling_ratio/min": 0.6191197037696838, "sampling/sampling_logp_difference/max": 0.47945666313171387, "sampling/sampling_logp_difference/mean": 0.011513352394104004, "step": 156 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 743.0, "completions/mean_length": 497.40625, "completions/mean_terminated_length": 479.36669921875, "completions/min_length": 255.0, "completions/min_terminated_length": 255.0, "entropy": 0.3987043537199497, "epoch": 0.08404710920770878, "frac_reward_zero_std": 0.25, "grad_norm": 0.0059003341011703014, "learning_rate": 1e-05, "loss": 0.011, "num_tokens": 3571382.0, "reward": 0.875, "reward_std": 0.292504221200943, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 1.3658850193023682, "sampling/importance_sampling_ratio/mean": 0.9999449849128723, "sampling/importance_sampling_ratio/min": 0.6775805354118347, "sampling/sampling_logp_difference/max": 0.38922691345214844, "sampling/sampling_logp_difference/mean": 0.011501464992761612, "step": 157 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.3125, "completions/max_length": 768.0, "completions/max_terminated_length": 636.0, "completions/mean_length": 567.5625, "completions/mean_terminated_length": 476.4545593261719, "completions/min_length": 306.0, "completions/min_terminated_length": 306.0, "entropy": 0.5364015437662601, "epoch": 0.08458244111349036, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0546, "num_tokens": 3593440.0, "reward": 0.6875, "reward_std": 0.1157275140285492, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.3599175214767456, "sampling/importance_sampling_ratio/mean": 0.9996451139450073, "sampling/importance_sampling_ratio/min": 0.5739397406578064, "sampling/sampling_logp_difference/max": 0.5552308559417725, "sampling/sampling_logp_difference/mean": 0.014400566928088665, "step": 158 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.833719867747277e-05, "clip_ratio/low_min": 4.833719867747277e-05, "clip_ratio/region_mean": 4.833719867747277e-05, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 762.0, "completions/mean_length": 568.8125, "completions/mean_terminated_length": 522.84619140625, "completions/min_length": 363.0, "completions/min_terminated_length": 363.0, "entropy": 0.3985172472894192, "epoch": 0.08511777301927195, "frac_reward_zero_std": 0.25, "grad_norm": 0.01399514265358448, "learning_rate": 1e-05, "loss": 0.0513, "num_tokens": 3615338.0, "reward": 0.75, "reward_std": 0.3514062464237213, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.3420157432556152, "sampling/importance_sampling_ratio/mean": 0.9997785091400146, "sampling/importance_sampling_ratio/min": 0.6207534074783325, "sampling/sampling_logp_difference/max": 0.4768214225769043, "sampling/sampling_logp_difference/mean": 0.0118543216958642, "step": 159 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0002292077333549969, "clip_ratio/low_min": 0.0002292077333549969, "clip_ratio/region_mean": 0.0002292077333549969, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 723.0, "completions/mean_length": 600.375, "completions/mean_terminated_length": 561.6923217773438, "completions/min_length": 404.0, "completions/min_terminated_length": 404.0, "entropy": 0.5832582972943783, "epoch": 0.08565310492505353, "frac_reward_zero_std": 0.0, "grad_norm": 0.009857095777988434, "learning_rate": 1e-05, "loss": 0.0666, "num_tokens": 3638150.0, "reward": 0.40625, "reward_std": 0.4628904461860657, "rewards/accuracy_reward/mean": 0.40625, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.2892735004425049, "sampling/importance_sampling_ratio/mean": 1.0000584125518799, "sampling/importance_sampling_ratio/min": 0.6991528868675232, "sampling/sampling_logp_difference/max": 0.35788583755493164, "sampling/sampling_logp_difference/mean": 0.014263789169490337, "step": 160 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0001376146828988567, "clip_ratio/low_min": 0.0001376146828988567, "clip_ratio/region_mean": 0.0001376146828988567, "completions/clipped_ratio": 0.4375, "completions/max_length": 768.0, "completions/max_terminated_length": 765.0, "completions/mean_length": 647.0625, "completions/mean_terminated_length": 553.0, "completions/min_length": 248.0, "completions/min_terminated_length": 248.0, "entropy": 0.5852552279829979, "epoch": 0.08618843683083512, "frac_reward_zero_std": 0.0, "grad_norm": 0.013069020584225655, "learning_rate": 1e-05, "loss": 0.0841, "num_tokens": 3662616.0, "reward": 0.40625, "reward_std": 0.4628904461860657, "rewards/accuracy_reward/mean": 0.40625, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.4001412391662598, "sampling/importance_sampling_ratio/mean": 0.9997668266296387, "sampling/importance_sampling_ratio/min": 0.6478813290596008, "sampling/sampling_logp_difference/max": 0.4340476989746094, "sampling/sampling_logp_difference/mean": 0.015113006345927715, "step": 161 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00016538630370632745, "clip_ratio/low_min": 0.00016538630370632745, "clip_ratio/region_mean": 0.00016538630370632745, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 724.0, "completions/mean_length": 579.875, "completions/mean_terminated_length": 536.4615478515625, "completions/min_length": 347.0, "completions/min_terminated_length": 347.0, "entropy": 0.5571671314537525, "epoch": 0.0867237687366167, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0287, "num_tokens": 3685020.0, "reward": 0.5625, "reward_std": 0.1157275140285492, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.4777604341506958, "sampling/importance_sampling_ratio/mean": 1.0005263090133667, "sampling/importance_sampling_ratio/min": 0.7564166784286499, "sampling/sampling_logp_difference/max": 0.39052772521972656, "sampling/sampling_logp_difference/mean": 0.014470912516117096, "step": 162 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.4375, "completions/max_length": 768.0, "completions/max_terminated_length": 678.0, "completions/mean_length": 611.03125, "completions/mean_terminated_length": 488.9444580078125, "completions/min_length": 250.0, "completions/min_terminated_length": 250.0, "entropy": 0.7517054751515388, "epoch": 0.0872591006423983, "frac_reward_zero_std": 0.75, "grad_norm": 0.0035496074706315994, "learning_rate": 1e-05, "loss": 0.0117, "num_tokens": 3708909.0, "reward": 0.28125, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.28125, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.523468255996704, "sampling/importance_sampling_ratio/mean": 1.0003571510314941, "sampling/importance_sampling_ratio/min": 0.5736925601959229, "sampling/sampling_logp_difference/max": 0.555661678314209, "sampling/sampling_logp_difference/mean": 0.018283069133758545, "step": 163 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.287647945806384e-05, "clip_ratio/low_min": 5.287647945806384e-05, "clip_ratio/region_mean": 5.287647945806384e-05, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 747.0, "completions/mean_length": 581.8125, "completions/mean_terminated_length": 538.84619140625, "completions/min_length": 322.0, "completions/min_terminated_length": 322.0, "entropy": 0.4150264039635658, "epoch": 0.08779443254817987, "frac_reward_zero_std": 0.5, "grad_norm": 0.0163887832313776, "learning_rate": 1e-05, "loss": 0.0044, "num_tokens": 3731159.0, "reward": 0.71875, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.367620587348938, "sampling/importance_sampling_ratio/mean": 0.9998429417610168, "sampling/importance_sampling_ratio/min": 0.6854574084281921, "sampling/sampling_logp_difference/max": 0.3776688575744629, "sampling/sampling_logp_difference/mean": 0.011968589387834072, "step": 164 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00011247417933191173, "clip_ratio/low_min": 0.00011247417933191173, "clip_ratio/region_mean": 0.00011247417933191173, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 758.0, "completions/mean_length": 578.125, "completions/mean_terminated_length": 542.9629516601562, "completions/min_length": 300.0, "completions/min_terminated_length": 300.0, "entropy": 0.3559522181749344, "epoch": 0.08832976445396146, "frac_reward_zero_std": 0.0, "grad_norm": 0.007052910048514605, "learning_rate": 1e-05, "loss": 0.0589, "num_tokens": 3752859.0, "reward": 0.625, "reward_std": 0.47655022144317627, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.373695969581604, "sampling/importance_sampling_ratio/mean": 1.0002590417861938, "sampling/importance_sampling_ratio/min": 0.7437906861305237, "sampling/sampling_logp_difference/max": 0.3175048828125, "sampling/sampling_logp_difference/mean": 0.010388054884970188, "step": 165 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 692.0, "completions/mean_length": 492.375, "completions/mean_terminated_length": 483.4838562011719, "completions/min_length": 283.0, "completions/min_terminated_length": 283.0, "entropy": 0.3500242754817009, "epoch": 0.08886509635974305, "frac_reward_zero_std": 0.5, "grad_norm": 0.017240503802895546, "learning_rate": 1e-05, "loss": 0.034, "num_tokens": 3772271.0, "reward": 0.8125, "reward_std": 0.249358132481575, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.460707664489746, "sampling/importance_sampling_ratio/mean": 1.0000114440917969, "sampling/importance_sampling_ratio/min": 0.7162594795227051, "sampling/sampling_logp_difference/max": 0.3789210319519043, "sampling/sampling_logp_difference/mean": 0.010518238879740238, "step": 166 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00010353789548389614, "clip_ratio/low_min": 0.00010353789548389614, "clip_ratio/region_mean": 0.00010353789548389614, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 745.0, "completions/mean_length": 566.5625, "completions/mean_terminated_length": 520.0769653320312, "completions/min_length": 300.0, "completions/min_terminated_length": 300.0, "entropy": 0.4458233341574669, "epoch": 0.08940042826552462, "frac_reward_zero_std": 0.0, "grad_norm": 0.010147130116820335, "learning_rate": 1e-05, "loss": 0.068, "num_tokens": 3793873.0, "reward": 0.6875, "reward_std": 0.4492306709289551, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.4302928447723389, "sampling/importance_sampling_ratio/mean": 1.000541090965271, "sampling/importance_sampling_ratio/min": 0.7610183954238892, "sampling/sampling_logp_difference/max": 0.3578791618347168, "sampling/sampling_logp_difference/mean": 0.012463034130632877, "step": 167 }, { "clip_ratio/high_max": 4.992012691218406e-05, "clip_ratio/high_mean": 4.992012691218406e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 4.992012691218406e-05, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 736.0, "completions/mean_length": 528.84375, "completions/mean_terminated_length": 494.6785888671875, "completions/min_length": 281.0, "completions/min_terminated_length": 281.0, "entropy": 0.41609687358140945, "epoch": 0.08993576017130621, "frac_reward_zero_std": 0.25, "grad_norm": 0.010141629725694656, "learning_rate": 1e-05, "loss": 0.0708, "num_tokens": 3814300.0, "reward": 0.84375, "reward_std": 0.3061639964580536, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.4766050577163696, "sampling/importance_sampling_ratio/mean": 0.9998204112052917, "sampling/importance_sampling_ratio/min": 0.6552395820617676, "sampling/sampling_logp_difference/max": 0.42275428771972656, "sampling/sampling_logp_difference/mean": 0.011525790207087994, "step": 168 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.98802874062676e-05, "clip_ratio/low_min": 4.98802874062676e-05, "clip_ratio/region_mean": 4.98802874062676e-05, "completions/clipped_ratio": 0.3125, "completions/max_length": 768.0, "completions/max_terminated_length": 706.0, "completions/mean_length": 534.78125, "completions/mean_terminated_length": 428.7727355957031, "completions/min_length": 233.0, "completions/min_terminated_length": 233.0, "entropy": 0.5607139207422733, "epoch": 0.0904710920770878, "frac_reward_zero_std": 0.5, "grad_norm": 0.0037553911097347736, "learning_rate": 1e-05, "loss": 0.039, "num_tokens": 3835629.0, "reward": 0.625, "reward_std": 0.2314550280570984, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.591864824295044, "sampling/importance_sampling_ratio/mean": 0.9997938871383667, "sampling/importance_sampling_ratio/min": 0.6814918518066406, "sampling/sampling_logp_difference/max": 0.4649062156677246, "sampling/sampling_logp_difference/mean": 0.013880348764359951, "step": 169 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 7.503000961150974e-05, "clip_ratio/low_min": 7.503000961150974e-05, "clip_ratio/region_mean": 7.503000961150974e-05, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 723.0, "completions/mean_length": 515.71875, "completions/mean_terminated_length": 479.6785888671875, "completions/min_length": 274.0, "completions/min_terminated_length": 274.0, "entropy": 0.4262724034488201, "epoch": 0.09100642398286937, "frac_reward_zero_std": 0.25, "grad_norm": 0.003561522578820586, "learning_rate": 1e-05, "loss": 0.0517, "num_tokens": 3855996.0, "reward": 0.71875, "reward_std": 0.3471629321575165, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.4927897453308105, "sampling/importance_sampling_ratio/mean": 1.0000567436218262, "sampling/importance_sampling_ratio/min": 0.7419149875640869, "sampling/sampling_logp_difference/max": 0.4006466865539551, "sampling/sampling_logp_difference/mean": 0.013346236199140549, "step": 170 }, { "clip_ratio/high_max": 4.8676010919734836e-05, "clip_ratio/high_mean": 4.8676010919734836e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 4.8676010919734836e-05, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 737.0, "completions/mean_length": 605.375, "completions/mean_terminated_length": 559.8399658203125, "completions/min_length": 307.0, "completions/min_terminated_length": 307.0, "entropy": 0.401042889803648, "epoch": 0.09154175588865096, "frac_reward_zero_std": 0.25, "grad_norm": 0.011611049063503742, "learning_rate": 1e-05, "loss": 0.0854, "num_tokens": 3879296.0, "reward": 0.53125, "reward_std": 0.3377464711666107, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9999179244041443, "sampling/importance_sampling_ratio/min": 0.49964797496795654, "sampling/sampling_logp_difference/max": 0.7910289764404297, "sampling/sampling_logp_difference/mean": 0.011209381744265556, "step": 171 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 686.0, "completions/max_terminated_length": 686.0, "completions/mean_length": 483.78125, "completions/mean_terminated_length": 483.78125, "completions/min_length": 237.0, "completions/min_terminated_length": 237.0, "entropy": 0.3809501491487026, "epoch": 0.09207708779443255, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 3898233.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.538259744644165, "sampling/importance_sampling_ratio/mean": 1.0003060102462769, "sampling/importance_sampling_ratio/min": 0.7762936949729919, "sampling/sampling_logp_difference/max": 0.4306516647338867, "sampling/sampling_logp_difference/mean": 0.010764282196760178, "step": 172 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 746.0, "completions/mean_length": 521.875, "completions/mean_terminated_length": 505.4667053222656, "completions/min_length": 271.0, "completions/min_terminated_length": 271.0, "entropy": 0.42339159548282623, "epoch": 0.09261241970021414, "frac_reward_zero_std": 0.5, "grad_norm": 0.005289682652801275, "learning_rate": 1e-05, "loss": 0.0558, "num_tokens": 3919013.0, "reward": 0.375, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.375, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.4113060235977173, "sampling/importance_sampling_ratio/mean": 0.999824583530426, "sampling/importance_sampling_ratio/min": 0.7252185940742493, "sampling/sampling_logp_difference/max": 0.3445155620574951, "sampling/sampling_logp_difference/mean": 0.012356176041066647, "step": 173 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 753.0, "completions/mean_length": 486.5625, "completions/mean_terminated_length": 457.4482727050781, "completions/min_length": 199.0, "completions/min_terminated_length": 199.0, "entropy": 0.33514219522476196, "epoch": 0.09314775160599571, "frac_reward_zero_std": 0.5, "grad_norm": 0.0060450229793787, "learning_rate": 1e-05, "loss": 0.0306, "num_tokens": 3938127.0, "reward": 0.78125, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.3566526174545288, "sampling/importance_sampling_ratio/mean": 1.0000343322753906, "sampling/importance_sampling_ratio/min": 0.6993787288665771, "sampling/sampling_logp_difference/max": 0.357562780380249, "sampling/sampling_logp_difference/mean": 0.010251290164887905, "step": 174 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 752.0, "completions/mean_length": 567.6875, "completions/mean_terminated_length": 530.5925903320312, "completions/min_length": 317.0, "completions/min_terminated_length": 317.0, "entropy": 0.37525275722146034, "epoch": 0.0936830835117773, "frac_reward_zero_std": 0.5, "grad_norm": 0.009321621619164944, "learning_rate": 1e-05, "loss": 0.0066, "num_tokens": 3960213.0, "reward": 0.53125, "reward_std": 0.2630178928375244, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.3685674667358398, "sampling/importance_sampling_ratio/mean": 1.0003997087478638, "sampling/importance_sampling_ratio/min": 0.7415798902511597, "sampling/sampling_logp_difference/max": 0.3137645721435547, "sampling/sampling_logp_difference/mean": 0.010577184148132801, "step": 175 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 7.626601291121915e-05, "clip_ratio/low_min": 7.626601291121915e-05, "clip_ratio/region_mean": 7.626601291121915e-05, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 761.0, "completions/mean_length": 521.6875, "completions/mean_terminated_length": 486.5000305175781, "completions/min_length": 315.0, "completions/min_terminated_length": 315.0, "entropy": 0.5321032479405403, "epoch": 0.09421841541755889, "frac_reward_zero_std": 0.5, "grad_norm": 0.010006904602050781, "learning_rate": 1e-05, "loss": -0.026, "num_tokens": 3981123.0, "reward": 0.46875, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.46875, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.6562020778656006, "sampling/importance_sampling_ratio/mean": 1.0004427433013916, "sampling/importance_sampling_ratio/min": 0.7053347229957581, "sampling/sampling_logp_difference/max": 0.5045270919799805, "sampling/sampling_logp_difference/mean": 0.014098327606916428, "step": 176 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00023155329836299643, "clip_ratio/low_min": 0.00023155329836299643, "clip_ratio/region_mean": 0.00023155329836299643, "completions/clipped_ratio": 0.34375, "completions/max_length": 768.0, "completions/max_terminated_length": 754.0, "completions/mean_length": 634.0625, "completions/mean_terminated_length": 563.90478515625, "completions/min_length": 398.0, "completions/min_terminated_length": 398.0, "entropy": 0.49840984120965004, "epoch": 0.09475374732334046, "frac_reward_zero_std": 0.0, "grad_norm": 0.0083969971165061, "learning_rate": 1e-05, "loss": 0.0934, "num_tokens": 4005893.0, "reward": 0.3125, "reward_std": 0.44403791427612305, "rewards/accuracy_reward/mean": 0.3125, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.4140453338623047, "sampling/importance_sampling_ratio/mean": 1.0002104043960571, "sampling/importance_sampling_ratio/min": 0.714652419090271, "sampling/sampling_logp_difference/max": 0.3464546203613281, "sampling/sampling_logp_difference/mean": 0.012913432903587818, "step": 177 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 757.0, "completions/mean_length": 471.875, "completions/mean_terminated_length": 403.5384826660156, "completions/min_length": 222.0, "completions/min_terminated_length": 222.0, "entropy": 0.6414377987384796, "epoch": 0.09528907922912205, "frac_reward_zero_std": 0.75, "grad_norm": 0.01471199095249176, "learning_rate": 1e-05, "loss": -0.005, "num_tokens": 4024449.0, "reward": 0.5625, "reward_std": 0.1157275140285492, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.4396910667419434, "sampling/importance_sampling_ratio/mean": 0.9999850988388062, "sampling/importance_sampling_ratio/min": 0.5289641618728638, "sampling/sampling_logp_difference/max": 0.6368346214294434, "sampling/sampling_logp_difference/mean": 0.015194835141301155, "step": 178 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 749.0, "completions/mean_length": 531.21875, "completions/mean_terminated_length": 515.433349609375, "completions/min_length": 287.0, "completions/min_terminated_length": 287.0, "entropy": 0.4419109709560871, "epoch": 0.09582441113490364, "frac_reward_zero_std": 0.0, "grad_norm": 0.02011791802942753, "learning_rate": 1e-05, "loss": 0.007, "num_tokens": 4045024.0, "reward": 0.59375, "reward_std": 0.4534739851951599, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.3287400007247925, "sampling/importance_sampling_ratio/mean": 0.9994969964027405, "sampling/importance_sampling_ratio/min": 0.7126895785331726, "sampling/sampling_logp_difference/max": 0.33870935440063477, "sampling/sampling_logp_difference/mean": 0.012954826466739178, "step": 179 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.9622867663856596e-05, "clip_ratio/low_min": 4.9622867663856596e-05, "clip_ratio/region_mean": 4.9622867663856596e-05, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 756.0, "completions/mean_length": 588.34375, "completions/mean_terminated_length": 546.8846435546875, "completions/min_length": 365.0, "completions/min_terminated_length": 365.0, "entropy": 0.32336973026394844, "epoch": 0.09635974304068523, "frac_reward_zero_std": 0.5, "grad_norm": 0.005477761849761009, "learning_rate": 1e-05, "loss": 0.0518, "num_tokens": 4067835.0, "reward": 0.8125, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.4798221588134766, "sampling/importance_sampling_ratio/mean": 1.000144362449646, "sampling/importance_sampling_ratio/min": 0.7506833672523499, "sampling/sampling_logp_difference/max": 0.3919219970703125, "sampling/sampling_logp_difference/mean": 0.009959114715456963, "step": 180 }, { "clip_ratio/high_max": 0.00011944577272515744, "clip_ratio/high_mean": 0.00011944577272515744, "clip_ratio/low_mean": 5.3949072025716305e-05, "clip_ratio/low_min": 5.3949072025716305e-05, "clip_ratio/region_mean": 0.00017339484475087374, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 716.0, "completions/mean_length": 478.5, "completions/mean_terminated_length": 448.5517272949219, "completions/min_length": 198.0, "completions/min_terminated_length": 198.0, "entropy": 0.46330392733216286, "epoch": 0.0968950749464668, "frac_reward_zero_std": 0.25, "grad_norm": 0.02045564167201519, "learning_rate": 1e-05, "loss": 0.0872, "num_tokens": 4086739.0, "reward": 0.8125, "reward_std": 0.3104073107242584, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.2771049737930298, "sampling/importance_sampling_ratio/mean": 0.9998090863227844, "sampling/importance_sampling_ratio/min": 0.7427229285240173, "sampling/sampling_logp_difference/max": 0.29743218421936035, "sampling/sampling_logp_difference/mean": 0.013459269888699055, "step": 181 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 9.229827264789492e-05, "clip_ratio/low_min": 9.229827264789492e-05, "clip_ratio/region_mean": 9.229827264789492e-05, "completions/clipped_ratio": 0.28125, "completions/max_length": 768.0, "completions/max_terminated_length": 707.0, "completions/mean_length": 633.03125, "completions/mean_terminated_length": 580.2174072265625, "completions/min_length": 299.0, "completions/min_terminated_length": 299.0, "entropy": 0.3913267273455858, "epoch": 0.0974304068522484, "frac_reward_zero_std": 0.25, "grad_norm": 0.007977542467415333, "learning_rate": 1e-05, "loss": 0.019, "num_tokens": 4110604.0, "reward": 0.65625, "reward_std": 0.3061639964580536, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.3955094814300537, "sampling/importance_sampling_ratio/mean": 0.9998458623886108, "sampling/importance_sampling_ratio/min": 0.6277883052825928, "sampling/sampling_logp_difference/max": 0.46555233001708984, "sampling/sampling_logp_difference/mean": 0.011498802341520786, "step": 182 }, { "clip_ratio/high_max": 5.3191488404991105e-05, "clip_ratio/high_mean": 5.3191488404991105e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 5.3191488404991105e-05, "completions/clipped_ratio": 0.34375, "completions/max_length": 768.0, "completions/max_terminated_length": 760.0, "completions/mean_length": 627.03125, "completions/mean_terminated_length": 553.1904907226562, "completions/min_length": 401.0, "completions/min_terminated_length": 401.0, "entropy": 0.40868574380874634, "epoch": 0.09796573875802998, "frac_reward_zero_std": 0.5, "grad_norm": 0.016252119094133377, "learning_rate": 1e-05, "loss": 0.0352, "num_tokens": 4135069.0, "reward": 0.6875, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.659118413925171, "sampling/importance_sampling_ratio/mean": 0.9998553395271301, "sampling/importance_sampling_ratio/min": 0.7253989577293396, "sampling/sampling_logp_difference/max": 0.5062863826751709, "sampling/sampling_logp_difference/mean": 0.011636482551693916, "step": 183 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00010872821803786792, "clip_ratio/low_min": 0.00010872821803786792, "clip_ratio/region_mean": 0.00010872821803786792, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 761.0, "completions/mean_length": 543.28125, "completions/mean_terminated_length": 480.3599853515625, "completions/min_length": 157.0, "completions/min_terminated_length": 157.0, "entropy": 0.4392571300268173, "epoch": 0.09850107066381156, "frac_reward_zero_std": 0.5, "grad_norm": 0.00900499802082777, "learning_rate": 1e-05, "loss": 0.0127, "num_tokens": 4156774.0, "reward": 0.75, "reward_std": 0.26726123690605164, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.7962998151779175, "sampling/importance_sampling_ratio/mean": 0.9998011589050293, "sampling/importance_sampling_ratio/min": 0.7309717535972595, "sampling/sampling_logp_difference/max": 0.5857288837432861, "sampling/sampling_logp_difference/mean": 0.012401177547872066, "step": 184 }, { "clip_ratio/high_max": 5.8520599850453436e-05, "clip_ratio/high_mean": 5.8520599850453436e-05, "clip_ratio/low_mean": 4.4867192627862096e-05, "clip_ratio/low_min": 4.4867192627862096e-05, "clip_ratio/region_mean": 0.00010338779247831553, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 733.0, "completions/mean_length": 549.0625, "completions/mean_terminated_length": 534.4666748046875, "completions/min_length": 315.0, "completions/min_terminated_length": 315.0, "entropy": 0.3790713008493185, "epoch": 0.09903640256959315, "frac_reward_zero_std": 0.5, "grad_norm": 0.003599144285544753, "learning_rate": 1e-05, "loss": 0.0731, "num_tokens": 4178104.0, "reward": 0.9375, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.9375, "rewards/accuracy_reward/std": 0.24593468010425568, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0004312992095947, "sampling/importance_sampling_ratio/min": 0.311521053314209, "sampling/sampling_logp_difference/max": 1.906259536743164, "sampling/sampling_logp_difference/mean": 0.011773568578064442, "step": 185 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00016008420789148659, "clip_ratio/low_min": 0.00016008420789148659, "clip_ratio/region_mean": 0.00016008420789148659, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 762.0, "completions/mean_length": 618.03125, "completions/mean_terminated_length": 568.0416870117188, "completions/min_length": 344.0, "completions/min_terminated_length": 344.0, "entropy": 0.3906097263097763, "epoch": 0.09957173447537473, "frac_reward_zero_std": 0.0, "grad_norm": 0.006143020000308752, "learning_rate": 1e-05, "loss": 0.0965, "num_tokens": 4201841.0, "reward": 0.65625, "reward_std": 0.4397946000099182, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.614789605140686, "sampling/importance_sampling_ratio/mean": 0.9999191164970398, "sampling/importance_sampling_ratio/min": 0.7043095827102661, "sampling/sampling_logp_difference/max": 0.4792046546936035, "sampling/sampling_logp_difference/mean": 0.011421887204051018, "step": 186 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00019992885063402355, "clip_ratio/low_min": 0.00019992885063402355, "clip_ratio/region_mean": 0.00019992885063402355, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 726.0, "completions/mean_length": 494.78125, "completions/mean_terminated_length": 444.1851806640625, "completions/min_length": 267.0, "completions/min_terminated_length": 267.0, "entropy": 0.4747123271226883, "epoch": 0.10010706638115632, "frac_reward_zero_std": 0.25, "grad_norm": 0.013716200366616249, "learning_rate": 1e-05, "loss": 0.0153, "num_tokens": 4220946.0, "reward": 0.21875, "reward_std": 0.3608423173427582, "rewards/accuracy_reward/mean": 0.21875, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.367693305015564, "sampling/importance_sampling_ratio/mean": 0.9998165369033813, "sampling/importance_sampling_ratio/min": 0.5654450058937073, "sampling/sampling_logp_difference/max": 0.5701421499252319, "sampling/sampling_logp_difference/mean": 0.012735454365611076, "step": 187 }, { "clip_ratio/high_max": 4.12133194913622e-05, "clip_ratio/high_mean": 4.12133194913622e-05, "clip_ratio/low_mean": 4.9446203775005415e-05, "clip_ratio/low_min": 4.9446203775005415e-05, "clip_ratio/region_mean": 9.065952326636761e-05, "completions/clipped_ratio": 0.3125, "completions/max_length": 768.0, "completions/max_terminated_length": 759.0, "completions/mean_length": 603.9375, "completions/mean_terminated_length": 529.3636474609375, "completions/min_length": 298.0, "completions/min_terminated_length": 298.0, "entropy": 0.4109209217131138, "epoch": 0.1006423982869379, "frac_reward_zero_std": 0.25, "grad_norm": 0.012013755738735199, "learning_rate": 1e-05, "loss": 0.0948, "num_tokens": 4244176.0, "reward": 0.5625, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.302966594696045, "sampling/importance_sampling_ratio/mean": 1.0002405643463135, "sampling/importance_sampling_ratio/min": 0.7807372212409973, "sampling/sampling_logp_difference/max": 0.26464366912841797, "sampling/sampling_logp_difference/mean": 0.01134407613426447, "step": 188 }, { "clip_ratio/high_max": 4.607445589499548e-05, "clip_ratio/high_mean": 4.607445589499548e-05, "clip_ratio/low_mean": 5.263157800072804e-05, "clip_ratio/low_min": 5.263157800072804e-05, "clip_ratio/region_mean": 9.870603389572352e-05, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 766.0, "completions/mean_length": 592.78125, "completions/mean_terminated_length": 543.719970703125, "completions/min_length": 282.0, "completions/min_terminated_length": 282.0, "entropy": 0.4264847170561552, "epoch": 0.10117773019271949, "frac_reward_zero_std": 0.0, "grad_norm": 0.020005634054541588, "learning_rate": 1e-05, "loss": 0.0307, "num_tokens": 4266921.0, "reward": 0.59375, "reward_std": 0.4534739851951599, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.2867344617843628, "sampling/importance_sampling_ratio/mean": 0.9999337792396545, "sampling/importance_sampling_ratio/min": 0.5134894847869873, "sampling/sampling_logp_difference/max": 0.6665257215499878, "sampling/sampling_logp_difference/mean": 0.012016260996460915, "step": 189 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00019170363157172687, "clip_ratio/low_min": 0.00019170363157172687, "clip_ratio/region_mean": 0.00019170363157172687, "completions/clipped_ratio": 0.53125, "completions/max_length": 768.0, "completions/max_terminated_length": 751.0, "completions/mean_length": 684.125, "completions/mean_terminated_length": 589.0667114257812, "completions/min_length": 392.0, "completions/min_terminated_length": 392.0, "entropy": 0.5222053229808807, "epoch": 0.10171306209850108, "frac_reward_zero_std": 0.0, "grad_norm": 0.0059653837233781815, "learning_rate": 1e-05, "loss": 0.0098, "num_tokens": 4293349.0, "reward": 0.15625, "reward_std": 0.3808925747871399, "rewards/accuracy_reward/mean": 0.15625, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.3785513639450073, "sampling/importance_sampling_ratio/mean": 0.9999143481254578, "sampling/importance_sampling_ratio/min": 0.5345979928970337, "sampling/sampling_logp_difference/max": 0.6262402534484863, "sampling/sampling_logp_difference/mean": 0.014318810775876045, "step": 190 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.402958802529611e-05, "clip_ratio/low_min": 4.402958802529611e-05, "clip_ratio/region_mean": 4.402958802529611e-05, "completions/clipped_ratio": 0.3125, "completions/max_length": 768.0, "completions/max_terminated_length": 731.0, "completions/mean_length": 668.5, "completions/mean_terminated_length": 623.2727661132812, "completions/min_length": 420.0, "completions/min_terminated_length": 420.0, "entropy": 0.3275398090481758, "epoch": 0.10224839400428265, "frac_reward_zero_std": 0.0, "grad_norm": 0.014578350819647312, "learning_rate": 1e-05, "loss": 0.0387, "num_tokens": 4319069.0, "reward": 0.53125, "reward_std": 0.4944729208946228, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.463234782218933, "sampling/importance_sampling_ratio/mean": 0.9999789595603943, "sampling/importance_sampling_ratio/min": 0.6955962181091309, "sampling/sampling_logp_difference/max": 0.3806495666503906, "sampling/sampling_logp_difference/mean": 0.010218731127679348, "step": 191 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00014937153537175618, "clip_ratio/low_min": 0.00014937153537175618, "clip_ratio/region_mean": 0.00014937153537175618, "completions/clipped_ratio": 0.34375, "completions/max_length": 768.0, "completions/max_terminated_length": 762.0, "completions/mean_length": 644.65625, "completions/mean_terminated_length": 580.047607421875, "completions/min_length": 343.0, "completions/min_terminated_length": 343.0, "entropy": 0.4292967952787876, "epoch": 0.10278372591006424, "frac_reward_zero_std": 0.25, "grad_norm": 0.012342764995992184, "learning_rate": 1e-05, "loss": 0.0349, "num_tokens": 4343602.0, "reward": 0.40625, "reward_std": 0.38816186785697937, "rewards/accuracy_reward/mean": 0.40625, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.5704963207244873, "sampling/importance_sampling_ratio/mean": 1.0000030994415283, "sampling/importance_sampling_ratio/min": 0.6687082052230835, "sampling/sampling_logp_difference/max": 0.45139169692993164, "sampling/sampling_logp_difference/mean": 0.012169972062110901, "step": 192 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0001649242258281447, "clip_ratio/low_min": 0.0001649242258281447, "clip_ratio/region_mean": 0.0001649242258281447, "completions/clipped_ratio": 0.375, "completions/max_length": 768.0, "completions/max_terminated_length": 697.0, "completions/mean_length": 602.21875, "completions/mean_terminated_length": 502.75, "completions/min_length": 240.0, "completions/min_terminated_length": 240.0, "entropy": 0.4525396376848221, "epoch": 0.10331905781584583, "frac_reward_zero_std": 0.0, "grad_norm": 0.010968129150569439, "learning_rate": 1e-05, "loss": 0.007, "num_tokens": 4366801.0, "reward": 0.46875, "reward_std": 0.4397946000099182, "rewards/accuracy_reward/mean": 0.46875, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.4232629537582397, "sampling/importance_sampling_ratio/mean": 0.999925971031189, "sampling/importance_sampling_ratio/min": 0.7066209316253662, "sampling/sampling_logp_difference/max": 0.3529520034790039, "sampling/sampling_logp_difference/mean": 0.013016062788665295, "step": 193 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00014899457528372295, "clip_ratio/low_min": 0.00014899457528372295, "clip_ratio/region_mean": 0.00014899457528372295, "completions/clipped_ratio": 0.5, "completions/max_length": 768.0, "completions/max_terminated_length": 751.0, "completions/mean_length": 678.59375, "completions/mean_terminated_length": 589.1875, "completions/min_length": 398.0, "completions/min_terminated_length": 398.0, "entropy": 0.5110501833260059, "epoch": 0.10385438972162742, "frac_reward_zero_std": 0.25, "grad_norm": 0.006947598420083523, "learning_rate": 1e-05, "loss": 0.0112, "num_tokens": 4392676.0, "reward": 0.34375, "reward_std": 0.3377464711666107, "rewards/accuracy_reward/mean": 0.34375, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.428097128868103, "sampling/importance_sampling_ratio/mean": 0.9999787211418152, "sampling/importance_sampling_ratio/min": 0.7371793985366821, "sampling/sampling_logp_difference/max": 0.35634279251098633, "sampling/sampling_logp_difference/mean": 0.013223528861999512, "step": 194 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 761.0, "completions/mean_length": 530.6875, "completions/mean_terminated_length": 514.86669921875, "completions/min_length": 179.0, "completions/min_terminated_length": 179.0, "entropy": 0.35680123791098595, "epoch": 0.10438972162740899, "frac_reward_zero_std": 0.25, "grad_norm": 0.004365036729723215, "learning_rate": 1e-05, "loss": 0.0164, "num_tokens": 4413218.0, "reward": 0.84375, "reward_std": 0.3198433816432953, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.3857086896896362, "sampling/importance_sampling_ratio/mean": 1.0003023147583008, "sampling/importance_sampling_ratio/min": 0.6768450736999512, "sampling/sampling_logp_difference/max": 0.39031291007995605, "sampling/sampling_logp_difference/mean": 0.010571901686489582, "step": 195 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00017099492833949625, "clip_ratio/low_min": 0.00017099492833949625, "clip_ratio/region_mean": 0.00017099492833949625, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 727.0, "completions/mean_length": 553.625, "completions/mean_terminated_length": 546.7096557617188, "completions/min_length": 353.0, "completions/min_terminated_length": 353.0, "entropy": 0.28742039389908314, "epoch": 0.10492505353319058, "frac_reward_zero_std": 0.0, "grad_norm": 0.014555180445313454, "learning_rate": 1e-05, "loss": 0.0595, "num_tokens": 4434622.0, "reward": 0.625, "reward_std": 0.4671337604522705, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.2978763580322266, "sampling/importance_sampling_ratio/mean": 0.9999411702156067, "sampling/importance_sampling_ratio/min": 0.6512435078620911, "sampling/sampling_logp_difference/max": 0.42887163162231445, "sampling/sampling_logp_difference/mean": 0.009337836876511574, "step": 196 }, { "clip_ratio/high_max": 5.4537522373721004e-05, "clip_ratio/high_mean": 5.4537522373721004e-05, "clip_ratio/low_mean": 9.76279770839028e-05, "clip_ratio/low_min": 9.76279770839028e-05, "clip_ratio/region_mean": 0.0001521654994576238, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 752.0, "completions/mean_length": 569.375, "completions/mean_terminated_length": 532.5925903320312, "completions/min_length": 325.0, "completions/min_terminated_length": 325.0, "entropy": 0.4544445499777794, "epoch": 0.10546038543897217, "frac_reward_zero_std": 0.25, "grad_norm": 0.003916038665920496, "learning_rate": 1e-05, "loss": 0.0349, "num_tokens": 4456538.0, "reward": 0.78125, "reward_std": 0.3061639666557312, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.6307182312011719, "sampling/importance_sampling_ratio/mean": 0.9999603629112244, "sampling/importance_sampling_ratio/min": 0.525356113910675, "sampling/sampling_logp_difference/max": 0.6436789035797119, "sampling/sampling_logp_difference/mean": 0.012684175744652748, "step": 197 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.24989482073579e-05, "clip_ratio/low_min": 5.24989482073579e-05, "clip_ratio/region_mean": 5.24989482073579e-05, "completions/clipped_ratio": 0.34375, "completions/max_length": 768.0, "completions/max_terminated_length": 746.0, "completions/mean_length": 632.15625, "completions/mean_terminated_length": 561.0, "completions/min_length": 389.0, "completions/min_terminated_length": 389.0, "entropy": 0.3937235027551651, "epoch": 0.10599571734475374, "frac_reward_zero_std": 0.5, "grad_norm": 0.008417624980211258, "learning_rate": 1e-05, "loss": -0.0076, "num_tokens": 4480863.0, "reward": 0.53125, "reward_std": 0.2630178928375244, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.525692105293274, "sampling/importance_sampling_ratio/mean": 0.9999752640724182, "sampling/importance_sampling_ratio/min": 0.5126069784164429, "sampling/sampling_logp_difference/max": 0.668245792388916, "sampling/sampling_logp_difference/mean": 0.011458354070782661, "step": 198 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.621072002919391e-05, "clip_ratio/low_min": 4.621072002919391e-05, "clip_ratio/region_mean": 4.621072002919391e-05, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 764.0, "completions/mean_length": 582.59375, "completions/mean_terminated_length": 520.7916870117188, "completions/min_length": 216.0, "completions/min_terminated_length": 216.0, "entropy": 0.34934471547603607, "epoch": 0.10653104925053533, "frac_reward_zero_std": 0.5, "grad_norm": 0.00955855194479227, "learning_rate": 1e-05, "loss": 0.0252, "num_tokens": 4502970.0, "reward": 0.71875, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.2755076885223389, "sampling/importance_sampling_ratio/mean": 1.0001780986785889, "sampling/importance_sampling_ratio/min": 0.6123645901679993, "sampling/sampling_logp_difference/max": 0.49042749404907227, "sampling/sampling_logp_difference/mean": 0.010630222037434578, "step": 199 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 676.0, "completions/mean_length": 498.28125, "completions/mean_terminated_length": 470.3793029785156, "completions/min_length": 284.0, "completions/min_terminated_length": 284.0, "entropy": 0.4826774448156357, "epoch": 0.10706638115631692, "frac_reward_zero_std": 0.25, "grad_norm": 0.007720374967902899, "learning_rate": 1e-05, "loss": 0.0542, "num_tokens": 4522443.0, "reward": 0.8125, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.374878168106079, "sampling/importance_sampling_ratio/mean": 0.9994122385978699, "sampling/importance_sampling_ratio/min": 0.7214857339859009, "sampling/sampling_logp_difference/max": 0.3264427185058594, "sampling/sampling_logp_difference/mean": 0.013857914134860039, "step": 200 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 759.0, "completions/mean_length": 515.90625, "completions/mean_terminated_length": 479.89288330078125, "completions/min_length": 155.0, "completions/min_terminated_length": 155.0, "entropy": 0.3860878758132458, "epoch": 0.1076017130620985, "frac_reward_zero_std": 0.25, "grad_norm": 0.004277211148291826, "learning_rate": 1e-05, "loss": 0.0945, "num_tokens": 4542248.0, "reward": 0.90625, "reward_std": 0.2651650309562683, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.2816795110702515, "sampling/importance_sampling_ratio/mean": 0.999394953250885, "sampling/importance_sampling_ratio/min": 0.4869225025177002, "sampling/sampling_logp_difference/max": 0.7196502685546875, "sampling/sampling_logp_difference/mean": 0.010670524090528488, "step": 201 }, { "clip_ratio/high_max": 4.669406189350411e-05, "clip_ratio/high_mean": 4.669406189350411e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 4.669406189350411e-05, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 766.0, "completions/mean_length": 563.375, "completions/mean_terminated_length": 525.4815063476562, "completions/min_length": 325.0, "completions/min_terminated_length": 325.0, "entropy": 0.3402336537837982, "epoch": 0.10813704496788008, "frac_reward_zero_std": 0.0, "grad_norm": 0.01772603578865528, "learning_rate": 1e-05, "loss": -0.0046, "num_tokens": 4563756.0, "reward": 0.8125, "reward_std": 0.3945523500442505, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.2836312055587769, "sampling/importance_sampling_ratio/mean": 1.0002542734146118, "sampling/importance_sampling_ratio/min": 0.6751722693443298, "sampling/sampling_logp_difference/max": 0.39278745651245117, "sampling/sampling_logp_difference/mean": 0.010218096897006035, "step": 202 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 694.0, "completions/mean_length": 522.1875, "completions/mean_terminated_length": 476.6666564941406, "completions/min_length": 285.0, "completions/min_terminated_length": 285.0, "entropy": 0.3548637293279171, "epoch": 0.10867237687366167, "frac_reward_zero_std": 0.25, "grad_norm": 0.011049263179302216, "learning_rate": 1e-05, "loss": 0.0161, "num_tokens": 4583898.0, "reward": 0.6875, "reward_std": 0.3514062762260437, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.6555202007293701, "sampling/importance_sampling_ratio/mean": 0.999890148639679, "sampling/importance_sampling_ratio/min": 0.5967310667037964, "sampling/sampling_logp_difference/max": 0.5162887573242188, "sampling/sampling_logp_difference/mean": 0.010576512664556503, "step": 203 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.813953612232581e-05, "clip_ratio/low_min": 5.813953612232581e-05, "clip_ratio/region_mean": 5.813953612232581e-05, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 736.0, "completions/mean_length": 607.25, "completions/mean_terminated_length": 570.1538696289062, "completions/min_length": 409.0, "completions/min_terminated_length": 409.0, "entropy": 0.37445712834596634, "epoch": 0.10920770877944326, "frac_reward_zero_std": 0.0, "grad_norm": 0.013399187475442886, "learning_rate": 1e-05, "loss": 0.0092, "num_tokens": 4607114.0, "reward": 0.53125, "reward_std": 0.4944729208946228, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.3712348937988281, "sampling/importance_sampling_ratio/mean": 1.0001106262207031, "sampling/importance_sampling_ratio/min": 0.7013443112373352, "sampling/sampling_logp_difference/max": 0.35475635528564453, "sampling/sampling_logp_difference/mean": 0.010567368939518929, "step": 204 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.3125, "completions/max_length": 768.0, "completions/max_terminated_length": 715.0, "completions/mean_length": 534.5625, "completions/mean_terminated_length": 428.4545593261719, "completions/min_length": 252.0, "completions/min_terminated_length": 252.0, "entropy": 0.7475202530622482, "epoch": 0.10974304068522484, "frac_reward_zero_std": 0.25, "grad_norm": 0.008707083761692047, "learning_rate": 1e-05, "loss": 0.0883, "num_tokens": 4628268.0, "reward": 0.65625, "reward_std": 0.3787454068660736, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.3773887157440186, "sampling/importance_sampling_ratio/mean": 0.9999659657478333, "sampling/importance_sampling_ratio/min": 0.7208519577980042, "sampling/sampling_logp_difference/max": 0.32732152938842773, "sampling/sampling_logp_difference/mean": 0.016627859324216843, "step": 205 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 751.0, "completions/mean_length": 535.71875, "completions/mean_terminated_length": 502.5357360839844, "completions/min_length": 224.0, "completions/min_terminated_length": 224.0, "entropy": 0.4045884683728218, "epoch": 0.11027837259100642, "frac_reward_zero_std": 0.0, "grad_norm": 0.02386460267007351, "learning_rate": 1e-05, "loss": 0.0739, "num_tokens": 4649387.0, "reward": 0.75, "reward_std": 0.4492306709289551, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.2934921979904175, "sampling/importance_sampling_ratio/mean": 1.0003174543380737, "sampling/importance_sampling_ratio/min": 0.5263379216194153, "sampling/sampling_logp_difference/max": 0.6418118476867676, "sampling/sampling_logp_difference/mean": 0.011270512826740742, "step": 206 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00011908754822798073, "clip_ratio/low_min": 0.00011908754822798073, "clip_ratio/region_mean": 0.00011908754822798073, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 734.0, "completions/mean_length": 535.6875, "completions/mean_terminated_length": 492.6666564941406, "completions/min_length": 174.0, "completions/min_terminated_length": 174.0, "entropy": 0.43413862586021423, "epoch": 0.11081370449678801, "frac_reward_zero_std": 0.0, "grad_norm": 0.021072428673505783, "learning_rate": 1e-05, "loss": -0.0282, "num_tokens": 4670121.0, "reward": 0.4375, "reward_std": 0.4671337604522705, "rewards/accuracy_reward/mean": 0.4375, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.2822644710540771, "sampling/importance_sampling_ratio/mean": 0.9997021555900574, "sampling/importance_sampling_ratio/min": 0.6542704105377197, "sampling/sampling_logp_difference/max": 0.424234539270401, "sampling/sampling_logp_difference/mean": 0.011866813525557518, "step": 207 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.28125, "completions/max_length": 768.0, "completions/max_terminated_length": 716.0, "completions/mean_length": 611.78125, "completions/mean_terminated_length": 550.6521606445312, "completions/min_length": 269.0, "completions/min_terminated_length": 269.0, "entropy": 0.4543142430484295, "epoch": 0.11134903640256959, "frac_reward_zero_std": 0.0, "grad_norm": 0.01737610250711441, "learning_rate": 1e-05, "loss": 0.1001, "num_tokens": 4693402.0, "reward": 0.59375, "reward_std": 0.4628904461860657, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.367607831954956, "sampling/importance_sampling_ratio/mean": 1.0001267194747925, "sampling/importance_sampling_ratio/min": 0.7402961254119873, "sampling/sampling_logp_difference/max": 0.3130631446838379, "sampling/sampling_logp_difference/mean": 0.012748447246849537, "step": 208 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 753.0, "completions/mean_length": 536.59375, "completions/mean_terminated_length": 521.1666870117188, "completions/min_length": 345.0, "completions/min_terminated_length": 345.0, "entropy": 0.41673338413238525, "epoch": 0.11188436830835118, "frac_reward_zero_std": 0.5, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0164, "num_tokens": 4714021.0, "reward": 0.375, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.375, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.2639020681381226, "sampling/importance_sampling_ratio/mean": 0.9998714327812195, "sampling/importance_sampling_ratio/min": 0.6510173082351685, "sampling/sampling_logp_difference/max": 0.42921900749206543, "sampling/sampling_logp_difference/mean": 0.01202225312590599, "step": 209 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 658.0, "completions/mean_length": 497.9375, "completions/mean_terminated_length": 435.6153869628906, "completions/min_length": 266.0, "completions/min_terminated_length": 266.0, "entropy": 0.4855153076350689, "epoch": 0.11241970021413276, "frac_reward_zero_std": 0.25, "grad_norm": 0.0029548266902565956, "learning_rate": 1e-05, "loss": 0.0709, "num_tokens": 4733395.0, "reward": 0.71875, "reward_std": 0.2651650309562683, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.374718189239502, "sampling/importance_sampling_ratio/mean": 1.0002045631408691, "sampling/importance_sampling_ratio/min": 0.7292927503585815, "sampling/sampling_logp_difference/max": 0.3182487487792969, "sampling/sampling_logp_difference/mean": 0.013204903341829777, "step": 210 }, { "clip_ratio/high_max": 4.588839874486439e-05, "clip_ratio/high_mean": 4.588839874486439e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 4.588839874486439e-05, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 708.0, "completions/mean_length": 573.03125, "completions/mean_terminated_length": 508.04168701171875, "completions/min_length": 324.0, "completions/min_terminated_length": 324.0, "entropy": 0.40894369408488274, "epoch": 0.11295503211991435, "frac_reward_zero_std": 0.0, "grad_norm": 0.011712045408785343, "learning_rate": 1e-05, "loss": 0.024, "num_tokens": 4755236.0, "reward": 0.59375, "reward_std": 0.4218915104866028, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.4561803340911865, "sampling/importance_sampling_ratio/mean": 0.9999224543571472, "sampling/importance_sampling_ratio/min": 0.5985782742500305, "sampling/sampling_logp_difference/max": 0.5131978988647461, "sampling/sampling_logp_difference/mean": 0.012139919213950634, "step": 211 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00011476811050670221, "clip_ratio/low_min": 0.00011476811050670221, "clip_ratio/region_mean": 0.00011476811050670221, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 748.0, "completions/mean_length": 544.3125, "completions/mean_terminated_length": 529.4000244140625, "completions/min_length": 336.0, "completions/min_terminated_length": 336.0, "entropy": 0.364752646535635, "epoch": 0.11349036402569593, "frac_reward_zero_std": 0.25, "grad_norm": 0.005543455947190523, "learning_rate": 1e-05, "loss": -0.002, "num_tokens": 4776406.0, "reward": 0.65625, "reward_std": 0.3198433816432953, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.4118499755859375, "sampling/importance_sampling_ratio/mean": 1.0000276565551758, "sampling/importance_sampling_ratio/min": 0.7016808390617371, "sampling/sampling_logp_difference/max": 0.3542766571044922, "sampling/sampling_logp_difference/mean": 0.011667272076010704, "step": 212 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00012984625573153608, "clip_ratio/low_min": 0.00012984625573153608, "clip_ratio/region_mean": 0.00012984625573153608, "completions/clipped_ratio": 0.5625, "completions/max_length": 768.0, "completions/max_terminated_length": 760.0, "completions/mean_length": 709.53125, "completions/mean_terminated_length": 634.357177734375, "completions/min_length": 482.0, "completions/min_terminated_length": 482.0, "entropy": 0.4654267206788063, "epoch": 0.11402569593147752, "frac_reward_zero_std": 0.0, "grad_norm": 0.0071574547328054905, "learning_rate": 1e-05, "loss": 0.0139, "num_tokens": 4802999.0, "reward": 0.46875, "reward_std": 0.4218915104866028, "rewards/accuracy_reward/mean": 0.46875, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.3884154558181763, "sampling/importance_sampling_ratio/mean": 1.0000523328781128, "sampling/importance_sampling_ratio/min": 0.697641134262085, "sampling/sampling_logp_difference/max": 0.3600504398345947, "sampling/sampling_logp_difference/mean": 0.012887127697467804, "step": 213 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 8.972283103503287e-05, "clip_ratio/low_min": 8.972283103503287e-05, "clip_ratio/region_mean": 8.972283103503287e-05, "completions/clipped_ratio": 0.28125, "completions/max_length": 768.0, "completions/max_terminated_length": 738.0, "completions/mean_length": 586.9375, "completions/mean_terminated_length": 516.0869750976562, "completions/min_length": 280.0, "completions/min_terminated_length": 280.0, "entropy": 0.4986208900809288, "epoch": 0.1145610278372591, "frac_reward_zero_std": 0.5, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0184, "num_tokens": 4825797.0, "reward": 0.46875, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.46875, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.4460728168487549, "sampling/importance_sampling_ratio/mean": 0.9999755620956421, "sampling/importance_sampling_ratio/min": 0.699849009513855, "sampling/sampling_logp_difference/max": 0.3688514232635498, "sampling/sampling_logp_difference/mean": 0.013791680335998535, "step": 214 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.395217911223881e-05, "clip_ratio/low_min": 4.395217911223881e-05, "clip_ratio/region_mean": 4.395217911223881e-05, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 747.0, "completions/mean_length": 535.59375, "completions/mean_terminated_length": 511.5517272949219, "completions/min_length": 278.0, "completions/min_terminated_length": 278.0, "entropy": 0.43015429750084877, "epoch": 0.11509635974304068, "frac_reward_zero_std": 0.25, "grad_norm": 0.004565034527331591, "learning_rate": 1e-05, "loss": 0.0801, "num_tokens": 4846912.0, "reward": 0.84375, "reward_std": 0.3198433816432953, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.3041958808898926, "sampling/importance_sampling_ratio/mean": 1.0001392364501953, "sampling/importance_sampling_ratio/min": 0.5819824934005737, "sampling/sampling_logp_difference/max": 0.541314959526062, "sampling/sampling_logp_difference/mean": 0.012156015262007713, "step": 215 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 725.0, "completions/mean_length": 534.84375, "completions/mean_terminated_length": 457.125, "completions/min_length": 231.0, "completions/min_terminated_length": 231.0, "entropy": 0.4418080672621727, "epoch": 0.11563169164882227, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0102, "num_tokens": 4867723.0, "reward": 0.71875, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.000184178352356, "sampling/importance_sampling_ratio/min": 0.7713487148284912, "sampling/sampling_logp_difference/max": 0.9640319347381592, "sampling/sampling_logp_difference/mean": 0.012921223416924477, "step": 216 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 743.0, "completions/mean_length": 549.71875, "completions/mean_terminated_length": 509.2962951660156, "completions/min_length": 305.0, "completions/min_terminated_length": 305.0, "entropy": 0.38898713886737823, "epoch": 0.11616702355460386, "frac_reward_zero_std": 0.5, "grad_norm": 0.013135960325598717, "learning_rate": 1e-05, "loss": 0.0207, "num_tokens": 4888914.0, "reward": 0.75, "reward_std": 0.2587745785713196, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.6967912912368774, "sampling/importance_sampling_ratio/mean": 0.9998115301132202, "sampling/importance_sampling_ratio/min": 0.41036003828048706, "sampling/sampling_logp_difference/max": 0.8907203674316406, "sampling/sampling_logp_difference/mean": 0.011933991685509682, "step": 217 }, { "clip_ratio/high_max": 4.873294528806582e-05, "clip_ratio/high_mean": 4.873294528806582e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 4.873294528806582e-05, "completions/clipped_ratio": 0.53125, "completions/max_length": 768.0, "completions/max_terminated_length": 736.0, "completions/mean_length": 707.0, "completions/mean_terminated_length": 637.86669921875, "completions/min_length": 516.0, "completions/min_terminated_length": 516.0, "entropy": 0.5665181279182434, "epoch": 0.11670235546038545, "frac_reward_zero_std": 0.5, "grad_norm": 0.00827651098370552, "learning_rate": 1e-05, "loss": 0.0344, "num_tokens": 4915986.0, "reward": 0.3125, "reward_std": 0.249358132481575, "rewards/accuracy_reward/mean": 0.3125, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.5402032136917114, "sampling/importance_sampling_ratio/mean": 0.9999552965164185, "sampling/importance_sampling_ratio/min": 0.6884464621543884, "sampling/sampling_logp_difference/max": 0.4319143295288086, "sampling/sampling_logp_difference/mean": 0.015448407270014286, "step": 218 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.53125, "completions/max_length": 768.0, "completions/max_terminated_length": 715.0, "completions/mean_length": 662.84375, "completions/mean_terminated_length": 543.6666870117188, "completions/min_length": 391.0, "completions/min_terminated_length": 391.0, "entropy": 0.6006809212267399, "epoch": 0.11723768736616702, "frac_reward_zero_std": 0.25, "grad_norm": 0.013638823293149471, "learning_rate": 1e-05, "loss": 0.0363, "num_tokens": 4941373.0, "reward": 0.4375, "reward_std": 0.3471825420856476, "rewards/accuracy_reward/mean": 0.4375, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.3881639242172241, "sampling/importance_sampling_ratio/mean": 0.9999839067459106, "sampling/importance_sampling_ratio/min": 0.612636148929596, "sampling/sampling_logp_difference/max": 0.48998403549194336, "sampling/sampling_logp_difference/mean": 0.015525132417678833, "step": 219 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.28125, "completions/max_length": 768.0, "completions/max_terminated_length": 740.0, "completions/mean_length": 611.78125, "completions/mean_terminated_length": 550.6521606445312, "completions/min_length": 313.0, "completions/min_terminated_length": 313.0, "entropy": 0.5649635568261147, "epoch": 0.11777301927194861, "frac_reward_zero_std": 0.5, "grad_norm": 0.00461383443325758, "learning_rate": 1e-05, "loss": 0.0299, "num_tokens": 4964814.0, "reward": 0.75, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.5734072923660278, "sampling/importance_sampling_ratio/mean": 1.0002329349517822, "sampling/importance_sampling_ratio/min": 0.6937447786331177, "sampling/sampling_logp_difference/max": 0.4532434940338135, "sampling/sampling_logp_difference/mean": 0.013195241801440716, "step": 220 }, { "clip_ratio/high_max": 6.300403038039804e-05, "clip_ratio/high_mean": 6.300403038039804e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 6.300403038039804e-05, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 756.0, "completions/mean_length": 578.96875, "completions/mean_terminated_length": 566.36669921875, "completions/min_length": 384.0, "completions/min_terminated_length": 384.0, "entropy": 0.37277746573090553, "epoch": 0.1183083511777302, "frac_reward_zero_std": 0.25, "grad_norm": 0.003993625286966562, "learning_rate": 1e-05, "loss": -0.0001, "num_tokens": 4986901.0, "reward": 0.84375, "reward_std": 0.3061639666557312, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.3628790378570557, "sampling/importance_sampling_ratio/mean": 1.0001591444015503, "sampling/importance_sampling_ratio/min": 0.5722905993461609, "sampling/sampling_logp_difference/max": 0.5581083297729492, "sampling/sampling_logp_difference/mean": 0.01126320380717516, "step": 221 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.494505603564903e-05, "clip_ratio/low_min": 5.494505603564903e-05, "clip_ratio/region_mean": 5.494505603564903e-05, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 748.0, "completions/mean_length": 553.8125, "completions/mean_terminated_length": 493.8399963378906, "completions/min_length": 269.0, "completions/min_terminated_length": 269.0, "entropy": 0.3825898841023445, "epoch": 0.11884368308351177, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": -0.0418, "num_tokens": 5008063.0, "reward": 0.59375, "reward_std": 0.1293872892856598, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.4533307552337646, "sampling/importance_sampling_ratio/mean": 1.0000147819519043, "sampling/importance_sampling_ratio/min": 0.6119077801704407, "sampling/sampling_logp_difference/max": 0.49117374420166016, "sampling/sampling_logp_difference/mean": 0.012249835766851902, "step": 222 }, { "clip_ratio/high_max": 5.3879310144111514e-05, "clip_ratio/high_mean": 5.3879310144111514e-05, "clip_ratio/low_mean": 4.9800797569332644e-05, "clip_ratio/low_min": 4.9800797569332644e-05, "clip_ratio/region_mean": 0.00010368010771344416, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 701.0, "completions/mean_length": 576.625, "completions/mean_terminated_length": 512.8333740234375, "completions/min_length": 264.0, "completions/min_terminated_length": 264.0, "entropy": 0.43691984564065933, "epoch": 0.11937901498929336, "frac_reward_zero_std": 0.0, "grad_norm": 0.020237060263752937, "learning_rate": 1e-05, "loss": 0.0335, "num_tokens": 5030339.0, "reward": 0.46875, "reward_std": 0.4397946000099182, "rewards/accuracy_reward/mean": 0.46875, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.6887725591659546, "sampling/importance_sampling_ratio/mean": 0.9997546672821045, "sampling/importance_sampling_ratio/min": 0.7662273645401001, "sampling/sampling_logp_difference/max": 0.524001955986023, "sampling/sampling_logp_difference/mean": 0.011880028061568737, "step": 223 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 758.0, "completions/mean_length": 590.84375, "completions/mean_terminated_length": 531.7916870117188, "completions/min_length": 201.0, "completions/min_terminated_length": 201.0, "entropy": 0.43852218613028526, "epoch": 0.11991434689507495, "frac_reward_zero_std": 0.0, "grad_norm": 0.004895425401628017, "learning_rate": 1e-05, "loss": 0.1202, "num_tokens": 5052390.0, "reward": 0.84375, "reward_std": 0.3808925747871399, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.330985426902771, "sampling/importance_sampling_ratio/mean": 0.9999130964279175, "sampling/importance_sampling_ratio/min": 0.5197319984436035, "sampling/sampling_logp_difference/max": 0.6544420719146729, "sampling/sampling_logp_difference/mean": 0.012208963744342327, "step": 224 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00018228655972052366, "clip_ratio/low_min": 0.00018228655972052366, "clip_ratio/region_mean": 0.00018228655972052366, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 735.0, "completions/mean_length": 548.65625, "completions/mean_terminated_length": 498.0384826660156, "completions/min_length": 221.0, "completions/min_terminated_length": 221.0, "entropy": 0.43273502588272095, "epoch": 0.12044967880085652, "frac_reward_zero_std": 0.25, "grad_norm": 0.015006604604423046, "learning_rate": 1e-05, "loss": 0.046, "num_tokens": 5074019.0, "reward": 0.53125, "reward_std": 0.2651650309562683, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.275411605834961, "sampling/importance_sampling_ratio/mean": 0.9996492266654968, "sampling/importance_sampling_ratio/min": 0.6270601153373718, "sampling/sampling_logp_difference/max": 0.46671295166015625, "sampling/sampling_logp_difference/mean": 0.012470901943743229, "step": 225 }, { "clip_ratio/high_max": 7.476076279999688e-05, "clip_ratio/high_mean": 7.476076279999688e-05, "clip_ratio/low_mean": 0.00012146871085860766, "clip_ratio/low_min": 0.00012146871085860766, "clip_ratio/region_mean": 0.00019622947365860455, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 757.0, "completions/mean_length": 559.96875, "completions/mean_terminated_length": 490.625, "completions/min_length": 142.0, "completions/min_terminated_length": 142.0, "entropy": 0.6108875423669815, "epoch": 0.12098501070663811, "frac_reward_zero_std": 0.0, "grad_norm": 0.014813713729381561, "learning_rate": 1e-05, "loss": 0.0931, "num_tokens": 5095994.0, "reward": 0.46875, "reward_std": 0.4628904461860657, "rewards/accuracy_reward/mean": 0.46875, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.648871660232544, "sampling/importance_sampling_ratio/mean": 1.000571846961975, "sampling/importance_sampling_ratio/min": 0.6507716774940491, "sampling/sampling_logp_difference/max": 0.5000911951065063, "sampling/sampling_logp_difference/mean": 0.014381887391209602, "step": 226 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.59375, "completions/max_length": 768.0, "completions/max_terminated_length": 705.0, "completions/mean_length": 696.03125, "completions/mean_terminated_length": 590.84619140625, "completions/min_length": 365.0, "completions/min_terminated_length": 365.0, "entropy": 0.8187565132975578, "epoch": 0.1215203426124197, "frac_reward_zero_std": 0.25, "grad_norm": 0.0037973658181726933, "learning_rate": 1e-05, "loss": 0.035, "num_tokens": 5123035.0, "reward": 0.15625, "reward_std": 0.3061639666557312, "rewards/accuracy_reward/mean": 0.15625, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.5762499570846558, "sampling/importance_sampling_ratio/mean": 0.999707818031311, "sampling/importance_sampling_ratio/min": 0.6020462512969971, "sampling/sampling_logp_difference/max": 0.5074210166931152, "sampling/sampling_logp_difference/mean": 0.017832642421126366, "step": 227 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.736642586067319e-05, "clip_ratio/low_min": 4.736642586067319e-05, "clip_ratio/region_mean": 4.736642586067319e-05, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 745.0, "completions/mean_length": 600.28125, "completions/mean_terminated_length": 561.5769653320312, "completions/min_length": 364.0, "completions/min_terminated_length": 364.0, "entropy": 0.3548762071877718, "epoch": 0.12205567451820129, "frac_reward_zero_std": 0.0, "grad_norm": 0.011822504922747612, "learning_rate": 1e-05, "loss": 0.0802, "num_tokens": 5145956.0, "reward": 0.625, "reward_std": 0.4492306709289551, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.3381006717681885, "sampling/importance_sampling_ratio/mean": 0.9998043775558472, "sampling/importance_sampling_ratio/min": 0.5986925363540649, "sampling/sampling_logp_difference/max": 0.5130071640014648, "sampling/sampling_logp_difference/mean": 0.01054322998970747, "step": 228 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.5737284381175414e-05, "clip_ratio/low_min": 4.5737284381175414e-05, "clip_ratio/region_mean": 4.5737284381175414e-05, "completions/clipped_ratio": 0.28125, "completions/max_length": 768.0, "completions/max_terminated_length": 766.0, "completions/mean_length": 640.3125, "completions/mean_terminated_length": 590.3478393554688, "completions/min_length": 285.0, "completions/min_terminated_length": 285.0, "entropy": 0.3537941761314869, "epoch": 0.12259100642398287, "frac_reward_zero_std": 0.25, "grad_norm": 0.006486724596470594, "learning_rate": 1e-05, "loss": 0.0692, "num_tokens": 5170182.0, "reward": 0.59375, "reward_std": 0.3787454068660736, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.8426074981689453, "sampling/importance_sampling_ratio/mean": 0.9999435544013977, "sampling/importance_sampling_ratio/min": 0.7162595391273499, "sampling/sampling_logp_difference/max": 0.6111817359924316, "sampling/sampling_logp_difference/mean": 0.011211688630282879, "step": 229 }, { "clip_ratio/high_max": 6.076811041566543e-05, "clip_ratio/high_mean": 6.076811041566543e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 6.076811041566543e-05, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 702.0, "completions/mean_length": 477.84375, "completions/mean_terminated_length": 468.4838562011719, "completions/min_length": 332.0, "completions/min_terminated_length": 332.0, "entropy": 0.33781515806913376, "epoch": 0.12312633832976445, "frac_reward_zero_std": 0.25, "grad_norm": 0.004910403862595558, "learning_rate": 1e-05, "loss": 0.0255, "num_tokens": 5189217.0, "reward": 0.84375, "reward_std": 0.3061639964580536, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.2802245616912842, "sampling/importance_sampling_ratio/mean": 0.9996927380561829, "sampling/importance_sampling_ratio/min": 0.705748975276947, "sampling/sampling_logp_difference/max": 0.3484957218170166, "sampling/sampling_logp_difference/mean": 0.01033768244087696, "step": 230 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.5163283832371235e-05, "clip_ratio/low_min": 5.5163283832371235e-05, "clip_ratio/region_mean": 5.5163283832371235e-05, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 763.0, "completions/mean_length": 554.9375, "completions/mean_terminated_length": 495.2799987792969, "completions/min_length": 235.0, "completions/min_terminated_length": 235.0, "entropy": 0.6406082026660442, "epoch": 0.12366167023554604, "frac_reward_zero_std": 0.5, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": -0.024, "num_tokens": 5210639.0, "reward": 0.5625, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9997336268424988, "sampling/importance_sampling_ratio/min": 0.7185488343238831, "sampling/sampling_logp_difference/max": 0.6950337886810303, "sampling/sampling_logp_difference/mean": 0.01410508994013071, "step": 231 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 710.0, "completions/mean_length": 517.15625, "completions/mean_terminated_length": 470.7037048339844, "completions/min_length": 254.0, "completions/min_terminated_length": 254.0, "entropy": 0.48460185527801514, "epoch": 0.12419700214132762, "frac_reward_zero_std": 0.5, "grad_norm": 0.0035355407744646072, "learning_rate": 1e-05, "loss": 0.0363, "num_tokens": 5230812.0, "reward": 0.75, "reward_std": 0.2314550280570984, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.6327197551727295, "sampling/importance_sampling_ratio/mean": 1.0002273321151733, "sampling/importance_sampling_ratio/min": 0.6186704039573669, "sampling/sampling_logp_difference/max": 0.4902472496032715, "sampling/sampling_logp_difference/mean": 0.013307410292327404, "step": 232 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 9.8251872259425e-05, "clip_ratio/low_min": 9.8251872259425e-05, "clip_ratio/region_mean": 9.8251872259425e-05, "completions/clipped_ratio": 0.59375, "completions/max_length": 768.0, "completions/max_terminated_length": 720.0, "completions/mean_length": 681.34375, "completions/mean_terminated_length": 554.6923217773438, "completions/min_length": 356.0, "completions/min_terminated_length": 356.0, "entropy": 0.41936226561665535, "epoch": 0.1247323340471092, "frac_reward_zero_std": 0.0, "grad_norm": 0.01071957778185606, "learning_rate": 1e-05, "loss": 0.0174, "num_tokens": 5256623.0, "reward": 0.53125, "reward_std": 0.5038893818855286, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.4164282083511353, "sampling/importance_sampling_ratio/mean": 1.0000749826431274, "sampling/importance_sampling_ratio/min": 0.5122416019439697, "sampling/sampling_logp_difference/max": 0.6689589023590088, "sampling/sampling_logp_difference/mean": 0.011418603360652924, "step": 233 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.785605051438324e-05, "clip_ratio/low_min": 4.785605051438324e-05, "clip_ratio/region_mean": 4.785605051438324e-05, "completions/clipped_ratio": 0.34375, "completions/max_length": 768.0, "completions/max_terminated_length": 748.0, "completions/mean_length": 607.78125, "completions/mean_terminated_length": 523.857177734375, "completions/min_length": 189.0, "completions/min_terminated_length": 189.0, "entropy": 0.5056234337389469, "epoch": 0.1252676659528908, "frac_reward_zero_std": 0.25, "grad_norm": 0.006079288199543953, "learning_rate": 1e-05, "loss": 0.0702, "num_tokens": 5279648.0, "reward": 0.625, "reward_std": 0.292504221200943, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.624884843826294, "sampling/importance_sampling_ratio/mean": 0.9995018839836121, "sampling/importance_sampling_ratio/min": 0.48730066418647766, "sampling/sampling_logp_difference/max": 0.7188739776611328, "sampling/sampling_logp_difference/mean": 0.0142142903059721, "step": 234 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 754.0, "completions/mean_length": 489.71875, "completions/mean_terminated_length": 480.7419128417969, "completions/min_length": 288.0, "completions/min_terminated_length": 288.0, "entropy": 0.5454281978309155, "epoch": 0.12580299785867238, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": -0.001, "num_tokens": 5299519.0, "reward": 0.71875, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.3391122817993164, "sampling/importance_sampling_ratio/mean": 1.000416874885559, "sampling/importance_sampling_ratio/min": 0.6362372636795044, "sampling/sampling_logp_difference/max": 0.45218372344970703, "sampling/sampling_logp_difference/mean": 0.013221533969044685, "step": 235 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.7129800552502275e-05, "clip_ratio/low_min": 5.7129800552502275e-05, "clip_ratio/region_mean": 5.7129800552502275e-05, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 734.0, "completions/mean_length": 549.75, "completions/mean_terminated_length": 499.3846435546875, "completions/min_length": 243.0, "completions/min_terminated_length": 243.0, "entropy": 0.4301159009337425, "epoch": 0.12633832976445397, "frac_reward_zero_std": 0.5, "grad_norm": 0.0030366797000169754, "learning_rate": 1e-05, "loss": 0.0918, "num_tokens": 5322119.0, "reward": 0.625, "reward_std": 0.2314550280570984, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.532578945159912, "sampling/importance_sampling_ratio/mean": 1.00002121925354, "sampling/importance_sampling_ratio/min": 0.7187860608100891, "sampling/sampling_logp_difference/max": 0.42695188522338867, "sampling/sampling_logp_difference/mean": 0.013005656190216541, "step": 236 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.723443246097304e-05, "clip_ratio/low_min": 5.723443246097304e-05, "clip_ratio/region_mean": 5.723443246097304e-05, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 709.0, "completions/mean_length": 516.0625, "completions/mean_terminated_length": 445.5199890136719, "completions/min_length": 273.0, "completions/min_terminated_length": 273.0, "entropy": 0.5875410586595535, "epoch": 0.12687366167023553, "frac_reward_zero_std": 0.5, "grad_norm": 0.008919088169932365, "learning_rate": 1e-05, "loss": 0.0629, "num_tokens": 5342737.0, "reward": 0.65625, "reward_std": 0.22201895713806152, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.5538220405578613, "sampling/importance_sampling_ratio/mean": 0.9994003772735596, "sampling/importance_sampling_ratio/min": 0.3847011625766754, "sampling/sampling_logp_difference/max": 0.9552884101867676, "sampling/sampling_logp_difference/mean": 0.014698689803481102, "step": 237 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.28125, "completions/max_length": 768.0, "completions/max_terminated_length": 749.0, "completions/mean_length": 602.25, "completions/mean_terminated_length": 537.3912963867188, "completions/min_length": 276.0, "completions/min_terminated_length": 276.0, "entropy": 0.43145033344626427, "epoch": 0.12740899357601712, "frac_reward_zero_std": 0.25, "grad_norm": 0.011445707641541958, "learning_rate": 1e-05, "loss": 0.0439, "num_tokens": 5365657.0, "reward": 0.59375, "reward_std": 0.3198433816432953, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.2852743864059448, "sampling/importance_sampling_ratio/mean": 0.9996131658554077, "sampling/importance_sampling_ratio/min": 0.7185183763504028, "sampling/sampling_logp_difference/max": 0.330564022064209, "sampling/sampling_logp_difference/mean": 0.012642141431570053, "step": 238 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.0100199587177485e-05, "clip_ratio/low_min": 5.0100199587177485e-05, "clip_ratio/region_mean": 5.0100199587177485e-05, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 756.0, "completions/mean_length": 569.34375, "completions/mean_terminated_length": 513.719970703125, "completions/min_length": 177.0, "completions/min_terminated_length": 177.0, "entropy": 0.4805685766041279, "epoch": 0.1279443254817987, "frac_reward_zero_std": 0.25, "grad_norm": 0.014481008984148502, "learning_rate": 1e-05, "loss": 0.021, "num_tokens": 5388156.0, "reward": 0.71875, "reward_std": 0.3471629321575165, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.4498913288116455, "sampling/importance_sampling_ratio/mean": 0.9998762607574463, "sampling/importance_sampling_ratio/min": 0.4585459530353546, "sampling/sampling_logp_difference/max": 0.7796947956085205, "sampling/sampling_logp_difference/mean": 0.013675138354301453, "step": 239 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 734.0, "completions/mean_length": 485.09375, "completions/mean_terminated_length": 455.82757568359375, "completions/min_length": 250.0, "completions/min_terminated_length": 250.0, "entropy": 0.5328178815543652, "epoch": 0.1284796573875803, "frac_reward_zero_std": 0.75, "grad_norm": 0.005874136462807655, "learning_rate": 1e-05, "loss": 0.0055, "num_tokens": 5407655.0, "reward": 0.53125, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.3096046447753906, "sampling/importance_sampling_ratio/mean": 0.999963641166687, "sampling/importance_sampling_ratio/min": 0.6940369606018066, "sampling/sampling_logp_difference/max": 0.36523008346557617, "sampling/sampling_logp_difference/mean": 0.014065473340451717, "step": 240 }, { "clip_ratio/high_max": 0.0001342642353847623, "clip_ratio/high_mean": 0.0001342642353847623, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0001342642353847623, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 746.0, "completions/mean_length": 565.71875, "completions/mean_terminated_length": 519.0384521484375, "completions/min_length": 237.0, "completions/min_terminated_length": 237.0, "entropy": 0.4152279533445835, "epoch": 0.1290149892933619, "frac_reward_zero_std": 0.0, "grad_norm": 0.007934553548693657, "learning_rate": 1e-05, "loss": 0.0746, "num_tokens": 5429494.0, "reward": 0.6875, "reward_std": 0.4082317352294922, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.441959023475647, "sampling/importance_sampling_ratio/mean": 1.000027060508728, "sampling/importance_sampling_ratio/min": 0.47246497869491577, "sampling/sampling_logp_difference/max": 0.7497916221618652, "sampling/sampling_logp_difference/mean": 0.011828385293483734, "step": 241 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.028157829656266e-05, "clip_ratio/low_min": 5.028157829656266e-05, "clip_ratio/region_mean": 5.028157829656266e-05, "completions/clipped_ratio": 0.34375, "completions/max_length": 768.0, "completions/max_terminated_length": 675.0, "completions/mean_length": 569.25, "completions/mean_terminated_length": 465.1428527832031, "completions/min_length": 273.0, "completions/min_terminated_length": 273.0, "entropy": 0.4523490183055401, "epoch": 0.12955032119914348, "frac_reward_zero_std": 0.5, "grad_norm": 0.012959744781255722, "learning_rate": 1e-05, "loss": 0.0527, "num_tokens": 5451662.0, "reward": 0.59375, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.3854886293411255, "sampling/importance_sampling_ratio/mean": 1.0002597570419312, "sampling/importance_sampling_ratio/min": 0.6795645952224731, "sampling/sampling_logp_difference/max": 0.3863029479980469, "sampling/sampling_logp_difference/mean": 0.012580934911966324, "step": 242 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 766.0, "completions/mean_length": 544.9375, "completions/mean_terminated_length": 513.0714721679688, "completions/min_length": 163.0, "completions/min_terminated_length": 163.0, "entropy": 0.4594997763633728, "epoch": 0.13008565310492506, "frac_reward_zero_std": 0.0, "grad_norm": 0.00827376265078783, "learning_rate": 1e-05, "loss": 0.0713, "num_tokens": 5472676.0, "reward": 0.71875, "reward_std": 0.4534739851951599, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0002453327178955, "sampling/importance_sampling_ratio/min": 0.3594430685043335, "sampling/sampling_logp_difference/max": 1.0231995582580566, "sampling/sampling_logp_difference/mean": 0.012782433070242405, "step": 243 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.627915768651292e-05, "clip_ratio/low_min": 4.627915768651292e-05, "clip_ratio/region_mean": 4.627915768651292e-05, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 747.0, "completions/mean_length": 525.90625, "completions/mean_terminated_length": 481.0740661621094, "completions/min_length": 285.0, "completions/min_terminated_length": 285.0, "entropy": 0.41479819267988205, "epoch": 0.13062098501070663, "frac_reward_zero_std": 0.5, "grad_norm": 0.012586255557835102, "learning_rate": 1e-05, "loss": 0.0274, "num_tokens": 5493249.0, "reward": 0.78125, "reward_std": 0.2630178928375244, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.3508623838424683, "sampling/importance_sampling_ratio/mean": 1.000050663948059, "sampling/importance_sampling_ratio/min": 0.46064114570617676, "sampling/sampling_logp_difference/max": 0.7751359939575195, "sampling/sampling_logp_difference/mean": 0.011814693920314312, "step": 244 }, { "clip_ratio/high_max": 4.813246050616726e-05, "clip_ratio/high_mean": 4.813246050616726e-05, "clip_ratio/low_mean": 5.118755143485032e-05, "clip_ratio/low_min": 5.118755143485032e-05, "clip_ratio/region_mean": 9.932001194101758e-05, "completions/clipped_ratio": 0.34375, "completions/max_length": 768.0, "completions/max_terminated_length": 707.0, "completions/mean_length": 632.21875, "completions/mean_terminated_length": 561.0952758789062, "completions/min_length": 384.0, "completions/min_terminated_length": 384.0, "entropy": 0.38899847120046616, "epoch": 0.1311563169164882, "frac_reward_zero_std": 0.25, "grad_norm": 0.018674733117222786, "learning_rate": 1e-05, "loss": 0.016, "num_tokens": 5517712.0, "reward": 0.4375, "reward_std": 0.3514062762260437, "rewards/accuracy_reward/mean": 0.4375, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.3634089231491089, "sampling/importance_sampling_ratio/mean": 1.0000673532485962, "sampling/importance_sampling_ratio/min": 0.5190722942352295, "sampling/sampling_logp_difference/max": 0.6557121276855469, "sampling/sampling_logp_difference/mean": 0.012373112142086029, "step": 245 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.645889905281365e-05, "clip_ratio/low_min": 5.645889905281365e-05, "clip_ratio/region_mean": 5.645889905281365e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 707.0, "completions/max_terminated_length": 707.0, "completions/mean_length": 506.78125, "completions/mean_terminated_length": 506.78125, "completions/min_length": 281.0, "completions/min_terminated_length": 281.0, "entropy": 0.4035028889775276, "epoch": 0.1316916488222698, "frac_reward_zero_std": 0.25, "grad_norm": 0.01403022464364767, "learning_rate": 1e-05, "loss": -0.0217, "num_tokens": 5537537.0, "reward": 0.65625, "reward_std": 0.3471629321575165, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.4795641899108887, "sampling/importance_sampling_ratio/mean": 1.0004591941833496, "sampling/importance_sampling_ratio/min": 0.6250762939453125, "sampling/sampling_logp_difference/max": 0.469881534576416, "sampling/sampling_logp_difference/mean": 0.011881633661687374, "step": 246 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 739.0, "completions/mean_length": 508.15625, "completions/mean_terminated_length": 490.8333740234375, "completions/min_length": 307.0, "completions/min_terminated_length": 307.0, "entropy": 0.4834870398044586, "epoch": 0.1322269807280514, "frac_reward_zero_std": 0.25, "grad_norm": 0.011544918641448021, "learning_rate": 1e-05, "loss": 0.0248, "num_tokens": 5557638.0, "reward": 0.6875, "reward_std": 0.3745020925998688, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.32330322265625, "sampling/importance_sampling_ratio/mean": 0.9998645782470703, "sampling/importance_sampling_ratio/min": 0.5559613108634949, "sampling/sampling_logp_difference/max": 0.5870565176010132, "sampling/sampling_logp_difference/mean": 0.01301555521786213, "step": 247 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 711.0, "completions/max_terminated_length": 711.0, "completions/mean_length": 497.0, "completions/mean_terminated_length": 497.0, "completions/min_length": 271.0, "completions/min_terminated_length": 271.0, "entropy": 0.31232312321662903, "epoch": 0.13276231263383298, "frac_reward_zero_std": 0.5, "grad_norm": 0.003026641672477126, "learning_rate": 1e-05, "loss": -0.0064, "num_tokens": 5576798.0, "reward": 0.9375, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.9375, "rewards/accuracy_reward/std": 0.24593468010425568, "sampling/importance_sampling_ratio/max": 1.2553433179855347, "sampling/importance_sampling_ratio/mean": 0.9997256398200989, "sampling/importance_sampling_ratio/min": 0.7074877023696899, "sampling/sampling_logp_difference/max": 0.3460350036621094, "sampling/sampling_logp_difference/mean": 0.010485081002116203, "step": 248 }, { "clip_ratio/high_max": 5.741846689488739e-05, "clip_ratio/high_mean": 5.741846689488739e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 5.741846689488739e-05, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 760.0, "completions/mean_length": 545.375, "completions/mean_terminated_length": 494.0000305175781, "completions/min_length": 236.0, "completions/min_terminated_length": 236.0, "entropy": 0.4944737106561661, "epoch": 0.13329764453961457, "frac_reward_zero_std": 0.25, "grad_norm": 0.017678584903478622, "learning_rate": 1e-05, "loss": 0.0744, "num_tokens": 5598242.0, "reward": 0.46875, "reward_std": 0.3608423173427582, "rewards/accuracy_reward/mean": 0.46875, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.4558478593826294, "sampling/importance_sampling_ratio/mean": 1.0001648664474487, "sampling/importance_sampling_ratio/min": 0.6514270901679993, "sampling/sampling_logp_difference/max": 0.4285898208618164, "sampling/sampling_logp_difference/mean": 0.012524483725428581, "step": 249 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00014562298747478053, "clip_ratio/low_min": 0.00014562298747478053, "clip_ratio/region_mean": 0.00014562298747478053, "completions/clipped_ratio": 0.3125, "completions/max_length": 768.0, "completions/max_terminated_length": 693.0, "completions/mean_length": 558.21875, "completions/mean_terminated_length": 462.8636474609375, "completions/min_length": 140.0, "completions/min_terminated_length": 140.0, "entropy": 0.6550707407295704, "epoch": 0.13383297644539616, "frac_reward_zero_std": 0.25, "grad_norm": 0.0033199263270944357, "learning_rate": 1e-05, "loss": 0.0529, "num_tokens": 5620249.0, "reward": 0.4375, "reward_std": 0.2925041913986206, "rewards/accuracy_reward/mean": 0.4375, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.5226207971572876, "sampling/importance_sampling_ratio/mean": 0.9997298717498779, "sampling/importance_sampling_ratio/min": 0.43849509954452515, "sampling/sampling_logp_difference/max": 0.824406623840332, "sampling/sampling_logp_difference/mean": 0.015646474435925484, "step": 250 }, { "clip_ratio/high_max": 4.826254735235125e-05, "clip_ratio/high_mean": 4.826254735235125e-05, "clip_ratio/low_mean": 0.00011380357318557799, "clip_ratio/low_min": 0.00011380357318557799, "clip_ratio/region_mean": 0.00016206612053792924, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 742.0, "completions/mean_length": 602.75, "completions/mean_terminated_length": 556.47998046875, "completions/min_length": 193.0, "completions/min_terminated_length": 193.0, "entropy": 0.4341139644384384, "epoch": 0.13436830835117772, "frac_reward_zero_std": 0.0, "grad_norm": 0.011149222031235695, "learning_rate": 1e-05, "loss": 0.0947, "num_tokens": 5643121.0, "reward": 0.53125, "reward_std": 0.4628904461860657, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.4957367181777954, "sampling/importance_sampling_ratio/mean": 0.9996941685676575, "sampling/importance_sampling_ratio/min": 0.6151829361915588, "sampling/sampling_logp_difference/max": 0.48583555221557617, "sampling/sampling_logp_difference/mean": 0.01273902878165245, "step": 251 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0001963456452358514, "clip_ratio/low_min": 0.0001963456452358514, "clip_ratio/region_mean": 0.0001963456452358514, "completions/clipped_ratio": 0.53125, "completions/max_length": 768.0, "completions/max_terminated_length": 698.0, "completions/mean_length": 675.4375, "completions/mean_terminated_length": 570.5333862304688, "completions/min_length": 326.0, "completions/min_terminated_length": 326.0, "entropy": 0.6026395335793495, "epoch": 0.1349036402569593, "frac_reward_zero_std": 0.0, "grad_norm": 0.0164803359657526, "learning_rate": 1e-05, "loss": 0.0027, "num_tokens": 5669247.0, "reward": 0.4375, "reward_std": 0.4850368797779083, "rewards/accuracy_reward/mean": 0.4375, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.4532537460327148, "sampling/importance_sampling_ratio/mean": 1.000057578086853, "sampling/importance_sampling_ratio/min": 0.6348453164100647, "sampling/sampling_logp_difference/max": 0.454373836517334, "sampling/sampling_logp_difference/mean": 0.01541457325220108, "step": 252 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 752.0, "completions/max_terminated_length": 752.0, "completions/mean_length": 484.40625, "completions/mean_terminated_length": 484.40625, "completions/min_length": 210.0, "completions/min_terminated_length": 210.0, "entropy": 0.2820499762892723, "epoch": 0.1354389721627409, "frac_reward_zero_std": 0.75, "grad_norm": 0.0022834998089820147, "learning_rate": 1e-05, "loss": -0.0083, "num_tokens": 5688100.0, "reward": 0.96875, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.96875, "rewards/accuracy_reward/std": 0.1767766922712326, "sampling/importance_sampling_ratio/max": 1.3358412981033325, "sampling/importance_sampling_ratio/mean": 0.9998927712440491, "sampling/importance_sampling_ratio/min": 0.4176056981086731, "sampling/sampling_logp_difference/max": 0.8732175827026367, "sampling/sampling_logp_difference/mean": 0.009565281681716442, "step": 253 }, { "clip_ratio/high_max": 5.2809464250458404e-05, "clip_ratio/high_mean": 5.2809464250458404e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 5.2809464250458404e-05, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 683.0, "completions/mean_length": 544.71875, "completions/mean_terminated_length": 503.370361328125, "completions/min_length": 331.0, "completions/min_terminated_length": 331.0, "entropy": 0.4243296906352043, "epoch": 0.13597430406852248, "frac_reward_zero_std": 0.25, "grad_norm": 0.015961753204464912, "learning_rate": 1e-05, "loss": -0.0206, "num_tokens": 5709331.0, "reward": 0.59375, "reward_std": 0.38816186785697937, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.3005475997924805, "sampling/importance_sampling_ratio/mean": 1.0003669261932373, "sampling/importance_sampling_ratio/min": 0.40930601954460144, "sampling/sampling_logp_difference/max": 0.8932921886444092, "sampling/sampling_logp_difference/mean": 0.012577049434185028, "step": 254 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00013144058175384998, "clip_ratio/low_min": 0.00013144058175384998, "clip_ratio/region_mean": 0.00013144058175384998, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 759.0, "completions/mean_length": 485.75, "completions/mean_terminated_length": 456.5517272949219, "completions/min_length": 229.0, "completions/min_terminated_length": 229.0, "entropy": 0.3989061266183853, "epoch": 0.13650963597430407, "frac_reward_zero_std": 0.0, "grad_norm": 0.029649488627910614, "learning_rate": 1e-05, "loss": 0.1649, "num_tokens": 5728131.0, "reward": 0.625, "reward_std": 0.4355512857437134, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.3780889511108398, "sampling/importance_sampling_ratio/mean": 1.0001741647720337, "sampling/importance_sampling_ratio/min": 0.4336349666118622, "sampling/sampling_logp_difference/max": 0.8355522155761719, "sampling/sampling_logp_difference/mean": 0.011804546229541302, "step": 255 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 747.0, "completions/mean_length": 535.21875, "completions/mean_terminated_length": 470.03997802734375, "completions/min_length": 246.0, "completions/min_terminated_length": 246.0, "entropy": 0.4749331511557102, "epoch": 0.13704496788008566, "frac_reward_zero_std": 0.5, "grad_norm": 0.007594781462103128, "learning_rate": 1e-05, "loss": 0.0409, "num_tokens": 5748826.0, "reward": 0.4375, "reward_std": 0.249358132481575, "rewards/accuracy_reward/mean": 0.4375, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.4648830890655518, "sampling/importance_sampling_ratio/mean": 0.9997185468673706, "sampling/importance_sampling_ratio/min": 0.6979843378067017, "sampling/sampling_logp_difference/max": 0.3817753791809082, "sampling/sampling_logp_difference/mean": 0.013496547006070614, "step": 256 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 723.0, "completions/mean_length": 500.84375, "completions/mean_terminated_length": 473.2069091796875, "completions/min_length": 335.0, "completions/min_terminated_length": 335.0, "entropy": 0.3384334407746792, "epoch": 0.13758029978586725, "frac_reward_zero_std": 0.5, "grad_norm": 0.003618735121563077, "learning_rate": 1e-05, "loss": 0.0123, "num_tokens": 5768053.0, "reward": 0.6875, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.320680022239685, "sampling/importance_sampling_ratio/mean": 0.9995749592781067, "sampling/importance_sampling_ratio/min": 0.703371524810791, "sampling/sampling_logp_difference/max": 0.351870059967041, "sampling/sampling_logp_difference/mean": 0.01065816730260849, "step": 257 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00013612463590106927, "clip_ratio/low_min": 0.00013612463590106927, "clip_ratio/region_mean": 0.00013612463590106927, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 720.0, "completions/mean_length": 495.46875, "completions/mean_terminated_length": 477.3000183105469, "completions/min_length": 291.0, "completions/min_terminated_length": 291.0, "entropy": 0.3730607181787491, "epoch": 0.1381156316916488, "frac_reward_zero_std": 0.5, "grad_norm": 0.005156353116035461, "learning_rate": 1e-05, "loss": 0.0488, "num_tokens": 5787364.0, "reward": 0.78125, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.3002409934997559, "sampling/importance_sampling_ratio/mean": 0.9996038675308228, "sampling/importance_sampling_ratio/min": 0.6701311469078064, "sampling/sampling_logp_difference/max": 0.4002819061279297, "sampling/sampling_logp_difference/mean": 0.011503715068101883, "step": 258 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.34375, "completions/max_length": 768.0, "completions/max_terminated_length": 768.0, "completions/mean_length": 641.21875, "completions/mean_terminated_length": 574.8095092773438, "completions/min_length": 360.0, "completions/min_terminated_length": 360.0, "entropy": 0.45424384996294975, "epoch": 0.1386509635974304, "frac_reward_zero_std": 0.5, "grad_norm": 0.0053697871044278145, "learning_rate": 1e-05, "loss": -0.0053, "num_tokens": 5811619.0, "reward": 0.25, "reward_std": 0.26726123690605164, "rewards/accuracy_reward/mean": 0.25, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.3962948322296143, "sampling/importance_sampling_ratio/mean": 1.0002083778381348, "sampling/importance_sampling_ratio/min": 0.6089805960655212, "sampling/sampling_logp_difference/max": 0.4959688186645508, "sampling/sampling_logp_difference/mean": 0.012901658192276955, "step": 259 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 744.0, "completions/mean_length": 531.65625, "completions/mean_terminated_length": 507.2069091796875, "completions/min_length": 203.0, "completions/min_terminated_length": 203.0, "entropy": 0.4479805454611778, "epoch": 0.139186295503212, "frac_reward_zero_std": 0.25, "grad_norm": 0.014216828159987926, "learning_rate": 1e-05, "loss": 0.0162, "num_tokens": 5832016.0, "reward": 0.59375, "reward_std": 0.3608423173427582, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.3912107944488525, "sampling/importance_sampling_ratio/mean": 1.0001329183578491, "sampling/importance_sampling_ratio/min": 0.24031956493854523, "sampling/sampling_logp_difference/max": 1.425785779953003, "sampling/sampling_logp_difference/mean": 0.012394886463880539, "step": 260 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.6357079301960766e-05, "clip_ratio/low_min": 5.6357079301960766e-05, "clip_ratio/region_mean": 5.6357079301960766e-05, "completions/clipped_ratio": 0.34375, "completions/max_length": 768.0, "completions/max_terminated_length": 742.0, "completions/mean_length": 635.46875, "completions/mean_terminated_length": 566.047607421875, "completions/min_length": 293.0, "completions/min_terminated_length": 293.0, "entropy": 0.6670098789036274, "epoch": 0.13972162740899358, "frac_reward_zero_std": 0.5, "grad_norm": 0.009135905653238297, "learning_rate": 1e-05, "loss": 0.0135, "num_tokens": 5856375.0, "reward": 0.3125, "reward_std": 0.249358132481575, "rewards/accuracy_reward/mean": 0.3125, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.457891583442688, "sampling/importance_sampling_ratio/mean": 1.0000693798065186, "sampling/importance_sampling_ratio/min": 0.33598586916923523, "sampling/sampling_logp_difference/max": 1.0906862020492554, "sampling/sampling_logp_difference/mean": 0.015408815816044807, "step": 261 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.9465768825029954e-05, "clip_ratio/low_min": 4.9465768825029954e-05, "clip_ratio/region_mean": 4.9465768825029954e-05, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 700.0, "completions/mean_length": 505.3125, "completions/mean_terminated_length": 456.6666564941406, "completions/min_length": 233.0, "completions/min_terminated_length": 233.0, "entropy": 0.4717660881578922, "epoch": 0.14025695931477516, "frac_reward_zero_std": 0.0, "grad_norm": 0.01704293303191662, "learning_rate": 1e-05, "loss": 0.0668, "num_tokens": 5875921.0, "reward": 0.71875, "reward_std": 0.4628904461860657, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.8099247217178345, "sampling/importance_sampling_ratio/mean": 1.0000708103179932, "sampling/importance_sampling_ratio/min": 0.6353431940078735, "sampling/sampling_logp_difference/max": 0.5932852029800415, "sampling/sampling_logp_difference/mean": 0.012012653052806854, "step": 262 }, { "clip_ratio/high_max": 4.839333996642381e-05, "clip_ratio/high_mean": 4.839333996642381e-05, "clip_ratio/low_mean": 0.0002190562845498789, "clip_ratio/low_min": 0.0002190562845498789, "clip_ratio/region_mean": 0.0002674496245163027, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 739.0, "completions/mean_length": 561.71875, "completions/mean_terminated_length": 492.9583435058594, "completions/min_length": 307.0, "completions/min_terminated_length": 307.0, "entropy": 0.47844066098332405, "epoch": 0.14079229122055675, "frac_reward_zero_std": 0.0, "grad_norm": 0.0119700338691473, "learning_rate": 1e-05, "loss": 0.0362, "num_tokens": 5897448.0, "reward": 0.5, "reward_std": 0.4671337604522705, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5080004930496216, "sampling/importance_sampling_ratio/max": 1.4810571670532227, "sampling/importance_sampling_ratio/mean": 0.9999778270721436, "sampling/importance_sampling_ratio/min": 0.5343120694160461, "sampling/sampling_logp_difference/max": 0.6267751455307007, "sampling/sampling_logp_difference/mean": 0.013196354731917381, "step": 263 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00014293238200480118, "clip_ratio/low_min": 0.00014293238200480118, "clip_ratio/region_mean": 0.00014293238200480118, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 715.0, "completions/mean_length": 524.28125, "completions/mean_terminated_length": 468.0384826660156, "completions/min_length": 179.0, "completions/min_terminated_length": 179.0, "entropy": 0.4351576343178749, "epoch": 0.14132762312633834, "frac_reward_zero_std": 0.25, "grad_norm": 0.005989194381982088, "learning_rate": 1e-05, "loss": -0.0366, "num_tokens": 5918177.0, "reward": 0.28125, "reward_std": 0.2651650309562683, "rewards/accuracy_reward/mean": 0.28125, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.2802380323410034, "sampling/importance_sampling_ratio/mean": 0.9998577237129211, "sampling/importance_sampling_ratio/min": 0.752738893032074, "sampling/sampling_logp_difference/max": 0.28403687477111816, "sampling/sampling_logp_difference/mean": 0.012446128763258457, "step": 264 }, { "clip_ratio/high_max": 7.413997809635475e-05, "clip_ratio/high_mean": 7.413997809635475e-05, "clip_ratio/low_mean": 4.7837733291089535e-05, "clip_ratio/low_min": 4.7837733291089535e-05, "clip_ratio/region_mean": 0.00012197771138744429, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 766.0, "completions/mean_length": 617.15625, "completions/mean_terminated_length": 582.34619140625, "completions/min_length": 384.0, "completions/min_terminated_length": 384.0, "entropy": 0.40651633590459824, "epoch": 0.1418629550321199, "frac_reward_zero_std": 0.25, "grad_norm": 0.0192613173276186, "learning_rate": 1e-05, "loss": 0.0216, "num_tokens": 5941598.0, "reward": 0.65625, "reward_std": 0.3061639666557312, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.7904272079467773, "sampling/importance_sampling_ratio/mean": 1.000046730041504, "sampling/importance_sampling_ratio/min": 0.5358311533927917, "sampling/sampling_logp_difference/max": 0.6239361763000488, "sampling/sampling_logp_difference/mean": 0.011971535161137581, "step": 265 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 9.424100062460639e-05, "clip_ratio/low_min": 9.424100062460639e-05, "clip_ratio/region_mean": 9.424100062460639e-05, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 683.0, "completions/mean_length": 591.5625, "completions/mean_terminated_length": 550.84619140625, "completions/min_length": 377.0, "completions/min_terminated_length": 377.0, "entropy": 0.4514969103038311, "epoch": 0.1423982869379015, "frac_reward_zero_std": 0.0, "grad_norm": 0.011818221770226955, "learning_rate": 1e-05, "loss": 0.0883, "num_tokens": 5964680.0, "reward": 0.71875, "reward_std": 0.4628904461860657, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.4639863967895508, "sampling/importance_sampling_ratio/mean": 1.0000672340393066, "sampling/importance_sampling_ratio/min": 0.7593649625778198, "sampling/sampling_logp_difference/max": 0.3811631202697754, "sampling/sampling_logp_difference/mean": 0.013523617759346962, "step": 266 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 614.0, "completions/mean_length": 422.21875, "completions/mean_terminated_length": 411.06451416015625, "completions/min_length": 159.0, "completions/min_terminated_length": 159.0, "entropy": 0.35408175364136696, "epoch": 0.14293361884368308, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.064, "num_tokens": 5981511.0, "reward": 0.9375, "reward_std": 0.1157275140285492, "rewards/accuracy_reward/mean": 0.9375, "rewards/accuracy_reward/std": 0.24593468010425568, "sampling/importance_sampling_ratio/max": 1.4102905988693237, "sampling/importance_sampling_ratio/mean": 0.999640941619873, "sampling/importance_sampling_ratio/min": 0.695646345615387, "sampling/sampling_logp_difference/max": 0.3629138469696045, "sampling/sampling_logp_difference/mean": 0.011139913462102413, "step": 267 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 723.0, "completions/mean_length": 543.8125, "completions/mean_terminated_length": 520.6206665039062, "completions/min_length": 264.0, "completions/min_terminated_length": 264.0, "entropy": 0.31492745503783226, "epoch": 0.14346895074946467, "frac_reward_zero_std": 0.5, "grad_norm": 0.00812839437276125, "learning_rate": 1e-05, "loss": 0.0057, "num_tokens": 6002537.0, "reward": 0.8125, "reward_std": 0.249358132481575, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.3698124885559082, "sampling/importance_sampling_ratio/mean": 0.9997535347938538, "sampling/importance_sampling_ratio/min": 0.6207072138786316, "sampling/sampling_logp_difference/max": 0.476895809173584, "sampling/sampling_logp_difference/mean": 0.009826837107539177, "step": 268 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00011529543917276897, "clip_ratio/low_min": 0.00011529543917276897, "clip_ratio/region_mean": 0.00011529543917276897, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 753.0, "completions/mean_length": 512.125, "completions/mean_terminated_length": 475.5714416503906, "completions/min_length": 252.0, "completions/min_terminated_length": 252.0, "entropy": 0.5743189193308353, "epoch": 0.14400428265524626, "frac_reward_zero_std": 0.25, "grad_norm": 0.007048206869512796, "learning_rate": 1e-05, "loss": 0.0477, "num_tokens": 6022277.0, "reward": 0.59375, "reward_std": 0.3198433816432953, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.3809080123901367, "sampling/importance_sampling_ratio/mean": 1.0001294612884521, "sampling/importance_sampling_ratio/min": 0.6060126423835754, "sampling/sampling_logp_difference/max": 0.5008544921875, "sampling/sampling_logp_difference/mean": 0.015097922645509243, "step": 269 }, { "clip_ratio/high_max": 6.565126386703923e-05, "clip_ratio/high_mean": 6.565126386703923e-05, "clip_ratio/low_mean": 0.00012793409041478299, "clip_ratio/low_min": 0.00012793409041478299, "clip_ratio/region_mean": 0.00019358535428182222, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 695.0, "completions/mean_length": 494.3125, "completions/mean_terminated_length": 466.0, "completions/min_length": 180.0, "completions/min_terminated_length": 180.0, "entropy": 0.4720449894666672, "epoch": 0.14453961456102785, "frac_reward_zero_std": 0.5, "grad_norm": 0.0026534381322562695, "learning_rate": 1e-05, "loss": -0.0283, "num_tokens": 6041383.0, "reward": 0.75, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.374011754989624, "sampling/importance_sampling_ratio/mean": 1.0000495910644531, "sampling/importance_sampling_ratio/min": 0.5556223392486572, "sampling/sampling_logp_difference/max": 0.5876665115356445, "sampling/sampling_logp_difference/mean": 0.013280272483825684, "step": 270 }, { "clip_ratio/high_max": 4.520795482676476e-05, "clip_ratio/high_mean": 4.520795482676476e-05, "clip_ratio/low_mean": 0.00014015863416716456, "clip_ratio/low_min": 0.00014015863416716456, "clip_ratio/region_mean": 0.00018536658899392933, "completions/clipped_ratio": 0.5, "completions/max_length": 768.0, "completions/max_terminated_length": 693.0, "completions/mean_length": 629.625, "completions/mean_terminated_length": 491.25, "completions/min_length": 10.0, "completions/min_terminated_length": 10.0, "entropy": 0.47507236525416374, "epoch": 0.14507494646680943, "frac_reward_zero_std": 0.0, "grad_norm": 0.01490703783929348, "learning_rate": 1e-05, "loss": -0.005, "num_tokens": 6064907.0, "reward": 0.4375, "reward_std": 0.4492306709289551, "rewards/accuracy_reward/mean": 0.4375, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.6046644449234009, "sampling/importance_sampling_ratio/mean": 1.0001014471054077, "sampling/importance_sampling_ratio/min": 0.46269458532333374, "sampling/sampling_logp_difference/max": 0.7706880569458008, "sampling/sampling_logp_difference/mean": 0.013661563396453857, "step": 271 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 745.0, "completions/mean_length": 544.53125, "completions/mean_terminated_length": 529.6333618164062, "completions/min_length": 362.0, "completions/min_terminated_length": 362.0, "entropy": 0.40128302201628685, "epoch": 0.145610278372591, "frac_reward_zero_std": 0.5, "grad_norm": 0.004165521822869778, "learning_rate": 1e-05, "loss": 0.0031, "num_tokens": 6086052.0, "reward": 0.75, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.3422083854675293, "sampling/importance_sampling_ratio/mean": 1.0000782012939453, "sampling/importance_sampling_ratio/min": 0.5597332119941711, "sampling/sampling_logp_difference/max": 0.5802950859069824, "sampling/sampling_logp_difference/mean": 0.011999118141829967, "step": 272 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0001749465081957169, "clip_ratio/low_min": 0.0001749465081957169, "clip_ratio/region_mean": 0.0001749465081957169, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 718.0, "completions/mean_length": 498.03125, "completions/mean_terminated_length": 422.44000244140625, "completions/min_length": 185.0, "completions/min_terminated_length": 185.0, "entropy": 0.49034508690238, "epoch": 0.14614561027837258, "frac_reward_zero_std": 0.0, "grad_norm": 0.022954454645514488, "learning_rate": 1e-05, "loss": 0.0649, "num_tokens": 6105285.0, "reward": 0.46875, "reward_std": 0.4628904461860657, "rewards/accuracy_reward/mean": 0.46875, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.000229835510254, "sampling/importance_sampling_ratio/min": 0.637057363986969, "sampling/sampling_logp_difference/max": 0.768197774887085, "sampling/sampling_logp_difference/mean": 0.01436677761375904, "step": 273 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 755.0, "completions/mean_length": 555.53125, "completions/mean_terminated_length": 506.5000305175781, "completions/min_length": 198.0, "completions/min_terminated_length": 198.0, "entropy": 0.35136451572179794, "epoch": 0.14668094218415417, "frac_reward_zero_std": 0.0, "grad_norm": 0.01199391856789589, "learning_rate": 1e-05, "loss": 0.0868, "num_tokens": 6126550.0, "reward": 0.40625, "reward_std": 0.4807935357093811, "rewards/accuracy_reward/mean": 0.40625, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.5508660078048706, "sampling/importance_sampling_ratio/mean": 0.9997656941413879, "sampling/importance_sampling_ratio/min": 0.5551180839538574, "sampling/sampling_logp_difference/max": 0.5885744094848633, "sampling/sampling_logp_difference/mean": 0.011597477830946445, "step": 274 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 740.0, "completions/mean_length": 505.34375, "completions/mean_terminated_length": 467.8214416503906, "completions/min_length": 292.0, "completions/min_terminated_length": 292.0, "entropy": 0.49765312299132347, "epoch": 0.14721627408993576, "frac_reward_zero_std": 0.25, "grad_norm": 0.009008870460093021, "learning_rate": 1e-05, "loss": 0.0428, "num_tokens": 6146449.0, "reward": 0.75, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.401552677154541, "sampling/importance_sampling_ratio/mean": 1.0001399517059326, "sampling/importance_sampling_ratio/min": 0.4989939332008362, "sampling/sampling_logp_difference/max": 0.6951613426208496, "sampling/sampling_logp_difference/mean": 0.015041221864521503, "step": 275 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 7.39207534934394e-05, "clip_ratio/low_min": 7.39207534934394e-05, "clip_ratio/region_mean": 7.39207534934394e-05, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 685.0, "completions/mean_length": 452.25, "completions/mean_terminated_length": 419.5862121582031, "completions/min_length": 249.0, "completions/min_terminated_length": 249.0, "entropy": 0.4714985527098179, "epoch": 0.14775160599571735, "frac_reward_zero_std": 0.5, "grad_norm": 0.0040002986788749695, "learning_rate": 1e-05, "loss": 0.0719, "num_tokens": 6164761.0, "reward": 0.8125, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.3116964101791382, "sampling/importance_sampling_ratio/mean": 1.0002880096435547, "sampling/importance_sampling_ratio/min": 0.6424389481544495, "sampling/sampling_logp_difference/max": 0.44248342514038086, "sampling/sampling_logp_difference/mean": 0.012760909274220467, "step": 276 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 765.0, "completions/max_terminated_length": 765.0, "completions/mean_length": 509.34375, "completions/mean_terminated_length": 509.34375, "completions/min_length": 196.0, "completions/min_terminated_length": 196.0, "entropy": 0.39188898727297783, "epoch": 0.14828693790149894, "frac_reward_zero_std": 0.25, "grad_norm": 0.00961736124008894, "learning_rate": 1e-05, "loss": 0.0513, "num_tokens": 6185116.0, "reward": 0.84375, "reward_std": 0.3061639666557312, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.2709670066833496, "sampling/importance_sampling_ratio/mean": 0.9999263286590576, "sampling/importance_sampling_ratio/min": 0.6830671429634094, "sampling/sampling_logp_difference/max": 0.381162166595459, "sampling/sampling_logp_difference/mean": 0.011551597155630589, "step": 277 }, { "clip_ratio/high_max": 5.6637971283635125e-05, "clip_ratio/high_mean": 5.6637971283635125e-05, "clip_ratio/low_mean": 0.00015031443399493583, "clip_ratio/low_min": 0.00015031443399493583, "clip_ratio/region_mean": 0.00020695240527857095, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 750.0, "completions/mean_length": 560.09375, "completions/mean_terminated_length": 530.3928833007812, "completions/min_length": 332.0, "completions/min_terminated_length": 332.0, "entropy": 0.3911485970020294, "epoch": 0.14882226980728053, "frac_reward_zero_std": 0.5, "grad_norm": 0.0049836356192827225, "learning_rate": 1e-05, "loss": 0.0352, "num_tokens": 6206807.0, "reward": 0.625, "reward_std": 0.2314550280570984, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.5188956260681152, "sampling/importance_sampling_ratio/mean": 1.000071406364441, "sampling/importance_sampling_ratio/min": 0.6432191133499146, "sampling/sampling_logp_difference/max": 0.4412698745727539, "sampling/sampling_logp_difference/mean": 0.01206052303314209, "step": 278 }, { "clip_ratio/high_max": 5.9045818488812074e-05, "clip_ratio/high_mean": 5.9045818488812074e-05, "clip_ratio/low_mean": 0.00018358248416916467, "clip_ratio/low_min": 0.00018358248416916467, "clip_ratio/region_mean": 0.00024262830265797675, "completions/clipped_ratio": 0.28125, "completions/max_length": 768.0, "completions/max_terminated_length": 749.0, "completions/mean_length": 603.8125, "completions/mean_terminated_length": 539.5652465820312, "completions/min_length": 370.0, "completions/min_terminated_length": 370.0, "entropy": 0.562998715788126, "epoch": 0.1493576017130621, "frac_reward_zero_std": 0.0, "grad_norm": 0.006244795396924019, "learning_rate": 1e-05, "loss": -0.0031, "num_tokens": 6230689.0, "reward": 0.5, "reward_std": 0.4082317352294922, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5080004930496216, "sampling/importance_sampling_ratio/max": 1.3814009428024292, "sampling/importance_sampling_ratio/mean": 1.0001212358474731, "sampling/importance_sampling_ratio/min": 0.4358620345592499, "sampling/sampling_logp_difference/max": 0.8304295539855957, "sampling/sampling_logp_difference/mean": 0.01555557455867529, "step": 279 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 714.0, "completions/max_terminated_length": 714.0, "completions/mean_length": 437.8125, "completions/mean_terminated_length": 437.8125, "completions/min_length": 239.0, "completions/min_terminated_length": 239.0, "entropy": 0.3339073471724987, "epoch": 0.14989293361884368, "frac_reward_zero_std": 0.5, "grad_norm": 0.006268746219575405, "learning_rate": 1e-05, "loss": 0.0473, "num_tokens": 6247787.0, "reward": 0.84375, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9997015595436096, "sampling/importance_sampling_ratio/min": 0.6794090867042542, "sampling/sampling_logp_difference/max": 0.9785534143447876, "sampling/sampling_logp_difference/mean": 0.01036280207335949, "step": 280 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 718.0, "completions/mean_length": 547.4375, "completions/mean_terminated_length": 540.3225708007812, "completions/min_length": 325.0, "completions/min_terminated_length": 325.0, "entropy": 0.44963525608181953, "epoch": 0.15042826552462527, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": -0.0197, "num_tokens": 6269177.0, "reward": 0.84375, "reward_std": 0.1293872892856598, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.5465196371078491, "sampling/importance_sampling_ratio/mean": 1.000325083732605, "sampling/importance_sampling_ratio/min": 0.5460077524185181, "sampling/sampling_logp_difference/max": 0.6051220893859863, "sampling/sampling_logp_difference/mean": 0.013183525763452053, "step": 281 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 7.344300684053451e-05, "clip_ratio/low_min": 7.344300684053451e-05, "clip_ratio/region_mean": 7.344300684053451e-05, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 744.0, "completions/mean_length": 544.25, "completions/mean_terminated_length": 521.1034545898438, "completions/min_length": 207.0, "completions/min_terminated_length": 207.0, "entropy": 0.3133871331810951, "epoch": 0.15096359743040685, "frac_reward_zero_std": 0.25, "grad_norm": 0.006081177853047848, "learning_rate": 1e-05, "loss": -0.007, "num_tokens": 6290025.0, "reward": 0.75, "reward_std": 0.3514062464237213, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.430967092514038, "sampling/importance_sampling_ratio/mean": 1.0002472400665283, "sampling/importance_sampling_ratio/min": 0.5997442603111267, "sampling/sampling_logp_difference/max": 0.5112519264221191, "sampling/sampling_logp_difference/mean": 0.010682223364710808, "step": 282 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.3125, "completions/max_length": 768.0, "completions/max_terminated_length": 757.0, "completions/mean_length": 560.25, "completions/mean_terminated_length": 465.8182067871094, "completions/min_length": 184.0, "completions/min_terminated_length": 184.0, "entropy": 0.5701764971017838, "epoch": 0.15149892933618844, "frac_reward_zero_std": 0.25, "grad_norm": 0.010730598121881485, "learning_rate": 1e-05, "loss": 0.1139, "num_tokens": 6312033.0, "reward": 0.375, "reward_std": 0.3104073107242584, "rewards/accuracy_reward/mean": 0.375, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.593160629272461, "sampling/importance_sampling_ratio/mean": 1.0000511407852173, "sampling/importance_sampling_ratio/min": 0.5179017186164856, "sampling/sampling_logp_difference/max": 0.6579697132110596, "sampling/sampling_logp_difference/mean": 0.014908690936863422, "step": 283 }, { "clip_ratio/high_max": 6.297229265328497e-05, "clip_ratio/high_mean": 6.297229265328497e-05, "clip_ratio/low_mean": 5.122950824443251e-05, "clip_ratio/low_min": 5.122950824443251e-05, "clip_ratio/region_mean": 0.00011420180089771748, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 748.0, "completions/mean_length": 544.75, "completions/mean_terminated_length": 503.40740966796875, "completions/min_length": 331.0, "completions/min_terminated_length": 331.0, "entropy": 0.4237790144979954, "epoch": 0.15203426124197003, "frac_reward_zero_std": 0.5, "grad_norm": 0.009094472974538803, "learning_rate": 1e-05, "loss": 0.0735, "num_tokens": 6333273.0, "reward": 0.875, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 1.4660954475402832, "sampling/importance_sampling_ratio/mean": 0.9999098777770996, "sampling/importance_sampling_ratio/min": 0.41424208879470825, "sampling/sampling_logp_difference/max": 0.8813047409057617, "sampling/sampling_logp_difference/mean": 0.012058655731379986, "step": 284 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00011835439363494515, "clip_ratio/low_min": 0.00011835439363494515, "clip_ratio/region_mean": 0.00011835439363494515, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 750.0, "completions/mean_length": 531.375, "completions/mean_terminated_length": 465.1199951171875, "completions/min_length": 292.0, "completions/min_terminated_length": 292.0, "entropy": 0.5237101577222347, "epoch": 0.15256959314775162, "frac_reward_zero_std": 0.25, "grad_norm": 0.003177138976752758, "learning_rate": 1e-05, "loss": 0.0363, "num_tokens": 6354237.0, "reward": 0.59375, "reward_std": 0.3198433816432953, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.5599786043167114, "sampling/importance_sampling_ratio/mean": 1.000277042388916, "sampling/importance_sampling_ratio/min": 0.5885826349258423, "sampling/sampling_logp_difference/max": 0.5300378799438477, "sampling/sampling_logp_difference/mean": 0.014514933340251446, "step": 285 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 721.0, "completions/mean_length": 496.375, "completions/mean_terminated_length": 487.6128845214844, "completions/min_length": 170.0, "completions/min_terminated_length": 170.0, "entropy": 0.3909755200147629, "epoch": 0.15310492505353318, "frac_reward_zero_std": 0.25, "grad_norm": 0.015391631983220577, "learning_rate": 1e-05, "loss": 0.0527, "num_tokens": 6373689.0, "reward": 0.8125, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.5359779596328735, "sampling/importance_sampling_ratio/mean": 0.9997937679290771, "sampling/importance_sampling_ratio/min": 0.6337550282478333, "sampling/sampling_logp_difference/max": 0.45609283447265625, "sampling/sampling_logp_difference/mean": 0.011829542927443981, "step": 286 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00015241292567225173, "clip_ratio/low_min": 0.00015241292567225173, "clip_ratio/region_mean": 0.00015241292567225173, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 659.0, "completions/mean_length": 581.25, "completions/mean_terminated_length": 519.0, "completions/min_length": 219.0, "completions/min_terminated_length": 219.0, "entropy": 0.5511211827397346, "epoch": 0.15364025695931477, "frac_reward_zero_std": 0.25, "grad_norm": 0.01204445119947195, "learning_rate": 1e-05, "loss": -0.0141, "num_tokens": 6395913.0, "reward": 0.21875, "reward_std": 0.3377464711666107, "rewards/accuracy_reward/mean": 0.21875, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.2857708930969238, "sampling/importance_sampling_ratio/mean": 1.0000673532485962, "sampling/importance_sampling_ratio/min": 0.6992047429084778, "sampling/sampling_logp_difference/max": 0.35781168937683105, "sampling/sampling_logp_difference/mean": 0.015487638302147388, "step": 287 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 762.0, "completions/mean_length": 603.21875, "completions/mean_terminated_length": 565.1923217773438, "completions/min_length": 289.0, "completions/min_terminated_length": 289.0, "entropy": 0.5171289891004562, "epoch": 0.15417558886509636, "frac_reward_zero_std": 0.25, "grad_norm": 0.008943334221839905, "learning_rate": 1e-05, "loss": -0.0328, "num_tokens": 6419656.0, "reward": 0.6875, "reward_std": 0.3514062464237213, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.4210548400878906, "sampling/importance_sampling_ratio/mean": 0.9999669790267944, "sampling/importance_sampling_ratio/min": 0.6377384066581726, "sampling/sampling_logp_difference/max": 0.4498271942138672, "sampling/sampling_logp_difference/mean": 0.013081030920147896, "step": 288 }, { "clip_ratio/high_max": 0.0001915773937071208, "clip_ratio/high_mean": 0.0001915773937071208, "clip_ratio/low_mean": 4.9682035751175135e-05, "clip_ratio/low_min": 4.9682035751175135e-05, "clip_ratio/region_mean": 0.00024125942945829593, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 744.0, "completions/mean_length": 536.0, "completions/mean_terminated_length": 471.03997802734375, "completions/min_length": 288.0, "completions/min_terminated_length": 288.0, "entropy": 0.4301992952823639, "epoch": 0.15471092077087795, "frac_reward_zero_std": 0.0, "grad_norm": 0.020020153373479843, "learning_rate": 1e-05, "loss": 0.103, "num_tokens": 6440480.0, "reward": 0.625, "reward_std": 0.4355512857437134, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.465025544166565, "sampling/importance_sampling_ratio/mean": 0.9999731183052063, "sampling/importance_sampling_ratio/min": 0.2902846038341522, "sampling/sampling_logp_difference/max": 1.2368934154510498, "sampling/sampling_logp_difference/mean": 0.013006285764276981, "step": 289 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.40625, "completions/max_length": 768.0, "completions/max_terminated_length": 731.0, "completions/mean_length": 627.90625, "completions/mean_terminated_length": 532.0526123046875, "completions/min_length": 275.0, "completions/min_terminated_length": 275.0, "entropy": 0.41944463923573494, "epoch": 0.15524625267665954, "frac_reward_zero_std": 0.5, "grad_norm": 0.005376975983381271, "learning_rate": 1e-05, "loss": 0.0432, "num_tokens": 6464445.0, "reward": 0.34375, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.34375, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.4394961595535278, "sampling/importance_sampling_ratio/mean": 1.0001652240753174, "sampling/importance_sampling_ratio/min": 0.63199383020401, "sampling/sampling_logp_difference/max": 0.4588756561279297, "sampling/sampling_logp_difference/mean": 0.01221837941557169, "step": 290 }, { "clip_ratio/high_max": 5.256518124951981e-05, "clip_ratio/high_mean": 5.256518124951981e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 5.256518124951981e-05, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 764.0, "completions/mean_length": 585.78125, "completions/mean_terminated_length": 543.7307739257812, "completions/min_length": 266.0, "completions/min_terminated_length": 266.0, "entropy": 0.2645743489265442, "epoch": 0.15578158458244112, "frac_reward_zero_std": 0.5, "grad_norm": 0.010447485372424126, "learning_rate": 1e-05, "loss": 0.0354, "num_tokens": 6486654.0, "reward": 0.84375, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.29339599609375, "sampling/importance_sampling_ratio/mean": 1.0000994205474854, "sampling/importance_sampling_ratio/min": 0.7572441697120667, "sampling/sampling_logp_difference/max": 0.27806949615478516, "sampling/sampling_logp_difference/mean": 0.008877474814653397, "step": 291 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 721.0, "completions/mean_length": 510.96875, "completions/mean_terminated_length": 502.6773986816406, "completions/min_length": 258.0, "completions/min_terminated_length": 258.0, "entropy": 0.37195832282304764, "epoch": 0.15631691648822268, "frac_reward_zero_std": 0.5, "grad_norm": 0.014043179340660572, "learning_rate": 1e-05, "loss": 0.043, "num_tokens": 6506885.0, "reward": 0.90625, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.4685640335083008, "sampling/importance_sampling_ratio/mean": 0.999987006187439, "sampling/importance_sampling_ratio/min": 0.7224671244621277, "sampling/sampling_logp_difference/max": 0.38428497314453125, "sampling/sampling_logp_difference/mean": 0.011961011216044426, "step": 292 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.1867220463464037e-05, "clip_ratio/low_min": 5.1867220463464037e-05, "clip_ratio/region_mean": 5.1867220463464037e-05, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 764.0, "completions/mean_length": 638.3125, "completions/mean_terminated_length": 595.0833740234375, "completions/min_length": 388.0, "completions/min_terminated_length": 388.0, "entropy": 0.5734780021011829, "epoch": 0.15685224839400427, "frac_reward_zero_std": 0.25, "grad_norm": 0.007615923881530762, "learning_rate": 1e-05, "loss": -0.003, "num_tokens": 6531583.0, "reward": 0.375, "reward_std": 0.3104073107242584, "rewards/accuracy_reward/mean": 0.375, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.8356448411941528, "sampling/importance_sampling_ratio/mean": 1.0000708103179932, "sampling/importance_sampling_ratio/min": 0.3261215090751648, "sampling/sampling_logp_difference/max": 1.1204851865768433, "sampling/sampling_logp_difference/mean": 0.014159752987325191, "step": 293 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 751.0, "completions/mean_length": 537.125, "completions/mean_terminated_length": 521.7333374023438, "completions/min_length": 327.0, "completions/min_terminated_length": 327.0, "entropy": 0.5348762422800064, "epoch": 0.15738758029978586, "frac_reward_zero_std": 0.25, "grad_norm": 0.00950230285525322, "learning_rate": 1e-05, "loss": -0.015, "num_tokens": 6552411.0, "reward": 0.6875, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.2969403266906738, "sampling/importance_sampling_ratio/mean": 1.0000113248825073, "sampling/importance_sampling_ratio/min": 0.5681097507476807, "sampling/sampling_logp_difference/max": 0.5654406547546387, "sampling/sampling_logp_difference/mean": 0.014715812169015408, "step": 294 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 740.0, "completions/mean_length": 494.78125, "completions/mean_terminated_length": 476.5666809082031, "completions/min_length": 304.0, "completions/min_terminated_length": 304.0, "entropy": 0.4981640614569187, "epoch": 0.15792291220556745, "frac_reward_zero_std": 0.5, "grad_norm": 0.013121116906404495, "learning_rate": 1e-05, "loss": 0.0058, "num_tokens": 6572004.0, "reward": 0.78125, "reward_std": 0.2630178928375244, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.3795262575149536, "sampling/importance_sampling_ratio/mean": 0.9999207258224487, "sampling/importance_sampling_ratio/min": 0.4757774770259857, "sampling/sampling_logp_difference/max": 0.742805004119873, "sampling/sampling_logp_difference/mean": 0.013673343695700169, "step": 295 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00012166906162747182, "clip_ratio/low_min": 0.00012166906162747182, "clip_ratio/region_mean": 0.00012166906162747182, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 721.0, "completions/mean_length": 535.09375, "completions/mean_terminated_length": 501.8214416503906, "completions/min_length": 283.0, "completions/min_terminated_length": 283.0, "entropy": 0.4476263076066971, "epoch": 0.15845824411134904, "frac_reward_zero_std": 0.25, "grad_norm": 0.017186393961310387, "learning_rate": 1e-05, "loss": 0.0379, "num_tokens": 6592431.0, "reward": 0.6875, "reward_std": 0.38298875093460083, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.4138660430908203, "sampling/importance_sampling_ratio/mean": 0.9998984336853027, "sampling/importance_sampling_ratio/min": 0.5262473225593567, "sampling/sampling_logp_difference/max": 0.6419839859008789, "sampling/sampling_logp_difference/mean": 0.012497576884925365, "step": 296 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00011944850120926276, "clip_ratio/low_min": 0.00011944850120926276, "clip_ratio/region_mean": 0.00011944850120926276, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 709.0, "completions/mean_length": 518.0625, "completions/mean_terminated_length": 448.0799865722656, "completions/min_length": 211.0, "completions/min_terminated_length": 211.0, "entropy": 0.41150325164198875, "epoch": 0.15899357601713063, "frac_reward_zero_std": 0.5, "grad_norm": 0.013153589330613613, "learning_rate": 1e-05, "loss": -0.0132, "num_tokens": 6612753.0, "reward": 0.5625, "reward_std": 0.249358132481575, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.374045729637146, "sampling/importance_sampling_ratio/mean": 0.9998302459716797, "sampling/importance_sampling_ratio/min": 0.4824265241622925, "sampling/sampling_logp_difference/max": 0.7289266586303711, "sampling/sampling_logp_difference/mean": 0.012472638860344887, "step": 297 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 758.0, "completions/max_terminated_length": 758.0, "completions/mean_length": 444.9375, "completions/mean_terminated_length": 444.9375, "completions/min_length": 209.0, "completions/min_terminated_length": 209.0, "entropy": 0.36902462504804134, "epoch": 0.15952890792291222, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 6630287.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.3408125638961792, "sampling/importance_sampling_ratio/mean": 1.0000786781311035, "sampling/importance_sampling_ratio/min": 0.26101750135421753, "sampling/sampling_logp_difference/max": 1.343167781829834, "sampling/sampling_logp_difference/mean": 0.01210569217801094, "step": 298 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.53125, "completions/max_length": 768.0, "completions/max_terminated_length": 722.0, "completions/mean_length": 634.9375, "completions/mean_terminated_length": 484.13336181640625, "completions/min_length": 333.0, "completions/min_terminated_length": 333.0, "entropy": 0.5509296171367168, "epoch": 0.16006423982869378, "frac_reward_zero_std": 0.5, "grad_norm": 0.01113807037472725, "learning_rate": 1e-05, "loss": 0.0254, "num_tokens": 6654581.0, "reward": 0.375, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.375, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.4714680910110474, "sampling/importance_sampling_ratio/mean": 0.9999656081199646, "sampling/importance_sampling_ratio/min": 0.7013254165649414, "sampling/sampling_logp_difference/max": 0.38626062870025635, "sampling/sampling_logp_difference/mean": 0.013161944225430489, "step": 299 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.3717230912297964e-05, "clip_ratio/low_min": 5.3717230912297964e-05, "clip_ratio/region_mean": 5.3717230912297964e-05, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 705.0, "completions/mean_length": 589.0625, "completions/mean_terminated_length": 538.9599609375, "completions/min_length": 308.0, "completions/min_terminated_length": 308.0, "entropy": 0.36450091376900673, "epoch": 0.16059957173447537, "frac_reward_zero_std": 0.0, "grad_norm": 0.007847234606742859, "learning_rate": 1e-05, "loss": 0.0477, "num_tokens": 6677431.0, "reward": 0.65625, "reward_std": 0.4628904461860657, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.3458384275436401, "sampling/importance_sampling_ratio/mean": 1.0003505945205688, "sampling/importance_sampling_ratio/min": 0.6132007241249084, "sampling/sampling_logp_difference/max": 0.489063024520874, "sampling/sampling_logp_difference/mean": 0.011224000714719296, "step": 300 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 766.0, "completions/mean_length": 504.15625, "completions/mean_terminated_length": 455.2962951660156, "completions/min_length": 238.0, "completions/min_terminated_length": 238.0, "entropy": 0.4634467475116253, "epoch": 0.16113490364025695, "frac_reward_zero_std": 0.0, "grad_norm": 0.02073357068002224, "learning_rate": 1e-05, "loss": 0.0323, "num_tokens": 6697276.0, "reward": 0.6875, "reward_std": 0.4082317352294922, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.8510899543762207, "sampling/importance_sampling_ratio/mean": 1.000158667564392, "sampling/importance_sampling_ratio/min": 0.5022976994514465, "sampling/sampling_logp_difference/max": 0.6885623931884766, "sampling/sampling_logp_difference/mean": 0.012787338346242905, "step": 301 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00016345419135177508, "clip_ratio/low_min": 0.00016345419135177508, "clip_ratio/region_mean": 0.00016345419135177508, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 732.0, "completions/mean_length": 560.15625, "completions/mean_terminated_length": 530.4642944335938, "completions/min_length": 229.0, "completions/min_terminated_length": 229.0, "entropy": 0.4693382754921913, "epoch": 0.16167023554603854, "frac_reward_zero_std": 0.25, "grad_norm": 0.013713146559894085, "learning_rate": 1e-05, "loss": 0.0335, "num_tokens": 6719025.0, "reward": 0.5625, "reward_std": 0.3924052119255066, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.7535113096237183, "sampling/importance_sampling_ratio/mean": 1.0005136728286743, "sampling/importance_sampling_ratio/min": 0.6734032034873962, "sampling/sampling_logp_difference/max": 0.5616202354431152, "sampling/sampling_logp_difference/mean": 0.013071180321276188, "step": 302 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00011687417281791568, "clip_ratio/low_min": 0.00011687417281791568, "clip_ratio/region_mean": 0.00011687417281791568, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 766.0, "completions/mean_length": 604.65625, "completions/mean_terminated_length": 558.9199829101562, "completions/min_length": 380.0, "completions/min_terminated_length": 380.0, "entropy": 0.49411723017692566, "epoch": 0.16220556745182013, "frac_reward_zero_std": 0.5, "grad_norm": 0.009031958878040314, "learning_rate": 1e-05, "loss": 0.0105, "num_tokens": 6741910.0, "reward": 0.53125, "reward_std": 0.2630178928375244, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.4679912328720093, "sampling/importance_sampling_ratio/mean": 0.999821126461029, "sampling/importance_sampling_ratio/min": 0.6487157344818115, "sampling/sampling_logp_difference/max": 0.43276071548461914, "sampling/sampling_logp_difference/mean": 0.014113925397396088, "step": 303 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00024655308516230434, "clip_ratio/low_min": 0.00024655308516230434, "clip_ratio/region_mean": 0.00024655308516230434, "completions/clipped_ratio": 0.28125, "completions/max_length": 768.0, "completions/max_terminated_length": 751.0, "completions/mean_length": 559.84375, "completions/mean_terminated_length": 478.3913269042969, "completions/min_length": 171.0, "completions/min_terminated_length": 171.0, "entropy": 0.719526868313551, "epoch": 0.16274089935760172, "frac_reward_zero_std": 0.25, "grad_norm": 0.006262456066906452, "learning_rate": 1e-05, "loss": -0.0247, "num_tokens": 6763609.0, "reward": 0.6875, "reward_std": 0.2925041913986206, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.457737922668457, "sampling/importance_sampling_ratio/mean": 1.0000252723693848, "sampling/importance_sampling_ratio/min": 0.2858038544654846, "sampling/sampling_logp_difference/max": 1.2524495124816895, "sampling/sampling_logp_difference/mean": 0.016774887219071388, "step": 304 }, { "clip_ratio/high_max": 5.439512460725382e-05, "clip_ratio/high_mean": 5.439512460725382e-05, "clip_ratio/low_mean": 0.00023651660740142688, "clip_ratio/low_min": 0.00023651660740142688, "clip_ratio/region_mean": 0.0002909117320086807, "completions/clipped_ratio": 0.28125, "completions/max_length": 768.0, "completions/max_terminated_length": 646.0, "completions/mean_length": 518.40625, "completions/mean_terminated_length": 420.7391357421875, "completions/min_length": 302.0, "completions/min_terminated_length": 302.0, "entropy": 0.6311408244073391, "epoch": 0.1632762312633833, "frac_reward_zero_std": 0.25, "grad_norm": 0.0065378895960748196, "learning_rate": 1e-05, "loss": 0.0192, "num_tokens": 6783854.0, "reward": 0.3125, "reward_std": 0.292504221200943, "rewards/accuracy_reward/mean": 0.3125, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.424057126045227, "sampling/importance_sampling_ratio/mean": 0.9998113512992859, "sampling/importance_sampling_ratio/min": 0.5049322843551636, "sampling/sampling_logp_difference/max": 0.6833310127258301, "sampling/sampling_logp_difference/mean": 0.017484525218605995, "step": 305 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.858142347075045e-05, "clip_ratio/low_min": 4.858142347075045e-05, "clip_ratio/region_mean": 4.858142347075045e-05, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 738.0, "completions/mean_length": 585.53125, "completions/mean_terminated_length": 524.7083740234375, "completions/min_length": 241.0, "completions/min_terminated_length": 241.0, "entropy": 0.4257782958447933, "epoch": 0.16381156316916487, "frac_reward_zero_std": 0.5, "grad_norm": 0.008525801822543144, "learning_rate": 1e-05, "loss": 0.0319, "num_tokens": 6806351.0, "reward": 0.5625, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.389258861541748, "sampling/importance_sampling_ratio/mean": 0.9999468922615051, "sampling/importance_sampling_ratio/min": 0.739719569683075, "sampling/sampling_logp_difference/max": 0.32877039909362793, "sampling/sampling_logp_difference/mean": 0.012768362648785114, "step": 306 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00011961722339037806, "clip_ratio/low_min": 0.00011961722339037806, "clip_ratio/region_mean": 0.00011961722339037806, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 681.0, "completions/mean_length": 551.625, "completions/mean_terminated_length": 520.7142944335938, "completions/min_length": 257.0, "completions/min_terminated_length": 257.0, "entropy": 0.36352822557091713, "epoch": 0.16434689507494646, "frac_reward_zero_std": 0.0, "grad_norm": 0.01421281136572361, "learning_rate": 1e-05, "loss": -0.0216, "num_tokens": 6828443.0, "reward": 0.6875, "reward_std": 0.44403791427612305, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.3255524635314941, "sampling/importance_sampling_ratio/mean": 0.9995919466018677, "sampling/importance_sampling_ratio/min": 0.6008109450340271, "sampling/sampling_logp_difference/max": 0.5094749927520752, "sampling/sampling_logp_difference/mean": 0.010572951287031174, "step": 307 }, { "clip_ratio/high_max": 5.427702853921801e-05, "clip_ratio/high_mean": 5.427702853921801e-05, "clip_ratio/low_mean": 4.069010537932627e-05, "clip_ratio/low_min": 4.069010537932627e-05, "clip_ratio/region_mean": 9.496713391854428e-05, "completions/clipped_ratio": 0.625, "completions/max_length": 768.0, "completions/max_terminated_length": 629.0, "completions/mean_length": 655.03125, "completions/mean_terminated_length": 466.75, "completions/min_length": 126.0, "completions/min_terminated_length": 126.0, "entropy": 0.547876700758934, "epoch": 0.16488222698072805, "frac_reward_zero_std": 0.25, "grad_norm": 0.009252430871129036, "learning_rate": 1e-05, "loss": -0.0019, "num_tokens": 6853996.0, "reward": 0.28125, "reward_std": 0.3471629321575165, "rewards/accuracy_reward/mean": 0.28125, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.3950695991516113, "sampling/importance_sampling_ratio/mean": 0.9997021555900574, "sampling/importance_sampling_ratio/min": 0.5319414138793945, "sampling/sampling_logp_difference/max": 0.6312220096588135, "sampling/sampling_logp_difference/mean": 0.013688534498214722, "step": 308 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 760.0, "completions/mean_length": 503.65625, "completions/mean_terminated_length": 495.1290283203125, "completions/min_length": 272.0, "completions/min_terminated_length": 272.0, "entropy": 0.31358828768134117, "epoch": 0.16541755888650964, "frac_reward_zero_std": 0.0, "grad_norm": 0.006506791803985834, "learning_rate": 1e-05, "loss": 0.0313, "num_tokens": 6873473.0, "reward": 0.84375, "reward_std": 0.3808925747871399, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.39066481590271, "sampling/importance_sampling_ratio/mean": 1.0000666379928589, "sampling/importance_sampling_ratio/min": 0.5800336599349976, "sampling/sampling_logp_difference/max": 0.5446691513061523, "sampling/sampling_logp_difference/mean": 0.010641239583492279, "step": 309 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.624491339200176e-05, "clip_ratio/low_min": 4.624491339200176e-05, "clip_ratio/region_mean": 4.624491339200176e-05, "completions/clipped_ratio": 0.3125, "completions/max_length": 768.0, "completions/max_terminated_length": 765.0, "completions/mean_length": 618.34375, "completions/mean_terminated_length": 550.3181762695312, "completions/min_length": 315.0, "completions/min_terminated_length": 315.0, "entropy": 0.4615786671638489, "epoch": 0.16595289079229122, "frac_reward_zero_std": 0.0, "grad_norm": 0.020220337435603142, "learning_rate": 1e-05, "loss": -0.0012, "num_tokens": 6897140.0, "reward": 0.5, "reward_std": 0.5081326961517334, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5080004930496216, "sampling/importance_sampling_ratio/max": 1.6202958822250366, "sampling/importance_sampling_ratio/mean": 1.0001581907272339, "sampling/importance_sampling_ratio/min": 0.7441466450691223, "sampling/sampling_logp_difference/max": 0.4826087951660156, "sampling/sampling_logp_difference/mean": 0.012216374278068542, "step": 310 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.2988554671173915e-05, "clip_ratio/low_min": 5.2988554671173915e-05, "clip_ratio/region_mean": 5.2988554671173915e-05, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 714.0, "completions/mean_length": 537.03125, "completions/mean_terminated_length": 472.3599853515625, "completions/min_length": 265.0, "completions/min_terminated_length": 265.0, "entropy": 0.406319011002779, "epoch": 0.1664882226980728, "frac_reward_zero_std": 0.0, "grad_norm": 0.019443826749920845, "learning_rate": 1e-05, "loss": 0.0616, "num_tokens": 6918861.0, "reward": 0.53125, "reward_std": 0.4534739851951599, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.585098385810852, "sampling/importance_sampling_ratio/mean": 0.9999993443489075, "sampling/importance_sampling_ratio/min": 0.589286744594574, "sampling/sampling_logp_difference/max": 0.5288424491882324, "sampling/sampling_logp_difference/mean": 0.012301858514547348, "step": 311 }, { "clip_ratio/high_max": 7.122506940504536e-05, "clip_ratio/high_mean": 7.122506940504536e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 7.122506940504536e-05, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 587.0, "completions/mean_length": 450.46875, "completions/mean_terminated_length": 429.3000183105469, "completions/min_length": 263.0, "completions/min_terminated_length": 263.0, "entropy": 0.465481735765934, "epoch": 0.1670235546038544, "frac_reward_zero_std": 0.25, "grad_norm": 0.00506245531141758, "learning_rate": 1e-05, "loss": 0.0612, "num_tokens": 6936260.0, "reward": 0.875, "reward_std": 0.292504221200943, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 1.5154260396957397, "sampling/importance_sampling_ratio/mean": 1.0000495910644531, "sampling/importance_sampling_ratio/min": 0.6307534575462341, "sampling/sampling_logp_difference/max": 0.46084022521972656, "sampling/sampling_logp_difference/mean": 0.013272913172841072, "step": 312 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00011962919234065339, "clip_ratio/low_min": 0.00011962919234065339, "clip_ratio/region_mean": 0.00011962919234065339, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 760.0, "completions/mean_length": 471.125, "completions/mean_terminated_length": 451.3333435058594, "completions/min_length": 268.0, "completions/min_terminated_length": 268.0, "entropy": 0.33092113956809044, "epoch": 0.16755888650963596, "frac_reward_zero_std": 0.5, "grad_norm": 0.006002717185765505, "learning_rate": 1e-05, "loss": 0.0025, "num_tokens": 6955368.0, "reward": 0.8125, "reward_std": 0.2587745785713196, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.358210563659668, "sampling/importance_sampling_ratio/mean": 0.9998170137405396, "sampling/importance_sampling_ratio/min": 0.7616143226623535, "sampling/sampling_logp_difference/max": 0.3061680793762207, "sampling/sampling_logp_difference/mean": 0.010767726227641106, "step": 313 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 746.0, "completions/max_terminated_length": 746.0, "completions/mean_length": 420.28125, "completions/mean_terminated_length": 420.28125, "completions/min_length": 235.0, "completions/min_terminated_length": 235.0, "entropy": 0.337965851649642, "epoch": 0.16809421841541755, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0177, "num_tokens": 6972241.0, "reward": 0.9375, "reward_std": 0.1157275140285492, "rewards/accuracy_reward/mean": 0.9375, "rewards/accuracy_reward/std": 0.24593468010425568, "sampling/importance_sampling_ratio/max": 1.4133011102676392, "sampling/importance_sampling_ratio/mean": 1.0003267526626587, "sampling/importance_sampling_ratio/min": 0.5993703603744507, "sampling/sampling_logp_difference/max": 0.5118756294250488, "sampling/sampling_logp_difference/mean": 0.011421999894082546, "step": 314 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 663.0, "completions/mean_length": 458.96875, "completions/mean_terminated_length": 449.0, "completions/min_length": 296.0, "completions/min_terminated_length": 296.0, "entropy": 0.38452618569135666, "epoch": 0.16862955032119914, "frac_reward_zero_std": 0.5, "grad_norm": 0.012224256992340088, "learning_rate": 1e-05, "loss": 0.0461, "num_tokens": 6990400.0, "reward": 0.8125, "reward_std": 0.249358132481575, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.2745556831359863, "sampling/importance_sampling_ratio/mean": 0.9997187256813049, "sampling/importance_sampling_ratio/min": 0.5163213610649109, "sampling/sampling_logp_difference/max": 0.6610260009765625, "sampling/sampling_logp_difference/mean": 0.012734955176711082, "step": 315 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.727664003847167e-05, "clip_ratio/low_min": 6.727664003847167e-05, "clip_ratio/region_mean": 6.727664003847167e-05, "completions/clipped_ratio": 0.34375, "completions/max_length": 768.0, "completions/max_terminated_length": 700.0, "completions/mean_length": 577.6875, "completions/mean_terminated_length": 478.0, "completions/min_length": 240.0, "completions/min_terminated_length": 240.0, "entropy": 0.6516948454082012, "epoch": 0.16916488222698073, "frac_reward_zero_std": 0.5, "grad_norm": 0.011612790636718273, "learning_rate": 1e-05, "loss": 0.0107, "num_tokens": 7012534.0, "reward": 0.3125, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.3125, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.5484830141067505, "sampling/importance_sampling_ratio/mean": 0.9997885823249817, "sampling/importance_sampling_ratio/min": 0.6952348947525024, "sampling/sampling_logp_difference/max": 0.437275767326355, "sampling/sampling_logp_difference/mean": 0.01592925749719143, "step": 316 }, { "clip_ratio/high_max": 6.845564348623157e-05, "clip_ratio/high_mean": 6.845564348623157e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 6.845564348623157e-05, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 768.0, "completions/mean_length": 565.28125, "completions/mean_terminated_length": 527.74072265625, "completions/min_length": 254.0, "completions/min_terminated_length": 254.0, "entropy": 0.3407766781747341, "epoch": 0.16970021413276232, "frac_reward_zero_std": 0.25, "grad_norm": 0.0070942766033113, "learning_rate": 1e-05, "loss": 0.0179, "num_tokens": 7034247.0, "reward": 0.6875, "reward_std": 0.38298875093460083, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.4793986082077026, "sampling/importance_sampling_ratio/mean": 1.0001431703567505, "sampling/importance_sampling_ratio/min": 0.46491867303848267, "sampling/sampling_logp_difference/max": 0.765892744064331, "sampling/sampling_logp_difference/mean": 0.01064304169267416, "step": 317 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.49350622552447e-05, "clip_ratio/low_min": 6.49350622552447e-05, "clip_ratio/region_mean": 6.49350622552447e-05, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 621.0, "completions/mean_length": 494.625, "completions/mean_terminated_length": 455.5714416503906, "completions/min_length": 276.0, "completions/min_terminated_length": 276.0, "entropy": 0.5356157831847668, "epoch": 0.1702355460385439, "frac_reward_zero_std": 0.25, "grad_norm": 0.020239029079675674, "learning_rate": 1e-05, "loss": 0.1042, "num_tokens": 7053795.0, "reward": 0.75, "reward_std": 0.3650856614112854, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.3353266716003418, "sampling/importance_sampling_ratio/mean": 0.9998921751976013, "sampling/importance_sampling_ratio/min": 0.5428999066352844, "sampling/sampling_logp_difference/max": 0.6108303070068359, "sampling/sampling_logp_difference/mean": 0.013654596172273159, "step": 318 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.65625, "completions/max_length": 768.0, "completions/max_terminated_length": 667.0, "completions/mean_length": 686.875, "completions/mean_terminated_length": 532.0, "completions/min_length": 424.0, "completions/min_terminated_length": 424.0, "entropy": 0.7220774479210377, "epoch": 0.1707708779443255, "frac_reward_zero_std": 0.75, "grad_norm": 0.003973943646997213, "learning_rate": 1e-05, "loss": 0.0373, "num_tokens": 7080335.0, "reward": 0.15625, "reward_std": 0.1293872892856598, "rewards/accuracy_reward/mean": 0.15625, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.607334017753601, "sampling/importance_sampling_ratio/mean": 1.0000230073928833, "sampling/importance_sampling_ratio/min": 0.5586801767349243, "sampling/sampling_logp_difference/max": 0.5821781158447266, "sampling/sampling_logp_difference/mean": 0.018136102706193924, "step": 319 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 760.0, "completions/mean_length": 587.96875, "completions/mean_terminated_length": 569.3448486328125, "completions/min_length": 323.0, "completions/min_terminated_length": 323.0, "entropy": 0.36708491668105125, "epoch": 0.17130620985010706, "frac_reward_zero_std": 0.5, "grad_norm": 0.003349228762090206, "learning_rate": 1e-05, "loss": -0.0248, "num_tokens": 7103398.0, "reward": 0.90625, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.4077260494232178, "sampling/importance_sampling_ratio/mean": 0.9999406337738037, "sampling/importance_sampling_ratio/min": 0.7000519633293152, "sampling/sampling_logp_difference/max": 0.3566007614135742, "sampling/sampling_logp_difference/mean": 0.011874967254698277, "step": 320 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00011507716408232227, "clip_ratio/low_min": 0.00011507716408232227, "clip_ratio/region_mean": 0.00011507716408232227, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 759.0, "completions/mean_length": 554.71875, "completions/mean_terminated_length": 524.25, "completions/min_length": 290.0, "completions/min_terminated_length": 290.0, "entropy": 0.3180145863443613, "epoch": 0.17184154175588864, "frac_reward_zero_std": 0.0, "grad_norm": 0.025451544672250748, "learning_rate": 1e-05, "loss": 0.011, "num_tokens": 7124637.0, "reward": 0.5, "reward_std": 0.44403791427612305, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5080004930496216, "sampling/importance_sampling_ratio/max": 1.3194462060928345, "sampling/importance_sampling_ratio/mean": 0.9997974634170532, "sampling/importance_sampling_ratio/min": 0.697174072265625, "sampling/sampling_logp_difference/max": 0.360720157623291, "sampling/sampling_logp_difference/mean": 0.010110925883054733, "step": 321 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.8934463595505804e-05, "clip_ratio/low_min": 5.8934463595505804e-05, "clip_ratio/region_mean": 5.8934463595505804e-05, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 735.0, "completions/mean_length": 539.21875, "completions/mean_terminated_length": 515.5516967773438, "completions/min_length": 77.0, "completions/min_terminated_length": 77.0, "entropy": 0.4827127531170845, "epoch": 0.17237687366167023, "frac_reward_zero_std": 0.25, "grad_norm": 0.013899889774620533, "learning_rate": 1e-05, "loss": -0.0222, "num_tokens": 7145948.0, "reward": 0.71875, "reward_std": 0.3198433816432953, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.398549199104309, "sampling/importance_sampling_ratio/mean": 0.9996058940887451, "sampling/importance_sampling_ratio/min": 0.5230233073234558, "sampling/sampling_logp_difference/max": 0.6481292247772217, "sampling/sampling_logp_difference/mean": 0.014207660220563412, "step": 322 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 722.0, "completions/mean_length": 561.4375, "completions/mean_terminated_length": 492.5833435058594, "completions/min_length": 203.0, "completions/min_terminated_length": 203.0, "entropy": 0.4749818481504917, "epoch": 0.17291220556745182, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": -0.0156, "num_tokens": 7167858.0, "reward": 0.65625, "reward_std": 0.1293872892856598, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.381294846534729, "sampling/importance_sampling_ratio/mean": 0.9998615384101868, "sampling/importance_sampling_ratio/min": 0.7124887704849243, "sampling/sampling_logp_difference/max": 0.3389911651611328, "sampling/sampling_logp_difference/mean": 0.014107558876276016, "step": 323 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 730.0, "completions/mean_length": 514.375, "completions/mean_terminated_length": 478.14288330078125, "completions/min_length": 201.0, "completions/min_terminated_length": 201.0, "entropy": 0.48403423465788364, "epoch": 0.1734475374732334, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": -0.0452, "num_tokens": 7188086.0, "reward": 0.71875, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.5673373937606812, "sampling/importance_sampling_ratio/mean": 0.9995380640029907, "sampling/importance_sampling_ratio/min": 0.5971093773841858, "sampling/sampling_logp_difference/max": 0.5156550407409668, "sampling/sampling_logp_difference/mean": 0.013777234591543674, "step": 324 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0001659023510001134, "clip_ratio/low_min": 0.0001659023510001134, "clip_ratio/region_mean": 0.0001659023510001134, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 697.0, "completions/mean_length": 549.6875, "completions/mean_terminated_length": 518.5, "completions/min_length": 352.0, "completions/min_terminated_length": 352.0, "entropy": 0.3863347228616476, "epoch": 0.173982869379015, "frac_reward_zero_std": 0.25, "grad_norm": 0.012430465780198574, "learning_rate": 1e-05, "loss": 0.0405, "num_tokens": 7209636.0, "reward": 0.65625, "reward_std": 0.3787454068660736, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.2858341932296753, "sampling/importance_sampling_ratio/mean": 1.0000137090682983, "sampling/importance_sampling_ratio/min": 0.774046003818512, "sampling/sampling_logp_difference/max": 0.25612401962280273, "sampling/sampling_logp_difference/mean": 0.011497581377625465, "step": 325 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 767.0, "completions/mean_length": 538.34375, "completions/mean_terminated_length": 530.9354858398438, "completions/min_length": 321.0, "completions/min_terminated_length": 321.0, "entropy": 0.386878427118063, "epoch": 0.1745182012847966, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0281, "num_tokens": 7230695.0, "reward": 0.71875, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.514595627784729, "sampling/importance_sampling_ratio/mean": 0.9998791217803955, "sampling/importance_sampling_ratio/min": 0.5104681849479675, "sampling/sampling_logp_difference/max": 0.6724269390106201, "sampling/sampling_logp_difference/mean": 0.012221625074744225, "step": 326 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.28125, "completions/max_length": 768.0, "completions/max_terminated_length": 739.0, "completions/mean_length": 534.40625, "completions/mean_terminated_length": 443.0, "completions/min_length": 265.0, "completions/min_terminated_length": 265.0, "entropy": 0.4374929815530777, "epoch": 0.17505353319057815, "frac_reward_zero_std": 0.5, "grad_norm": 0.005102497525513172, "learning_rate": 1e-05, "loss": 0.0637, "num_tokens": 7251404.0, "reward": 0.625, "reward_std": 0.2314550280570984, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.3603565692901611, "sampling/importance_sampling_ratio/mean": 0.9999849200248718, "sampling/importance_sampling_ratio/min": 0.7341492176055908, "sampling/sampling_logp_difference/max": 0.30904293060302734, "sampling/sampling_logp_difference/mean": 0.0129553759470582, "step": 327 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 757.0, "completions/mean_length": 567.21875, "completions/mean_terminated_length": 530.0370483398438, "completions/min_length": 255.0, "completions/min_terminated_length": 255.0, "entropy": 0.3481616973876953, "epoch": 0.17558886509635974, "frac_reward_zero_std": 0.5, "grad_norm": 0.0023771924898028374, "learning_rate": 1e-05, "loss": -0.0104, "num_tokens": 7273675.0, "reward": 0.78125, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.4590474367141724, "sampling/importance_sampling_ratio/mean": 0.9999516010284424, "sampling/importance_sampling_ratio/min": 0.6946262121200562, "sampling/sampling_logp_difference/max": 0.37778377532958984, "sampling/sampling_logp_difference/mean": 0.011197168380022049, "step": 328 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00011354553862474859, "clip_ratio/low_min": 0.00011354553862474859, "clip_ratio/region_mean": 0.00011354553862474859, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 701.0, "completions/mean_length": 528.65625, "completions/mean_terminated_length": 503.89654541015625, "completions/min_length": 310.0, "completions/min_terminated_length": 310.0, "entropy": 0.42359526082873344, "epoch": 0.17612419700214133, "frac_reward_zero_std": 0.25, "grad_norm": 0.011642787605524063, "learning_rate": 1e-05, "loss": 0.0282, "num_tokens": 7294440.0, "reward": 0.71875, "reward_std": 0.378745436668396, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.340375304222107, "sampling/importance_sampling_ratio/mean": 1.0001251697540283, "sampling/importance_sampling_ratio/min": 0.6960331797599792, "sampling/sampling_logp_difference/max": 0.36235785484313965, "sampling/sampling_logp_difference/mean": 0.012815607711672783, "step": 329 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 750.0, "completions/mean_length": 563.75, "completions/mean_terminated_length": 534.5714721679688, "completions/min_length": 317.0, "completions/min_terminated_length": 317.0, "entropy": 0.4971456937491894, "epoch": 0.1766595289079229, "frac_reward_zero_std": 0.25, "grad_norm": 0.009611795656383038, "learning_rate": 1e-05, "loss": 0.0238, "num_tokens": 7316760.0, "reward": 0.75, "reward_std": 0.3514062762260437, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.4389238357543945, "sampling/importance_sampling_ratio/mean": 0.9995952248573303, "sampling/importance_sampling_ratio/min": 0.7226351499557495, "sampling/sampling_logp_difference/max": 0.3638954162597656, "sampling/sampling_logp_difference/mean": 0.014337463304400444, "step": 330 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 705.0, "completions/max_terminated_length": 705.0, "completions/mean_length": 452.0625, "completions/mean_terminated_length": 452.0625, "completions/min_length": 294.0, "completions/min_terminated_length": 294.0, "entropy": 0.39480288326740265, "epoch": 0.1771948608137045, "frac_reward_zero_std": 0.5, "grad_norm": 0.005923368968069553, "learning_rate": 1e-05, "loss": 0.0117, "num_tokens": 7334642.0, "reward": 0.71875, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.5162053108215332, "sampling/importance_sampling_ratio/mean": 1.0008217096328735, "sampling/importance_sampling_ratio/min": 0.6893972754478455, "sampling/sampling_logp_difference/max": 0.4162106513977051, "sampling/sampling_logp_difference/mean": 0.013177015818655491, "step": 331 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 760.0, "completions/mean_length": 506.34375, "completions/mean_terminated_length": 488.9000244140625, "completions/min_length": 289.0, "completions/min_terminated_length": 289.0, "entropy": 0.42032600566744804, "epoch": 0.1777301927194861, "frac_reward_zero_std": 0.5, "grad_norm": 0.0068185231648385525, "learning_rate": 1e-05, "loss": 0.0139, "num_tokens": 7354253.0, "reward": 0.84375, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.5562868118286133, "sampling/importance_sampling_ratio/mean": 0.9997234344482422, "sampling/importance_sampling_ratio/min": 0.5751549005508423, "sampling/sampling_logp_difference/max": 0.5531158447265625, "sampling/sampling_logp_difference/mean": 0.012651457451283932, "step": 332 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 9.495236372458749e-05, "clip_ratio/low_min": 9.495236372458749e-05, "clip_ratio/region_mean": 9.495236372458749e-05, "completions/clipped_ratio": 0.3125, "completions/max_length": 768.0, "completions/max_terminated_length": 749.0, "completions/mean_length": 617.3125, "completions/mean_terminated_length": 548.8181762695312, "completions/min_length": 326.0, "completions/min_terminated_length": 326.0, "entropy": 0.36886048689484596, "epoch": 0.17826552462526768, "frac_reward_zero_std": 0.25, "grad_norm": 0.008584854193031788, "learning_rate": 1e-05, "loss": 0.0608, "num_tokens": 7377591.0, "reward": 0.5, "reward_std": 0.3650856614112854, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5080004930496216, "sampling/importance_sampling_ratio/max": 1.3923873901367188, "sampling/importance_sampling_ratio/mean": 1.0001944303512573, "sampling/importance_sampling_ratio/min": 0.7187380790710449, "sampling/sampling_logp_difference/max": 0.33101987838745117, "sampling/sampling_logp_difference/mean": 0.012307781726121902, "step": 333 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00017662672325968742, "clip_ratio/low_min": 0.00017662672325968742, "clip_ratio/region_mean": 0.00017662672325968742, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 728.0, "completions/mean_length": 524.3125, "completions/mean_terminated_length": 499.10345458984375, "completions/min_length": 220.0, "completions/min_terminated_length": 220.0, "entropy": 0.6167640164494514, "epoch": 0.17880085653104924, "frac_reward_zero_std": 0.25, "grad_norm": 0.009452517144382, "learning_rate": 1e-05, "loss": 0.0273, "num_tokens": 7398289.0, "reward": 0.6875, "reward_std": 0.3924052119255066, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.431172251701355, "sampling/importance_sampling_ratio/mean": 0.999972939491272, "sampling/importance_sampling_ratio/min": 0.5996284484863281, "sampling/sampling_logp_difference/max": 0.5114450454711914, "sampling/sampling_logp_difference/mean": 0.01580020599067211, "step": 334 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00010184981510974467, "clip_ratio/low_min": 0.00010184981510974467, "clip_ratio/region_mean": 0.00010184981510974467, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 636.0, "completions/mean_length": 539.15625, "completions/mean_terminated_length": 496.77777099609375, "completions/min_length": 178.0, "completions/min_terminated_length": 178.0, "entropy": 0.3947272449731827, "epoch": 0.17933618843683083, "frac_reward_zero_std": 0.5, "grad_norm": 0.028039462864398956, "learning_rate": 1e-05, "loss": 0.086, "num_tokens": 7419398.0, "reward": 0.84375, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.4883170127868652, "sampling/importance_sampling_ratio/mean": 0.9996607303619385, "sampling/importance_sampling_ratio/min": 0.7330660820007324, "sampling/sampling_logp_difference/max": 0.3976459503173828, "sampling/sampling_logp_difference/mean": 0.012055269442498684, "step": 335 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.106497312430292e-05, "clip_ratio/low_min": 6.106497312430292e-05, "clip_ratio/region_mean": 6.106497312430292e-05, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 738.0, "completions/mean_length": 496.03125, "completions/mean_terminated_length": 477.9000244140625, "completions/min_length": 163.0, "completions/min_terminated_length": 163.0, "entropy": 0.3668104074895382, "epoch": 0.17987152034261242, "frac_reward_zero_std": 0.5, "grad_norm": 0.005759675987064838, "learning_rate": 1e-05, "loss": -0.0024, "num_tokens": 7439207.0, "reward": 0.84375, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.5150907039642334, "sampling/importance_sampling_ratio/mean": 1.0002535581588745, "sampling/importance_sampling_ratio/min": 0.5885434150695801, "sampling/sampling_logp_difference/max": 0.5301046371459961, "sampling/sampling_logp_difference/mean": 0.010901729576289654, "step": 336 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00029001983057241887, "clip_ratio/low_min": 0.00029001983057241887, "clip_ratio/region_mean": 0.00029001983057241887, "completions/clipped_ratio": 0.34375, "completions/max_length": 768.0, "completions/max_terminated_length": 741.0, "completions/mean_length": 624.09375, "completions/mean_terminated_length": 548.7142944335938, "completions/min_length": 322.0, "completions/min_terminated_length": 322.0, "entropy": 0.4067046120762825, "epoch": 0.180406852248394, "frac_reward_zero_std": 0.25, "grad_norm": 0.007753439247608185, "learning_rate": 1e-05, "loss": 0.0364, "num_tokens": 7463946.0, "reward": 0.34375, "reward_std": 0.3377464711666107, "rewards/accuracy_reward/mean": 0.34375, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.5898274183273315, "sampling/importance_sampling_ratio/mean": 1.0000672340393066, "sampling/importance_sampling_ratio/min": 0.5347333550453186, "sampling/sampling_logp_difference/max": 0.6259870529174805, "sampling/sampling_logp_difference/mean": 0.012336867861449718, "step": 337 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 753.0, "completions/mean_length": 525.25, "completions/mean_terminated_length": 517.4193115234375, "completions/min_length": 290.0, "completions/min_terminated_length": 290.0, "entropy": 0.3610754832625389, "epoch": 0.1809421841541756, "frac_reward_zero_std": 0.5, "grad_norm": 0.002317107515409589, "learning_rate": 1e-05, "loss": -0.0514, "num_tokens": 7484298.0, "reward": 0.90625, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.3228033781051636, "sampling/importance_sampling_ratio/mean": 0.9998362064361572, "sampling/importance_sampling_ratio/min": 0.4087929427623749, "sampling/sampling_logp_difference/max": 0.8945465087890625, "sampling/sampling_logp_difference/mean": 0.011356317438185215, "step": 338 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.8112505939789116e-05, "clip_ratio/low_min": 5.8112505939789116e-05, "clip_ratio/region_mean": 5.8112505939789116e-05, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 679.0, "completions/mean_length": 492.75, "completions/mean_terminated_length": 474.4000244140625, "completions/min_length": 267.0, "completions/min_terminated_length": 267.0, "entropy": 0.34434760361909866, "epoch": 0.18147751605995718, "frac_reward_zero_std": 0.5, "grad_norm": 0.00666041299700737, "learning_rate": 1e-05, "loss": 0.0381, "num_tokens": 7503642.0, "reward": 0.84375, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.3360539674758911, "sampling/importance_sampling_ratio/mean": 0.9999451041221619, "sampling/importance_sampling_ratio/min": 0.5476760864257812, "sampling/sampling_logp_difference/max": 0.6020712852478027, "sampling/sampling_logp_difference/mean": 0.011382617987692356, "step": 339 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00022654929489362985, "clip_ratio/low_min": 0.00022654929489362985, "clip_ratio/region_mean": 0.00022654929489362985, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 714.0, "completions/mean_length": 530.15625, "completions/mean_terminated_length": 463.55999755859375, "completions/min_length": 183.0, "completions/min_terminated_length": 183.0, "entropy": 0.3601557370275259, "epoch": 0.18201284796573874, "frac_reward_zero_std": 0.25, "grad_norm": 0.01300281286239624, "learning_rate": 1e-05, "loss": 0.029, "num_tokens": 7523927.0, "reward": 0.75, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.5163910388946533, "sampling/importance_sampling_ratio/mean": 1.0000921487808228, "sampling/importance_sampling_ratio/min": 0.6854645013809204, "sampling/sampling_logp_difference/max": 0.4163331985473633, "sampling/sampling_logp_difference/mean": 0.01064501516520977, "step": 340 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00018614684086060151, "clip_ratio/low_min": 0.00018614684086060151, "clip_ratio/region_mean": 0.00018614684086060151, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 749.0, "completions/mean_length": 563.03125, "completions/mean_terminated_length": 505.6399841308594, "completions/min_length": 292.0, "completions/min_terminated_length": 292.0, "entropy": 0.3186303861439228, "epoch": 0.18254817987152033, "frac_reward_zero_std": 0.0, "grad_norm": 0.006939996499568224, "learning_rate": 1e-05, "loss": 0.0407, "num_tokens": 7546128.0, "reward": 0.5, "reward_std": 0.4082317352294922, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5080004930496216, "sampling/importance_sampling_ratio/max": 1.3860214948654175, "sampling/importance_sampling_ratio/mean": 0.999988317489624, "sampling/importance_sampling_ratio/min": 0.6332712769508362, "sampling/sampling_logp_difference/max": 0.45685648918151855, "sampling/sampling_logp_difference/mean": 0.010448703542351723, "step": 341 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 675.0, "completions/max_terminated_length": 675.0, "completions/mean_length": 487.75, "completions/mean_terminated_length": 487.75, "completions/min_length": 281.0, "completions/min_terminated_length": 281.0, "entropy": 0.3758729547262192, "epoch": 0.18308351177730192, "frac_reward_zero_std": 0.75, "grad_norm": 0.0025084316730499268, "learning_rate": 1e-05, "loss": -0.0077, "num_tokens": 7565600.0, "reward": 0.96875, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.96875, "rewards/accuracy_reward/std": 0.1767766922712326, "sampling/importance_sampling_ratio/max": 1.6143831014633179, "sampling/importance_sampling_ratio/mean": 1.0002031326293945, "sampling/importance_sampling_ratio/min": 0.6292134523391724, "sampling/sampling_logp_difference/max": 0.47895288467407227, "sampling/sampling_logp_difference/mean": 0.012416450306773186, "step": 342 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 7.977025961736217e-05, "clip_ratio/low_min": 7.977025961736217e-05, "clip_ratio/region_mean": 7.977025961736217e-05, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 757.0, "completions/mean_length": 472.5625, "completions/mean_terminated_length": 463.0322570800781, "completions/min_length": 272.0, "completions/min_terminated_length": 272.0, "entropy": 0.4267321713268757, "epoch": 0.1836188436830835, "frac_reward_zero_std": 0.75, "grad_norm": 0.0029031119775027037, "learning_rate": 1e-05, "loss": -0.0016, "num_tokens": 7584178.0, "reward": 0.96875, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.96875, "rewards/accuracy_reward/std": 0.1767766922712326, "sampling/importance_sampling_ratio/max": 1.7095082998275757, "sampling/importance_sampling_ratio/mean": 0.9997206926345825, "sampling/importance_sampling_ratio/min": 0.6359575986862183, "sampling/sampling_logp_difference/max": 0.5362057685852051, "sampling/sampling_logp_difference/mean": 0.012804127298295498, "step": 343 }, { "clip_ratio/high_max": 6.503642362076789e-05, "clip_ratio/high_mean": 6.503642362076789e-05, "clip_ratio/low_mean": 9.452578160562553e-05, "clip_ratio/low_min": 9.452578160562553e-05, "clip_ratio/region_mean": 0.00015956220522639342, "completions/clipped_ratio": 0.375, "completions/max_length": 768.0, "completions/max_terminated_length": 660.0, "completions/mean_length": 581.375, "completions/mean_terminated_length": 469.3999938964844, "completions/min_length": 333.0, "completions/min_terminated_length": 333.0, "entropy": 0.49340501241385937, "epoch": 0.1841541755888651, "frac_reward_zero_std": 0.25, "grad_norm": 0.002044502180069685, "learning_rate": 1e-05, "loss": 0.073, "num_tokens": 7607190.0, "reward": 0.46875, "reward_std": 0.3198433816432953, "rewards/accuracy_reward/mean": 0.46875, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.4086116552352905, "sampling/importance_sampling_ratio/mean": 1.0002012252807617, "sampling/importance_sampling_ratio/min": 0.6478887796401978, "sampling/sampling_logp_difference/max": 0.4340362548828125, "sampling/sampling_logp_difference/mean": 0.014010047540068626, "step": 344 }, { "clip_ratio/high_max": 5.1250513934064656e-05, "clip_ratio/high_mean": 5.1250513934064656e-05, "clip_ratio/low_mean": 0.00010124410255230032, "clip_ratio/low_min": 0.00010124410255230032, "clip_ratio/region_mean": 0.00015249461648636498, "completions/clipped_ratio": 0.28125, "completions/max_length": 768.0, "completions/max_terminated_length": 739.0, "completions/mean_length": 609.5625, "completions/mean_terminated_length": 547.5652465820312, "completions/min_length": 299.0, "completions/min_terminated_length": 299.0, "entropy": 0.40680334717035294, "epoch": 0.1846895074946467, "frac_reward_zero_std": 0.5, "grad_norm": 0.01422973908483982, "learning_rate": 1e-05, "loss": 0.0472, "num_tokens": 7630456.0, "reward": 0.5, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5080004930496216, "sampling/importance_sampling_ratio/max": 1.3167784214019775, "sampling/importance_sampling_ratio/mean": 0.9998512268066406, "sampling/importance_sampling_ratio/min": 0.7650836110115051, "sampling/sampling_logp_difference/max": 0.2751882076263428, "sampling/sampling_logp_difference/mean": 0.012928473763167858, "step": 345 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00010970633593387902, "clip_ratio/low_min": 0.00010970633593387902, "clip_ratio/region_mean": 0.00010970633593387902, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 697.0, "completions/mean_length": 494.5625, "completions/mean_terminated_length": 466.2758483886719, "completions/min_length": 245.0, "completions/min_terminated_length": 245.0, "entropy": 0.5458939038217068, "epoch": 0.18522483940042828, "frac_reward_zero_std": 0.5, "grad_norm": 0.003950346261262894, "learning_rate": 1e-05, "loss": 0.0319, "num_tokens": 7650042.0, "reward": 0.75, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.413759469985962, "sampling/importance_sampling_ratio/mean": 1.0001903772354126, "sampling/importance_sampling_ratio/min": 0.6996801495552063, "sampling/sampling_logp_difference/max": 0.3571319580078125, "sampling/sampling_logp_difference/mean": 0.013414536602795124, "step": 346 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00017880662926472723, "clip_ratio/low_min": 0.00017880662926472723, "clip_ratio/region_mean": 0.00017880662926472723, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 754.0, "completions/mean_length": 579.625, "completions/mean_terminated_length": 526.8800048828125, "completions/min_length": 316.0, "completions/min_terminated_length": 316.0, "entropy": 0.4780072160065174, "epoch": 0.18576017130620984, "frac_reward_zero_std": 0.0, "grad_norm": 0.025157634168863297, "learning_rate": 1e-05, "loss": 0.0936, "num_tokens": 7672518.0, "reward": 0.5625, "reward_std": 0.49022960662841797, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.3807069063186646, "sampling/importance_sampling_ratio/mean": 1.0001271963119507, "sampling/importance_sampling_ratio/min": 0.44348034262657166, "sampling/sampling_logp_difference/max": 0.8131017684936523, "sampling/sampling_logp_difference/mean": 0.013579275459051132, "step": 347 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 729.0, "completions/max_terminated_length": 729.0, "completions/mean_length": 442.15625, "completions/mean_terminated_length": 442.15625, "completions/min_length": 274.0, "completions/min_terminated_length": 274.0, "entropy": 0.4018973298370838, "epoch": 0.18629550321199143, "frac_reward_zero_std": 0.75, "grad_norm": 0.003933882340788841, "learning_rate": 1e-05, "loss": 0.001, "num_tokens": 7690275.0, "reward": 0.9375, "reward_std": 0.1157275140285492, "rewards/accuracy_reward/mean": 0.9375, "rewards/accuracy_reward/std": 0.24593468010425568, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0001379251480103, "sampling/importance_sampling_ratio/min": 0.6150002479553223, "sampling/sampling_logp_difference/max": 0.7210378646850586, "sampling/sampling_logp_difference/mean": 0.012157607823610306, "step": 348 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00014049775927560404, "clip_ratio/low_min": 0.00014049775927560404, "clip_ratio/region_mean": 0.00014049775927560404, "completions/clipped_ratio": 0.28125, "completions/max_length": 768.0, "completions/max_terminated_length": 576.0, "completions/mean_length": 499.40625, "completions/mean_terminated_length": 394.3043518066406, "completions/min_length": 216.0, "completions/min_terminated_length": 216.0, "entropy": 0.42569514364004135, "epoch": 0.18683083511777301, "frac_reward_zero_std": 0.25, "grad_norm": 0.010466511361300945, "learning_rate": 1e-05, "loss": -0.0032, "num_tokens": 7709648.0, "reward": 0.3125, "reward_std": 0.38298875093460083, "rewards/accuracy_reward/mean": 0.3125, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.4525471925735474, "sampling/importance_sampling_ratio/mean": 0.9996471405029297, "sampling/importance_sampling_ratio/min": 0.7371333241462708, "sampling/sampling_logp_difference/max": 0.3733186721801758, "sampling/sampling_logp_difference/mean": 0.013134589418768883, "step": 349 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 659.0, "completions/mean_length": 502.78125, "completions/mean_terminated_length": 475.3448181152344, "completions/min_length": 227.0, "completions/min_terminated_length": 227.0, "entropy": 0.42978648096323013, "epoch": 0.1873661670235546, "frac_reward_zero_std": 0.5, "grad_norm": 0.0021181588526815176, "learning_rate": 1e-05, "loss": 0.0603, "num_tokens": 7729569.0, "reward": 0.65625, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.4324434995651245, "sampling/importance_sampling_ratio/mean": 0.999704897403717, "sampling/importance_sampling_ratio/min": 0.5219557285308838, "sampling/sampling_logp_difference/max": 0.6501724720001221, "sampling/sampling_logp_difference/mean": 0.013171912170946598, "step": 350 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 8.967001485871151e-05, "clip_ratio/low_min": 8.967001485871151e-05, "clip_ratio/region_mean": 8.967001485871151e-05, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 764.0, "completions/mean_length": 512.3125, "completions/mean_terminated_length": 453.3077087402344, "completions/min_length": 252.0, "completions/min_terminated_length": 252.0, "entropy": 0.3851243816316128, "epoch": 0.1879014989293362, "frac_reward_zero_std": 0.5, "grad_norm": 0.003343230113387108, "learning_rate": 1e-05, "loss": 0.0088, "num_tokens": 7749731.0, "reward": 0.5625, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.4046403169631958, "sampling/importance_sampling_ratio/mean": 0.9997744560241699, "sampling/importance_sampling_ratio/min": 0.7431578040122986, "sampling/sampling_logp_difference/max": 0.3397812843322754, "sampling/sampling_logp_difference/mean": 0.011805604211986065, "step": 351 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.163708167150617e-05, "clip_ratio/low_min": 6.163708167150617e-05, "clip_ratio/region_mean": 6.163708167150617e-05, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 721.0, "completions/mean_length": 526.75, "completions/mean_terminated_length": 510.66668701171875, "completions/min_length": 352.0, "completions/min_terminated_length": 352.0, "entropy": 0.3494218699634075, "epoch": 0.18843683083511778, "frac_reward_zero_std": 0.25, "grad_norm": 0.027358848601579666, "learning_rate": 1e-05, "loss": 0.0382, "num_tokens": 7770339.0, "reward": 0.78125, "reward_std": 0.3377464711666107, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.3417854309082031, "sampling/importance_sampling_ratio/mean": 1.0001251697540283, "sampling/importance_sampling_ratio/min": 0.744340181350708, "sampling/sampling_logp_difference/max": 0.2952570915222168, "sampling/sampling_logp_difference/mean": 0.011354874819517136, "step": 352 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.0813006964745e-05, "clip_ratio/low_min": 5.0813006964745e-05, "clip_ratio/region_mean": 5.0813006964745e-05, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 714.0, "completions/mean_length": 532.90625, "completions/mean_terminated_length": 467.0799865722656, "completions/min_length": 355.0, "completions/min_terminated_length": 355.0, "entropy": 0.5631093829870224, "epoch": 0.18897216274089937, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0375, "num_tokens": 7791168.0, "reward": 0.59375, "reward_std": 0.1293872892856598, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.44841468334198, "sampling/importance_sampling_ratio/mean": 1.0001357793807983, "sampling/importance_sampling_ratio/min": 0.6426133513450623, "sampling/sampling_logp_difference/max": 0.4422121047973633, "sampling/sampling_logp_difference/mean": 0.01514861173927784, "step": 353 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 683.0, "completions/mean_length": 543.875, "completions/mean_terminated_length": 502.370361328125, "completions/min_length": 261.0, "completions/min_terminated_length": 261.0, "entropy": 0.36577605083584785, "epoch": 0.18950749464668093, "frac_reward_zero_std": 0.25, "grad_norm": 0.0052385940216481686, "learning_rate": 1e-05, "loss": 0.1193, "num_tokens": 7812028.0, "reward": 0.78125, "reward_std": 0.3061639666557312, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.2856734991073608, "sampling/importance_sampling_ratio/mean": 0.9999237060546875, "sampling/importance_sampling_ratio/min": 0.6599866151809692, "sampling/sampling_logp_difference/max": 0.41553568840026855, "sampling/sampling_logp_difference/mean": 0.011133652180433273, "step": 354 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 9.308270819019526e-05, "clip_ratio/low_min": 9.308270819019526e-05, "clip_ratio/region_mean": 9.308270819019526e-05, "completions/clipped_ratio": 0.34375, "completions/max_length": 768.0, "completions/max_terminated_length": 739.0, "completions/mean_length": 626.34375, "completions/mean_terminated_length": 552.1428833007812, "completions/min_length": 321.0, "completions/min_terminated_length": 321.0, "entropy": 0.46222998946905136, "epoch": 0.19004282655246252, "frac_reward_zero_std": 0.0, "grad_norm": 0.02112583816051483, "learning_rate": 1e-05, "loss": -0.014, "num_tokens": 7835935.0, "reward": 0.46875, "reward_std": 0.4218915104866028, "rewards/accuracy_reward/mean": 0.46875, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.4092257022857666, "sampling/importance_sampling_ratio/mean": 0.9999447464942932, "sampling/importance_sampling_ratio/min": 0.6687667369842529, "sampling/sampling_logp_difference/max": 0.40231990814208984, "sampling/sampling_logp_difference/mean": 0.013156254775822163, "step": 355 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00011775788880186155, "clip_ratio/low_min": 0.00011775788880186155, "clip_ratio/region_mean": 0.00011775788880186155, "completions/clipped_ratio": 0.375, "completions/max_length": 768.0, "completions/max_terminated_length": 736.0, "completions/mean_length": 639.21875, "completions/mean_terminated_length": 561.9500122070312, "completions/min_length": 428.0, "completions/min_terminated_length": 428.0, "entropy": 0.5517634004354477, "epoch": 0.1905781584582441, "frac_reward_zero_std": 0.5, "grad_norm": 0.008774088695645332, "learning_rate": 1e-05, "loss": 0.0435, "num_tokens": 7861310.0, "reward": 0.53125, "reward_std": 0.2630178928375244, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.445049524307251, "sampling/importance_sampling_ratio/mean": 1.000304102897644, "sampling/importance_sampling_ratio/min": 0.641190230846405, "sampling/sampling_logp_difference/max": 0.44442909955978394, "sampling/sampling_logp_difference/mean": 0.014857380650937557, "step": 356 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 763.0, "completions/mean_length": 507.8125, "completions/mean_terminated_length": 490.4667053222656, "completions/min_length": 297.0, "completions/min_terminated_length": 297.0, "entropy": 0.31233030930161476, "epoch": 0.1911134903640257, "frac_reward_zero_std": 0.5, "grad_norm": 0.0022226155269891024, "learning_rate": 1e-05, "loss": 0.0188, "num_tokens": 7880672.0, "reward": 0.875, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 1.4607328176498413, "sampling/importance_sampling_ratio/mean": 1.0001425743103027, "sampling/importance_sampling_ratio/min": 0.6651771068572998, "sampling/sampling_logp_difference/max": 0.4077019691467285, "sampling/sampling_logp_difference/mean": 0.010300527326762676, "step": 357 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.28125, "completions/max_length": 768.0, "completions/max_terminated_length": 754.0, "completions/mean_length": 602.5625, "completions/mean_terminated_length": 537.8261108398438, "completions/min_length": 284.0, "completions/min_terminated_length": 284.0, "entropy": 0.3632350265979767, "epoch": 0.19164882226980728, "frac_reward_zero_std": 0.75, "grad_norm": 0.0023946890141814947, "learning_rate": 1e-05, "loss": 0.0213, "num_tokens": 7903482.0, "reward": 0.46875, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.46875, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.4644660949707031, "sampling/importance_sampling_ratio/mean": 1.0002481937408447, "sampling/importance_sampling_ratio/min": 0.6679155826568604, "sampling/sampling_logp_difference/max": 0.4035935401916504, "sampling/sampling_logp_difference/mean": 0.010992293246090412, "step": 358 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 631.0, "completions/max_terminated_length": 631.0, "completions/mean_length": 427.28125, "completions/mean_terminated_length": 427.28125, "completions/min_length": 186.0, "completions/min_terminated_length": 186.0, "entropy": 0.48370204120874405, "epoch": 0.19218415417558887, "frac_reward_zero_std": 0.75, "grad_norm": 0.016941169276833534, "learning_rate": 1e-05, "loss": -0.0292, "num_tokens": 7921179.0, "reward": 0.90625, "reward_std": 0.1293872892856598, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.3823636770248413, "sampling/importance_sampling_ratio/mean": 0.9999827742576599, "sampling/importance_sampling_ratio/min": 0.4540562331676483, "sampling/sampling_logp_difference/max": 0.7895342111587524, "sampling/sampling_logp_difference/mean": 0.014404737390577793, "step": 359 }, { "clip_ratio/high_max": 5.577866977546364e-05, "clip_ratio/high_mean": 5.577866977546364e-05, "clip_ratio/low_mean": 5.6844019127311185e-05, "clip_ratio/low_min": 5.6844019127311185e-05, "clip_ratio/region_mean": 0.00011262268890277483, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 760.0, "completions/mean_length": 522.625, "completions/mean_terminated_length": 487.5714416503906, "completions/min_length": 256.0, "completions/min_terminated_length": 256.0, "entropy": 0.4285016618669033, "epoch": 0.19271948608137046, "frac_reward_zero_std": 0.25, "grad_norm": 0.019180461764335632, "learning_rate": 1e-05, "loss": 0.0479, "num_tokens": 7941479.0, "reward": 0.71875, "reward_std": 0.3198433816432953, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.766838550567627, "sampling/importance_sampling_ratio/mean": 1.0003491640090942, "sampling/importance_sampling_ratio/min": 0.5647155046463013, "sampling/sampling_logp_difference/max": 0.5714331865310669, "sampling/sampling_logp_difference/mean": 0.012375672347843647, "step": 360 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00011393861859687604, "clip_ratio/low_min": 0.00011393861859687604, "clip_ratio/region_mean": 0.00011393861859687604, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 767.0, "completions/mean_length": 546.9375, "completions/mean_terminated_length": 515.357177734375, "completions/min_length": 278.0, "completions/min_terminated_length": 278.0, "entropy": 0.3968591131269932, "epoch": 0.19325481798715202, "frac_reward_zero_std": 0.5, "grad_norm": 0.010016828775405884, "learning_rate": 1e-05, "loss": 0.0204, "num_tokens": 7962981.0, "reward": 0.75, "reward_std": 0.2587745785713196, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.5805957317352295, "sampling/importance_sampling_ratio/mean": 1.0001254081726074, "sampling/importance_sampling_ratio/min": 0.4869549870491028, "sampling/sampling_logp_difference/max": 0.7195836305618286, "sampling/sampling_logp_difference/mean": 0.012024071998894215, "step": 361 }, { "clip_ratio/high_max": 6.400409620255232e-05, "clip_ratio/high_mean": 6.400409620255232e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 6.400409620255232e-05, "completions/clipped_ratio": 0.28125, "completions/max_length": 768.0, "completions/max_terminated_length": 643.0, "completions/mean_length": 535.0625, "completions/mean_terminated_length": 443.9130554199219, "completions/min_length": 281.0, "completions/min_terminated_length": 281.0, "entropy": 0.4211147539317608, "epoch": 0.1937901498929336, "frac_reward_zero_std": 0.25, "grad_norm": 0.004181662108749151, "learning_rate": 1e-05, "loss": 0.0112, "num_tokens": 7984343.0, "reward": 0.71875, "reward_std": 0.2651650309562683, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.58782160282135, "sampling/importance_sampling_ratio/mean": 1.0001400709152222, "sampling/importance_sampling_ratio/min": 0.5990554690361023, "sampling/sampling_logp_difference/max": 0.5124011039733887, "sampling/sampling_logp_difference/mean": 0.01206454448401928, "step": 362 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.7339449995197356e-05, "clip_ratio/low_min": 5.7339449995197356e-05, "clip_ratio/region_mean": 5.7339449995197356e-05, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 760.0, "completions/mean_length": 562.0, "completions/mean_terminated_length": 548.2667236328125, "completions/min_length": 312.0, "completions/min_terminated_length": 312.0, "entropy": 0.38405920192599297, "epoch": 0.1943254817987152, "frac_reward_zero_std": 0.0, "grad_norm": 0.015265021473169327, "learning_rate": 1e-05, "loss": 0.0195, "num_tokens": 8006415.0, "reward": 0.59375, "reward_std": 0.4218915104866028, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.5048727989196777, "sampling/importance_sampling_ratio/mean": 1.0001049041748047, "sampling/importance_sampling_ratio/min": 0.6782993078231812, "sampling/sampling_logp_difference/max": 0.4087083339691162, "sampling/sampling_logp_difference/mean": 0.01102854497730732, "step": 363 }, { "clip_ratio/high_max": 5.002000762033276e-05, "clip_ratio/high_mean": 5.002000762033276e-05, "clip_ratio/low_mean": 0.00029838310729246587, "clip_ratio/low_min": 0.00029838310729246587, "clip_ratio/region_mean": 0.00034840311491279863, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 736.0, "completions/mean_length": 558.65625, "completions/mean_terminated_length": 528.75, "completions/min_length": 265.0, "completions/min_terminated_length": 265.0, "entropy": 0.38297540321946144, "epoch": 0.1948608137044968, "frac_reward_zero_std": 0.0, "grad_norm": 0.008084873668849468, "learning_rate": 1e-05, "loss": -0.0243, "num_tokens": 8028484.0, "reward": 0.5625, "reward_std": 0.4355512857437134, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.3734850883483887, "sampling/importance_sampling_ratio/mean": 1.0000678300857544, "sampling/importance_sampling_ratio/min": 0.6157987117767334, "sampling/sampling_logp_difference/max": 0.484835147857666, "sampling/sampling_logp_difference/mean": 0.01192574854940176, "step": 364 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00010874668441829272, "clip_ratio/low_min": 0.00010874668441829272, "clip_ratio/region_mean": 0.00010874668441829272, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 736.0, "completions/mean_length": 536.25, "completions/mean_terminated_length": 528.774169921875, "completions/min_length": 282.0, "completions/min_terminated_length": 282.0, "entropy": 0.40858972258865833, "epoch": 0.19539614561027838, "frac_reward_zero_std": 0.5, "grad_norm": 0.010102041997015476, "learning_rate": 1e-05, "loss": -0.0444, "num_tokens": 8049372.0, "reward": 0.625, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.7266088724136353, "sampling/importance_sampling_ratio/mean": 1.000022053718567, "sampling/importance_sampling_ratio/min": 0.6679679751396179, "sampling/sampling_logp_difference/max": 0.5461592674255371, "sampling/sampling_logp_difference/mean": 0.011561091057956219, "step": 365 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 711.0, "completions/mean_length": 440.5, "completions/mean_terminated_length": 418.66668701171875, "completions/min_length": 227.0, "completions/min_terminated_length": 227.0, "entropy": 0.41376325115561485, "epoch": 0.19593147751605997, "frac_reward_zero_std": 0.5, "grad_norm": 0.012588988989591599, "learning_rate": 1e-05, "loss": -0.029, "num_tokens": 8067092.0, "reward": 0.84375, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.3792674541473389, "sampling/importance_sampling_ratio/mean": 0.9999568462371826, "sampling/importance_sampling_ratio/min": 0.7405104637145996, "sampling/sampling_logp_difference/max": 0.3215525150299072, "sampling/sampling_logp_difference/mean": 0.011990740895271301, "step": 366 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00042268036486348137, "clip_ratio/low_min": 0.00042268036486348137, "clip_ratio/region_mean": 0.00042268036486348137, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 728.0, "completions/mean_length": 508.625, "completions/mean_terminated_length": 471.5714416503906, "completions/min_length": 232.0, "completions/min_terminated_length": 232.0, "entropy": 0.6101974248886108, "epoch": 0.19646680942184155, "frac_reward_zero_std": 0.25, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": -0.0023, "num_tokens": 8087536.0, "reward": 0.625, "reward_std": 0.3104073107242584, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.925344467163086, "sampling/importance_sampling_ratio/mean": 0.9996810555458069, "sampling/importance_sampling_ratio/min": 0.6387473940849304, "sampling/sampling_logp_difference/max": 0.6551048755645752, "sampling/sampling_logp_difference/mean": 0.015741897746920586, "step": 367 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00012450199574232101, "clip_ratio/low_min": 0.00012450199574232101, "clip_ratio/region_mean": 0.00012450199574232101, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 702.0, "completions/mean_length": 488.65625, "completions/mean_terminated_length": 479.6451416015625, "completions/min_length": 230.0, "completions/min_terminated_length": 230.0, "entropy": 0.44869812577962875, "epoch": 0.19700214132762311, "frac_reward_zero_std": 0.5, "grad_norm": 0.005796290468424559, "learning_rate": 1e-05, "loss": 0.0044, "num_tokens": 8106949.0, "reward": 0.78125, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.3189603090286255, "sampling/importance_sampling_ratio/mean": 0.9999547600746155, "sampling/importance_sampling_ratio/min": 0.4622354805469513, "sampling/sampling_logp_difference/max": 0.7716808319091797, "sampling/sampling_logp_difference/mean": 0.013602005317807198, "step": 368 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 642.0, "completions/max_terminated_length": 642.0, "completions/mean_length": 407.28125, "completions/mean_terminated_length": 407.28125, "completions/min_length": 180.0, "completions/min_terminated_length": 180.0, "entropy": 0.35991888865828514, "epoch": 0.1975374732334047, "frac_reward_zero_std": 0.75, "grad_norm": 0.019617488607764244, "learning_rate": 1e-05, "loss": 0.0227, "num_tokens": 8123366.0, "reward": 0.96875, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.96875, "rewards/accuracy_reward/std": 0.1767766922712326, "sampling/importance_sampling_ratio/max": 1.4685968160629272, "sampling/importance_sampling_ratio/mean": 1.0000109672546387, "sampling/importance_sampling_ratio/min": 0.5388011336326599, "sampling/sampling_logp_difference/max": 0.6184086799621582, "sampling/sampling_logp_difference/mean": 0.011547592468559742, "step": 369 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00012916622290504165, "clip_ratio/low_min": 0.00012916622290504165, "clip_ratio/region_mean": 0.00012916622290504165, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 758.0, "completions/mean_length": 492.9375, "completions/mean_terminated_length": 453.64288330078125, "completions/min_length": 247.0, "completions/min_terminated_length": 247.0, "entropy": 0.45084620639681816, "epoch": 0.1980728051391863, "frac_reward_zero_std": 0.0, "grad_norm": 0.00818666908890009, "learning_rate": 1e-05, "loss": 0.0584, "num_tokens": 8142916.0, "reward": 0.40625, "reward_std": 0.4765698313713074, "rewards/accuracy_reward/mean": 0.40625, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.4255330562591553, "sampling/importance_sampling_ratio/mean": 0.9999929666519165, "sampling/importance_sampling_ratio/min": 0.7036133408546448, "sampling/sampling_logp_difference/max": 0.35454583168029785, "sampling/sampling_logp_difference/mean": 0.01342479046434164, "step": 370 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 749.0, "completions/mean_length": 492.28125, "completions/mean_terminated_length": 473.9000244140625, "completions/min_length": 255.0, "completions/min_terminated_length": 255.0, "entropy": 0.4124649167060852, "epoch": 0.19860813704496788, "frac_reward_zero_std": 0.25, "grad_norm": 0.004048778675496578, "learning_rate": 1e-05, "loss": 0.0159, "num_tokens": 8162861.0, "reward": 0.71875, "reward_std": 0.3608423173427582, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.3044075965881348, "sampling/importance_sampling_ratio/mean": 0.9998164772987366, "sampling/importance_sampling_ratio/min": 0.5544739961624146, "sampling/sampling_logp_difference/max": 0.5897352695465088, "sampling/sampling_logp_difference/mean": 0.012232414446771145, "step": 371 }, { "clip_ratio/high_max": 0.00011063877900596708, "clip_ratio/high_mean": 0.00011063877900596708, "clip_ratio/low_mean": 0.00010116694829775952, "clip_ratio/low_min": 0.00010116694829775952, "clip_ratio/region_mean": 0.0002118057273037266, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 692.0, "completions/mean_length": 573.34375, "completions/mean_terminated_length": 537.2963256835938, "completions/min_length": 359.0, "completions/min_terminated_length": 359.0, "entropy": 0.3547341823577881, "epoch": 0.19914346895074947, "frac_reward_zero_std": 0.0, "grad_norm": 0.01210623700171709, "learning_rate": 1e-05, "loss": 0.0644, "num_tokens": 8184624.0, "reward": 0.625, "reward_std": 0.4492306709289551, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.350893259048462, "sampling/importance_sampling_ratio/mean": 0.9997089505195618, "sampling/importance_sampling_ratio/min": 0.44091445207595825, "sampling/sampling_logp_difference/max": 0.8189043998718262, "sampling/sampling_logp_difference/mean": 0.01156165823340416, "step": 372 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 750.0, "completions/mean_length": 538.21875, "completions/mean_terminated_length": 505.39288330078125, "completions/min_length": 320.0, "completions/min_terminated_length": 320.0, "entropy": 0.4700101502239704, "epoch": 0.19967880085653106, "frac_reward_zero_std": 0.5, "grad_norm": 0.01357709988951683, "learning_rate": 1e-05, "loss": 0.0461, "num_tokens": 8206159.0, "reward": 0.8125, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.4291284084320068, "sampling/importance_sampling_ratio/mean": 1.0000104904174805, "sampling/importance_sampling_ratio/min": 0.5418497920036316, "sampling/sampling_logp_difference/max": 0.6127665042877197, "sampling/sampling_logp_difference/mean": 0.012792416848242283, "step": 373 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 7.130633457563818e-05, "clip_ratio/low_min": 7.130633457563818e-05, "clip_ratio/region_mean": 7.130633457563818e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 756.0, "completions/max_terminated_length": 756.0, "completions/mean_length": 509.53125, "completions/mean_terminated_length": 509.53125, "completions/min_length": 263.0, "completions/min_terminated_length": 263.0, "entropy": 0.3493783734738827, "epoch": 0.20021413276231265, "frac_reward_zero_std": 0.5, "grad_norm": 0.004631737247109413, "learning_rate": 1e-05, "loss": -0.0205, "num_tokens": 8225888.0, "reward": 0.90625, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.3596831560134888, "sampling/importance_sampling_ratio/mean": 1.000054121017456, "sampling/importance_sampling_ratio/min": 0.5772933959960938, "sampling/sampling_logp_difference/max": 0.5494046211242676, "sampling/sampling_logp_difference/mean": 0.01120574027299881, "step": 374 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 619.0, "completions/mean_length": 537.21875, "completions/mean_terminated_length": 460.29168701171875, "completions/min_length": 257.0, "completions/min_terminated_length": 257.0, "entropy": 0.5183845311403275, "epoch": 0.2007494646680942, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": -0.0054, "num_tokens": 8247015.0, "reward": 0.71875, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.4426804780960083, "sampling/importance_sampling_ratio/mean": 1.0001507997512817, "sampling/importance_sampling_ratio/min": 0.7089053988456726, "sampling/sampling_logp_difference/max": 0.3665027618408203, "sampling/sampling_logp_difference/mean": 0.015082940459251404, "step": 375 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 765.0, "completions/mean_length": 535.8125, "completions/mean_terminated_length": 511.7930908203125, "completions/min_length": 228.0, "completions/min_terminated_length": 228.0, "entropy": 0.445853628218174, "epoch": 0.2012847965738758, "frac_reward_zero_std": 0.75, "grad_norm": 0.0038687344640493393, "learning_rate": 1e-05, "loss": 0.0125, "num_tokens": 8268049.0, "reward": 0.96875, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.96875, "rewards/accuracy_reward/std": 0.1767766922712326, "sampling/importance_sampling_ratio/max": 1.3863552808761597, "sampling/importance_sampling_ratio/mean": 1.0003184080123901, "sampling/importance_sampling_ratio/min": 0.6021171808242798, "sampling/sampling_logp_difference/max": 0.5073032379150391, "sampling/sampling_logp_difference/mean": 0.012995378114283085, "step": 376 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00017787283650250174, "clip_ratio/low_min": 0.00017787283650250174, "clip_ratio/region_mean": 0.00017787283650250174, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 713.0, "completions/mean_length": 565.46875, "completions/mean_terminated_length": 536.5357666015625, "completions/min_length": 337.0, "completions/min_terminated_length": 337.0, "entropy": 0.39838796481490135, "epoch": 0.20182012847965738, "frac_reward_zero_std": 0.25, "grad_norm": 0.005153874401003122, "learning_rate": 1e-05, "loss": 0.0363, "num_tokens": 8290520.0, "reward": 0.6875, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.5002710819244385, "sampling/importance_sampling_ratio/mean": 0.999914288520813, "sampling/importance_sampling_ratio/min": 0.6235157251358032, "sampling/sampling_logp_difference/max": 0.4723813533782959, "sampling/sampling_logp_difference/mean": 0.012132040224969387, "step": 377 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.0100199587177485e-05, "clip_ratio/low_min": 5.0100199587177485e-05, "clip_ratio/region_mean": 5.0100199587177485e-05, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 718.0, "completions/mean_length": 564.71875, "completions/mean_terminated_length": 517.8077392578125, "completions/min_length": 331.0, "completions/min_terminated_length": 331.0, "entropy": 0.4018013924360275, "epoch": 0.20235546038543897, "frac_reward_zero_std": 0.0, "grad_norm": 0.02573443576693535, "learning_rate": 1e-05, "loss": -0.0692, "num_tokens": 8312071.0, "reward": 0.625, "reward_std": 0.4492306709289551, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.3948887586593628, "sampling/importance_sampling_ratio/mean": 0.9999281167984009, "sampling/importance_sampling_ratio/min": 0.6977291107177734, "sampling/sampling_logp_difference/max": 0.35992431640625, "sampling/sampling_logp_difference/mean": 0.012433131225407124, "step": 378 }, { "clip_ratio/high_max": 5.2388935728231445e-05, "clip_ratio/high_mean": 5.2388935728231445e-05, "clip_ratio/low_mean": 5.921364208916202e-05, "clip_ratio/low_min": 5.921364208916202e-05, "clip_ratio/region_mean": 0.00011160257781739347, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 737.0, "completions/mean_length": 557.71875, "completions/mean_terminated_length": 487.625, "completions/min_length": 14.0, "completions/min_terminated_length": 14.0, "entropy": 0.4676593616604805, "epoch": 0.20289079229122056, "frac_reward_zero_std": 0.0, "grad_norm": 0.007939019240438938, "learning_rate": 1e-05, "loss": -0.0319, "num_tokens": 8333726.0, "reward": 0.65625, "reward_std": 0.4355708956718445, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.4016294479370117, "sampling/importance_sampling_ratio/mean": 1.0002610683441162, "sampling/importance_sampling_ratio/min": 0.6747454404830933, "sampling/sampling_logp_difference/max": 0.39341986179351807, "sampling/sampling_logp_difference/mean": 0.013635436072945595, "step": 379 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 744.0, "completions/mean_length": 550.40625, "completions/mean_terminated_length": 519.3214721679688, "completions/min_length": 301.0, "completions/min_terminated_length": 301.0, "entropy": 0.5052536204457283, "epoch": 0.20342612419700215, "frac_reward_zero_std": 0.25, "grad_norm": 0.014165861532092094, "learning_rate": 1e-05, "loss": 0.0005, "num_tokens": 8354883.0, "reward": 0.71875, "reward_std": 0.3787454068660736, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.4219069480895996, "sampling/importance_sampling_ratio/mean": 1.0001037120819092, "sampling/importance_sampling_ratio/min": 0.6856846213340759, "sampling/sampling_logp_difference/max": 0.3773374557495117, "sampling/sampling_logp_difference/mean": 0.014261284843087196, "step": 380 }, { "clip_ratio/high_max": 0.00015106108912732452, "clip_ratio/high_mean": 0.00015106108912732452, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00015106108912732452, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 659.0, "completions/mean_length": 424.6875, "completions/mean_terminated_length": 413.6128845214844, "completions/min_length": 157.0, "completions/min_terminated_length": 157.0, "entropy": 0.4332553744316101, "epoch": 0.20396145610278374, "frac_reward_zero_std": 0.25, "grad_norm": 0.00642042700201273, "learning_rate": 1e-05, "loss": 0.0391, "num_tokens": 8372161.0, "reward": 0.84375, "reward_std": 0.3061639964580536, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.2888432741165161, "sampling/importance_sampling_ratio/mean": 0.9997114539146423, "sampling/importance_sampling_ratio/min": 0.7245350480079651, "sampling/sampling_logp_difference/max": 0.32222509384155273, "sampling/sampling_logp_difference/mean": 0.012700242921710014, "step": 381 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 719.0, "completions/mean_length": 436.65625, "completions/mean_terminated_length": 402.3793029785156, "completions/min_length": 141.0, "completions/min_terminated_length": 141.0, "entropy": 0.3294503465294838, "epoch": 0.2044967880085653, "frac_reward_zero_std": 0.5, "grad_norm": 0.018209051340818405, "learning_rate": 1e-05, "loss": 0.0114, "num_tokens": 8389646.0, "reward": 0.78125, "reward_std": 0.2630178928375244, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.3683762550354004, "sampling/importance_sampling_ratio/mean": 0.9996477961540222, "sampling/importance_sampling_ratio/min": 0.22582344710826874, "sampling/sampling_logp_difference/max": 1.488001823425293, "sampling/sampling_logp_difference/mean": 0.011063874699175358, "step": 382 }, { "clip_ratio/high_max": 5.980861169518903e-05, "clip_ratio/high_mean": 5.980861169518903e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 5.980861169518903e-05, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 714.0, "completions/mean_length": 546.28125, "completions/mean_terminated_length": 523.3448486328125, "completions/min_length": 284.0, "completions/min_terminated_length": 284.0, "entropy": 0.37299900874495506, "epoch": 0.2050321199143469, "frac_reward_zero_std": 0.25, "grad_norm": 0.008004746399819851, "learning_rate": 1e-05, "loss": 0.0667, "num_tokens": 8410639.0, "reward": 0.78125, "reward_std": 0.3608423173427582, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.4790222644805908, "sampling/importance_sampling_ratio/mean": 0.9999213218688965, "sampling/importance_sampling_ratio/min": 0.7226288914680481, "sampling/sampling_logp_difference/max": 0.39138126373291016, "sampling/sampling_logp_difference/mean": 0.012188081629574299, "step": 383 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 9.745671559358016e-05, "clip_ratio/low_min": 9.745671559358016e-05, "clip_ratio/region_mean": 9.745671559358016e-05, "completions/clipped_ratio": 0.3125, "completions/max_length": 768.0, "completions/max_terminated_length": 722.0, "completions/mean_length": 602.0, "completions/mean_terminated_length": 526.5454711914062, "completions/min_length": 315.0, "completions/min_terminated_length": 315.0, "entropy": 0.5234112665057182, "epoch": 0.20556745182012848, "frac_reward_zero_std": 0.25, "grad_norm": 0.005902193021029234, "learning_rate": 1e-05, "loss": -0.0292, "num_tokens": 8433935.0, "reward": 0.5625, "reward_std": 0.2925041913986206, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.4153141975402832, "sampling/importance_sampling_ratio/mean": 1.0001457929611206, "sampling/importance_sampling_ratio/min": 0.7190697193145752, "sampling/sampling_logp_difference/max": 0.3473515510559082, "sampling/sampling_logp_difference/mean": 0.015038399957120419, "step": 384 }, { "clip_ratio/high_max": 4.990019806427881e-05, "clip_ratio/high_mean": 4.990019806427881e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 4.990019806427881e-05, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 746.0, "completions/mean_length": 597.0, "completions/mean_terminated_length": 579.3103637695312, "completions/min_length": 372.0, "completions/min_terminated_length": 372.0, "entropy": 0.35777005180716515, "epoch": 0.20610278372591007, "frac_reward_zero_std": 0.0, "grad_norm": 0.027439260855317116, "learning_rate": 1e-05, "loss": 0.0501, "num_tokens": 8456887.0, "reward": 0.5, "reward_std": 0.4082317352294922, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5080004930496216, "sampling/importance_sampling_ratio/max": 1.449072241783142, "sampling/importance_sampling_ratio/mean": 1.0000883340835571, "sampling/importance_sampling_ratio/min": 0.6497271060943604, "sampling/sampling_logp_difference/max": 0.43120288848876953, "sampling/sampling_logp_difference/mean": 0.011739862151443958, "step": 385 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.02008042531088e-05, "clip_ratio/low_min": 5.02008042531088e-05, "clip_ratio/region_mean": 5.02008042531088e-05, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 753.0, "completions/mean_length": 532.0, "completions/mean_terminated_length": 498.2857360839844, "completions/min_length": 312.0, "completions/min_terminated_length": 312.0, "entropy": 0.37218380346894264, "epoch": 0.20663811563169165, "frac_reward_zero_std": 0.5, "grad_norm": 0.020878946408629417, "learning_rate": 1e-05, "loss": 0.012, "num_tokens": 8477519.0, "reward": 0.5, "reward_std": 0.2314550280570984, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5080004930496216, "sampling/importance_sampling_ratio/max": 1.658609390258789, "sampling/importance_sampling_ratio/mean": 1.000179648399353, "sampling/importance_sampling_ratio/min": 0.6999051570892334, "sampling/sampling_logp_difference/max": 0.5059795379638672, "sampling/sampling_logp_difference/mean": 0.012145856395363808, "step": 386 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.3602059779223055e-05, "clip_ratio/low_min": 5.3602059779223055e-05, "clip_ratio/region_mean": 5.3602059779223055e-05, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 744.0, "completions/mean_length": 557.21875, "completions/mean_terminated_length": 508.5769348144531, "completions/min_length": 339.0, "completions/min_terminated_length": 339.0, "entropy": 0.3212047778069973, "epoch": 0.20717344753747324, "frac_reward_zero_std": 0.0, "grad_norm": 0.010054142214357853, "learning_rate": 1e-05, "loss": -0.0094, "num_tokens": 8499038.0, "reward": 0.4375, "reward_std": 0.4492306709289551, "rewards/accuracy_reward/mean": 0.4375, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.4544676542282104, "sampling/importance_sampling_ratio/mean": 1.000232219696045, "sampling/importance_sampling_ratio/min": 0.6854678988456726, "sampling/sampling_logp_difference/max": 0.3776535987854004, "sampling/sampling_logp_difference/mean": 0.010407916270196438, "step": 387 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 700.0, "completions/mean_length": 562.21875, "completions/mean_terminated_length": 493.625, "completions/min_length": 241.0, "completions/min_terminated_length": 241.0, "entropy": 0.417966291308403, "epoch": 0.20770877944325483, "frac_reward_zero_std": 0.0, "grad_norm": 0.006222824100404978, "learning_rate": 1e-05, "loss": 0.0263, "num_tokens": 8520893.0, "reward": 0.65625, "reward_std": 0.3808925747871399, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.4255802631378174, "sampling/importance_sampling_ratio/mean": 0.9999366998672485, "sampling/importance_sampling_ratio/min": 0.5939368605613708, "sampling/sampling_logp_difference/max": 0.5209822654724121, "sampling/sampling_logp_difference/mean": 0.013065759092569351, "step": 388 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.98658989474643e-05, "clip_ratio/low_min": 5.98658989474643e-05, "clip_ratio/region_mean": 5.98658989474643e-05, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 757.0, "completions/mean_length": 508.78125, "completions/mean_terminated_length": 481.96551513671875, "completions/min_length": 302.0, "completions/min_terminated_length": 302.0, "entropy": 0.39041562750935555, "epoch": 0.2082441113490364, "frac_reward_zero_std": 0.5, "grad_norm": 0.012411758303642273, "learning_rate": 1e-05, "loss": 0.0185, "num_tokens": 8540494.0, "reward": 0.6875, "reward_std": 0.2587745785713196, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.4231623411178589, "sampling/importance_sampling_ratio/mean": 1.0004234313964844, "sampling/importance_sampling_ratio/min": 0.6063537001609802, "sampling/sampling_logp_difference/max": 0.5002918243408203, "sampling/sampling_logp_difference/mean": 0.012109454721212387, "step": 389 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.585879600606859e-05, "clip_ratio/low_min": 6.585879600606859e-05, "clip_ratio/region_mean": 6.585879600606859e-05, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 694.0, "completions/mean_length": 522.5625, "completions/mean_terminated_length": 465.923095703125, "completions/min_length": 313.0, "completions/min_terminated_length": 313.0, "entropy": 0.3886766582727432, "epoch": 0.20877944325481798, "frac_reward_zero_std": 0.75, "grad_norm": 0.005338247865438461, "learning_rate": 1e-05, "loss": -0.0307, "num_tokens": 8561016.0, "reward": 0.625, "reward_std": 0.13363061845302582, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.4371895790100098, "sampling/importance_sampling_ratio/mean": 0.9998371601104736, "sampling/importance_sampling_ratio/min": 0.4145455062389374, "sampling/sampling_logp_difference/max": 0.8805725574493408, "sampling/sampling_logp_difference/mean": 0.012448754161596298, "step": 390 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 693.0, "completions/mean_length": 472.0625, "completions/mean_terminated_length": 417.25927734375, "completions/min_length": 187.0, "completions/min_terminated_length": 187.0, "entropy": 0.42797714844346046, "epoch": 0.20931477516059957, "frac_reward_zero_std": 0.5, "grad_norm": 0.00887267291545868, "learning_rate": 1e-05, "loss": -0.0061, "num_tokens": 8579570.0, "reward": 0.46875, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.46875, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.3618123531341553, "sampling/importance_sampling_ratio/mean": 0.9998660683631897, "sampling/importance_sampling_ratio/min": 0.5906755924224854, "sampling/sampling_logp_difference/max": 0.5264883041381836, "sampling/sampling_logp_difference/mean": 0.012660915963351727, "step": 391 }, { "clip_ratio/high_max": 6.541077891597524e-05, "clip_ratio/high_mean": 6.541077891597524e-05, "clip_ratio/low_mean": 5.401901580626145e-05, "clip_ratio/low_min": 5.401901580626145e-05, "clip_ratio/region_mean": 0.00011942979472223669, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 728.0, "completions/mean_length": 478.34375, "completions/mean_terminated_length": 469.0, "completions/min_length": 224.0, "completions/min_terminated_length": 224.0, "entropy": 0.4162847027182579, "epoch": 0.20985010706638116, "frac_reward_zero_std": 0.5, "grad_norm": 0.01660206727683544, "learning_rate": 1e-05, "loss": -0.0402, "num_tokens": 8598445.0, "reward": 0.8125, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.3681479692459106, "sampling/importance_sampling_ratio/mean": 1.0001448392868042, "sampling/importance_sampling_ratio/min": 0.5606738328933716, "sampling/sampling_logp_difference/max": 0.5786159038543701, "sampling/sampling_logp_difference/mean": 0.012906822375953197, "step": 392 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00010018170360126533, "clip_ratio/low_min": 0.00010018170360126533, "clip_ratio/region_mean": 0.00010018170360126533, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 746.0, "completions/mean_length": 556.0, "completions/mean_terminated_length": 496.6399841308594, "completions/min_length": 308.0, "completions/min_terminated_length": 308.0, "entropy": 0.4442379027605057, "epoch": 0.21038543897216275, "frac_reward_zero_std": 0.75, "grad_norm": 0.005422597285360098, "learning_rate": 1e-05, "loss": 0.0329, "num_tokens": 8620069.0, "reward": 0.8125, "reward_std": 0.1157275140285492, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.4322624206542969, "sampling/importance_sampling_ratio/mean": 0.9999589920043945, "sampling/importance_sampling_ratio/min": 0.6889292001724243, "sampling/sampling_logp_difference/max": 0.3726167678833008, "sampling/sampling_logp_difference/mean": 0.013623659498989582, "step": 393 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.234413740457967e-05, "clip_ratio/low_min": 6.234413740457967e-05, "clip_ratio/region_mean": 6.234413740457967e-05, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 721.0, "completions/mean_length": 533.46875, "completions/mean_terminated_length": 455.29168701171875, "completions/min_length": 246.0, "completions/min_terminated_length": 246.0, "entropy": 0.48644712567329407, "epoch": 0.21092077087794434, "frac_reward_zero_std": 0.0, "grad_norm": 0.010294018313288689, "learning_rate": 1e-05, "loss": 0.0317, "num_tokens": 8641044.0, "reward": 0.5, "reward_std": 0.4355512857437134, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5080004930496216, "sampling/importance_sampling_ratio/max": 1.4253495931625366, "sampling/importance_sampling_ratio/mean": 0.999750018119812, "sampling/importance_sampling_ratio/min": 0.697155773639679, "sampling/sampling_logp_difference/max": 0.3607463836669922, "sampling/sampling_logp_difference/mean": 0.014992058277130127, "step": 394 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.477651211549528e-05, "clip_ratio/low_min": 5.477651211549528e-05, "clip_ratio/region_mean": 5.477651211549528e-05, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 692.0, "completions/mean_length": 554.40625, "completions/mean_terminated_length": 532.3103637695312, "completions/min_length": 302.0, "completions/min_terminated_length": 302.0, "entropy": 0.3426280617713928, "epoch": 0.2114561027837259, "frac_reward_zero_std": 0.25, "grad_norm": 0.021630527451634407, "learning_rate": 1e-05, "loss": 0.0869, "num_tokens": 8663057.0, "reward": 0.875, "reward_std": 0.2925041913986206, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 1.4331676959991455, "sampling/importance_sampling_ratio/mean": 0.9998255968093872, "sampling/importance_sampling_ratio/min": 0.37173134088516235, "sampling/sampling_logp_difference/max": 0.9895839691162109, "sampling/sampling_logp_difference/mean": 0.011631157249212265, "step": 395 }, { "clip_ratio/high_max": 5.0545895646791905e-05, "clip_ratio/high_mean": 5.0545895646791905e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 5.0545895646791905e-05, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 758.0, "completions/mean_length": 539.03125, "completions/mean_terminated_length": 531.6451416015625, "completions/min_length": 309.0, "completions/min_terminated_length": 309.0, "entropy": 0.2946431804448366, "epoch": 0.21199143468950749, "frac_reward_zero_std": 0.5, "grad_norm": 0.008164488710463047, "learning_rate": 1e-05, "loss": 0.0102, "num_tokens": 8684106.0, "reward": 0.5625, "reward_std": 0.249358132481575, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.480309247970581, "sampling/importance_sampling_ratio/mean": 1.0002437829971313, "sampling/importance_sampling_ratio/min": 0.7011746168136597, "sampling/sampling_logp_difference/max": 0.39225101470947266, "sampling/sampling_logp_difference/mean": 0.010684709995985031, "step": 396 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.2105042414041236e-05, "clip_ratio/low_min": 5.2105042414041236e-05, "clip_ratio/region_mean": 5.2105042414041236e-05, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 759.0, "completions/mean_length": 487.59375, "completions/mean_terminated_length": 447.5357360839844, "completions/min_length": 201.0, "completions/min_terminated_length": 201.0, "entropy": 0.39214953407645226, "epoch": 0.21252676659528907, "frac_reward_zero_std": 0.25, "grad_norm": 0.004930248949676752, "learning_rate": 1e-05, "loss": 0.0399, "num_tokens": 8703669.0, "reward": 0.625, "reward_std": 0.3104073107242584, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.3939827680587769, "sampling/importance_sampling_ratio/mean": 0.9999846816062927, "sampling/importance_sampling_ratio/min": 0.6818623542785645, "sampling/sampling_logp_difference/max": 0.38292741775512695, "sampling/sampling_logp_difference/mean": 0.012797117233276367, "step": 397 }, { "clip_ratio/high_max": 5.763024455518462e-05, "clip_ratio/high_mean": 5.763024455518462e-05, "clip_ratio/low_mean": 0.00011622617967077531, "clip_ratio/low_min": 0.00011622617967077531, "clip_ratio/region_mean": 0.00017385642422595993, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 711.0, "completions/mean_length": 519.78125, "completions/mean_terminated_length": 473.8148193359375, "completions/min_length": 229.0, "completions/min_terminated_length": 229.0, "entropy": 0.4568267725408077, "epoch": 0.21306209850107066, "frac_reward_zero_std": 0.0, "grad_norm": 0.0210589449852705, "learning_rate": 1e-05, "loss": 0.0104, "num_tokens": 8724918.0, "reward": 0.71875, "reward_std": 0.4218915104866028, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.5084317922592163, "sampling/importance_sampling_ratio/mean": 1.0004701614379883, "sampling/importance_sampling_ratio/min": 0.5377944707870483, "sampling/sampling_logp_difference/max": 0.6202788352966309, "sampling/sampling_logp_difference/mean": 0.015271781012415886, "step": 398 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.910165418754332e-05, "clip_ratio/low_min": 5.910165418754332e-05, "clip_ratio/region_mean": 5.910165418754332e-05, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 735.0, "completions/mean_length": 551.8125, "completions/mean_terminated_length": 529.4483032226562, "completions/min_length": 318.0, "completions/min_terminated_length": 318.0, "entropy": 0.4376138709485531, "epoch": 0.21359743040685225, "frac_reward_zero_std": 0.5, "grad_norm": 0.004334732424467802, "learning_rate": 1e-05, "loss": 0.0595, "num_tokens": 8746432.0, "reward": 0.75, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.5721514225006104, "sampling/importance_sampling_ratio/mean": 1.0001081228256226, "sampling/importance_sampling_ratio/min": 0.5356160402297974, "sampling/sampling_logp_difference/max": 0.6243376731872559, "sampling/sampling_logp_difference/mean": 0.012872467748820782, "step": 399 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 673.0, "completions/mean_length": 513.5, "completions/mean_terminated_length": 496.5333557128906, "completions/min_length": 246.0, "completions/min_terminated_length": 246.0, "entropy": 0.33190320059657097, "epoch": 0.21413276231263384, "frac_reward_zero_std": 0.25, "grad_norm": 0.009642883203923702, "learning_rate": 1e-05, "loss": -0.0026, "num_tokens": 8766456.0, "reward": 0.71875, "reward_std": 0.3787454068660736, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.4526475667953491, "sampling/importance_sampling_ratio/mean": 0.9997441172599792, "sampling/importance_sampling_ratio/min": 0.6310703754425049, "sampling/sampling_logp_difference/max": 0.46033787727355957, "sampling/sampling_logp_difference/mean": 0.011378666386008263, "step": 400 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.5580258049303666e-05, "clip_ratio/low_min": 5.5580258049303666e-05, "clip_ratio/region_mean": 5.5580258049303666e-05, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 691.0, "completions/mean_length": 524.09375, "completions/mean_terminated_length": 489.2500305175781, "completions/min_length": 227.0, "completions/min_terminated_length": 227.0, "entropy": 0.27389667369425297, "epoch": 0.21466809421841543, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0188, "num_tokens": 8786691.0, "reward": 0.90625, "reward_std": 0.1293872892856598, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.4428799152374268, "sampling/importance_sampling_ratio/mean": 0.9998692274093628, "sampling/importance_sampling_ratio/min": 0.6590561270713806, "sampling/sampling_logp_difference/max": 0.4169466495513916, "sampling/sampling_logp_difference/mean": 0.009783259592950344, "step": 401 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 762.0, "completions/mean_length": 452.6875, "completions/mean_terminated_length": 431.66668701171875, "completions/min_length": 210.0, "completions/min_terminated_length": 210.0, "entropy": 0.39985427632927895, "epoch": 0.215203426124197, "frac_reward_zero_std": 0.25, "grad_norm": 0.019520267844200134, "learning_rate": 1e-05, "loss": 0.0171, "num_tokens": 8804393.0, "reward": 0.625, "reward_std": 0.3104073107242584, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.4345918893814087, "sampling/importance_sampling_ratio/mean": 0.9998717904090881, "sampling/importance_sampling_ratio/min": 0.7071976661682129, "sampling/sampling_logp_difference/max": 0.36088037490844727, "sampling/sampling_logp_difference/mean": 0.013089355081319809, "step": 402 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 760.0, "completions/mean_length": 574.75, "completions/mean_terminated_length": 530.1538696289062, "completions/min_length": 351.0, "completions/min_terminated_length": 351.0, "entropy": 0.3019810635596514, "epoch": 0.21573875802997858, "frac_reward_zero_std": 0.5, "grad_norm": 0.017643939703702927, "learning_rate": 1e-05, "loss": 0.0298, "num_tokens": 8826433.0, "reward": 0.65625, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.4424341917037964, "sampling/importance_sampling_ratio/mean": 1.0000219345092773, "sampling/importance_sampling_ratio/min": 0.6996686458587646, "sampling/sampling_logp_difference/max": 0.3663320541381836, "sampling/sampling_logp_difference/mean": 0.010206487029790878, "step": 403 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 687.0, "completions/mean_length": 449.3125, "completions/mean_terminated_length": 439.0322570800781, "completions/min_length": 198.0, "completions/min_terminated_length": 198.0, "entropy": 0.39225055649876595, "epoch": 0.21627408993576017, "frac_reward_zero_std": 0.25, "grad_norm": 0.008761139586567879, "learning_rate": 1e-05, "loss": 0.0823, "num_tokens": 8844227.0, "reward": 0.8125, "reward_std": 0.3104073107242584, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.352560043334961, "sampling/importance_sampling_ratio/mean": 1.0001022815704346, "sampling/importance_sampling_ratio/min": 0.7114943265914917, "sampling/sampling_logp_difference/max": 0.34038782119750977, "sampling/sampling_logp_difference/mean": 0.012354631908237934, "step": 404 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 755.0, "completions/mean_length": 474.25, "completions/mean_terminated_length": 454.66668701171875, "completions/min_length": 156.0, "completions/min_terminated_length": 156.0, "entropy": 0.4077269211411476, "epoch": 0.21680942184154176, "frac_reward_zero_std": 0.25, "grad_norm": 0.01683603599667549, "learning_rate": 1e-05, "loss": 0.0527, "num_tokens": 8863171.0, "reward": 0.84375, "reward_std": 0.3198433816432953, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.4718608856201172, "sampling/importance_sampling_ratio/mean": 1.0002131462097168, "sampling/importance_sampling_ratio/min": 0.6866901516914368, "sampling/sampling_logp_difference/max": 0.38652753829956055, "sampling/sampling_logp_difference/mean": 0.01278675813227892, "step": 405 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.924170545767993e-05, "clip_ratio/low_min": 5.924170545767993e-05, "clip_ratio/region_mean": 5.924170545767993e-05, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 749.0, "completions/mean_length": 474.8125, "completions/mean_terminated_length": 465.3548278808594, "completions/min_length": 258.0, "completions/min_terminated_length": 258.0, "entropy": 0.39252786710858345, "epoch": 0.21734475374732334, "frac_reward_zero_std": 0.25, "grad_norm": 0.008180202916264534, "learning_rate": 1e-05, "loss": 0.0355, "num_tokens": 8881981.0, "reward": 0.46875, "reward_std": 0.3198433816432953, "rewards/accuracy_reward/mean": 0.46875, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.37120521068573, "sampling/importance_sampling_ratio/mean": 1.0004533529281616, "sampling/importance_sampling_ratio/min": 0.6885886788368225, "sampling/sampling_logp_difference/max": 0.3731112480163574, "sampling/sampling_logp_difference/mean": 0.012994229793548584, "step": 406 }, { "clip_ratio/high_max": 5.3236795793054625e-05, "clip_ratio/high_mean": 5.3236795793054625e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 5.3236795793054625e-05, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 724.0, "completions/mean_length": 540.8125, "completions/mean_terminated_length": 508.357177734375, "completions/min_length": 201.0, "completions/min_terminated_length": 201.0, "entropy": 0.3133581168949604, "epoch": 0.21788008565310493, "frac_reward_zero_std": 0.0, "grad_norm": 0.00873376801609993, "learning_rate": 1e-05, "loss": 0.0343, "num_tokens": 8903383.0, "reward": 0.59375, "reward_std": 0.4218915104866028, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.5257455110549927, "sampling/importance_sampling_ratio/mean": 1.0000054836273193, "sampling/importance_sampling_ratio/min": 0.7207877039909363, "sampling/sampling_logp_difference/max": 0.4224832057952881, "sampling/sampling_logp_difference/mean": 0.010808401741087437, "step": 407 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00023761633565300144, "clip_ratio/low_min": 0.00023761633565300144, "clip_ratio/region_mean": 0.00023761633565300144, "completions/clipped_ratio": 0.28125, "completions/max_length": 768.0, "completions/max_terminated_length": 756.0, "completions/mean_length": 616.0625, "completions/mean_terminated_length": 556.6087036132812, "completions/min_length": 304.0, "completions/min_terminated_length": 304.0, "entropy": 0.35527442395687103, "epoch": 0.21841541755888652, "frac_reward_zero_std": 0.0, "grad_norm": 0.00962072890251875, "learning_rate": 1e-05, "loss": 0.095, "num_tokens": 8926441.0, "reward": 0.625, "reward_std": 0.3945523500442505, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.4273719787597656, "sampling/importance_sampling_ratio/mean": 1.000439167022705, "sampling/importance_sampling_ratio/min": 0.6599928140640259, "sampling/sampling_logp_difference/max": 0.4155263900756836, "sampling/sampling_logp_difference/mean": 0.011201596818864346, "step": 408 }, { "clip_ratio/high_max": 5.763024455518462e-05, "clip_ratio/high_mean": 5.763024455518462e-05, "clip_ratio/low_mean": 5.325947859091684e-05, "clip_ratio/low_min": 5.325947859091684e-05, "clip_ratio/region_mean": 0.00011088972314610146, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 767.0, "completions/mean_length": 564.90625, "completions/mean_terminated_length": 508.03997802734375, "completions/min_length": 277.0, "completions/min_terminated_length": 277.0, "entropy": 0.6263220384716988, "epoch": 0.21895074946466808, "frac_reward_zero_std": 0.0, "grad_norm": 0.03098428063094616, "learning_rate": 1e-05, "loss": 0.0041, "num_tokens": 8948174.0, "reward": 0.4375, "reward_std": 0.47655022144317627, "rewards/accuracy_reward/mean": 0.4375, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.4989752769470215, "sampling/importance_sampling_ratio/mean": 1.0001471042633057, "sampling/importance_sampling_ratio/min": 0.6510981917381287, "sampling/sampling_logp_difference/max": 0.4290947914123535, "sampling/sampling_logp_difference/mean": 0.016983481124043465, "step": 409 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 697.0, "completions/mean_length": 583.84375, "completions/mean_terminated_length": 532.2799682617188, "completions/min_length": 320.0, "completions/min_terminated_length": 320.0, "entropy": 0.4311000406742096, "epoch": 0.21948608137044967, "frac_reward_zero_std": 0.5, "grad_norm": 0.005811337381601334, "learning_rate": 1e-05, "loss": 0.0042, "num_tokens": 8970945.0, "reward": 0.8125, "reward_std": 0.249358132481575, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.6212395429611206, "sampling/importance_sampling_ratio/mean": 1.0001956224441528, "sampling/importance_sampling_ratio/min": 0.6533339619636536, "sampling/sampling_logp_difference/max": 0.48319101333618164, "sampling/sampling_logp_difference/mean": 0.013583813793957233, "step": 410 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00014990774798206985, "clip_ratio/low_min": 0.00014990774798206985, "clip_ratio/region_mean": 0.00014990774798206985, "completions/clipped_ratio": 0.34375, "completions/max_length": 768.0, "completions/max_terminated_length": 695.0, "completions/mean_length": 582.28125, "completions/mean_terminated_length": 485.0, "completions/min_length": 264.0, "completions/min_terminated_length": 264.0, "entropy": 0.6309629529714584, "epoch": 0.22002141327623126, "frac_reward_zero_std": 0.5, "grad_norm": 0.022864552214741707, "learning_rate": 1e-05, "loss": 0.0471, "num_tokens": 8994002.0, "reward": 0.4375, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.4375, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.4099698066711426, "sampling/importance_sampling_ratio/mean": 0.9998086094856262, "sampling/importance_sampling_ratio/min": 0.6662465333938599, "sampling/sampling_logp_difference/max": 0.4060955047607422, "sampling/sampling_logp_difference/mean": 0.01729053072631359, "step": 411 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.397236600401811e-05, "clip_ratio/low_min": 5.397236600401811e-05, "clip_ratio/region_mean": 5.397236600401811e-05, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 690.0, "completions/mean_length": 565.9375, "completions/mean_terminated_length": 537.0714721679688, "completions/min_length": 183.0, "completions/min_terminated_length": 183.0, "entropy": 0.3992793597280979, "epoch": 0.22055674518201285, "frac_reward_zero_std": 0.5, "grad_norm": 0.014462416060268879, "learning_rate": 1e-05, "loss": 0.0598, "num_tokens": 9015656.0, "reward": 0.5625, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.4708422422409058, "sampling/importance_sampling_ratio/mean": 1.0002230405807495, "sampling/importance_sampling_ratio/min": 0.712653398513794, "sampling/sampling_logp_difference/max": 0.3858351707458496, "sampling/sampling_logp_difference/mean": 0.011997113935649395, "step": 412 }, { "clip_ratio/high_max": 4.3706295400625095e-05, "clip_ratio/high_mean": 4.3706295400625095e-05, "clip_ratio/low_mean": 9.409108315594494e-05, "clip_ratio/low_min": 9.409108315594494e-05, "clip_ratio/region_mean": 0.00013779737855657004, "completions/clipped_ratio": 0.28125, "completions/max_length": 768.0, "completions/max_terminated_length": 754.0, "completions/mean_length": 629.9375, "completions/mean_terminated_length": 575.9130249023438, "completions/min_length": 341.0, "completions/min_terminated_length": 341.0, "entropy": 0.4415524788200855, "epoch": 0.22109207708779444, "frac_reward_zero_std": 0.0, "grad_norm": 0.004228921141475439, "learning_rate": 1e-05, "loss": 0.0496, "num_tokens": 9040142.0, "reward": 0.65625, "reward_std": 0.3808925747871399, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.423226237297058, "sampling/importance_sampling_ratio/mean": 0.9998692274093628, "sampling/importance_sampling_ratio/min": 0.6089670062065125, "sampling/sampling_logp_difference/max": 0.49599123001098633, "sampling/sampling_logp_difference/mean": 0.012853759340941906, "step": 413 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 7.271669892361388e-05, "clip_ratio/low_min": 7.271669892361388e-05, "clip_ratio/region_mean": 7.271669892361388e-05, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 662.0, "completions/mean_length": 504.4375, "completions/mean_terminated_length": 443.6153869628906, "completions/min_length": 227.0, "completions/min_terminated_length": 227.0, "entropy": 0.3778429850935936, "epoch": 0.22162740899357602, "frac_reward_zero_std": 0.25, "grad_norm": 0.0022418948356062174, "learning_rate": 1e-05, "loss": 0.022, "num_tokens": 9059884.0, "reward": 0.5, "reward_std": 0.292504221200943, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5080004930496216, "sampling/importance_sampling_ratio/max": 1.359606146812439, "sampling/importance_sampling_ratio/mean": 1.0003730058670044, "sampling/importance_sampling_ratio/min": 0.7151662111282349, "sampling/sampling_logp_difference/max": 0.33524036407470703, "sampling/sampling_logp_difference/mean": 0.01248636282980442, "step": 414 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 748.0, "completions/mean_length": 628.84375, "completions/mean_terminated_length": 582.4583740234375, "completions/min_length": 105.0, "completions/min_terminated_length": 105.0, "entropy": 0.521563958376646, "epoch": 0.2221627408993576, "frac_reward_zero_std": 0.25, "grad_norm": 0.005306879058480263, "learning_rate": 1e-05, "loss": -0.0034, "num_tokens": 9084127.0, "reward": 0.40625, "reward_std": 0.3198433816432953, "rewards/accuracy_reward/mean": 0.40625, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.612446665763855, "sampling/importance_sampling_ratio/mean": 1.000000238418579, "sampling/importance_sampling_ratio/min": 0.6933175325393677, "sampling/sampling_logp_difference/max": 0.477752685546875, "sampling/sampling_logp_difference/mean": 0.013742045499384403, "step": 415 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.0730519433273e-05, "clip_ratio/low_min": 5.0730519433273e-05, "clip_ratio/region_mean": 5.0730519433273e-05, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 737.0, "completions/mean_length": 602.8125, "completions/mean_terminated_length": 564.6923217773438, "completions/min_length": 328.0, "completions/min_terminated_length": 328.0, "entropy": 0.5645763855427504, "epoch": 0.22269807280513917, "frac_reward_zero_std": 0.5, "grad_norm": 0.020786359906196594, "learning_rate": 1e-05, "loss": -0.0038, "num_tokens": 9106849.0, "reward": 0.6875, "reward_std": 0.249358132481575, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.4992916584014893, "sampling/importance_sampling_ratio/mean": 0.999614417552948, "sampling/importance_sampling_ratio/min": 0.641618549823761, "sampling/sampling_logp_difference/max": 0.44376134872436523, "sampling/sampling_logp_difference/mean": 0.014872213825583458, "step": 416 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.28125, "completions/max_length": 768.0, "completions/max_terminated_length": 740.0, "completions/mean_length": 602.5, "completions/mean_terminated_length": 537.7391357421875, "completions/min_length": 258.0, "completions/min_terminated_length": 258.0, "entropy": 0.5658843778073788, "epoch": 0.22323340471092076, "frac_reward_zero_std": 0.5, "grad_norm": 0.02242957055568695, "learning_rate": 1e-05, "loss": 0.0509, "num_tokens": 9130089.0, "reward": 0.65625, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.4205304384231567, "sampling/importance_sampling_ratio/mean": 1.0001039505004883, "sampling/importance_sampling_ratio/min": 0.5605683922767639, "sampling/sampling_logp_difference/max": 0.5788040161132812, "sampling/sampling_logp_difference/mean": 0.015003115870058537, "step": 417 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00010851299521164037, "clip_ratio/low_min": 0.00010851299521164037, "clip_ratio/region_mean": 0.00010851299521164037, "completions/clipped_ratio": 0.28125, "completions/max_length": 768.0, "completions/max_terminated_length": 582.0, "completions/mean_length": 536.1875, "completions/mean_terminated_length": 445.478271484375, "completions/min_length": 296.0, "completions/min_terminated_length": 296.0, "entropy": 0.7728952057659626, "epoch": 0.22376873661670235, "frac_reward_zero_std": 0.5, "grad_norm": 0.01875445991754532, "learning_rate": 1e-05, "loss": 0.0335, "num_tokens": 9151927.0, "reward": 0.59375, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.2919591665267944, "sampling/importance_sampling_ratio/mean": 0.9998198747634888, "sampling/importance_sampling_ratio/min": 0.5904240012168884, "sampling/sampling_logp_difference/max": 0.5269143581390381, "sampling/sampling_logp_difference/mean": 0.019147329032421112, "step": 418 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.28125, "completions/max_length": 768.0, "completions/max_terminated_length": 765.0, "completions/mean_length": 599.15625, "completions/mean_terminated_length": 533.0869750976562, "completions/min_length": 238.0, "completions/min_terminated_length": 238.0, "entropy": 0.44611238315701485, "epoch": 0.22430406852248394, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0302, "num_tokens": 9174892.0, "reward": 0.65625, "reward_std": 0.1293872892856598, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.000004768371582, "sampling/importance_sampling_ratio/min": 0.5909232497215271, "sampling/sampling_logp_difference/max": 0.6977920532226562, "sampling/sampling_logp_difference/mean": 0.013043554499745369, "step": 419 }, { "clip_ratio/high_max": 4.763719334732741e-05, "clip_ratio/high_mean": 4.763719334732741e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 4.763719334732741e-05, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 733.0, "completions/mean_length": 580.875, "completions/mean_terminated_length": 546.2222290039062, "completions/min_length": 353.0, "completions/min_terminated_length": 353.0, "entropy": 0.3297699876129627, "epoch": 0.22483940042826553, "frac_reward_zero_std": 0.25, "grad_norm": 0.008070514537394047, "learning_rate": 1e-05, "loss": 0.0289, "num_tokens": 9197112.0, "reward": 0.65625, "reward_std": 0.3377464711666107, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.5710424184799194, "sampling/importance_sampling_ratio/mean": 1.0000263452529907, "sampling/importance_sampling_ratio/min": 0.5443271398544312, "sampling/sampling_logp_difference/max": 0.6082048416137695, "sampling/sampling_logp_difference/mean": 0.010427650064229965, "step": 420 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00010435800868435763, "clip_ratio/low_min": 0.00010435800868435763, "clip_ratio/region_mean": 0.00010435800868435763, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 758.0, "completions/mean_length": 561.1875, "completions/mean_terminated_length": 547.4000244140625, "completions/min_length": 326.0, "completions/min_terminated_length": 326.0, "entropy": 0.3939843326807022, "epoch": 0.22537473233404712, "frac_reward_zero_std": 0.25, "grad_norm": 0.01451626606285572, "learning_rate": 1e-05, "loss": 0.0222, "num_tokens": 9218894.0, "reward": 0.6875, "reward_std": 0.3514062464237213, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9999934434890747, "sampling/importance_sampling_ratio/min": 0.6974787712097168, "sampling/sampling_logp_difference/max": 0.7706835269927979, "sampling/sampling_logp_difference/mean": 0.012921259738504887, "step": 421 }, { "clip_ratio/high_max": 5.197505379328504e-05, "clip_ratio/high_mean": 5.197505379328504e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 5.197505379328504e-05, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 691.0, "completions/mean_length": 567.25, "completions/mean_terminated_length": 538.5714721679688, "completions/min_length": 314.0, "completions/min_terminated_length": 314.0, "entropy": 0.4246158041059971, "epoch": 0.2259100642398287, "frac_reward_zero_std": 0.5, "grad_norm": 0.016051743179559708, "learning_rate": 1e-05, "loss": 0.0089, "num_tokens": 9241310.0, "reward": 0.59375, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.4534480571746826, "sampling/importance_sampling_ratio/mean": 1.0003355741500854, "sampling/importance_sampling_ratio/min": 0.7063543200492859, "sampling/sampling_logp_difference/max": 0.3739386796951294, "sampling/sampling_logp_difference/mean": 0.012058069929480553, "step": 422 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 766.0, "completions/mean_length": 478.28125, "completions/mean_terminated_length": 448.3103332519531, "completions/min_length": 270.0, "completions/min_terminated_length": 270.0, "entropy": 0.4556988328695297, "epoch": 0.22644539614561027, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 9260031.0, "reward": 0.75, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.4392606019973755, "sampling/importance_sampling_ratio/mean": 1.0001574754714966, "sampling/importance_sampling_ratio/min": 0.6109625101089478, "sampling/sampling_logp_difference/max": 0.4927196502685547, "sampling/sampling_logp_difference/mean": 0.013808070681989193, "step": 423 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0001522822967672255, "clip_ratio/low_min": 0.0001522822967672255, "clip_ratio/region_mean": 0.0001522822967672255, "completions/clipped_ratio": 0.34375, "completions/max_length": 768.0, "completions/max_terminated_length": 663.0, "completions/mean_length": 559.84375, "completions/mean_terminated_length": 450.8095397949219, "completions/min_length": 256.0, "completions/min_terminated_length": 256.0, "entropy": 0.59726956859231, "epoch": 0.22698072805139186, "frac_reward_zero_std": 0.5, "grad_norm": 0.010211768560111523, "learning_rate": 1e-05, "loss": 0.0221, "num_tokens": 9282122.0, "reward": 0.5625, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.3962732553482056, "sampling/importance_sampling_ratio/mean": 1.0002508163452148, "sampling/importance_sampling_ratio/min": 0.4065399765968323, "sampling/sampling_logp_difference/max": 0.9000730514526367, "sampling/sampling_logp_difference/mean": 0.014666518196463585, "step": 424 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.362505544326268e-05, "clip_ratio/low_min": 5.362505544326268e-05, "clip_ratio/region_mean": 5.362505544326268e-05, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 676.0, "completions/mean_length": 495.75, "completions/mean_terminated_length": 456.857177734375, "completions/min_length": 193.0, "completions/min_terminated_length": 193.0, "entropy": 0.3560310807079077, "epoch": 0.22751605995717344, "frac_reward_zero_std": 0.5, "grad_norm": 0.010207312181591988, "learning_rate": 1e-05, "loss": 0.0621, "num_tokens": 9301418.0, "reward": 0.65625, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.383007526397705, "sampling/importance_sampling_ratio/mean": 0.9999959468841553, "sampling/importance_sampling_ratio/min": 0.6719864010810852, "sampling/sampling_logp_difference/max": 0.39751720428466797, "sampling/sampling_logp_difference/mean": 0.012058280408382416, "step": 425 }, { "clip_ratio/high_max": 5.763024455518462e-05, "clip_ratio/high_mean": 5.763024455518462e-05, "clip_ratio/low_mean": 0.0001055040156643372, "clip_ratio/low_min": 0.0001055040156643372, "clip_ratio/region_mean": 0.00016313426021952182, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 725.0, "completions/mean_length": 552.5, "completions/mean_terminated_length": 512.5925903320312, "completions/min_length": 288.0, "completions/min_terminated_length": 288.0, "entropy": 0.2971346490085125, "epoch": 0.22805139186295503, "frac_reward_zero_std": 0.25, "grad_norm": 0.011103782802820206, "learning_rate": 1e-05, "loss": 0.037, "num_tokens": 9322850.0, "reward": 0.78125, "reward_std": 0.3377465009689331, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.6555261611938477, "sampling/importance_sampling_ratio/mean": 1.0000243186950684, "sampling/importance_sampling_ratio/min": 0.5366583466529846, "sampling/sampling_logp_difference/max": 0.6223936080932617, "sampling/sampling_logp_difference/mean": 0.010465702041983604, "step": 426 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.375, "completions/max_length": 768.0, "completions/max_terminated_length": 687.0, "completions/mean_length": 577.09375, "completions/mean_terminated_length": 462.5500183105469, "completions/min_length": 284.0, "completions/min_terminated_length": 284.0, "entropy": 0.5005332380533218, "epoch": 0.22858672376873662, "frac_reward_zero_std": 0.25, "grad_norm": 0.004219317343086004, "learning_rate": 1e-05, "loss": 0.0215, "num_tokens": 9345213.0, "reward": 0.59375, "reward_std": 0.3061639964580536, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.4410640001296997, "sampling/importance_sampling_ratio/mean": 1.0001318454742432, "sampling/importance_sampling_ratio/min": 0.6258665323257446, "sampling/sampling_logp_difference/max": 0.46861815452575684, "sampling/sampling_logp_difference/mean": 0.011728247627615929, "step": 427 }, { "clip_ratio/high_max": 6.253126775845885e-05, "clip_ratio/high_mean": 6.253126775845885e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 6.253126775845885e-05, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 684.0, "completions/mean_length": 465.03125, "completions/mean_terminated_length": 421.7500305175781, "completions/min_length": 269.0, "completions/min_terminated_length": 269.0, "entropy": 0.42955965362489223, "epoch": 0.2291220556745182, "frac_reward_zero_std": 0.0, "grad_norm": 0.03251678869128227, "learning_rate": 1e-05, "loss": 0.0921, "num_tokens": 9363638.0, "reward": 0.6875, "reward_std": 0.49022960662841797, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.634706735610962, "sampling/importance_sampling_ratio/mean": 1.0000828504562378, "sampling/importance_sampling_ratio/min": 0.6828658580780029, "sampling/sampling_logp_difference/max": 0.49146342277526855, "sampling/sampling_logp_difference/mean": 0.012613005936145782, "step": 428 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00013609144662041217, "clip_ratio/low_min": 0.00013609144662041217, "clip_ratio/region_mean": 0.00013609144662041217, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 758.0, "completions/mean_length": 575.75, "completions/mean_terminated_length": 531.3846435546875, "completions/min_length": 335.0, "completions/min_terminated_length": 335.0, "entropy": 0.3703045956790447, "epoch": 0.2296573875802998, "frac_reward_zero_std": 0.5, "grad_norm": 0.010236953385174274, "learning_rate": 1e-05, "loss": 0.0438, "num_tokens": 9385950.0, "reward": 0.53125, "reward_std": 0.2630178928375244, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.4611130952835083, "sampling/importance_sampling_ratio/mean": 0.999873161315918, "sampling/importance_sampling_ratio/min": 0.6329577565193176, "sampling/sampling_logp_difference/max": 0.4573516845703125, "sampling/sampling_logp_difference/mean": 0.011274333111941814, "step": 429 }, { "clip_ratio/high_max": 4.978096330887638e-05, "clip_ratio/high_mean": 4.978096330887638e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 4.978096330887638e-05, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 627.0, "completions/mean_length": 425.21875, "completions/mean_terminated_length": 414.1612854003906, "completions/min_length": 114.0, "completions/min_terminated_length": 114.0, "entropy": 0.4106295108795166, "epoch": 0.23019271948608136, "frac_reward_zero_std": 0.25, "grad_norm": 0.007003496866673231, "learning_rate": 1e-05, "loss": 0.0523, "num_tokens": 9403085.0, "reward": 0.84375, "reward_std": 0.3198433816432953, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.3398635387420654, "sampling/importance_sampling_ratio/mean": 0.9995890855789185, "sampling/importance_sampling_ratio/min": 0.572006344795227, "sampling/sampling_logp_difference/max": 0.5586051940917969, "sampling/sampling_logp_difference/mean": 0.012640136294066906, "step": 430 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00013014055730309337, "clip_ratio/low_min": 0.00013014055730309337, "clip_ratio/region_mean": 0.00013014055730309337, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 628.0, "completions/mean_length": 415.375, "completions/mean_terminated_length": 391.86669921875, "completions/min_length": 141.0, "completions/min_terminated_length": 141.0, "entropy": 0.40977718308568, "epoch": 0.23072805139186295, "frac_reward_zero_std": 0.25, "grad_norm": 0.022661225870251656, "learning_rate": 1e-05, "loss": 0.1111, "num_tokens": 9420009.0, "reward": 0.8125, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.524586796760559, "sampling/importance_sampling_ratio/mean": 1.0001529455184937, "sampling/importance_sampling_ratio/min": 0.7325661778450012, "sampling/sampling_logp_difference/max": 0.4217233657836914, "sampling/sampling_logp_difference/mean": 0.012729874812066555, "step": 431 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.335040623322129e-05, "clip_ratio/low_min": 5.335040623322129e-05, "clip_ratio/region_mean": 5.335040623322129e-05, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 734.0, "completions/mean_length": 541.03125, "completions/mean_terminated_length": 499.0, "completions/min_length": 276.0, "completions/min_terminated_length": 276.0, "entropy": 0.395013727247715, "epoch": 0.23126338329764454, "frac_reward_zero_std": 0.25, "grad_norm": 0.018038636073470116, "learning_rate": 1e-05, "loss": 0.0501, "num_tokens": 9441250.0, "reward": 0.78125, "reward_std": 0.3061639666557312, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.3602145910263062, "sampling/importance_sampling_ratio/mean": 1.0005038976669312, "sampling/importance_sampling_ratio/min": 0.6148450970649719, "sampling/sampling_logp_difference/max": 0.48638486862182617, "sampling/sampling_logp_difference/mean": 0.012256108224391937, "step": 432 }, { "clip_ratio/high_max": 0.0001048838093993254, "clip_ratio/high_mean": 0.0001048838093993254, "clip_ratio/low_mean": 4.8468398745171726e-05, "clip_ratio/low_min": 4.8468398745171726e-05, "clip_ratio/region_mean": 0.00015335220814449713, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 748.0, "completions/mean_length": 610.71875, "completions/mean_terminated_length": 558.2916870117188, "completions/min_length": 383.0, "completions/min_terminated_length": 383.0, "entropy": 0.40235445462167263, "epoch": 0.23179871520342613, "frac_reward_zero_std": 0.0, "grad_norm": 0.02189856767654419, "learning_rate": 1e-05, "loss": 0.0304, "num_tokens": 9464625.0, "reward": 0.65625, "reward_std": 0.4628904461860657, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.4339468479156494, "sampling/importance_sampling_ratio/mean": 0.9997170567512512, "sampling/importance_sampling_ratio/min": 0.6185896992683411, "sampling/sampling_logp_difference/max": 0.4803130626678467, "sampling/sampling_logp_difference/mean": 0.011577612720429897, "step": 433 }, { "clip_ratio/high_max": 6.979340832913294e-05, "clip_ratio/high_mean": 6.979340832913294e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 6.979340832913294e-05, "completions/clipped_ratio": 0.40625, "completions/max_length": 768.0, "completions/max_terminated_length": 683.0, "completions/mean_length": 593.8125, "completions/mean_terminated_length": 474.631591796875, "completions/min_length": 220.0, "completions/min_terminated_length": 220.0, "entropy": 0.7491941712796688, "epoch": 0.23233404710920771, "frac_reward_zero_std": 0.5, "grad_norm": 0.005339367780834436, "learning_rate": 1e-05, "loss": 0.0305, "num_tokens": 9487267.0, "reward": 0.3125, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.3125, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9995450973510742, "sampling/importance_sampling_ratio/min": 0.30289116501808167, "sampling/sampling_logp_difference/max": 1.1943817138671875, "sampling/sampling_logp_difference/mean": 0.017373323440551758, "step": 434 }, { "clip_ratio/high_max": 6.963788473512977e-05, "clip_ratio/high_mean": 6.963788473512977e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 6.963788473512977e-05, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 681.0, "completions/mean_length": 447.96875, "completions/mean_terminated_length": 437.6451416015625, "completions/min_length": 202.0, "completions/min_terminated_length": 202.0, "entropy": 0.433709230273962, "epoch": 0.2328693790149893, "frac_reward_zero_std": 0.25, "grad_norm": 0.004871928598731756, "learning_rate": 1e-05, "loss": 0.1021, "num_tokens": 9505362.0, "reward": 0.90625, "reward_std": 0.2651650309562683, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.43455970287323, "sampling/importance_sampling_ratio/mean": 1.0003327131271362, "sampling/importance_sampling_ratio/min": 0.7222458720207214, "sampling/sampling_logp_difference/max": 0.3608579635620117, "sampling/sampling_logp_difference/mean": 0.01311936043202877, "step": 435 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00021607899543596432, "clip_ratio/low_min": 0.00021607899543596432, "clip_ratio/region_mean": 0.00021607899543596432, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 741.0, "completions/mean_length": 565.25, "completions/mean_terminated_length": 508.47998046875, "completions/min_length": 271.0, "completions/min_terminated_length": 271.0, "entropy": 0.43687858432531357, "epoch": 0.2334047109207709, "frac_reward_zero_std": 0.25, "grad_norm": 0.01967741549015045, "learning_rate": 1e-05, "loss": 0.0628, "num_tokens": 9527178.0, "reward": 0.46875, "reward_std": 0.3377464711666107, "rewards/accuracy_reward/mean": 0.46875, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.352242112159729, "sampling/importance_sampling_ratio/mean": 1.000245451927185, "sampling/importance_sampling_ratio/min": 0.6261184215545654, "sampling/sampling_logp_difference/max": 0.4682157635688782, "sampling/sampling_logp_difference/mean": 0.012557344511151314, "step": 436 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.835667616338469e-05, "clip_ratio/low_min": 5.835667616338469e-05, "clip_ratio/region_mean": 5.835667616338469e-05, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 743.0, "completions/mean_length": 504.28125, "completions/mean_terminated_length": 486.70001220703125, "completions/min_length": 274.0, "completions/min_terminated_length": 274.0, "entropy": 0.3174806609749794, "epoch": 0.23394004282655245, "frac_reward_zero_std": 0.0, "grad_norm": 0.009594975039362907, "learning_rate": 1e-05, "loss": -0.0158, "num_tokens": 9546547.0, "reward": 0.6875, "reward_std": 0.4492306709289551, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.6969773769378662, "sampling/importance_sampling_ratio/mean": 1.0000183582305908, "sampling/importance_sampling_ratio/min": 0.4709964990615845, "sampling/sampling_logp_difference/max": 0.7529046535491943, "sampling/sampling_logp_difference/mean": 0.010570117272436619, "step": 437 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00013532026423490606, "clip_ratio/low_min": 0.00013532026423490606, "clip_ratio/region_mean": 0.00013532026423490606, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 766.0, "completions/mean_length": 524.0, "completions/mean_terminated_length": 478.8148193359375, "completions/min_length": 234.0, "completions/min_terminated_length": 234.0, "entropy": 0.399887278676033, "epoch": 0.23447537473233404, "frac_reward_zero_std": 0.5, "grad_norm": 0.00976628065109253, "learning_rate": 1e-05, "loss": 0.0289, "num_tokens": 9567579.0, "reward": 0.71875, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.7815918922424316, "sampling/importance_sampling_ratio/mean": 0.9996789693832397, "sampling/importance_sampling_ratio/min": 0.47193360328674316, "sampling/sampling_logp_difference/max": 0.7509169578552246, "sampling/sampling_logp_difference/mean": 0.011319872923195362, "step": 438 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.673785537714139e-05, "clip_ratio/low_min": 6.673785537714139e-05, "clip_ratio/region_mean": 6.673785537714139e-05, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 737.0, "completions/mean_length": 460.8125, "completions/mean_terminated_length": 450.9031982421875, "completions/min_length": 281.0, "completions/min_terminated_length": 281.0, "entropy": 0.41155969724059105, "epoch": 0.23501070663811563, "frac_reward_zero_std": 0.5, "grad_norm": 0.020215319469571114, "learning_rate": 1e-05, "loss": -0.0528, "num_tokens": 9585845.0, "reward": 0.75, "reward_std": 0.2314550280570984, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9997062683105469, "sampling/importance_sampling_ratio/min": 0.6609448194503784, "sampling/sampling_logp_difference/max": 0.7664585113525391, "sampling/sampling_logp_difference/mean": 0.012989513576030731, "step": 439 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00031872427643975243, "clip_ratio/low_min": 0.00031872427643975243, "clip_ratio/region_mean": 0.00031872427643975243, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 746.0, "completions/mean_length": 545.0, "completions/mean_terminated_length": 503.7037048339844, "completions/min_length": 289.0, "completions/min_terminated_length": 289.0, "entropy": 0.4743258059024811, "epoch": 0.23554603854389722, "frac_reward_zero_std": 0.0, "grad_norm": 0.021723873913288116, "learning_rate": 1e-05, "loss": 0.0247, "num_tokens": 9607157.0, "reward": 0.53125, "reward_std": 0.4628904461860657, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.456073522567749, "sampling/importance_sampling_ratio/mean": 1.0003944635391235, "sampling/importance_sampling_ratio/min": 0.5540905594825745, "sampling/sampling_logp_difference/max": 0.5904271006584167, "sampling/sampling_logp_difference/mean": 0.01416078768670559, "step": 440 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 664.0, "completions/mean_length": 417.6875, "completions/mean_terminated_length": 406.3870849609375, "completions/min_length": 229.0, "completions/min_terminated_length": 229.0, "entropy": 0.4558962471783161, "epoch": 0.2360813704496788, "frac_reward_zero_std": 0.5, "grad_norm": 0.0036255482118576765, "learning_rate": 1e-05, "loss": -0.0207, "num_tokens": 9623923.0, "reward": 0.65625, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.5447536706924438, "sampling/importance_sampling_ratio/mean": 0.9993640780448914, "sampling/importance_sampling_ratio/min": 0.43391481041908264, "sampling/sampling_logp_difference/max": 0.834907054901123, "sampling/sampling_logp_difference/mean": 0.013908558525145054, "step": 441 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00022008095402270555, "clip_ratio/low_min": 0.00022008095402270555, "clip_ratio/region_mean": 0.00022008095402270555, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 709.0, "completions/mean_length": 476.25, "completions/mean_terminated_length": 456.8000183105469, "completions/min_length": 201.0, "completions/min_terminated_length": 201.0, "entropy": 0.41936802864074707, "epoch": 0.2366167023554604, "frac_reward_zero_std": 0.25, "grad_norm": 0.028359320014715195, "learning_rate": 1e-05, "loss": -0.0155, "num_tokens": 9643083.0, "reward": 0.65625, "reward_std": 0.3608423173427582, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.4420297145843506, "sampling/importance_sampling_ratio/mean": 1.0000584125518799, "sampling/importance_sampling_ratio/min": 0.5562911033630371, "sampling/sampling_logp_difference/max": 0.5864635705947876, "sampling/sampling_logp_difference/mean": 0.013299894519150257, "step": 442 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.620503725367598e-05, "clip_ratio/low_min": 5.620503725367598e-05, "clip_ratio/region_mean": 5.620503725367598e-05, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 747.0, "completions/mean_length": 469.4375, "completions/mean_terminated_length": 459.8064270019531, "completions/min_length": 274.0, "completions/min_terminated_length": 274.0, "entropy": 0.3731671664863825, "epoch": 0.23715203426124196, "frac_reward_zero_std": 0.75, "grad_norm": 0.008341815322637558, "learning_rate": 1e-05, "loss": 0.0316, "num_tokens": 9661465.0, "reward": 0.8125, "reward_std": 0.1157275140285492, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.4305990934371948, "sampling/importance_sampling_ratio/mean": 0.9999405741691589, "sampling/importance_sampling_ratio/min": 0.6110574007034302, "sampling/sampling_logp_difference/max": 0.49256443977355957, "sampling/sampling_logp_difference/mean": 0.011082973331212997, "step": 443 }, { "clip_ratio/high_max": 7.237985118990764e-05, "clip_ratio/high_mean": 7.237985118990764e-05, "clip_ratio/low_mean": 9.527438669465482e-05, "clip_ratio/low_min": 9.527438669465482e-05, "clip_ratio/region_mean": 0.00016765423788456246, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 708.0, "completions/mean_length": 423.46875, "completions/mean_terminated_length": 400.5000305175781, "completions/min_length": 202.0, "completions/min_terminated_length": 202.0, "entropy": 0.5332957282662392, "epoch": 0.23768736616702354, "frac_reward_zero_std": 0.5, "grad_norm": 0.002445866586640477, "learning_rate": 1e-05, "loss": 0.0121, "num_tokens": 9678536.0, "reward": 0.75, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.525464415550232, "sampling/importance_sampling_ratio/mean": 0.9999432563781738, "sampling/importance_sampling_ratio/min": 0.3964083790779114, "sampling/sampling_logp_difference/max": 0.9253103733062744, "sampling/sampling_logp_difference/mean": 0.014590022154152393, "step": 444 }, { "clip_ratio/high_max": 7.678132533328608e-05, "clip_ratio/high_mean": 7.678132533328608e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 7.678132533328608e-05, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 724.0, "completions/mean_length": 511.0, "completions/mean_terminated_length": 484.4137878417969, "completions/min_length": 243.0, "completions/min_terminated_length": 243.0, "entropy": 0.3169939760118723, "epoch": 0.23822269807280513, "frac_reward_zero_std": 0.0, "grad_norm": 0.008223761804401875, "learning_rate": 1e-05, "loss": 0.0289, "num_tokens": 9698456.0, "reward": 0.75, "reward_std": 0.3945523500442505, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.3793164491653442, "sampling/importance_sampling_ratio/mean": 1.0003175735473633, "sampling/importance_sampling_ratio/min": 0.4191257655620575, "sampling/sampling_logp_difference/max": 0.869584321975708, "sampling/sampling_logp_difference/mean": 0.010988960973918438, "step": 445 }, { "clip_ratio/high_max": 6.603274960070848e-05, "clip_ratio/high_mean": 6.603274960070848e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 6.603274960070848e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 637.0, "completions/max_terminated_length": 637.0, "completions/mean_length": 498.21875, "completions/mean_terminated_length": 498.21875, "completions/min_length": 322.0, "completions/min_terminated_length": 322.0, "entropy": 0.2948935627937317, "epoch": 0.23875802997858672, "frac_reward_zero_std": 0.25, "grad_norm": 0.007870606146752834, "learning_rate": 1e-05, "loss": -0.005, "num_tokens": 9718255.0, "reward": 0.8125, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.6618587970733643, "sampling/importance_sampling_ratio/mean": 0.9999118447303772, "sampling/importance_sampling_ratio/min": 0.5116075277328491, "sampling/sampling_logp_difference/max": 0.6701974868774414, "sampling/sampling_logp_difference/mean": 0.01068306714296341, "step": 446 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 686.0, "completions/mean_length": 499.96875, "completions/mean_terminated_length": 491.32257080078125, "completions/min_length": 258.0, "completions/min_terminated_length": 258.0, "entropy": 0.31847131066024303, "epoch": 0.2392933618843683, "frac_reward_zero_std": 0.25, "grad_norm": 0.003491860581561923, "learning_rate": 1e-05, "loss": 0.0373, "num_tokens": 9738102.0, "reward": 0.75, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.5237727165222168, "sampling/importance_sampling_ratio/mean": 0.9999576807022095, "sampling/importance_sampling_ratio/min": 0.642872154712677, "sampling/sampling_logp_difference/max": 0.44180941581726074, "sampling/sampling_logp_difference/mean": 0.011166163720190525, "step": 447 }, { "clip_ratio/high_max": 8.278145833173767e-05, "clip_ratio/high_mean": 8.278145833173767e-05, "clip_ratio/low_mean": 0.0002440301250317134, "clip_ratio/low_min": 0.0002440301250317134, "clip_ratio/region_mean": 0.00032681158336345106, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 757.0, "completions/mean_length": 503.8125, "completions/mean_terminated_length": 442.8461608886719, "completions/min_length": 302.0, "completions/min_terminated_length": 302.0, "entropy": 0.46436743438243866, "epoch": 0.2398286937901499, "frac_reward_zero_std": 0.0, "grad_norm": 0.0266990065574646, "learning_rate": 1e-05, "loss": 0.0961, "num_tokens": 9758248.0, "reward": 0.59375, "reward_std": 0.4218915104866028, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.5394748449325562, "sampling/importance_sampling_ratio/mean": 1.0004838705062866, "sampling/importance_sampling_ratio/min": 0.37726452946662903, "sampling/sampling_logp_difference/max": 0.9748086929321289, "sampling/sampling_logp_difference/mean": 0.013636860065162182, "step": 448 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.736576349590905e-05, "clip_ratio/low_min": 5.736576349590905e-05, "clip_ratio/region_mean": 5.736576349590905e-05, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 709.0, "completions/mean_length": 532.5, "completions/mean_terminated_length": 508.137939453125, "completions/min_length": 314.0, "completions/min_terminated_length": 314.0, "entropy": 0.5246148034930229, "epoch": 0.2403640256959315, "frac_reward_zero_std": 0.25, "grad_norm": 0.01972244307398796, "learning_rate": 1e-05, "loss": -0.0051, "num_tokens": 9778928.0, "reward": 0.59375, "reward_std": 0.38816186785697937, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.4355343580245972, "sampling/importance_sampling_ratio/mean": 0.9999771118164062, "sampling/importance_sampling_ratio/min": 0.6700218319892883, "sampling/sampling_logp_difference/max": 0.40044498443603516, "sampling/sampling_logp_difference/mean": 0.014444036409258842, "step": 449 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 736.0, "completions/mean_length": 521.21875, "completions/mean_terminated_length": 475.5185241699219, "completions/min_length": 255.0, "completions/min_terminated_length": 255.0, "entropy": 0.3715777900069952, "epoch": 0.24089935760171305, "frac_reward_zero_std": 0.5, "grad_norm": 0.015243914909660816, "learning_rate": 1e-05, "loss": 0.0501, "num_tokens": 9798983.0, "reward": 0.84375, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9999101758003235, "sampling/importance_sampling_ratio/min": 0.6141006350517273, "sampling/sampling_logp_difference/max": 0.6950243711471558, "sampling/sampling_logp_difference/mean": 0.012243429198861122, "step": 450 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00011904113853233866, "clip_ratio/low_min": 0.00011904113853233866, "clip_ratio/region_mean": 0.00011904113853233866, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 653.0, "completions/mean_length": 494.53125, "completions/mean_terminated_length": 455.46429443359375, "completions/min_length": 198.0, "completions/min_terminated_length": 198.0, "entropy": 0.49071974121034145, "epoch": 0.24143468950749464, "frac_reward_zero_std": 0.25, "grad_norm": 0.019020816311240196, "learning_rate": 1e-05, "loss": 0.0911, "num_tokens": 9818416.0, "reward": 0.65625, "reward_std": 0.3471629321575165, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.493733525276184, "sampling/importance_sampling_ratio/mean": 1.0001856088638306, "sampling/importance_sampling_ratio/min": 0.6480744481086731, "sampling/sampling_logp_difference/max": 0.4337496757507324, "sampling/sampling_logp_difference/mean": 0.013320188038051128, "step": 451 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 705.0, "completions/mean_length": 442.71875, "completions/mean_terminated_length": 421.0333557128906, "completions/min_length": 167.0, "completions/min_terminated_length": 167.0, "entropy": 0.3883431516587734, "epoch": 0.24197002141327623, "frac_reward_zero_std": 0.5, "grad_norm": 0.023118771612644196, "learning_rate": 1e-05, "loss": 0.0155, "num_tokens": 9835903.0, "reward": 0.65625, "reward_std": 0.22201895713806152, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.4577070474624634, "sampling/importance_sampling_ratio/mean": 1.0001904964447021, "sampling/importance_sampling_ratio/min": 0.7477656602859497, "sampling/sampling_logp_difference/max": 0.3768646717071533, "sampling/sampling_logp_difference/mean": 0.012309646233916283, "step": 452 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00011099191760877147, "clip_ratio/low_min": 0.00011099191760877147, "clip_ratio/region_mean": 0.00011099191760877147, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 699.0, "completions/mean_length": 474.65625, "completions/mean_terminated_length": 455.10003662109375, "completions/min_length": 214.0, "completions/min_terminated_length": 214.0, "entropy": 0.447996161878109, "epoch": 0.24250535331905781, "frac_reward_zero_std": 0.25, "grad_norm": 0.01568351499736309, "learning_rate": 1e-05, "loss": 0.0393, "num_tokens": 9854684.0, "reward": 0.65625, "reward_std": 0.3787454068660736, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.2972382307052612, "sampling/importance_sampling_ratio/mean": 0.9994772672653198, "sampling/importance_sampling_ratio/min": 0.6115005612373352, "sampling/sampling_logp_difference/max": 0.4918394088745117, "sampling/sampling_logp_difference/mean": 0.013990916311740875, "step": 453 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00011640253069344908, "clip_ratio/low_min": 0.00011640253069344908, "clip_ratio/region_mean": 0.00011640253069344908, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 738.0, "completions/mean_length": 584.40625, "completions/mean_terminated_length": 533.0, "completions/min_length": 318.0, "completions/min_terminated_length": 318.0, "entropy": 0.41890430450439453, "epoch": 0.2430406852248394, "frac_reward_zero_std": 0.25, "grad_norm": 0.009600254707038403, "learning_rate": 1e-05, "loss": 0.014, "num_tokens": 9877497.0, "reward": 0.40625, "reward_std": 0.3608423173427582, "rewards/accuracy_reward/mean": 0.40625, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.6659141778945923, "sampling/importance_sampling_ratio/mean": 0.9993649125099182, "sampling/importance_sampling_ratio/min": 0.07411067187786102, "sampling/sampling_logp_difference/max": 2.6021957397460938, "sampling/sampling_logp_difference/mean": 0.013599834404885769, "step": 454 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00020961598784197122, "clip_ratio/low_min": 0.00020961598784197122, "clip_ratio/region_mean": 0.00020961598784197122, "completions/clipped_ratio": 0.375, "completions/max_length": 768.0, "completions/max_terminated_length": 755.0, "completions/mean_length": 608.25, "completions/mean_terminated_length": 512.4000244140625, "completions/min_length": 252.0, "completions/min_terminated_length": 252.0, "entropy": 0.5103533044457436, "epoch": 0.243576017130621, "frac_reward_zero_std": 0.5, "grad_norm": 0.002938179299235344, "learning_rate": 1e-05, "loss": 0.0594, "num_tokens": 9901201.0, "reward": 0.5, "reward_std": 0.2314550280570984, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5080004930496216, "sampling/importance_sampling_ratio/max": 1.438673973083496, "sampling/importance_sampling_ratio/mean": 1.0000073909759521, "sampling/importance_sampling_ratio/min": 0.5488241314888, "sampling/sampling_logp_difference/max": 0.5999772548675537, "sampling/sampling_logp_difference/mean": 0.014897631481289864, "step": 455 }, { "clip_ratio/high_max": 6.073858094168827e-05, "clip_ratio/high_mean": 6.073858094168827e-05, "clip_ratio/low_mean": 0.0001444907029508613, "clip_ratio/low_min": 0.0001444907029508613, "clip_ratio/region_mean": 0.00020522928389254957, "completions/clipped_ratio": 0.375, "completions/max_length": 768.0, "completions/max_terminated_length": 711.0, "completions/mean_length": 560.6875, "completions/mean_terminated_length": 436.3000183105469, "completions/min_length": 198.0, "completions/min_terminated_length": 198.0, "entropy": 0.6812451519072056, "epoch": 0.24411134903640258, "frac_reward_zero_std": 0.25, "grad_norm": 0.0035004240926355124, "learning_rate": 1e-05, "loss": 0.0904, "num_tokens": 9922999.0, "reward": 0.4375, "reward_std": 0.2925041913986206, "rewards/accuracy_reward/mean": 0.4375, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.5718141794204712, "sampling/importance_sampling_ratio/mean": 1.0002167224884033, "sampling/importance_sampling_ratio/min": 0.6040858030319214, "sampling/sampling_logp_difference/max": 0.5040390491485596, "sampling/sampling_logp_difference/mean": 0.016588600352406502, "step": 456 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 7.788161747157574e-05, "clip_ratio/low_min": 7.788161747157574e-05, "clip_ratio/region_mean": 7.788161747157574e-05, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 717.0, "completions/mean_length": 464.0625, "completions/mean_terminated_length": 454.258056640625, "completions/min_length": 218.0, "completions/min_terminated_length": 218.0, "entropy": 0.40892244130373, "epoch": 0.24464668094218414, "frac_reward_zero_std": 0.0, "grad_norm": 0.016150210052728653, "learning_rate": 1e-05, "loss": 0.0923, "num_tokens": 9941793.0, "reward": 0.6875, "reward_std": 0.44403791427612305, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.5257829427719116, "sampling/importance_sampling_ratio/mean": 0.9999562501907349, "sampling/importance_sampling_ratio/min": 0.6359580159187317, "sampling/sampling_logp_difference/max": 0.452622652053833, "sampling/sampling_logp_difference/mean": 0.01319899596273899, "step": 457 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.416961019160226e-05, "clip_ratio/low_min": 4.416961019160226e-05, "clip_ratio/region_mean": 4.416961019160226e-05, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 751.0, "completions/mean_length": 502.78125, "completions/mean_terminated_length": 441.5769348144531, "completions/min_length": 252.0, "completions/min_terminated_length": 252.0, "entropy": 0.5654423013329506, "epoch": 0.24518201284796573, "frac_reward_zero_std": 0.25, "grad_norm": 0.037923622876405716, "learning_rate": 1e-05, "loss": 0.1189, "num_tokens": 9961354.0, "reward": 0.6875, "reward_std": 0.3471825420856476, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.4707679748535156, "sampling/importance_sampling_ratio/mean": 1.0002282857894897, "sampling/importance_sampling_ratio/min": 0.6000503897666931, "sampling/sampling_logp_difference/max": 0.5107417106628418, "sampling/sampling_logp_difference/mean": 0.015262113884091377, "step": 458 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.724044942529872e-05, "clip_ratio/low_min": 6.724044942529872e-05, "clip_ratio/region_mean": 6.724044942529872e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 611.0, "completions/max_terminated_length": 611.0, "completions/mean_length": 419.28125, "completions/mean_terminated_length": 419.28125, "completions/min_length": 175.0, "completions/min_terminated_length": 175.0, "entropy": 0.4602057747542858, "epoch": 0.24571734475374732, "frac_reward_zero_std": 0.25, "grad_norm": 0.010243136435747147, "learning_rate": 1e-05, "loss": -0.0229, "num_tokens": 9978419.0, "reward": 0.8125, "reward_std": 0.3104073107242584, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.4284471273422241, "sampling/importance_sampling_ratio/mean": 0.9997591376304626, "sampling/importance_sampling_ratio/min": 0.6500998735427856, "sampling/sampling_logp_difference/max": 0.43062925338745117, "sampling/sampling_logp_difference/mean": 0.012870880775153637, "step": 459 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 642.0, "completions/mean_length": 451.0, "completions/mean_terminated_length": 429.86669921875, "completions/min_length": 145.0, "completions/min_terminated_length": 145.0, "entropy": 0.41782718896865845, "epoch": 0.2462526766595289, "frac_reward_zero_std": 0.5, "grad_norm": 0.0030655178707093, "learning_rate": 1e-05, "loss": 0.0751, "num_tokens": 9996075.0, "reward": 0.8125, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.4146220684051514, "sampling/importance_sampling_ratio/mean": 0.9997550249099731, "sampling/importance_sampling_ratio/min": 0.6687881350517273, "sampling/sampling_logp_difference/max": 0.40228796005249023, "sampling/sampling_logp_difference/mean": 0.01258467324078083, "step": 460 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.967670196900144e-05, "clip_ratio/low_min": 6.967670196900144e-05, "clip_ratio/region_mean": 6.967670196900144e-05, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 698.0, "completions/mean_length": 467.4375, "completions/mean_terminated_length": 457.7419128417969, "completions/min_length": 236.0, "completions/min_terminated_length": 236.0, "entropy": 0.3530072644352913, "epoch": 0.2467880085653105, "frac_reward_zero_std": 0.75, "grad_norm": 0.005839026998728514, "learning_rate": 1e-05, "loss": 0.0341, "num_tokens": 10014713.0, "reward": 0.9375, "reward_std": 0.1157275140285492, "rewards/accuracy_reward/mean": 0.9375, "rewards/accuracy_reward/std": 0.24593468010425568, "sampling/importance_sampling_ratio/max": 1.3702119588851929, "sampling/importance_sampling_ratio/mean": 1.0002683401107788, "sampling/importance_sampling_ratio/min": 0.3436853587627411, "sampling/sampling_logp_difference/max": 1.0680286884307861, "sampling/sampling_logp_difference/mean": 0.011217878200113773, "step": 461 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 767.0, "completions/max_terminated_length": 767.0, "completions/mean_length": 504.3125, "completions/mean_terminated_length": 504.3125, "completions/min_length": 291.0, "completions/min_terminated_length": 291.0, "entropy": 0.3424733020365238, "epoch": 0.24732334047109208, "frac_reward_zero_std": 0.25, "grad_norm": 0.0069100093096494675, "learning_rate": 1e-05, "loss": -0.0329, "num_tokens": 10034699.0, "reward": 0.875, "reward_std": 0.292504221200943, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 1.4882044792175293, "sampling/importance_sampling_ratio/mean": 0.9999819397926331, "sampling/importance_sampling_ratio/min": 0.6906961798667908, "sampling/sampling_logp_difference/max": 0.3975703716278076, "sampling/sampling_logp_difference/mean": 0.011526926420629025, "step": 462 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00012753055852954276, "clip_ratio/low_min": 0.00012753055852954276, "clip_ratio/region_mean": 0.00012753055852954276, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 697.0, "completions/mean_length": 564.5, "completions/mean_terminated_length": 496.66668701171875, "completions/min_length": 240.0, "completions/min_terminated_length": 240.0, "entropy": 0.526708697900176, "epoch": 0.24785867237687367, "frac_reward_zero_std": 0.0, "grad_norm": 0.014757676981389523, "learning_rate": 1e-05, "loss": 0.0036, "num_tokens": 10056763.0, "reward": 0.53125, "reward_std": 0.3808925747871399, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.4526184797286987, "sampling/importance_sampling_ratio/mean": 1.000228762626648, "sampling/importance_sampling_ratio/min": 0.6797546744346619, "sampling/sampling_logp_difference/max": 0.38602328300476074, "sampling/sampling_logp_difference/mean": 0.014186311513185501, "step": 463 }, { "clip_ratio/high_max": 4.722327139461413e-05, "clip_ratio/high_mean": 4.722327139461413e-05, "clip_ratio/low_mean": 5.024115671403706e-05, "clip_ratio/low_min": 5.024115671403706e-05, "clip_ratio/region_mean": 9.746442810865119e-05, "completions/clipped_ratio": 0.4375, "completions/max_length": 768.0, "completions/max_terminated_length": 765.0, "completions/mean_length": 633.15625, "completions/mean_terminated_length": 528.2777709960938, "completions/min_length": 340.0, "completions/min_terminated_length": 340.0, "entropy": 0.503486780449748, "epoch": 0.24839400428265523, "frac_reward_zero_std": 0.25, "grad_norm": 0.009431887418031693, "learning_rate": 1e-05, "loss": 0.0467, "num_tokens": 10081448.0, "reward": 0.4375, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.4375, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.4104869365692139, "sampling/importance_sampling_ratio/mean": 0.9998577237129211, "sampling/importance_sampling_ratio/min": 0.6300022602081299, "sampling/sampling_logp_difference/max": 0.46203184127807617, "sampling/sampling_logp_difference/mean": 0.013622620142996311, "step": 464 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 693.0, "completions/mean_length": 444.625, "completions/mean_terminated_length": 434.19354248046875, "completions/min_length": 161.0, "completions/min_terminated_length": 161.0, "entropy": 0.3349144458770752, "epoch": 0.24892933618843682, "frac_reward_zero_std": 0.5, "grad_norm": 0.017284739762544632, "learning_rate": 1e-05, "loss": -0.0071, "num_tokens": 10099316.0, "reward": 0.75, "reward_std": 0.2587745785713196, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.4152506589889526, "sampling/importance_sampling_ratio/mean": 1.0000320672988892, "sampling/importance_sampling_ratio/min": 0.5610714554786682, "sampling/sampling_logp_difference/max": 0.5779069662094116, "sampling/sampling_logp_difference/mean": 0.012320639565587044, "step": 465 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00010665528679965064, "clip_ratio/low_min": 0.00010665528679965064, "clip_ratio/region_mean": 0.00010665528679965064, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 732.0, "completions/mean_length": 506.84375, "completions/mean_terminated_length": 458.4814758300781, "completions/min_length": 210.0, "completions/min_terminated_length": 210.0, "entropy": 0.45612189546227455, "epoch": 0.2494646680942184, "frac_reward_zero_std": 0.0, "grad_norm": 0.008770694024860859, "learning_rate": 1e-05, "loss": 0.1441, "num_tokens": 10119375.0, "reward": 0.40625, "reward_std": 0.4807935357093811, "rewards/accuracy_reward/mean": 0.40625, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.6200143098831177, "sampling/importance_sampling_ratio/mean": 1.000049352645874, "sampling/importance_sampling_ratio/min": 0.696904182434082, "sampling/sampling_logp_difference/max": 0.4824349880218506, "sampling/sampling_logp_difference/mean": 0.014635104686021805, "step": 466 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.778741953894496e-05, "clip_ratio/low_min": 6.778741953894496e-05, "clip_ratio/region_mean": 6.778741953894496e-05, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 763.0, "completions/mean_length": 538.875, "completions/mean_terminated_length": 515.1724243164062, "completions/min_length": 139.0, "completions/min_terminated_length": 139.0, "entropy": 0.38099303282797337, "epoch": 0.25, "frac_reward_zero_std": 0.0, "grad_norm": 0.005283311475068331, "learning_rate": 1e-05, "loss": 0.0439, "num_tokens": 10140091.0, "reward": 0.8125, "reward_std": 0.4082317352294922, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.3786720037460327, "sampling/importance_sampling_ratio/mean": 1.0000300407409668, "sampling/importance_sampling_ratio/min": 0.6642163991928101, "sampling/sampling_logp_difference/max": 0.4091472625732422, "sampling/sampling_logp_difference/mean": 0.011428258381783962, "step": 467 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 724.0, "completions/mean_length": 568.125, "completions/mean_terminated_length": 512.1599731445312, "completions/min_length": 352.0, "completions/min_terminated_length": 352.0, "entropy": 0.45564014092087746, "epoch": 0.2505353319057816, "frac_reward_zero_std": 0.25, "grad_norm": 0.01017637737095356, "learning_rate": 1e-05, "loss": 0.0776, "num_tokens": 10162127.0, "reward": 0.375, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.375, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.3832758665084839, "sampling/importance_sampling_ratio/mean": 1.0002566576004028, "sampling/importance_sampling_ratio/min": 0.6669967770576477, "sampling/sampling_logp_difference/max": 0.40497004985809326, "sampling/sampling_logp_difference/mean": 0.013177342712879181, "step": 468 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00011625924889813177, "clip_ratio/low_min": 0.00011625924889813177, "clip_ratio/region_mean": 0.00011625924889813177, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 687.0, "completions/mean_length": 463.3125, "completions/mean_terminated_length": 431.7930908203125, "completions/min_length": 263.0, "completions/min_terminated_length": 263.0, "entropy": 0.5970989130437374, "epoch": 0.2510706638115632, "frac_reward_zero_std": 0.25, "grad_norm": 0.02166728861629963, "learning_rate": 1e-05, "loss": 0.0339, "num_tokens": 10180289.0, "reward": 0.53125, "reward_std": 0.3377465009689331, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.4432886838912964, "sampling/importance_sampling_ratio/mean": 0.9999966025352478, "sampling/importance_sampling_ratio/min": 0.5740286111831665, "sampling/sampling_logp_difference/max": 0.5550761222839355, "sampling/sampling_logp_difference/mean": 0.01610528491437435, "step": 469 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 8.02825961727649e-05, "clip_ratio/low_min": 8.02825961727649e-05, "clip_ratio/region_mean": 8.02825961727649e-05, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 742.0, "completions/mean_length": 540.1875, "completions/mean_terminated_length": 476.3999938964844, "completions/min_length": 201.0, "completions/min_terminated_length": 201.0, "entropy": 0.436391057446599, "epoch": 0.25160599571734477, "frac_reward_zero_std": 0.0, "grad_norm": 0.013399085029959679, "learning_rate": 1e-05, "loss": -0.0402, "num_tokens": 10201591.0, "reward": 0.625, "reward_std": 0.4355512857437134, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.8025364875793457, "sampling/importance_sampling_ratio/mean": 0.9997819662094116, "sampling/importance_sampling_ratio/min": 0.4784715473651886, "sampling/sampling_logp_difference/max": 0.7371585369110107, "sampling/sampling_logp_difference/mean": 0.013918759301304817, "step": 470 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.796623034053482e-05, "clip_ratio/low_min": 4.796623034053482e-05, "clip_ratio/region_mean": 4.796623034053482e-05, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 766.0, "completions/mean_length": 528.1875, "completions/mean_terminated_length": 472.8461608886719, "completions/min_length": 250.0, "completions/min_terminated_length": 250.0, "entropy": 0.413949154317379, "epoch": 0.25214132762312635, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.077, "num_tokens": 10221965.0, "reward": 0.8125, "reward_std": 0.1157275140285492, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.5112911462783813, "sampling/importance_sampling_ratio/mean": 1.000109076499939, "sampling/importance_sampling_ratio/min": 0.6991905570030212, "sampling/sampling_logp_difference/max": 0.4129643440246582, "sampling/sampling_logp_difference/mean": 0.01309647411108017, "step": 471 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.834335363237187e-05, "clip_ratio/low_min": 6.834335363237187e-05, "clip_ratio/region_mean": 6.834335363237187e-05, "completions/clipped_ratio": 0.40625, "completions/max_length": 768.0, "completions/max_terminated_length": 673.0, "completions/mean_length": 613.6875, "completions/mean_terminated_length": 508.1052551269531, "completions/min_length": 192.0, "completions/min_terminated_length": 192.0, "entropy": 0.3977750465273857, "epoch": 0.25267665952890794, "frac_reward_zero_std": 0.5, "grad_norm": 0.010896327905356884, "learning_rate": 1e-05, "loss": 0.0613, "num_tokens": 10245251.0, "reward": 0.53125, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.2770696878433228, "sampling/importance_sampling_ratio/mean": 0.9999854564666748, "sampling/importance_sampling_ratio/min": 0.6412532925605774, "sampling/sampling_logp_difference/max": 0.44433069229125977, "sampling/sampling_logp_difference/mean": 0.012365834787487984, "step": 472 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 549.0, "completions/mean_length": 386.0625, "completions/mean_terminated_length": 373.7419128417969, "completions/min_length": 110.0, "completions/min_terminated_length": 110.0, "entropy": 0.3356468752026558, "epoch": 0.25321199143468953, "frac_reward_zero_std": 0.25, "grad_norm": 0.014588293619453907, "learning_rate": 1e-05, "loss": -0.0603, "num_tokens": 10261013.0, "reward": 0.625, "reward_std": 0.3745020925998688, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.375716209411621, "sampling/importance_sampling_ratio/mean": 0.9994784593582153, "sampling/importance_sampling_ratio/min": 0.5615304112434387, "sampling/sampling_logp_difference/max": 0.5770893096923828, "sampling/sampling_logp_difference/mean": 0.01205011922866106, "step": 473 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 765.0, "completions/mean_length": 487.5, "completions/mean_terminated_length": 458.4827575683594, "completions/min_length": 173.0, "completions/min_terminated_length": 173.0, "entropy": 0.3404757045209408, "epoch": 0.25374732334047106, "frac_reward_zero_std": 0.25, "grad_norm": 0.023921577259898186, "learning_rate": 1e-05, "loss": 0.0021, "num_tokens": 10280109.0, "reward": 0.84375, "reward_std": 0.3198433816432953, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.5478626489639282, "sampling/importance_sampling_ratio/mean": 1.000269889831543, "sampling/importance_sampling_ratio/min": 0.5776863694190979, "sampling/sampling_logp_difference/max": 0.5487241744995117, "sampling/sampling_logp_difference/mean": 0.01192216295748949, "step": 474 }, { "clip_ratio/high_max": 6.053268589312211e-05, "clip_ratio/high_mean": 6.053268589312211e-05, "clip_ratio/low_mean": 0.00011320862176944502, "clip_ratio/low_min": 0.00011320862176944502, "clip_ratio/region_mean": 0.00017374130766256712, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 709.0, "completions/mean_length": 518.84375, "completions/mean_terminated_length": 449.0799865722656, "completions/min_length": 234.0, "completions/min_terminated_length": 234.0, "entropy": 0.4688952974975109, "epoch": 0.25428265524625265, "frac_reward_zero_std": 0.0, "grad_norm": 0.01842525042593479, "learning_rate": 1e-05, "loss": 0.0407, "num_tokens": 10300696.0, "reward": 0.625, "reward_std": 0.4261348247528076, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.379124402999878, "sampling/importance_sampling_ratio/mean": 0.9997978806495667, "sampling/importance_sampling_ratio/min": 0.5780776739120483, "sampling/sampling_logp_difference/max": 0.5480470657348633, "sampling/sampling_logp_difference/mean": 0.015245016664266586, "step": 475 }, { "clip_ratio/high_max": 7.030371489236131e-05, "clip_ratio/high_mean": 7.030371489236131e-05, "clip_ratio/low_mean": 0.00011832231757580303, "clip_ratio/low_min": 0.00011832231757580303, "clip_ratio/region_mean": 0.00018862603246816434, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 673.0, "completions/mean_length": 449.78125, "completions/mean_terminated_length": 428.5666809082031, "completions/min_length": 212.0, "completions/min_terminated_length": 212.0, "entropy": 0.3439778834581375, "epoch": 0.25481798715203424, "frac_reward_zero_std": 0.25, "grad_norm": 0.021267594769597054, "learning_rate": 1e-05, "loss": 0.0405, "num_tokens": 10318849.0, "reward": 0.59375, "reward_std": 0.38816186785697937, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.5494940280914307, "sampling/importance_sampling_ratio/mean": 0.9998381733894348, "sampling/importance_sampling_ratio/min": 0.5941833257675171, "sampling/sampling_logp_difference/max": 0.5205674171447754, "sampling/sampling_logp_difference/mean": 0.012644737027585506, "step": 476 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00011962609642068855, "clip_ratio/low_min": 0.00011962609642068855, "clip_ratio/region_mean": 0.00011962609642068855, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 767.0, "completions/mean_length": 517.96875, "completions/mean_terminated_length": 501.3000183105469, "completions/min_length": 227.0, "completions/min_terminated_length": 227.0, "entropy": 0.3871957026422024, "epoch": 0.25535331905781583, "frac_reward_zero_std": 0.25, "grad_norm": 0.02794320322573185, "learning_rate": 1e-05, "loss": 0.005, "num_tokens": 10339488.0, "reward": 0.6875, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.59406578540802, "sampling/importance_sampling_ratio/mean": 1.0002950429916382, "sampling/importance_sampling_ratio/min": 0.683752715587616, "sampling/sampling_logp_difference/max": 0.46628785133361816, "sampling/sampling_logp_difference/mean": 0.012846510857343674, "step": 477 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00010789814405143261, "clip_ratio/low_min": 0.00010789814405143261, "clip_ratio/region_mean": 0.00010789814405143261, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 662.0, "completions/mean_length": 484.21875, "completions/mean_terminated_length": 465.3000183105469, "completions/min_length": 267.0, "completions/min_terminated_length": 267.0, "entropy": 0.5373753607273102, "epoch": 0.2558886509635974, "frac_reward_zero_std": 0.25, "grad_norm": 0.0144814969971776, "learning_rate": 1e-05, "loss": 0.0599, "num_tokens": 10359303.0, "reward": 0.65625, "reward_std": 0.3061639964580536, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.3996108770370483, "sampling/importance_sampling_ratio/mean": 1.0001202821731567, "sampling/importance_sampling_ratio/min": 0.7214785218238831, "sampling/sampling_logp_difference/max": 0.3361942768096924, "sampling/sampling_logp_difference/mean": 0.016105568036437035, "step": 478 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.062075772206299e-05, "clip_ratio/low_min": 6.062075772206299e-05, "clip_ratio/region_mean": 6.062075772206299e-05, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 717.0, "completions/mean_length": 493.65625, "completions/mean_terminated_length": 454.46429443359375, "completions/min_length": 283.0, "completions/min_terminated_length": 283.0, "entropy": 0.43290482461452484, "epoch": 0.256423982869379, "frac_reward_zero_std": 0.25, "grad_norm": 0.01961589977145195, "learning_rate": 1e-05, "loss": 0.1068, "num_tokens": 10379420.0, "reward": 0.78125, "reward_std": 0.3608423173427582, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.4752286672592163, "sampling/importance_sampling_ratio/mean": 1.0003256797790527, "sampling/importance_sampling_ratio/min": 0.6372998356819153, "sampling/sampling_logp_difference/max": 0.4505150318145752, "sampling/sampling_logp_difference/mean": 0.014734323136508465, "step": 479 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.002000762033276e-05, "clip_ratio/low_min": 5.002000762033276e-05, "clip_ratio/region_mean": 5.002000762033276e-05, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 743.0, "completions/mean_length": 542.46875, "completions/mean_terminated_length": 490.423095703125, "completions/min_length": 196.0, "completions/min_terminated_length": 196.0, "entropy": 0.4133027270436287, "epoch": 0.2569593147751606, "frac_reward_zero_std": 0.75, "grad_norm": 0.010949130170047283, "learning_rate": 1e-05, "loss": 0.0148, "num_tokens": 10400483.0, "reward": 0.84375, "reward_std": 0.1293872892856598, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.5021144151687622, "sampling/importance_sampling_ratio/mean": 0.9995373487472534, "sampling/importance_sampling_ratio/min": 0.648211658000946, "sampling/sampling_logp_difference/max": 0.43353796005249023, "sampling/sampling_logp_difference/mean": 0.012802211567759514, "step": 480 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00011310099216643721, "clip_ratio/low_min": 0.00011310099216643721, "clip_ratio/region_mean": 0.00011310099216643721, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 713.0, "completions/mean_length": 448.71875, "completions/mean_terminated_length": 438.4193420410156, "completions/min_length": 196.0, "completions/min_terminated_length": 196.0, "entropy": 0.5019905753433704, "epoch": 0.2574946466809422, "frac_reward_zero_std": 0.5, "grad_norm": 0.002961214166134596, "learning_rate": 1e-05, "loss": 0.0949, "num_tokens": 10418186.0, "reward": 0.90625, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.394699215888977, "sampling/importance_sampling_ratio/mean": 1.0002309083938599, "sampling/importance_sampling_ratio/min": 0.6880372762680054, "sampling/sampling_logp_difference/max": 0.3739122152328491, "sampling/sampling_logp_difference/mean": 0.013774196617305279, "step": 481 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.575486622750759e-05, "clip_ratio/low_min": 6.575486622750759e-05, "clip_ratio/region_mean": 6.575486622750759e-05, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 698.0, "completions/mean_length": 457.59375, "completions/mean_terminated_length": 447.58062744140625, "completions/min_length": 274.0, "completions/min_terminated_length": 274.0, "entropy": 0.2937863077968359, "epoch": 0.2580299785867238, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": -0.0006, "num_tokens": 10436397.0, "reward": 0.9375, "reward_std": 0.1157275140285492, "rewards/accuracy_reward/mean": 0.9375, "rewards/accuracy_reward/std": 0.24593468010425568, "sampling/importance_sampling_ratio/max": 1.4217981100082397, "sampling/importance_sampling_ratio/mean": 0.9999369382858276, "sampling/importance_sampling_ratio/min": 0.7290858030319214, "sampling/sampling_logp_difference/max": 0.35192227363586426, "sampling/sampling_logp_difference/mean": 0.01051220390945673, "step": 482 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00019248343232902698, "clip_ratio/low_min": 0.00019248343232902698, "clip_ratio/region_mean": 0.00019248343232902698, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 731.0, "completions/mean_length": 499.15625, "completions/mean_terminated_length": 460.7500305175781, "completions/min_length": 191.0, "completions/min_terminated_length": 191.0, "entropy": 0.42333074286580086, "epoch": 0.25856531049250536, "frac_reward_zero_std": 0.0, "grad_norm": 0.016791101545095444, "learning_rate": 1e-05, "loss": 0.11, "num_tokens": 10456634.0, "reward": 0.75, "reward_std": 0.4629100561141968, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.538956642150879, "sampling/importance_sampling_ratio/mean": 1.0002506971359253, "sampling/importance_sampling_ratio/min": 0.6009966135025024, "sampling/sampling_logp_difference/max": 0.5091660022735596, "sampling/sampling_logp_difference/mean": 0.013416146859526634, "step": 483 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 768.0, "completions/mean_length": 510.875, "completions/mean_terminated_length": 493.7333679199219, "completions/min_length": 273.0, "completions/min_terminated_length": 273.0, "entropy": 0.3170407898724079, "epoch": 0.25910064239828695, "frac_reward_zero_std": 0.75, "grad_norm": 0.014164317399263382, "learning_rate": 1e-05, "loss": 0.0088, "num_tokens": 10476630.0, "reward": 0.90625, "reward_std": 0.1293872892856598, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.4614593982696533, "sampling/importance_sampling_ratio/mean": 1.0002671480178833, "sampling/importance_sampling_ratio/min": 0.6998748183250427, "sampling/sampling_logp_difference/max": 0.37943553924560547, "sampling/sampling_logp_difference/mean": 0.011068416759371758, "step": 484 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0001888966144178994, "clip_ratio/low_min": 0.0001888966144178994, "clip_ratio/region_mean": 0.0001888966144178994, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 706.0, "completions/mean_length": 522.0625, "completions/mean_terminated_length": 486.9285888671875, "completions/min_length": 216.0, "completions/min_terminated_length": 216.0, "entropy": 0.4183397702872753, "epoch": 0.25963597430406854, "frac_reward_zero_std": 0.5, "grad_norm": 0.00877167098224163, "learning_rate": 1e-05, "loss": 0.0498, "num_tokens": 10497280.0, "reward": 0.78125, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.3947205543518066, "sampling/importance_sampling_ratio/mean": 0.9999744892120361, "sampling/importance_sampling_ratio/min": 0.6607444286346436, "sampling/sampling_logp_difference/max": 0.41438817977905273, "sampling/sampling_logp_difference/mean": 0.012642388232052326, "step": 485 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 613.0, "completions/max_terminated_length": 613.0, "completions/mean_length": 398.96875, "completions/mean_terminated_length": 398.96875, "completions/min_length": 221.0, "completions/min_terminated_length": 221.0, "entropy": 0.3845636360347271, "epoch": 0.26017130620985013, "frac_reward_zero_std": 0.5, "grad_norm": 0.005070340819656849, "learning_rate": 1e-05, "loss": 0.0548, "num_tokens": 10513551.0, "reward": 0.75, "reward_std": 0.2314550280570984, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.287777066230774, "sampling/importance_sampling_ratio/mean": 0.9999028444290161, "sampling/importance_sampling_ratio/min": 0.32589948177337646, "sampling/sampling_logp_difference/max": 1.1211662292480469, "sampling/sampling_logp_difference/mean": 0.01249685138463974, "step": 486 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 704.0, "completions/mean_length": 547.3125, "completions/mean_terminated_length": 540.1935424804688, "completions/min_length": 345.0, "completions/min_terminated_length": 345.0, "entropy": 0.33053289353847504, "epoch": 0.2607066381156317, "frac_reward_zero_std": 0.25, "grad_norm": 0.02196439541876316, "learning_rate": 1e-05, "loss": 0.0042, "num_tokens": 10535313.0, "reward": 0.71875, "reward_std": 0.3198433816432953, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.381263256072998, "sampling/importance_sampling_ratio/mean": 0.9999964237213135, "sampling/importance_sampling_ratio/min": 0.6867998242378235, "sampling/sampling_logp_difference/max": 0.37571239471435547, "sampling/sampling_logp_difference/mean": 0.01102694682776928, "step": 487 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.091617979109287e-05, "clip_ratio/low_min": 6.091617979109287e-05, "clip_ratio/region_mean": 6.091617979109287e-05, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 711.0, "completions/mean_length": 467.875, "completions/mean_terminated_length": 425.0000305175781, "completions/min_length": 207.0, "completions/min_terminated_length": 207.0, "entropy": 0.4783867262303829, "epoch": 0.26124197002141325, "frac_reward_zero_std": 0.25, "grad_norm": 0.009078850038349628, "learning_rate": 1e-05, "loss": 0.059, "num_tokens": 10554277.0, "reward": 0.40625, "reward_std": 0.3608423173427582, "rewards/accuracy_reward/mean": 0.40625, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.3827059268951416, "sampling/importance_sampling_ratio/mean": 0.9999626278877258, "sampling/importance_sampling_ratio/min": 0.7200254201889038, "sampling/sampling_logp_difference/max": 0.32846879959106445, "sampling/sampling_logp_difference/mean": 0.013202628120779991, "step": 488 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 747.0, "completions/mean_length": 595.65625, "completions/mean_terminated_length": 555.8846435546875, "completions/min_length": 280.0, "completions/min_terminated_length": 280.0, "entropy": 0.42200541496276855, "epoch": 0.26177730192719484, "frac_reward_zero_std": 0.0, "grad_norm": 0.02131091058254242, "learning_rate": 1e-05, "loss": 0.0481, "num_tokens": 10577498.0, "reward": 0.59375, "reward_std": 0.3987956643104553, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.4412873983383179, "sampling/importance_sampling_ratio/mean": 1.000063180923462, "sampling/importance_sampling_ratio/min": 0.6986035108566284, "sampling/sampling_logp_difference/max": 0.3655366897583008, "sampling/sampling_logp_difference/mean": 0.012317468412220478, "step": 489 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.620762724196538e-05, "clip_ratio/low_min": 6.620762724196538e-05, "clip_ratio/region_mean": 6.620762724196538e-05, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 721.0, "completions/mean_length": 452.375, "completions/mean_terminated_length": 442.19354248046875, "completions/min_length": 226.0, "completions/min_terminated_length": 226.0, "entropy": 0.3676196113228798, "epoch": 0.2623126338329764, "frac_reward_zero_std": 0.25, "grad_norm": 0.007502678781747818, "learning_rate": 1e-05, "loss": -0.0272, "num_tokens": 10595846.0, "reward": 0.59375, "reward_std": 0.3061639964580536, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.403680443763733, "sampling/importance_sampling_ratio/mean": 0.9999904036521912, "sampling/importance_sampling_ratio/min": 0.6795672178268433, "sampling/sampling_logp_difference/max": 0.38629913330078125, "sampling/sampling_logp_difference/mean": 0.012061580084264278, "step": 490 }, { "clip_ratio/high_max": 5.1082959544146433e-05, "clip_ratio/high_mean": 5.1082959544146433e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 5.1082959544146433e-05, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 728.0, "completions/mean_length": 558.71875, "completions/mean_terminated_length": 510.423095703125, "completions/min_length": 300.0, "completions/min_terminated_length": 300.0, "entropy": 0.3995843194425106, "epoch": 0.262847965738758, "frac_reward_zero_std": 0.25, "grad_norm": 0.006878616288304329, "learning_rate": 1e-05, "loss": -0.0421, "num_tokens": 10617973.0, "reward": 0.625, "reward_std": 0.2925041913986206, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0006136894226074, "sampling/importance_sampling_ratio/min": 0.671826183795929, "sampling/sampling_logp_difference/max": 0.7320971488952637, "sampling/sampling_logp_difference/mean": 0.01277911476790905, "step": 491 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.5285272537730634e-05, "clip_ratio/low_min": 5.5285272537730634e-05, "clip_ratio/region_mean": 5.5285272537730634e-05, "completions/clipped_ratio": 0.28125, "completions/max_length": 768.0, "completions/max_terminated_length": 730.0, "completions/mean_length": 612.25, "completions/mean_terminated_length": 551.3043823242188, "completions/min_length": 334.0, "completions/min_terminated_length": 334.0, "entropy": 0.3280426822602749, "epoch": 0.2633832976445396, "frac_reward_zero_std": 0.25, "grad_norm": 0.008638107217848301, "learning_rate": 1e-05, "loss": -0.0026, "num_tokens": 10641469.0, "reward": 0.59375, "reward_std": 0.3061639666557312, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.3695597648620605, "sampling/importance_sampling_ratio/mean": 0.9999805688858032, "sampling/importance_sampling_ratio/min": 0.7360080480575562, "sampling/sampling_logp_difference/max": 0.31448936462402344, "sampling/sampling_logp_difference/mean": 0.01082125399261713, "step": 492 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 755.0, "completions/mean_length": 552.125, "completions/mean_terminated_length": 529.7930908203125, "completions/min_length": 227.0, "completions/min_terminated_length": 227.0, "entropy": 0.46098942682147026, "epoch": 0.2639186295503212, "frac_reward_zero_std": 0.0, "grad_norm": 0.014518959447741508, "learning_rate": 1e-05, "loss": 0.0251, "num_tokens": 10662673.0, "reward": 0.625, "reward_std": 0.4671337604522705, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.3212416172027588, "sampling/importance_sampling_ratio/mean": 1.00014328956604, "sampling/importance_sampling_ratio/min": 0.5942538976669312, "sampling/sampling_logp_difference/max": 0.5204486846923828, "sampling/sampling_logp_difference/mean": 0.013670578598976135, "step": 493 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00015996973525034264, "clip_ratio/low_min": 0.00015996973525034264, "clip_ratio/region_mean": 0.00015996973525034264, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 763.0, "completions/mean_length": 577.9375, "completions/mean_terminated_length": 542.74072265625, "completions/min_length": 321.0, "completions/min_terminated_length": 321.0, "entropy": 0.44146905466914177, "epoch": 0.2644539614561028, "frac_reward_zero_std": 0.0, "grad_norm": 0.017010698094964027, "learning_rate": 1e-05, "loss": 0.0374, "num_tokens": 10684887.0, "reward": 0.53125, "reward_std": 0.521792471408844, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.4384315013885498, "sampling/importance_sampling_ratio/mean": 0.99985671043396, "sampling/importance_sampling_ratio/min": 0.6885712742805481, "sampling/sampling_logp_difference/max": 0.3731365203857422, "sampling/sampling_logp_difference/mean": 0.013539792969822884, "step": 494 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 757.0, "completions/mean_length": 570.65625, "completions/mean_terminated_length": 525.1154174804688, "completions/min_length": 241.0, "completions/min_terminated_length": 241.0, "entropy": 0.6237023696303368, "epoch": 0.26498929336188437, "frac_reward_zero_std": 0.5, "grad_norm": 0.012915457598865032, "learning_rate": 1e-05, "loss": -0.0322, "num_tokens": 10707324.0, "reward": 0.40625, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.40625, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.434587836265564, "sampling/importance_sampling_ratio/mean": 1.0001200437545776, "sampling/importance_sampling_ratio/min": 0.618630588054657, "sampling/sampling_logp_difference/max": 0.48024702072143555, "sampling/sampling_logp_difference/mean": 0.015508514828979969, "step": 495 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 669.0, "completions/mean_length": 410.125, "completions/mean_terminated_length": 386.2666931152344, "completions/min_length": 198.0, "completions/min_terminated_length": 198.0, "entropy": 0.372609905898571, "epoch": 0.26552462526766596, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0203, "num_tokens": 10723712.0, "reward": 0.625, "reward_std": 0.13363061845302582, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.5183675289154053, "sampling/importance_sampling_ratio/mean": 0.9996618628501892, "sampling/importance_sampling_ratio/min": 0.6053699851036072, "sampling/sampling_logp_difference/max": 0.501915454864502, "sampling/sampling_logp_difference/mean": 0.01172690000385046, "step": 496 }, { "clip_ratio/high_max": 5.87406029808335e-05, "clip_ratio/high_mean": 5.87406029808335e-05, "clip_ratio/low_mean": 5.9297912230249494e-05, "clip_ratio/low_min": 5.9297912230249494e-05, "clip_ratio/region_mean": 0.000118038515211083, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 734.0, "completions/mean_length": 593.09375, "completions/mean_terminated_length": 581.433349609375, "completions/min_length": 415.0, "completions/min_terminated_length": 415.0, "entropy": 0.2715241611003876, "epoch": 0.26605995717344755, "frac_reward_zero_std": 0.25, "grad_norm": 0.005119976121932268, "learning_rate": 1e-05, "loss": 0.0096, "num_tokens": 10746419.0, "reward": 0.875, "reward_std": 0.2925041913986206, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 1.4020519256591797, "sampling/importance_sampling_ratio/mean": 1.0000591278076172, "sampling/importance_sampling_ratio/min": 0.6005457043647766, "sampling/sampling_logp_difference/max": 0.5099165439605713, "sampling/sampling_logp_difference/mean": 0.00956046488136053, "step": 497 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 731.0, "completions/mean_length": 542.53125, "completions/mean_terminated_length": 519.2069091796875, "completions/min_length": 314.0, "completions/min_terminated_length": 314.0, "entropy": 0.3512185513973236, "epoch": 0.26659528907922914, "frac_reward_zero_std": 0.5, "grad_norm": 0.005529796238988638, "learning_rate": 1e-05, "loss": 0.0624, "num_tokens": 10767300.0, "reward": 0.875, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 1.3655188083648682, "sampling/importance_sampling_ratio/mean": 1.0005236864089966, "sampling/importance_sampling_ratio/min": 0.5821472406387329, "sampling/sampling_logp_difference/max": 0.5410318374633789, "sampling/sampling_logp_difference/mean": 0.012006034143269062, "step": 498 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.9980804508086294e-05, "clip_ratio/low_min": 5.9980804508086294e-05, "clip_ratio/region_mean": 5.9980804508086294e-05, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 737.0, "completions/mean_length": 492.125, "completions/mean_terminated_length": 463.5862121582031, "completions/min_length": 156.0, "completions/min_terminated_length": 156.0, "entropy": 0.38408940471708775, "epoch": 0.2671306209850107, "frac_reward_zero_std": 0.5, "grad_norm": 0.016117999330163002, "learning_rate": 1e-05, "loss": 0.0756, "num_tokens": 10786568.0, "reward": 0.78125, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.4392329454421997, "sampling/importance_sampling_ratio/mean": 1.0000207424163818, "sampling/importance_sampling_ratio/min": 0.6839957237243652, "sampling/sampling_logp_difference/max": 0.3798036575317383, "sampling/sampling_logp_difference/mean": 0.013170016929507256, "step": 499 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00014782911966904067, "clip_ratio/low_min": 0.00014782911966904067, "clip_ratio/region_mean": 0.00014782911966904067, "completions/clipped_ratio": 0.375, "completions/max_length": 768.0, "completions/max_terminated_length": 691.0, "completions/mean_length": 587.78125, "completions/mean_terminated_length": 479.6499938964844, "completions/min_length": 283.0, "completions/min_terminated_length": 283.0, "entropy": 0.5249118953943253, "epoch": 0.2676659528907923, "frac_reward_zero_std": 0.5, "grad_norm": 0.014432675205171108, "learning_rate": 1e-05, "loss": 0.0496, "num_tokens": 10809961.0, "reward": 0.5, "reward_std": 0.2587745785713196, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5080004930496216, "sampling/importance_sampling_ratio/max": 1.4473928213119507, "sampling/importance_sampling_ratio/mean": 0.9997637271881104, "sampling/importance_sampling_ratio/min": 0.7013431191444397, "sampling/sampling_logp_difference/max": 0.3697638511657715, "sampling/sampling_logp_difference/mean": 0.014930631965398788, "step": 500 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00014360740169649944, "clip_ratio/low_min": 0.00014360740169649944, "clip_ratio/region_mean": 0.00014360740169649944, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 700.0, "completions/mean_length": 450.0625, "completions/mean_terminated_length": 417.17242431640625, "completions/min_length": 255.0, "completions/min_terminated_length": 255.0, "entropy": 0.450850535184145, "epoch": 0.2682012847965739, "frac_reward_zero_std": 0.0, "grad_norm": 0.020207157358527184, "learning_rate": 1e-05, "loss": 0.025, "num_tokens": 10828395.0, "reward": 0.59375, "reward_std": 0.4218915104866028, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.3967061042785645, "sampling/importance_sampling_ratio/mean": 0.9997287392616272, "sampling/importance_sampling_ratio/min": 0.6463490724563599, "sampling/sampling_logp_difference/max": 0.4364156723022461, "sampling/sampling_logp_difference/mean": 0.014574073255062103, "step": 501 }, { "clip_ratio/high_max": 0.00012582465933519416, "clip_ratio/high_mean": 0.00012582465933519416, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00012582465933519416, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 655.0, "completions/mean_length": 436.125, "completions/mean_terminated_length": 401.7930908203125, "completions/min_length": 207.0, "completions/min_terminated_length": 207.0, "entropy": 0.3665753863751888, "epoch": 0.26873661670235544, "frac_reward_zero_std": 0.5, "grad_norm": 0.004430126864463091, "learning_rate": 1e-05, "loss": 0.023, "num_tokens": 10846223.0, "reward": 0.90625, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.431652545928955, "sampling/importance_sampling_ratio/mean": 0.9997940063476562, "sampling/importance_sampling_ratio/min": 0.7723632454872131, "sampling/sampling_logp_difference/max": 0.3588293790817261, "sampling/sampling_logp_difference/mean": 0.013006269931793213, "step": 502 }, { "clip_ratio/high_max": 6.599789048777893e-05, "clip_ratio/high_mean": 6.599789048777893e-05, "clip_ratio/low_mean": 8.007687574718148e-05, "clip_ratio/low_min": 8.007687574718148e-05, "clip_ratio/region_mean": 0.0001460747662349604, "completions/clipped_ratio": 0.0, "completions/max_length": 682.0, "completions/max_terminated_length": 682.0, "completions/mean_length": 446.28125, "completions/mean_terminated_length": 446.28125, "completions/min_length": 227.0, "completions/min_terminated_length": 227.0, "entropy": 0.39169222488999367, "epoch": 0.269271948608137, "frac_reward_zero_std": 0.25, "grad_norm": 0.02893698774278164, "learning_rate": 1e-05, "loss": 0.0664, "num_tokens": 10864640.0, "reward": 0.78125, "reward_std": 0.3471629321575165, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.367005467414856, "sampling/importance_sampling_ratio/mean": 0.9998120069503784, "sampling/importance_sampling_ratio/min": 0.7178518176078796, "sampling/sampling_logp_difference/max": 0.33149218559265137, "sampling/sampling_logp_difference/mean": 0.012990551069378853, "step": 503 }, { "clip_ratio/high_max": 5.475251964526251e-05, "clip_ratio/high_mean": 5.475251964526251e-05, "clip_ratio/low_mean": 6.091617979109287e-05, "clip_ratio/low_min": 6.091617979109287e-05, "clip_ratio/region_mean": 0.00011566869943635538, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 758.0, "completions/mean_length": 512.59375, "completions/mean_terminated_length": 465.2962951660156, "completions/min_length": 182.0, "completions/min_terminated_length": 182.0, "entropy": 0.4896366633474827, "epoch": 0.2698072805139186, "frac_reward_zero_std": 0.25, "grad_norm": 0.00998708140105009, "learning_rate": 1e-05, "loss": 0.0244, "num_tokens": 10884603.0, "reward": 0.8125, "reward_std": 0.3104073107242584, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.4698736667633057, "sampling/importance_sampling_ratio/mean": 1.000044584274292, "sampling/importance_sampling_ratio/min": 0.6605985760688782, "sampling/sampling_logp_difference/max": 0.4146089553833008, "sampling/sampling_logp_difference/mean": 0.014566963538527489, "step": 504 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0002147885134036187, "clip_ratio/low_min": 0.0002147885134036187, "clip_ratio/region_mean": 0.0002147885134036187, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 759.0, "completions/mean_length": 571.65625, "completions/mean_terminated_length": 516.6799926757812, "completions/min_length": 148.0, "completions/min_terminated_length": 148.0, "entropy": 0.46576593816280365, "epoch": 0.2703426124197002, "frac_reward_zero_std": 0.0, "grad_norm": 0.0218172799795866, "learning_rate": 1e-05, "loss": 0.0564, "num_tokens": 10906936.0, "reward": 0.375, "reward_std": 0.49022960662841797, "rewards/accuracy_reward/mean": 0.375, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.4341405630111694, "sampling/importance_sampling_ratio/mean": 0.9999771118164062, "sampling/importance_sampling_ratio/min": 0.7091548442840576, "sampling/sampling_logp_difference/max": 0.3605656623840332, "sampling/sampling_logp_difference/mean": 0.013464457355439663, "step": 505 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.041565939085558e-05, "clip_ratio/low_min": 6.041565939085558e-05, "clip_ratio/region_mean": 6.041565939085558e-05, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 740.0, "completions/mean_length": 482.3125, "completions/mean_terminated_length": 463.2666931152344, "completions/min_length": 293.0, "completions/min_terminated_length": 293.0, "entropy": 0.39628337137401104, "epoch": 0.2708779443254818, "frac_reward_zero_std": 0.5, "grad_norm": 0.009350677952170372, "learning_rate": 1e-05, "loss": -0.0288, "num_tokens": 10926402.0, "reward": 0.6875, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.4072039127349854, "sampling/importance_sampling_ratio/mean": 1.0000338554382324, "sampling/importance_sampling_ratio/min": 0.6879214644432068, "sampling/sampling_logp_difference/max": 0.3740806579589844, "sampling/sampling_logp_difference/mean": 0.012697096914052963, "step": 506 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.656108623952605e-05, "clip_ratio/low_min": 5.656108623952605e-05, "clip_ratio/region_mean": 5.656108623952605e-05, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 753.0, "completions/mean_length": 499.5625, "completions/mean_terminated_length": 461.21429443359375, "completions/min_length": 204.0, "completions/min_terminated_length": 204.0, "entropy": 0.36180321499705315, "epoch": 0.2714132762312634, "frac_reward_zero_std": 0.25, "grad_norm": 0.01136671844869852, "learning_rate": 1e-05, "loss": -0.0064, "num_tokens": 10946476.0, "reward": 0.78125, "reward_std": 0.3377465009689331, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.2846935987472534, "sampling/importance_sampling_ratio/mean": 1.000289797782898, "sampling/importance_sampling_ratio/min": 0.7727760076522827, "sampling/sampling_logp_difference/max": 0.2577660083770752, "sampling/sampling_logp_difference/mean": 0.011764943599700928, "step": 507 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 628.0, "completions/mean_length": 499.75, "completions/mean_terminated_length": 461.4285888671875, "completions/min_length": 240.0, "completions/min_terminated_length": 240.0, "entropy": 0.49245674163103104, "epoch": 0.27194860813704497, "frac_reward_zero_std": 0.75, "grad_norm": 0.018738845363259315, "learning_rate": 1e-05, "loss": -0.0012, "num_tokens": 10966012.0, "reward": 0.4375, "reward_std": 0.1157275140285492, "rewards/accuracy_reward/mean": 0.4375, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.424360752105713, "sampling/importance_sampling_ratio/mean": 0.9999735951423645, "sampling/importance_sampling_ratio/min": 0.6477689146995544, "sampling/sampling_logp_difference/max": 0.4342212677001953, "sampling/sampling_logp_difference/mean": 0.015169242396950722, "step": 508 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.218905764399096e-05, "clip_ratio/low_min": 6.218905764399096e-05, "clip_ratio/region_mean": 6.218905764399096e-05, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 620.0, "completions/mean_length": 480.0625, "completions/mean_terminated_length": 460.86669921875, "completions/min_length": 222.0, "completions/min_terminated_length": 222.0, "entropy": 0.2804887369275093, "epoch": 0.27248394004282656, "frac_reward_zero_std": 0.5, "grad_norm": 0.007586286403238773, "learning_rate": 1e-05, "loss": 0.0221, "num_tokens": 10985150.0, "reward": 0.84375, "reward_std": 0.22201895713806152, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.8020237684249878, "sampling/importance_sampling_ratio/mean": 0.9999838471412659, "sampling/importance_sampling_ratio/min": 0.6451939940452576, "sampling/sampling_logp_difference/max": 0.5889103412628174, "sampling/sampling_logp_difference/mean": 0.01012120209634304, "step": 509 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.674080966855399e-05, "clip_ratio/low_min": 5.674080966855399e-05, "clip_ratio/region_mean": 5.674080966855399e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 715.0, "completions/max_terminated_length": 715.0, "completions/mean_length": 442.46875, "completions/mean_terminated_length": 442.46875, "completions/min_length": 193.0, "completions/min_terminated_length": 193.0, "entropy": 0.3159977290779352, "epoch": 0.27301927194860814, "frac_reward_zero_std": 0.25, "grad_norm": 0.018495755270123482, "learning_rate": 1e-05, "loss": -0.0744, "num_tokens": 11002733.0, "reward": 0.84375, "reward_std": 0.3198433816432953, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.3573119640350342, "sampling/importance_sampling_ratio/mean": 0.9999318718910217, "sampling/importance_sampling_ratio/min": 0.6412972807884216, "sampling/sampling_logp_difference/max": 0.44426214694976807, "sampling/sampling_logp_difference/mean": 0.010570033453404903, "step": 510 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.400409620255232e-05, "clip_ratio/low_min": 6.400409620255232e-05, "clip_ratio/region_mean": 6.400409620255232e-05, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 759.0, "completions/mean_length": 531.8125, "completions/mean_terminated_length": 516.0667114257812, "completions/min_length": 295.0, "completions/min_terminated_length": 295.0, "entropy": 0.367650642991066, "epoch": 0.27355460385438973, "frac_reward_zero_std": 0.25, "grad_norm": 0.02325468882918358, "learning_rate": 1e-05, "loss": 0.0268, "num_tokens": 11023591.0, "reward": 0.6875, "reward_std": 0.3745020925998688, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.4159274101257324, "sampling/importance_sampling_ratio/mean": 0.9999678730964661, "sampling/importance_sampling_ratio/min": 0.42176902294158936, "sampling/sampling_logp_difference/max": 0.8632974624633789, "sampling/sampling_logp_difference/mean": 0.012066773138940334, "step": 511 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 8.283631905214861e-05, "clip_ratio/low_min": 8.283631905214861e-05, "clip_ratio/region_mean": 8.283631905214861e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 548.0, "completions/max_terminated_length": 548.0, "completions/mean_length": 359.75, "completions/mean_terminated_length": 359.75, "completions/min_length": 181.0, "completions/min_terminated_length": 181.0, "entropy": 0.4486824981868267, "epoch": 0.2740899357601713, "frac_reward_zero_std": 0.5, "grad_norm": 0.011279053054749966, "learning_rate": 1e-05, "loss": -0.0761, "num_tokens": 11038895.0, "reward": 0.625, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.352524757385254, "sampling/importance_sampling_ratio/mean": 0.9998483657836914, "sampling/importance_sampling_ratio/min": 0.714945912361145, "sampling/sampling_logp_difference/max": 0.33554840087890625, "sampling/sampling_logp_difference/mean": 0.011539670638740063, "step": 512 }, { "clip_ratio/high_max": 5.058680835645646e-05, "clip_ratio/high_mean": 5.058680835645646e-05, "clip_ratio/low_mean": 6.909894727868959e-05, "clip_ratio/low_min": 6.909894727868959e-05, "clip_ratio/region_mean": 0.00011968575563514605, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 756.0, "completions/mean_length": 474.78125, "completions/mean_terminated_length": 465.32257080078125, "completions/min_length": 221.0, "completions/min_terminated_length": 221.0, "entropy": 0.3655603341758251, "epoch": 0.2746252676659529, "frac_reward_zero_std": 0.0, "grad_norm": 0.009271195158362389, "learning_rate": 1e-05, "loss": -0.0099, "num_tokens": 11058024.0, "reward": 0.65625, "reward_std": 0.4807935357093811, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0000110864639282, "sampling/importance_sampling_ratio/min": 0.6234405636787415, "sampling/sampling_logp_difference/max": 1.103832721710205, "sampling/sampling_logp_difference/mean": 0.011671015061438084, "step": 513 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0001488558918936178, "clip_ratio/low_min": 0.0001488558918936178, "clip_ratio/region_mean": 0.0001488558918936178, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 760.0, "completions/mean_length": 562.125, "completions/mean_terminated_length": 514.6154174804688, "completions/min_length": 244.0, "completions/min_terminated_length": 244.0, "entropy": 0.41012832522392273, "epoch": 0.2751605995717345, "frac_reward_zero_std": 0.5, "grad_norm": 0.02393708936870098, "learning_rate": 1e-05, "loss": -0.013, "num_tokens": 11080140.0, "reward": 0.46875, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.46875, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.4002383947372437, "sampling/importance_sampling_ratio/mean": 1.0002782344818115, "sampling/importance_sampling_ratio/min": 0.716558575630188, "sampling/sampling_logp_difference/max": 0.3366425037384033, "sampling/sampling_logp_difference/mean": 0.011899859644472599, "step": 514 }, { "clip_ratio/high_max": 5.752416109316982e-05, "clip_ratio/high_mean": 5.752416109316982e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 5.752416109316982e-05, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 760.0, "completions/mean_length": 538.96875, "completions/mean_terminated_length": 496.5555725097656, "completions/min_length": 111.0, "completions/min_terminated_length": 111.0, "entropy": 0.37249042466282845, "epoch": 0.2756959314775161, "frac_reward_zero_std": 0.5, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": -0.0269, "num_tokens": 11101347.0, "reward": 0.75, "reward_std": 0.26726123690605164, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.5325285196304321, "sampling/importance_sampling_ratio/mean": 0.9997504353523254, "sampling/importance_sampling_ratio/min": 0.5773702263832092, "sampling/sampling_logp_difference/max": 0.5492715835571289, "sampling/sampling_logp_difference/mean": 0.010609548538923264, "step": 515 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.5587163185700774e-05, "clip_ratio/low_min": 4.5587163185700774e-05, "clip_ratio/region_mean": 4.5587163185700774e-05, "completions/clipped_ratio": 0.34375, "completions/max_length": 768.0, "completions/max_terminated_length": 694.0, "completions/mean_length": 616.8125, "completions/mean_terminated_length": 537.6190795898438, "completions/min_length": 357.0, "completions/min_terminated_length": 357.0, "entropy": 0.37951890006661415, "epoch": 0.2762312633832976, "frac_reward_zero_std": 0.0, "grad_norm": 0.018385717645287514, "learning_rate": 1e-05, "loss": 0.0467, "num_tokens": 11125005.0, "reward": 0.4375, "reward_std": 0.4261348247528076, "rewards/accuracy_reward/mean": 0.4375, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.4259604215621948, "sampling/importance_sampling_ratio/mean": 1.000199556350708, "sampling/importance_sampling_ratio/min": 0.6429257392883301, "sampling/sampling_logp_difference/max": 0.44172608852386475, "sampling/sampling_logp_difference/mean": 0.011684532277286053, "step": 516 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 7.832080154912546e-05, "clip_ratio/low_min": 7.832080154912546e-05, "clip_ratio/region_mean": 7.832080154912546e-05, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 765.0, "completions/mean_length": 532.03125, "completions/mean_terminated_length": 477.5769348144531, "completions/min_length": 213.0, "completions/min_terminated_length": 213.0, "entropy": 0.4765149876475334, "epoch": 0.2767665952890792, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": -0.0042, "num_tokens": 11146350.0, "reward": 0.625, "reward_std": 0.13363061845302582, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.5157736539840698, "sampling/importance_sampling_ratio/mean": 0.9999045133590698, "sampling/importance_sampling_ratio/min": 0.706811785697937, "sampling/sampling_logp_difference/max": 0.4159259796142578, "sampling/sampling_logp_difference/mean": 0.014530723914504051, "step": 517 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.325259578763507e-05, "clip_ratio/low_min": 4.325259578763507e-05, "clip_ratio/region_mean": 4.325259578763507e-05, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 762.0, "completions/mean_length": 570.875, "completions/mean_terminated_length": 505.16668701171875, "completions/min_length": 224.0, "completions/min_terminated_length": 224.0, "entropy": 0.36934390664100647, "epoch": 0.2773019271948608, "frac_reward_zero_std": 0.0, "grad_norm": 0.013133304193615913, "learning_rate": 1e-05, "loss": 0.0272, "num_tokens": 11168498.0, "reward": 0.5625, "reward_std": 0.5081326961517334, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.383011817932129, "sampling/importance_sampling_ratio/mean": 1.0003128051757812, "sampling/importance_sampling_ratio/min": 0.699653148651123, "sampling/sampling_logp_difference/max": 0.35717058181762695, "sampling/sampling_logp_difference/mean": 0.011757852509617805, "step": 518 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00016539677380933426, "clip_ratio/low_min": 0.00016539677380933426, "clip_ratio/region_mean": 0.00016539677380933426, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 716.0, "completions/mean_length": 542.65625, "completions/mean_terminated_length": 500.9259338378906, "completions/min_length": 272.0, "completions/min_terminated_length": 272.0, "entropy": 0.41028618440032005, "epoch": 0.2778372591006424, "frac_reward_zero_std": 0.25, "grad_norm": 0.009345927275717258, "learning_rate": 1e-05, "loss": 0.0006, "num_tokens": 11189655.0, "reward": 0.75, "reward_std": 0.3514062464237213, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.4565088748931885, "sampling/importance_sampling_ratio/mean": 0.9998155832290649, "sampling/importance_sampling_ratio/min": 0.662328839302063, "sampling/sampling_logp_difference/max": 0.411993145942688, "sampling/sampling_logp_difference/mean": 0.013395563699305058, "step": 519 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 719.0, "completions/mean_length": 461.40625, "completions/mean_terminated_length": 451.51611328125, "completions/min_length": 300.0, "completions/min_terminated_length": 300.0, "entropy": 0.296026648953557, "epoch": 0.278372591006424, "frac_reward_zero_std": 0.5, "grad_norm": 0.01672695390880108, "learning_rate": 1e-05, "loss": -0.0005, "num_tokens": 11207996.0, "reward": 0.71875, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.3332693576812744, "sampling/importance_sampling_ratio/mean": 1.0001620054244995, "sampling/importance_sampling_ratio/min": 0.5548195838928223, "sampling/sampling_logp_difference/max": 0.5891122817993164, "sampling/sampling_logp_difference/mean": 0.009919258765876293, "step": 520 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.658963734982535e-05, "clip_ratio/low_min": 4.658963734982535e-05, "clip_ratio/region_mean": 4.658963734982535e-05, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 699.0, "completions/mean_length": 545.03125, "completions/mean_terminated_length": 493.5769348144531, "completions/min_length": 239.0, "completions/min_terminated_length": 239.0, "entropy": 0.4593871794641018, "epoch": 0.27890792291220556, "frac_reward_zero_std": 0.5, "grad_norm": 0.012729750014841557, "learning_rate": 1e-05, "loss": 0.0783, "num_tokens": 11229597.0, "reward": 0.8125, "reward_std": 0.249358132481575, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.302427053451538, "sampling/importance_sampling_ratio/mean": 1.0001894235610962, "sampling/importance_sampling_ratio/min": 0.723248302936554, "sampling/sampling_logp_difference/max": 0.324002742767334, "sampling/sampling_logp_difference/mean": 0.012064204551279545, "step": 521 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00010246843157801777, "clip_ratio/low_min": 0.00010246843157801777, "clip_ratio/region_mean": 0.00010246843157801777, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 735.0, "completions/mean_length": 587.96875, "completions/mean_terminated_length": 546.423095703125, "completions/min_length": 228.0, "completions/min_terminated_length": 228.0, "entropy": 0.41242023557424545, "epoch": 0.27944325481798715, "frac_reward_zero_std": 0.0, "grad_norm": 0.01993529498577118, "learning_rate": 1e-05, "loss": 0.0912, "num_tokens": 11252292.0, "reward": 0.59375, "reward_std": 0.4807935357093811, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.5058708190917969, "sampling/importance_sampling_ratio/mean": 0.999801754951477, "sampling/importance_sampling_ratio/min": 0.6496912837028503, "sampling/sampling_logp_difference/max": 0.431257963180542, "sampling/sampling_logp_difference/mean": 0.012509185820817947, "step": 522 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00013470427074935287, "clip_ratio/low_min": 0.00013470427074935287, "clip_ratio/region_mean": 0.00013470427074935287, "completions/clipped_ratio": 0.0, "completions/max_length": 723.0, "completions/max_terminated_length": 723.0, "completions/mean_length": 471.6875, "completions/mean_terminated_length": 471.6875, "completions/min_length": 248.0, "completions/min_terminated_length": 248.0, "entropy": 0.345854677259922, "epoch": 0.27997858672376874, "frac_reward_zero_std": 0.5, "grad_norm": 0.016254199668765068, "learning_rate": 1e-05, "loss": -0.0161, "num_tokens": 11270570.0, "reward": 0.59375, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.3566155433654785, "sampling/importance_sampling_ratio/mean": 1.0004968643188477, "sampling/importance_sampling_ratio/min": 0.6889597773551941, "sampling/sampling_logp_difference/max": 0.3725724220275879, "sampling/sampling_logp_difference/mean": 0.011251438409090042, "step": 523 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 760.0, "completions/mean_length": 551.71875, "completions/mean_terminated_length": 511.6666564941406, "completions/min_length": 235.0, "completions/min_terminated_length": 235.0, "entropy": 0.3851979784667492, "epoch": 0.28051391862955033, "frac_reward_zero_std": 0.0, "grad_norm": 0.012225509621202946, "learning_rate": 1e-05, "loss": 0.1003, "num_tokens": 11292073.0, "reward": 0.5625, "reward_std": 0.5175491571426392, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.6538488864898682, "sampling/importance_sampling_ratio/mean": 0.9999245405197144, "sampling/importance_sampling_ratio/min": 0.6564745903015137, "sampling/sampling_logp_difference/max": 0.5031051635742188, "sampling/sampling_logp_difference/mean": 0.012781694531440735, "step": 524 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 588.0, "completions/max_terminated_length": 588.0, "completions/mean_length": 413.65625, "completions/mean_terminated_length": 413.65625, "completions/min_length": 96.0, "completions/min_terminated_length": 96.0, "entropy": 0.3201225623488426, "epoch": 0.2810492505353319, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0018, "num_tokens": 11308606.0, "reward": 0.9375, "reward_std": 0.1157275140285492, "rewards/accuracy_reward/mean": 0.9375, "rewards/accuracy_reward/std": 0.24593468010425568, "sampling/importance_sampling_ratio/max": 1.4716546535491943, "sampling/importance_sampling_ratio/mean": 0.999687910079956, "sampling/importance_sampling_ratio/min": 0.6867811679840088, "sampling/sampling_logp_difference/max": 0.38638734817504883, "sampling/sampling_logp_difference/mean": 0.011294263415038586, "step": 525 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00011418146823416464, "clip_ratio/low_min": 0.00011418146823416464, "clip_ratio/region_mean": 0.00011418146823416464, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 721.0, "completions/mean_length": 587.09375, "completions/mean_terminated_length": 545.34619140625, "completions/min_length": 318.0, "completions/min_terminated_length": 318.0, "entropy": 0.5572668500244617, "epoch": 0.2815845824411135, "frac_reward_zero_std": 0.0, "grad_norm": 0.01513946894556284, "learning_rate": 1e-05, "loss": 0.0349, "num_tokens": 11330937.0, "reward": 0.5, "reward_std": 0.49022960662841797, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5080004930496216, "sampling/importance_sampling_ratio/max": 1.393357753753662, "sampling/importance_sampling_ratio/mean": 1.0000786781311035, "sampling/importance_sampling_ratio/min": 0.6948387622833252, "sampling/sampling_logp_difference/max": 0.3640754222869873, "sampling/sampling_logp_difference/mean": 0.015769880264997482, "step": 526 }, { "clip_ratio/high_max": 6.325911090243608e-05, "clip_ratio/high_mean": 6.325911090243608e-05, "clip_ratio/low_mean": 5.083367068436928e-05, "clip_ratio/low_min": 5.083367068436928e-05, "clip_ratio/region_mean": 0.00011409278158680536, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 751.0, "completions/mean_length": 488.90625, "completions/mean_terminated_length": 479.9031982421875, "completions/min_length": 289.0, "completions/min_terminated_length": 289.0, "entropy": 0.316387178376317, "epoch": 0.2821199143468951, "frac_reward_zero_std": 0.25, "grad_norm": 0.007906363345682621, "learning_rate": 1e-05, "loss": -0.0239, "num_tokens": 11350190.0, "reward": 0.78125, "reward_std": 0.3377465009689331, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.6339191198349, "sampling/importance_sampling_ratio/mean": 1.000378131866455, "sampling/importance_sampling_ratio/min": 0.6571432948112488, "sampling/sampling_logp_difference/max": 0.4909815788269043, "sampling/sampling_logp_difference/mean": 0.010797815397381783, "step": 527 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.088650843594223e-05, "clip_ratio/low_min": 6.088650843594223e-05, "clip_ratio/region_mean": 6.088650843594223e-05, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 724.0, "completions/mean_length": 468.59375, "completions/mean_terminated_length": 448.63336181640625, "completions/min_length": 284.0, "completions/min_terminated_length": 284.0, "entropy": 0.36522257700562477, "epoch": 0.2826552462526767, "frac_reward_zero_std": 0.75, "grad_norm": 0.007435937412083149, "learning_rate": 1e-05, "loss": 0.0232, "num_tokens": 11368633.0, "reward": 0.9375, "reward_std": 0.1157275140285492, "rewards/accuracy_reward/mean": 0.9375, "rewards/accuracy_reward/std": 0.24593468010425568, "sampling/importance_sampling_ratio/max": 1.558573603630066, "sampling/importance_sampling_ratio/mean": 1.0000553131103516, "sampling/importance_sampling_ratio/min": 0.5700126886367798, "sampling/sampling_logp_difference/max": 0.5620965957641602, "sampling/sampling_logp_difference/mean": 0.012128496542572975, "step": 528 }, { "clip_ratio/high_max": 6.169793050503358e-05, "clip_ratio/high_mean": 6.169793050503358e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 6.169793050503358e-05, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 734.0, "completions/mean_length": 487.75, "completions/mean_terminated_length": 458.75860595703125, "completions/min_length": 215.0, "completions/min_terminated_length": 215.0, "entropy": 0.3328167535364628, "epoch": 0.2831905781584582, "frac_reward_zero_std": 0.25, "grad_norm": 0.006445335689932108, "learning_rate": 1e-05, "loss": -0.0293, "num_tokens": 11387961.0, "reward": 0.84375, "reward_std": 0.3061639964580536, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.6161160469055176, "sampling/importance_sampling_ratio/mean": 1.0002354383468628, "sampling/importance_sampling_ratio/min": 0.6984847784042358, "sampling/sampling_logp_difference/max": 0.4800257682800293, "sampling/sampling_logp_difference/mean": 0.01108283456414938, "step": 529 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 742.0, "completions/mean_length": 544.78125, "completions/mean_terminated_length": 493.2692565917969, "completions/min_length": 346.0, "completions/min_terminated_length": 346.0, "entropy": 0.3023095689713955, "epoch": 0.2837259100642398, "frac_reward_zero_std": 0.75, "grad_norm": 0.0020286289509385824, "learning_rate": 1e-05, "loss": -0.0327, "num_tokens": 11409378.0, "reward": 0.71875, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.5844956636428833, "sampling/importance_sampling_ratio/mean": 0.999405562877655, "sampling/importance_sampling_ratio/min": 0.7147829532623291, "sampling/sampling_logp_difference/max": 0.46026611328125, "sampling/sampling_logp_difference/mean": 0.010206337086856365, "step": 530 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 616.0, "completions/max_terminated_length": 616.0, "completions/mean_length": 408.78125, "completions/mean_terminated_length": 408.78125, "completions/min_length": 256.0, "completions/min_terminated_length": 256.0, "entropy": 0.35411177203059196, "epoch": 0.2842612419700214, "frac_reward_zero_std": 0.5, "grad_norm": 0.006431852467358112, "learning_rate": 1e-05, "loss": 0.0022, "num_tokens": 11426059.0, "reward": 0.9375, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.9375, "rewards/accuracy_reward/std": 0.24593468010425568, "sampling/importance_sampling_ratio/max": 1.5280834436416626, "sampling/importance_sampling_ratio/mean": 1.000044822692871, "sampling/importance_sampling_ratio/min": 0.6985319256782532, "sampling/sampling_logp_difference/max": 0.4240143299102783, "sampling/sampling_logp_difference/mean": 0.011911740526556969, "step": 531 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.489679460879415e-05, "clip_ratio/low_min": 5.489679460879415e-05, "clip_ratio/region_mean": 5.489679460879415e-05, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 726.0, "completions/mean_length": 543.09375, "completions/mean_terminated_length": 501.4444580078125, "completions/min_length": 291.0, "completions/min_terminated_length": 291.0, "entropy": 0.5014722589403391, "epoch": 0.284796573875803, "frac_reward_zero_std": 0.0, "grad_norm": 0.027082255110144615, "learning_rate": 1e-05, "loss": 0.0419, "num_tokens": 11447094.0, "reward": 0.5625, "reward_std": 0.4492306709289551, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.3744487762451172, "sampling/importance_sampling_ratio/mean": 1.000076413154602, "sampling/importance_sampling_ratio/min": 0.5975451469421387, "sampling/sampling_logp_difference/max": 0.514925479888916, "sampling/sampling_logp_difference/mean": 0.015390015207231045, "step": 532 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 693.0, "completions/mean_length": 533.3125, "completions/mean_terminated_length": 479.15386962890625, "completions/min_length": 295.0, "completions/min_terminated_length": 295.0, "entropy": 0.3969234135001898, "epoch": 0.28533190578158457, "frac_reward_zero_std": 0.25, "grad_norm": 0.004750375170260668, "learning_rate": 1e-05, "loss": 0.0717, "num_tokens": 11467984.0, "reward": 0.53125, "reward_std": 0.3471629321575165, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.3498871326446533, "sampling/importance_sampling_ratio/mean": 0.9999984502792358, "sampling/importance_sampling_ratio/min": 0.6877531409263611, "sampling/sampling_logp_difference/max": 0.3743252754211426, "sampling/sampling_logp_difference/mean": 0.012613819912075996, "step": 533 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00012495110422605649, "clip_ratio/low_min": 0.00012495110422605649, "clip_ratio/region_mean": 0.00012495110422605649, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 704.0, "completions/mean_length": 463.09375, "completions/mean_terminated_length": 442.7666931152344, "completions/min_length": 209.0, "completions/min_terminated_length": 209.0, "entropy": 0.38495583832263947, "epoch": 0.28586723768736616, "frac_reward_zero_std": 0.5, "grad_norm": 0.011860489845275879, "learning_rate": 1e-05, "loss": 0.0607, "num_tokens": 11486547.0, "reward": 0.6875, "reward_std": 0.249358132481575, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.4120491743087769, "sampling/importance_sampling_ratio/mean": 0.9995814561843872, "sampling/importance_sampling_ratio/min": 0.7352058291435242, "sampling/sampling_logp_difference/max": 0.34504199028015137, "sampling/sampling_logp_difference/mean": 0.012138399295508862, "step": 534 }, { "clip_ratio/high_max": 6.358087557600811e-05, "clip_ratio/high_mean": 6.358087557600811e-05, "clip_ratio/low_mean": 0.00012862501534982584, "clip_ratio/low_min": 0.00012862501534982584, "clip_ratio/region_mean": 0.00019220589092583396, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 696.0, "completions/mean_length": 464.0625, "completions/mean_terminated_length": 443.8000183105469, "completions/min_length": 244.0, "completions/min_terminated_length": 244.0, "entropy": 0.33858613669872284, "epoch": 0.28640256959314775, "frac_reward_zero_std": 0.5, "grad_norm": 0.004619220271706581, "learning_rate": 1e-05, "loss": 0.0571, "num_tokens": 11505253.0, "reward": 0.78125, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.473806619644165, "sampling/importance_sampling_ratio/mean": 0.9999437928199768, "sampling/importance_sampling_ratio/min": 0.44408559799194336, "sampling/sampling_logp_difference/max": 0.8117380142211914, "sampling/sampling_logp_difference/mean": 0.011063523590564728, "step": 535 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 765.0, "completions/max_terminated_length": 765.0, "completions/mean_length": 462.53125, "completions/mean_terminated_length": 462.53125, "completions/min_length": 302.0, "completions/min_terminated_length": 302.0, "entropy": 0.3775027021765709, "epoch": 0.28693790149892934, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0003, "num_tokens": 11523758.0, "reward": 0.96875, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.96875, "rewards/accuracy_reward/std": 0.1767766922712326, "sampling/importance_sampling_ratio/max": 1.5902752876281738, "sampling/importance_sampling_ratio/mean": 0.9999961256980896, "sampling/importance_sampling_ratio/min": 0.5626102685928345, "sampling/sampling_logp_difference/max": 0.5751681327819824, "sampling/sampling_logp_difference/mean": 0.012702928856015205, "step": 536 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00011407211786718108, "clip_ratio/low_min": 0.00011407211786718108, "clip_ratio/region_mean": 0.00011407211786718108, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 766.0, "completions/mean_length": 604.375, "completions/mean_terminated_length": 599.0967407226562, "completions/min_length": 344.0, "completions/min_terminated_length": 344.0, "entropy": 0.2872885260730982, "epoch": 0.2874732334047109, "frac_reward_zero_std": 0.25, "grad_norm": 0.02618037350475788, "learning_rate": 1e-05, "loss": 0.0025, "num_tokens": 11547162.0, "reward": 0.59375, "reward_std": 0.3061639964580536, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.4447325468063354, "sampling/importance_sampling_ratio/mean": 1.0000284910202026, "sampling/importance_sampling_ratio/min": 0.6939098834991455, "sampling/sampling_logp_difference/max": 0.36792421340942383, "sampling/sampling_logp_difference/mean": 0.011039305478334427, "step": 537 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00010052443030872382, "clip_ratio/low_min": 0.00010052443030872382, "clip_ratio/region_mean": 0.00010052443030872382, "completions/clipped_ratio": 0.375, "completions/max_length": 768.0, "completions/max_terminated_length": 717.0, "completions/mean_length": 619.8125, "completions/mean_terminated_length": 530.9000244140625, "completions/min_length": 352.0, "completions/min_terminated_length": 352.0, "entropy": 0.7173309400677681, "epoch": 0.2880085653104925, "frac_reward_zero_std": 0.5, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0337, "num_tokens": 11571476.0, "reward": 0.4375, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.4375, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.4322186708450317, "sampling/importance_sampling_ratio/mean": 0.9997522234916687, "sampling/importance_sampling_ratio/min": 0.39511144161224365, "sampling/sampling_logp_difference/max": 0.9285874366760254, "sampling/sampling_logp_difference/mean": 0.01736968383193016, "step": 538 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.804572331020609e-05, "clip_ratio/low_min": 6.804572331020609e-05, "clip_ratio/region_mean": 6.804572331020609e-05, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 679.0, "completions/mean_length": 483.21875, "completions/mean_terminated_length": 474.0322570800781, "completions/min_length": 329.0, "completions/min_terminated_length": 329.0, "entropy": 0.4886215142905712, "epoch": 0.2885438972162741, "frac_reward_zero_std": 0.25, "grad_norm": 0.012145275250077248, "learning_rate": 1e-05, "loss": -0.0286, "num_tokens": 11590651.0, "reward": 0.59375, "reward_std": 0.3608423173427582, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.3283801078796387, "sampling/importance_sampling_ratio/mean": 1.0001471042633057, "sampling/importance_sampling_ratio/min": 0.7307401895523071, "sampling/sampling_logp_difference/max": 0.31369733810424805, "sampling/sampling_logp_difference/mean": 0.014218015596270561, "step": 539 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.335040623322129e-05, "clip_ratio/low_min": 5.335040623322129e-05, "clip_ratio/region_mean": 5.335040623322129e-05, "completions/clipped_ratio": 0.34375, "completions/max_length": 768.0, "completions/max_terminated_length": 746.0, "completions/mean_length": 589.9375, "completions/mean_terminated_length": 496.66668701171875, "completions/min_length": 349.0, "completions/min_terminated_length": 349.0, "entropy": 0.41439638659358025, "epoch": 0.2890792291220557, "frac_reward_zero_std": 0.5, "grad_norm": 0.01842697151005268, "learning_rate": 1e-05, "loss": 0.0346, "num_tokens": 11613337.0, "reward": 0.625, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.6582441329956055, "sampling/importance_sampling_ratio/mean": 0.9999056458473206, "sampling/importance_sampling_ratio/min": 0.5047377347946167, "sampling/sampling_logp_difference/max": 0.6837162971496582, "sampling/sampling_logp_difference/mean": 0.013271604664623737, "step": 540 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 768.0, "completions/mean_length": 541.09375, "completions/mean_terminated_length": 517.6206665039062, "completions/min_length": 345.0, "completions/min_terminated_length": 345.0, "entropy": 0.3945649564266205, "epoch": 0.2896145610278373, "frac_reward_zero_std": 0.5, "grad_norm": 0.004135321360081434, "learning_rate": 1e-05, "loss": 0.0425, "num_tokens": 11634492.0, "reward": 0.6875, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.6172866821289062, "sampling/importance_sampling_ratio/mean": 0.9996880292892456, "sampling/importance_sampling_ratio/min": 0.7439208030700684, "sampling/sampling_logp_difference/max": 0.48074984550476074, "sampling/sampling_logp_difference/mean": 0.012405997142195702, "step": 541 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.000125376129290089, "clip_ratio/low_min": 0.000125376129290089, "clip_ratio/region_mean": 0.000125376129290089, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 713.0, "completions/mean_length": 523.125, "completions/mean_terminated_length": 477.77777099609375, "completions/min_length": 238.0, "completions/min_terminated_length": 238.0, "entropy": 0.3971831537783146, "epoch": 0.29014989293361887, "frac_reward_zero_std": 0.0, "grad_norm": 0.01960701122879982, "learning_rate": 1e-05, "loss": 0.0508, "num_tokens": 11655424.0, "reward": 0.59375, "reward_std": 0.4628904461860657, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.477827787399292, "sampling/importance_sampling_ratio/mean": 0.999897837638855, "sampling/importance_sampling_ratio/min": 0.7256606817245483, "sampling/sampling_logp_difference/max": 0.39057326316833496, "sampling/sampling_logp_difference/mean": 0.012579919770359993, "step": 542 }, { "clip_ratio/high_max": 5.307855462888256e-05, "clip_ratio/high_mean": 5.307855462888256e-05, "clip_ratio/low_mean": 0.00015782160335220397, "clip_ratio/low_min": 0.00015782160335220397, "clip_ratio/region_mean": 0.00021090015798108652, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 754.0, "completions/mean_length": 586.8125, "completions/mean_terminated_length": 536.0800170898438, "completions/min_length": 168.0, "completions/min_terminated_length": 168.0, "entropy": 0.37633199617266655, "epoch": 0.2906852248394004, "frac_reward_zero_std": 0.0, "grad_norm": 0.022214163094758987, "learning_rate": 1e-05, "loss": 0.0017, "num_tokens": 11678346.0, "reward": 0.5, "reward_std": 0.4492306709289551, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5080004930496216, "sampling/importance_sampling_ratio/max": 1.7390252351760864, "sampling/importance_sampling_ratio/mean": 1.0002113580703735, "sampling/importance_sampling_ratio/min": 0.6801731586456299, "sampling/sampling_logp_difference/max": 0.5533246994018555, "sampling/sampling_logp_difference/mean": 0.012878337875008583, "step": 543 }, { "clip_ratio/high_max": 6.397133984137326e-05, "clip_ratio/high_mean": 6.397133984137326e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 6.397133984137326e-05, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 632.0, "completions/mean_length": 456.21875, "completions/mean_terminated_length": 446.1612854003906, "completions/min_length": 309.0, "completions/min_terminated_length": 309.0, "entropy": 0.3101335186511278, "epoch": 0.291220556745182, "frac_reward_zero_std": 0.5, "grad_norm": 0.02585303783416748, "learning_rate": 1e-05, "loss": 0.0347, "num_tokens": 11696337.0, "reward": 0.90625, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.4308266639709473, "sampling/importance_sampling_ratio/mean": 1.0001417398452759, "sampling/importance_sampling_ratio/min": 0.7768503427505493, "sampling/sampling_logp_difference/max": 0.35825228691101074, "sampling/sampling_logp_difference/mean": 0.011320236138999462, "step": 544 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 743.0, "completions/mean_length": 559.71875, "completions/mean_terminated_length": 511.65386962890625, "completions/min_length": 294.0, "completions/min_terminated_length": 294.0, "entropy": 0.3975968435406685, "epoch": 0.2917558886509636, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 11718216.0, "reward": 0.75, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.4286057949066162, "sampling/importance_sampling_ratio/mean": 0.9998687505722046, "sampling/importance_sampling_ratio/min": 0.6326331496238708, "sampling/sampling_logp_difference/max": 0.4578646421432495, "sampling/sampling_logp_difference/mean": 0.012749411165714264, "step": 545 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00012626725947484374, "clip_ratio/low_min": 0.00012626725947484374, "clip_ratio/region_mean": 0.00012626725947484374, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 734.0, "completions/mean_length": 514.71875, "completions/mean_terminated_length": 506.5483703613281, "completions/min_length": 193.0, "completions/min_terminated_length": 193.0, "entropy": 0.3637736365199089, "epoch": 0.29229122055674517, "frac_reward_zero_std": 0.5, "grad_norm": 0.011513571254909039, "learning_rate": 1e-05, "loss": -0.0048, "num_tokens": 11738639.0, "reward": 0.8125, "reward_std": 0.249358132481575, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.3561745882034302, "sampling/importance_sampling_ratio/mean": 1.0005443096160889, "sampling/importance_sampling_ratio/min": 0.7002918124198914, "sampling/sampling_logp_difference/max": 0.3562581539154053, "sampling/sampling_logp_difference/mean": 0.011963626369833946, "step": 546 }, { "clip_ratio/high_max": 5.9045818488812074e-05, "clip_ratio/high_mean": 5.9045818488812074e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 5.9045818488812074e-05, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 732.0, "completions/mean_length": 549.375, "completions/mean_terminated_length": 526.7586059570312, "completions/min_length": 384.0, "completions/min_terminated_length": 384.0, "entropy": 0.2855883315205574, "epoch": 0.29282655246252676, "frac_reward_zero_std": 0.25, "grad_norm": 0.009666687808930874, "learning_rate": 1e-05, "loss": 0.0505, "num_tokens": 11759867.0, "reward": 0.84375, "reward_std": 0.3061639666557312, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.4756367206573486, "sampling/importance_sampling_ratio/mean": 1.0005191564559937, "sampling/importance_sampling_ratio/min": 0.7696874737739563, "sampling/sampling_logp_difference/max": 0.38908958435058594, "sampling/sampling_logp_difference/mean": 0.010424262844026089, "step": 547 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 752.0, "completions/max_terminated_length": 752.0, "completions/mean_length": 533.09375, "completions/mean_terminated_length": 533.09375, "completions/min_length": 310.0, "completions/min_terminated_length": 310.0, "entropy": 0.30539813078939915, "epoch": 0.29336188436830835, "frac_reward_zero_std": 0.75, "grad_norm": 0.005505862180143595, "learning_rate": 1e-05, "loss": -0.0045, "num_tokens": 11781382.0, "reward": 0.96875, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.96875, "rewards/accuracy_reward/std": 0.1767766922712326, "sampling/importance_sampling_ratio/max": 1.3387318849563599, "sampling/importance_sampling_ratio/mean": 1.0000278949737549, "sampling/importance_sampling_ratio/min": 0.727968156337738, "sampling/sampling_logp_difference/max": 0.31749796867370605, "sampling/sampling_logp_difference/mean": 0.010298791341483593, "step": 548 }, { "clip_ratio/high_max": 7.896398892626166e-05, "clip_ratio/high_mean": 7.896398892626166e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 7.896398892626166e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 728.0, "completions/max_terminated_length": 728.0, "completions/mean_length": 419.03125, "completions/mean_terminated_length": 419.03125, "completions/min_length": 247.0, "completions/min_terminated_length": 247.0, "entropy": 0.3250376991927624, "epoch": 0.29389721627408993, "frac_reward_zero_std": 0.5, "grad_norm": 0.013827790506184101, "learning_rate": 1e-05, "loss": -0.0034, "num_tokens": 11798767.0, "reward": 0.84375, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.3619909286499023, "sampling/importance_sampling_ratio/mean": 0.9998846054077148, "sampling/importance_sampling_ratio/min": 0.623042643070221, "sampling/sampling_logp_difference/max": 0.47314023971557617, "sampling/sampling_logp_difference/mean": 0.011378364637494087, "step": 549 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 703.0, "completions/mean_length": 446.78125, "completions/mean_terminated_length": 425.36669921875, "completions/min_length": 231.0, "completions/min_terminated_length": 231.0, "entropy": 0.32534400187432766, "epoch": 0.2944325481798715, "frac_reward_zero_std": 0.5, "grad_norm": 0.003956167493015528, "learning_rate": 1e-05, "loss": 0.0734, "num_tokens": 11816336.0, "reward": 0.84375, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.4408214092254639, "sampling/importance_sampling_ratio/mean": 1.0000100135803223, "sampling/importance_sampling_ratio/min": 0.678433358669281, "sampling/sampling_logp_difference/max": 0.3879690170288086, "sampling/sampling_logp_difference/mean": 0.01156111154705286, "step": 550 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00020651722297770903, "clip_ratio/low_min": 0.00020651722297770903, "clip_ratio/region_mean": 0.00020651722297770903, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 708.0, "completions/mean_length": 478.625, "completions/mean_terminated_length": 469.2903137207031, "completions/min_length": 250.0, "completions/min_terminated_length": 250.0, "entropy": 0.42275893315672874, "epoch": 0.2949678800856531, "frac_reward_zero_std": 0.25, "grad_norm": 0.02261337637901306, "learning_rate": 1e-05, "loss": -0.0115, "num_tokens": 11835292.0, "reward": 0.78125, "reward_std": 0.3377464711666107, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.6950032711029053, "sampling/importance_sampling_ratio/mean": 1.0003259181976318, "sampling/importance_sampling_ratio/min": 0.5934163331985474, "sampling/sampling_logp_difference/max": 0.5276846885681152, "sampling/sampling_logp_difference/mean": 0.012762945145368576, "step": 551 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 677.0, "completions/mean_length": 499.34375, "completions/mean_terminated_length": 481.433349609375, "completions/min_length": 221.0, "completions/min_terminated_length": 221.0, "entropy": 0.3455977477133274, "epoch": 0.2955032119914347, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0321, "num_tokens": 11855127.0, "reward": 0.71875, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.6328233480453491, "sampling/importance_sampling_ratio/mean": 1.0001987218856812, "sampling/importance_sampling_ratio/min": 0.7042146325111389, "sampling/sampling_logp_difference/max": 0.4903106689453125, "sampling/sampling_logp_difference/mean": 0.011651122011244297, "step": 552 }, { "clip_ratio/high_max": 7.427213131450117e-05, "clip_ratio/high_mean": 7.427213131450117e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 7.427213131450117e-05, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 619.0, "completions/mean_length": 471.5, "completions/mean_terminated_length": 461.9354553222656, "completions/min_length": 308.0, "completions/min_terminated_length": 308.0, "entropy": 0.3185413535684347, "epoch": 0.2960385438972163, "frac_reward_zero_std": 0.25, "grad_norm": 0.005406456999480724, "learning_rate": 1e-05, "loss": 0.0352, "num_tokens": 11873735.0, "reward": 0.84375, "reward_std": 0.3198433816432953, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.4264329671859741, "sampling/importance_sampling_ratio/mean": 0.9997532963752747, "sampling/importance_sampling_ratio/min": 0.5113204717636108, "sampling/sampling_logp_difference/max": 0.6707587242126465, "sampling/sampling_logp_difference/mean": 0.011310521513223648, "step": 553 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 710.0, "completions/mean_length": 468.28125, "completions/mean_terminated_length": 437.2758483886719, "completions/min_length": 171.0, "completions/min_terminated_length": 171.0, "entropy": 0.396459287032485, "epoch": 0.2965738758029979, "frac_reward_zero_std": 0.25, "grad_norm": 0.004435537848621607, "learning_rate": 1e-05, "loss": 0.0326, "num_tokens": 11892320.0, "reward": 0.90625, "reward_std": 0.2651650309562683, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.6119393110275269, "sampling/importance_sampling_ratio/mean": 1.0003647804260254, "sampling/importance_sampling_ratio/min": 0.4504607319831848, "sampling/sampling_logp_difference/max": 0.7974843978881836, "sampling/sampling_logp_difference/mean": 0.011878822930157185, "step": 554 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00010394055425422266, "clip_ratio/low_min": 0.00010394055425422266, "clip_ratio/region_mean": 0.00010394055425422266, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 733.0, "completions/mean_length": 519.375, "completions/mean_terminated_length": 483.857177734375, "completions/min_length": 272.0, "completions/min_terminated_length": 272.0, "entropy": 0.41442643851041794, "epoch": 0.29710920770877947, "frac_reward_zero_std": 0.25, "grad_norm": 0.02032538317143917, "learning_rate": 1e-05, "loss": 0.0576, "num_tokens": 11912996.0, "reward": 0.65625, "reward_std": 0.3377465009689331, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.4642575979232788, "sampling/importance_sampling_ratio/mean": 1.0001085996627808, "sampling/importance_sampling_ratio/min": 0.7119376063346863, "sampling/sampling_logp_difference/max": 0.3813483715057373, "sampling/sampling_logp_difference/mean": 0.013506373390555382, "step": 555 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 747.0, "completions/mean_length": 474.40625, "completions/mean_terminated_length": 464.9354553222656, "completions/min_length": 274.0, "completions/min_terminated_length": 274.0, "entropy": 0.34534524008631706, "epoch": 0.29764453961456105, "frac_reward_zero_std": 0.5, "grad_norm": 0.02208186872303486, "learning_rate": 1e-05, "loss": 0.0163, "num_tokens": 11931689.0, "reward": 0.8125, "reward_std": 0.2587745785713196, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.4568241834640503, "sampling/importance_sampling_ratio/mean": 0.999871015548706, "sampling/importance_sampling_ratio/min": 0.648076593875885, "sampling/sampling_logp_difference/max": 0.433746337890625, "sampling/sampling_logp_difference/mean": 0.012376468628644943, "step": 556 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00011861355960718356, "clip_ratio/low_min": 0.00011861355960718356, "clip_ratio/region_mean": 0.00011861355960718356, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 711.0, "completions/mean_length": 557.0625, "completions/mean_terminated_length": 486.75, "completions/min_length": 355.0, "completions/min_terminated_length": 355.0, "entropy": 0.3613390251994133, "epoch": 0.2981798715203426, "frac_reward_zero_std": 0.25, "grad_norm": 0.017086103558540344, "learning_rate": 1e-05, "loss": 0.0929, "num_tokens": 11953243.0, "reward": 0.53125, "reward_std": 0.2651650309562683, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.4256588220596313, "sampling/importance_sampling_ratio/mean": 1.0003442764282227, "sampling/importance_sampling_ratio/min": 0.7358534932136536, "sampling/sampling_logp_difference/max": 0.35463404655456543, "sampling/sampling_logp_difference/mean": 0.011376078240573406, "step": 557 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.771006544819102e-05, "clip_ratio/low_min": 5.771006544819102e-05, "clip_ratio/region_mean": 5.771006544819102e-05, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 663.0, "completions/mean_length": 559.71875, "completions/mean_terminated_length": 490.29168701171875, "completions/min_length": 324.0, "completions/min_terminated_length": 324.0, "entropy": 0.32203043811023235, "epoch": 0.2987152034261242, "frac_reward_zero_std": 0.5, "grad_norm": 0.012001988478004932, "learning_rate": 1e-05, "loss": 0.034, "num_tokens": 11974778.0, "reward": 0.3125, "reward_std": 0.2587745785713196, "rewards/accuracy_reward/mean": 0.3125, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.4094603061676025, "sampling/importance_sampling_ratio/mean": 0.9996496438980103, "sampling/importance_sampling_ratio/min": 0.6508768796920776, "sampling/sampling_logp_difference/max": 0.42943477630615234, "sampling/sampling_logp_difference/mean": 0.010747031308710575, "step": 558 }, { "clip_ratio/high_max": 5.5506217904621735e-05, "clip_ratio/high_mean": 5.5506217904621735e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 5.5506217904621735e-05, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 718.0, "completions/mean_length": 564.34375, "completions/mean_terminated_length": 550.7667236328125, "completions/min_length": 342.0, "completions/min_terminated_length": 342.0, "entropy": 0.32396041601896286, "epoch": 0.29925053533190576, "frac_reward_zero_std": 0.0, "grad_norm": 0.009842682629823685, "learning_rate": 1e-05, "loss": 0.0229, "num_tokens": 11996669.0, "reward": 0.8125, "reward_std": 0.3945523500442505, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.4444060325622559, "sampling/importance_sampling_ratio/mean": 1.000019907951355, "sampling/importance_sampling_ratio/min": 0.6250066161155701, "sampling/sampling_logp_difference/max": 0.46999311447143555, "sampling/sampling_logp_difference/mean": 0.011722080409526825, "step": 559 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 724.0, "completions/max_terminated_length": 724.0, "completions/mean_length": 437.09375, "completions/mean_terminated_length": 437.09375, "completions/min_length": 256.0, "completions/min_terminated_length": 256.0, "entropy": 0.3068154342472553, "epoch": 0.29978586723768735, "frac_reward_zero_std": 0.5, "grad_norm": 0.007331756874918938, "learning_rate": 1e-05, "loss": 0.0035, "num_tokens": 12014216.0, "reward": 0.84375, "reward_std": 0.22201895713806152, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.427247166633606, "sampling/importance_sampling_ratio/mean": 0.9997251033782959, "sampling/importance_sampling_ratio/min": 0.7263378500938416, "sampling/sampling_logp_difference/max": 0.3557474613189697, "sampling/sampling_logp_difference/mean": 0.010692532174289227, "step": 560 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00026017974596470594, "clip_ratio/low_min": 0.00026017974596470594, "clip_ratio/region_mean": 0.00026017974596470594, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 766.0, "completions/mean_length": 587.03125, "completions/mean_terminated_length": 545.2692260742188, "completions/min_length": 314.0, "completions/min_terminated_length": 314.0, "entropy": 0.337619349360466, "epoch": 0.30032119914346894, "frac_reward_zero_std": 0.5, "grad_norm": 0.016035636886954308, "learning_rate": 1e-05, "loss": -0.0047, "num_tokens": 12036193.0, "reward": 0.71875, "reward_std": 0.2630178928375244, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.426025629043579, "sampling/importance_sampling_ratio/mean": 0.9999319911003113, "sampling/importance_sampling_ratio/min": 0.7283347249031067, "sampling/sampling_logp_difference/max": 0.354891300201416, "sampling/sampling_logp_difference/mean": 0.011244530789554119, "step": 561 }, { "clip_ratio/high_max": 4.9212598241865635e-05, "clip_ratio/high_mean": 4.9212598241865635e-05, "clip_ratio/low_mean": 5.139802669873461e-05, "clip_ratio/low_min": 5.139802669873461e-05, "clip_ratio/region_mean": 0.00010061062494060025, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 758.0, "completions/mean_length": 583.25, "completions/mean_terminated_length": 549.0370483398438, "completions/min_length": 292.0, "completions/min_terminated_length": 292.0, "entropy": 0.2893373966217041, "epoch": 0.30085653104925053, "frac_reward_zero_std": 0.25, "grad_norm": 0.008920936845242977, "learning_rate": 1e-05, "loss": 0.0779, "num_tokens": 12058841.0, "reward": 0.65625, "reward_std": 0.3377465009689331, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.3797690868377686, "sampling/importance_sampling_ratio/mean": 1.000322699546814, "sampling/importance_sampling_ratio/min": 0.5136542320251465, "sampling/sampling_logp_difference/max": 0.6662049293518066, "sampling/sampling_logp_difference/mean": 0.010499080643057823, "step": 562 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.17812768521253e-05, "clip_ratio/low_min": 5.17812768521253e-05, "clip_ratio/region_mean": 5.17812768521253e-05, "completions/clipped_ratio": 0.3125, "completions/max_length": 768.0, "completions/max_terminated_length": 759.0, "completions/mean_length": 582.375, "completions/mean_terminated_length": 498.0, "completions/min_length": 285.0, "completions/min_terminated_length": 285.0, "entropy": 0.3608653359115124, "epoch": 0.3013918629550321, "frac_reward_zero_std": 0.25, "grad_norm": 0.018555885180830956, "learning_rate": 1e-05, "loss": 0.0752, "num_tokens": 12081013.0, "reward": 0.625, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.4223147630691528, "sampling/importance_sampling_ratio/mean": 1.0002999305725098, "sampling/importance_sampling_ratio/min": 0.6105955839157104, "sampling/sampling_logp_difference/max": 0.4933204650878906, "sampling/sampling_logp_difference/mean": 0.01181753445416689, "step": 563 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 7.365940109593794e-05, "clip_ratio/low_min": 7.365940109593794e-05, "clip_ratio/region_mean": 7.365940109593794e-05, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 621.0, "completions/mean_length": 531.3125, "completions/mean_terminated_length": 465.03997802734375, "completions/min_length": 225.0, "completions/min_terminated_length": 225.0, "entropy": 0.536725327372551, "epoch": 0.3019271948608137, "frac_reward_zero_std": 0.5, "grad_norm": 0.008478255942463875, "learning_rate": 1e-05, "loss": -0.0273, "num_tokens": 12102159.0, "reward": 0.53125, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.4203286170959473, "sampling/importance_sampling_ratio/mean": 1.000118613243103, "sampling/importance_sampling_ratio/min": 0.6662923693656921, "sampling/sampling_logp_difference/max": 0.4060267210006714, "sampling/sampling_logp_difference/mean": 0.01727110892534256, "step": 564 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00034745450830087066, "clip_ratio/low_min": 0.00034745450830087066, "clip_ratio/region_mean": 0.00034745450830087066, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 603.0, "completions/mean_length": 445.59375, "completions/mean_terminated_length": 424.10003662109375, "completions/min_length": 233.0, "completions/min_terminated_length": 233.0, "entropy": 0.3727577328681946, "epoch": 0.3024625267665953, "frac_reward_zero_std": 0.25, "grad_norm": 0.02334926649928093, "learning_rate": 1e-05, "loss": 0.0033, "num_tokens": 12119946.0, "reward": 0.65625, "reward_std": 0.3608423173427582, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.4262263774871826, "sampling/importance_sampling_ratio/mean": 1.0003966093063354, "sampling/importance_sampling_ratio/min": 0.4430871903896332, "sampling/sampling_logp_difference/max": 0.8139886856079102, "sampling/sampling_logp_difference/mean": 0.012455431744456291, "step": 565 }, { "clip_ratio/high_max": 6.032818419043906e-05, "clip_ratio/high_mean": 6.032818419043906e-05, "clip_ratio/low_mean": 0.00020465075795073062, "clip_ratio/low_min": 0.00020465075795073062, "clip_ratio/region_mean": 0.0002649789421411697, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 682.0, "completions/mean_length": 483.5625, "completions/mean_terminated_length": 474.3870849609375, "completions/min_length": 277.0, "completions/min_terminated_length": 277.0, "entropy": 0.27930397167801857, "epoch": 0.3029978586723769, "frac_reward_zero_std": 0.25, "grad_norm": 0.029391754418611526, "learning_rate": 1e-05, "loss": 0.0408, "num_tokens": 12139068.0, "reward": 0.75, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.3648651838302612, "sampling/importance_sampling_ratio/mean": 0.9999411106109619, "sampling/importance_sampling_ratio/min": 0.6107367873191833, "sampling/sampling_logp_difference/max": 0.4930891990661621, "sampling/sampling_logp_difference/mean": 0.009847821667790413, "step": 566 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 668.0, "completions/max_terminated_length": 668.0, "completions/mean_length": 463.5, "completions/mean_terminated_length": 463.5, "completions/min_length": 248.0, "completions/min_terminated_length": 248.0, "entropy": 0.28906029276549816, "epoch": 0.3035331905781585, "frac_reward_zero_std": 0.5, "grad_norm": 0.008024390786886215, "learning_rate": 1e-05, "loss": 0.0037, "num_tokens": 12157588.0, "reward": 0.8125, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.297465443611145, "sampling/importance_sampling_ratio/mean": 1.000006914138794, "sampling/importance_sampling_ratio/min": 0.6893514394760132, "sampling/sampling_logp_difference/max": 0.37200403213500977, "sampling/sampling_logp_difference/mean": 0.010215293616056442, "step": 567 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00016838796727824956, "clip_ratio/low_min": 0.00016838796727824956, "clip_ratio/region_mean": 0.00016838796727824956, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 652.0, "completions/mean_length": 464.53125, "completions/mean_terminated_length": 444.3000183105469, "completions/min_length": 256.0, "completions/min_terminated_length": 256.0, "entropy": 0.3061659652739763, "epoch": 0.30406852248394006, "frac_reward_zero_std": 0.5, "grad_norm": 0.005570350680500269, "learning_rate": 1e-05, "loss": 0.0573, "num_tokens": 12175701.0, "reward": 0.90625, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.4560657739639282, "sampling/importance_sampling_ratio/mean": 1.0002351999282837, "sampling/importance_sampling_ratio/min": 0.6175297498703003, "sampling/sampling_logp_difference/max": 0.4820280075073242, "sampling/sampling_logp_difference/mean": 0.01099972240626812, "step": 568 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 691.0, "completions/mean_length": 480.75, "completions/mean_terminated_length": 471.4838562011719, "completions/min_length": 157.0, "completions/min_terminated_length": 157.0, "entropy": 0.31722648069262505, "epoch": 0.30460385438972165, "frac_reward_zero_std": 0.5, "grad_norm": 0.01537348609417677, "learning_rate": 1e-05, "loss": 0.0188, "num_tokens": 12194621.0, "reward": 0.78125, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.3345178365707397, "sampling/importance_sampling_ratio/mean": 0.9995911717414856, "sampling/importance_sampling_ratio/min": 0.507928192615509, "sampling/sampling_logp_difference/max": 0.677415132522583, "sampling/sampling_logp_difference/mean": 0.011245360597968102, "step": 569 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 7.617306255269796e-05, "clip_ratio/low_min": 7.617306255269796e-05, "clip_ratio/region_mean": 7.617306255269796e-05, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 722.0, "completions/mean_length": 487.96875, "completions/mean_terminated_length": 459.0, "completions/min_length": 224.0, "completions/min_terminated_length": 224.0, "entropy": 0.45248181000351906, "epoch": 0.30513918629550324, "frac_reward_zero_std": 0.25, "grad_norm": 0.025909526273608208, "learning_rate": 1e-05, "loss": 0.0343, "num_tokens": 12214460.0, "reward": 0.65625, "reward_std": 0.3608423173427582, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.428450584411621, "sampling/importance_sampling_ratio/mean": 1.000129222869873, "sampling/importance_sampling_ratio/min": 0.7096893191337585, "sampling/sampling_logp_difference/max": 0.35659027099609375, "sampling/sampling_logp_difference/mean": 0.014986622147262096, "step": 570 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 752.0, "completions/mean_length": 517.46875, "completions/mean_terminated_length": 500.7666931152344, "completions/min_length": 220.0, "completions/min_terminated_length": 220.0, "entropy": 0.32898028939962387, "epoch": 0.3056745182012848, "frac_reward_zero_std": 0.5, "grad_norm": 0.006802439223974943, "learning_rate": 1e-05, "loss": -0.05, "num_tokens": 12235019.0, "reward": 0.625, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.435111165046692, "sampling/importance_sampling_ratio/mean": 0.9999232292175293, "sampling/importance_sampling_ratio/min": 0.6228591203689575, "sampling/sampling_logp_difference/max": 0.4734349250793457, "sampling/sampling_logp_difference/mean": 0.011875735595822334, "step": 571 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.849315104773268e-05, "clip_ratio/low_min": 6.849315104773268e-05, "clip_ratio/region_mean": 6.849315104773268e-05, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 662.0, "completions/mean_length": 481.6875, "completions/mean_terminated_length": 472.45159912109375, "completions/min_length": 236.0, "completions/min_terminated_length": 236.0, "entropy": 0.38885802403092384, "epoch": 0.30620985010706636, "frac_reward_zero_std": 0.25, "grad_norm": 0.030769284814596176, "learning_rate": 1e-05, "loss": 0.0703, "num_tokens": 12253921.0, "reward": 0.625, "reward_std": 0.2925041913986206, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.6335504055023193, "sampling/importance_sampling_ratio/mean": 0.9997223019599915, "sampling/importance_sampling_ratio/min": 0.7307669520378113, "sampling/sampling_logp_difference/max": 0.4907557964324951, "sampling/sampling_logp_difference/mean": 0.01296333596110344, "step": 572 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 640.0, "completions/max_terminated_length": 640.0, "completions/mean_length": 450.75, "completions/mean_terminated_length": 450.75, "completions/min_length": 240.0, "completions/min_terminated_length": 240.0, "entropy": 0.359542403370142, "epoch": 0.30674518201284795, "frac_reward_zero_std": 0.75, "grad_norm": 0.016131862998008728, "learning_rate": 1e-05, "loss": -0.0383, "num_tokens": 12271545.0, "reward": 0.4375, "reward_std": 0.1157275140285492, "rewards/accuracy_reward/mean": 0.4375, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.374092936515808, "sampling/importance_sampling_ratio/mean": 0.9999517202377319, "sampling/importance_sampling_ratio/min": 0.6975948810577393, "sampling/sampling_logp_difference/max": 0.36011672019958496, "sampling/sampling_logp_difference/mean": 0.01208482775837183, "step": 573 }, { "clip_ratio/high_max": 5.8768218877958134e-05, "clip_ratio/high_mean": 5.8768218877958134e-05, "clip_ratio/low_mean": 0.00011150757927680388, "clip_ratio/low_min": 0.00011150757927680388, "clip_ratio/region_mean": 0.00017027579815476201, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 759.0, "completions/mean_length": 534.625, "completions/mean_terminated_length": 491.40740966796875, "completions/min_length": 270.0, "completions/min_terminated_length": 270.0, "entropy": 0.3541809841990471, "epoch": 0.30728051391862954, "frac_reward_zero_std": 0.25, "grad_norm": 0.007234027609229088, "learning_rate": 1e-05, "loss": 0.0549, "num_tokens": 12292645.0, "reward": 0.78125, "reward_std": 0.3377464711666107, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.4642210006713867, "sampling/importance_sampling_ratio/mean": 1.0002000331878662, "sampling/importance_sampling_ratio/min": 0.5763978362083435, "sampling/sampling_logp_difference/max": 0.550957202911377, "sampling/sampling_logp_difference/mean": 0.01204842422157526, "step": 574 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0002030145114986226, "clip_ratio/low_min": 0.0002030145114986226, "clip_ratio/region_mean": 0.0002030145114986226, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 664.0, "completions/mean_length": 456.6875, "completions/mean_terminated_length": 446.6451416015625, "completions/min_length": 247.0, "completions/min_terminated_length": 247.0, "entropy": 0.440885741263628, "epoch": 0.3078158458244111, "frac_reward_zero_std": 0.0, "grad_norm": 0.016579922288656235, "learning_rate": 1e-05, "loss": -0.0194, "num_tokens": 12311067.0, "reward": 0.625, "reward_std": 0.4355512857437134, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.327857255935669, "sampling/importance_sampling_ratio/mean": 1.0000910758972168, "sampling/importance_sampling_ratio/min": 0.6185588836669922, "sampling/sampling_logp_difference/max": 0.4803628921508789, "sampling/sampling_logp_difference/mean": 0.0136201661080122, "step": 575 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 734.0, "completions/mean_length": 489.46875, "completions/mean_terminated_length": 449.6785888671875, "completions/min_length": 291.0, "completions/min_terminated_length": 291.0, "entropy": 0.6463162414729595, "epoch": 0.3083511777301927, "frac_reward_zero_std": 0.75, "grad_norm": 0.01171574555337429, "learning_rate": 1e-05, "loss": -0.0162, "num_tokens": 12330650.0, "reward": 0.625, "reward_std": 0.13363061845302582, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.3853341341018677, "sampling/importance_sampling_ratio/mean": 0.9996410012245178, "sampling/importance_sampling_ratio/min": 0.6901805996894836, "sampling/sampling_logp_difference/max": 0.3708019256591797, "sampling/sampling_logp_difference/mean": 0.01636696606874466, "step": 576 }, { "clip_ratio/high_max": 0.00012325304851401597, "clip_ratio/high_mean": 0.00012325304851401597, "clip_ratio/low_mean": 0.00012293529289308935, "clip_ratio/low_min": 0.00012293529289308935, "clip_ratio/region_mean": 0.0002461883414071053, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 762.0, "completions/mean_length": 487.3125, "completions/mean_terminated_length": 458.2758483886719, "completions/min_length": 227.0, "completions/min_terminated_length": 227.0, "entropy": 0.4137922339141369, "epoch": 0.3088865096359743, "frac_reward_zero_std": 0.25, "grad_norm": 0.009602822363376617, "learning_rate": 1e-05, "loss": -0.0041, "num_tokens": 12350116.0, "reward": 0.6875, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.6735979318618774, "sampling/importance_sampling_ratio/mean": 1.0003747940063477, "sampling/importance_sampling_ratio/min": 0.6221232414245605, "sampling/sampling_logp_difference/max": 0.5149757862091064, "sampling/sampling_logp_difference/mean": 0.013964244164526463, "step": 577 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00019415884162299335, "clip_ratio/low_min": 0.00019415884162299335, "clip_ratio/region_mean": 0.00019415884162299335, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 662.0, "completions/mean_length": 456.78125, "completions/mean_terminated_length": 446.7419128417969, "completions/min_length": 186.0, "completions/min_terminated_length": 186.0, "entropy": 0.3068028539419174, "epoch": 0.3094218415417559, "frac_reward_zero_std": 0.5, "grad_norm": 0.004635384771972895, "learning_rate": 1e-05, "loss": 0.041, "num_tokens": 12368085.0, "reward": 0.75, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.3456618785858154, "sampling/importance_sampling_ratio/mean": 1.0001176595687866, "sampling/importance_sampling_ratio/min": 0.717003345489502, "sampling/sampling_logp_difference/max": 0.3326747417449951, "sampling/sampling_logp_difference/mean": 0.011704741045832634, "step": 578 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.05033892497886e-05, "clip_ratio/low_min": 6.05033892497886e-05, "clip_ratio/region_mean": 6.05033892497886e-05, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 700.0, "completions/mean_length": 500.65625, "completions/mean_terminated_length": 462.46429443359375, "completions/min_length": 193.0, "completions/min_terminated_length": 193.0, "entropy": 0.3540695309638977, "epoch": 0.3099571734475375, "frac_reward_zero_std": 0.5, "grad_norm": 0.01123807393014431, "learning_rate": 1e-05, "loss": 0.0147, "num_tokens": 12388354.0, "reward": 0.875, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 1.404441475868225, "sampling/importance_sampling_ratio/mean": 1.0000264644622803, "sampling/importance_sampling_ratio/min": 0.47998878359794617, "sampling/sampling_logp_difference/max": 0.7339925765991211, "sampling/sampling_logp_difference/mean": 0.01219634898006916, "step": 579 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 704.0, "completions/max_terminated_length": 704.0, "completions/mean_length": 493.875, "completions/mean_terminated_length": 493.875, "completions/min_length": 215.0, "completions/min_terminated_length": 215.0, "entropy": 0.4227006174623966, "epoch": 0.31049250535331907, "frac_reward_zero_std": 0.0, "grad_norm": 0.0075199357233941555, "learning_rate": 1e-05, "loss": -0.0693, "num_tokens": 12408502.0, "reward": 0.84375, "reward_std": 0.3808925747871399, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.411729097366333, "sampling/importance_sampling_ratio/mean": 1.0003435611724854, "sampling/importance_sampling_ratio/min": 0.5519125461578369, "sampling/sampling_logp_difference/max": 0.5943655967712402, "sampling/sampling_logp_difference/mean": 0.013516896404325962, "step": 580 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 608.0, "completions/max_terminated_length": 608.0, "completions/mean_length": 458.53125, "completions/mean_terminated_length": 458.53125, "completions/min_length": 157.0, "completions/min_terminated_length": 157.0, "entropy": 0.32681285217404366, "epoch": 0.31102783725910066, "frac_reward_zero_std": 0.75, "grad_norm": 0.01620262674987316, "learning_rate": 1e-05, "loss": 0.0047, "num_tokens": 12426999.0, "reward": 0.9375, "reward_std": 0.1157275140285492, "rewards/accuracy_reward/mean": 0.9375, "rewards/accuracy_reward/std": 0.24593468010425568, "sampling/importance_sampling_ratio/max": 1.4409444332122803, "sampling/importance_sampling_ratio/mean": 1.0002988576889038, "sampling/importance_sampling_ratio/min": 0.5771945714950562, "sampling/sampling_logp_difference/max": 0.5495758056640625, "sampling/sampling_logp_difference/mean": 0.010949954390525818, "step": 581 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 689.0, "completions/max_terminated_length": 689.0, "completions/mean_length": 510.34375, "completions/mean_terminated_length": 510.34375, "completions/min_length": 334.0, "completions/min_terminated_length": 334.0, "entropy": 0.3204137049615383, "epoch": 0.31156316916488225, "frac_reward_zero_std": 0.0, "grad_norm": 0.012267018668353558, "learning_rate": 1e-05, "loss": 0.0017, "num_tokens": 12446650.0, "reward": 0.78125, "reward_std": 0.4218915104866028, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.4486991167068481, "sampling/importance_sampling_ratio/mean": 1.000304102897644, "sampling/importance_sampling_ratio/min": 0.47284069657325745, "sampling/sampling_logp_difference/max": 0.7489967346191406, "sampling/sampling_logp_difference/mean": 0.011372043751180172, "step": 582 }, { "clip_ratio/high_max": 7.797878788551316e-05, "clip_ratio/high_mean": 7.797878788551316e-05, "clip_ratio/low_mean": 7.896398892626166e-05, "clip_ratio/low_min": 7.896398892626166e-05, "clip_ratio/region_mean": 0.00015694277681177482, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 752.0, "completions/mean_length": 486.5, "completions/mean_terminated_length": 446.2857360839844, "completions/min_length": 212.0, "completions/min_terminated_length": 212.0, "entropy": 0.4158746749162674, "epoch": 0.31209850107066384, "frac_reward_zero_std": 0.25, "grad_norm": 0.010495089925825596, "learning_rate": 1e-05, "loss": 0.115, "num_tokens": 12465634.0, "reward": 0.59375, "reward_std": 0.38816186785697937, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.4466509819030762, "sampling/importance_sampling_ratio/mean": 0.9998987913131714, "sampling/importance_sampling_ratio/min": 0.53195720911026, "sampling/sampling_logp_difference/max": 0.6311922073364258, "sampling/sampling_logp_difference/mean": 0.01349297072738409, "step": 583 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 718.0, "completions/max_terminated_length": 718.0, "completions/mean_length": 373.5625, "completions/mean_terminated_length": 373.5625, "completions/min_length": 216.0, "completions/min_terminated_length": 216.0, "entropy": 0.32849518209695816, "epoch": 0.31263383297644537, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": -0.0198, "num_tokens": 12480948.0, "reward": 0.84375, "reward_std": 0.1293872892856598, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.2808761596679688, "sampling/importance_sampling_ratio/mean": 1.0000447034835815, "sampling/importance_sampling_ratio/min": 0.7090793251991272, "sampling/sampling_logp_difference/max": 0.34378790855407715, "sampling/sampling_logp_difference/mean": 0.011789319105446339, "step": 584 }, { "clip_ratio/high_max": 8.947744936449453e-05, "clip_ratio/high_mean": 8.947744936449453e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 8.947744936449453e-05, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 589.0, "completions/mean_length": 427.34375, "completions/mean_terminated_length": 416.3548278808594, "completions/min_length": 224.0, "completions/min_terminated_length": 224.0, "entropy": 0.3590961582958698, "epoch": 0.31316916488222696, "frac_reward_zero_std": 0.75, "grad_norm": 0.015608078800141811, "learning_rate": 1e-05, "loss": -0.0022, "num_tokens": 12498159.0, "reward": 0.875, "reward_std": 0.13363061845302582, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 1.4217698574066162, "sampling/importance_sampling_ratio/mean": 1.0004570484161377, "sampling/importance_sampling_ratio/min": 0.5954433679580688, "sampling/sampling_logp_difference/max": 0.518449068069458, "sampling/sampling_logp_difference/mean": 0.012804794125258923, "step": 585 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.706008571200073e-05, "clip_ratio/low_min": 6.706008571200073e-05, "clip_ratio/region_mean": 6.706008571200073e-05, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 718.0, "completions/mean_length": 519.84375, "completions/mean_terminated_length": 494.17242431640625, "completions/min_length": 293.0, "completions/min_terminated_length": 293.0, "entropy": 0.3184284344315529, "epoch": 0.31370449678800855, "frac_reward_zero_std": 0.5, "grad_norm": 0.005449289456009865, "learning_rate": 1e-05, "loss": 0.0778, "num_tokens": 12518298.0, "reward": 0.875, "reward_std": 0.2314550280570984, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 1.4463058710098267, "sampling/importance_sampling_ratio/mean": 0.9996153712272644, "sampling/importance_sampling_ratio/min": 0.6053465008735657, "sampling/sampling_logp_difference/max": 0.5019543170928955, "sampling/sampling_logp_difference/mean": 0.011470277793705463, "step": 586 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00015746803546790034, "clip_ratio/low_min": 0.00015746803546790034, "clip_ratio/region_mean": 0.00015746803546790034, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 717.0, "completions/mean_length": 527.28125, "completions/mean_terminated_length": 471.73077392578125, "completions/min_length": 227.0, "completions/min_terminated_length": 227.0, "entropy": 0.3616977296769619, "epoch": 0.31423982869379014, "frac_reward_zero_std": 0.0, "grad_norm": 0.02209935523569584, "learning_rate": 1e-05, "loss": 0.0553, "num_tokens": 12538611.0, "reward": 0.625, "reward_std": 0.4671337604522705, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.3193449974060059, "sampling/importance_sampling_ratio/mean": 0.9998812675476074, "sampling/importance_sampling_ratio/min": 0.5874658226966858, "sampling/sampling_logp_difference/max": 0.5319371223449707, "sampling/sampling_logp_difference/mean": 0.011441144160926342, "step": 587 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 670.0, "completions/max_terminated_length": 670.0, "completions/mean_length": 404.53125, "completions/mean_terminated_length": 404.53125, "completions/min_length": 235.0, "completions/min_terminated_length": 235.0, "entropy": 0.3217569924890995, "epoch": 0.3147751605995717, "frac_reward_zero_std": 0.25, "grad_norm": 0.03017519973218441, "learning_rate": 1e-05, "loss": 0.0006, "num_tokens": 12554620.0, "reward": 0.90625, "reward_std": 0.2651650309562683, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.2829689979553223, "sampling/importance_sampling_ratio/mean": 1.0002161264419556, "sampling/importance_sampling_ratio/min": 0.6986616253852844, "sampling/sampling_logp_difference/max": 0.35858869552612305, "sampling/sampling_logp_difference/mean": 0.011221968568861485, "step": 588 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 740.0, "completions/max_terminated_length": 740.0, "completions/mean_length": 369.84375, "completions/mean_terminated_length": 369.84375, "completions/min_length": 208.0, "completions/min_terminated_length": 208.0, "entropy": 0.32443469017744064, "epoch": 0.3153104925053533, "frac_reward_zero_std": 0.25, "grad_norm": 0.014641523361206055, "learning_rate": 1e-05, "loss": -0.0353, "num_tokens": 12570047.0, "reward": 0.71875, "reward_std": 0.3787454068660736, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.469570517539978, "sampling/importance_sampling_ratio/mean": 1.0000818967819214, "sampling/importance_sampling_ratio/min": 0.7451204061508179, "sampling/sampling_logp_difference/max": 0.38497018814086914, "sampling/sampling_logp_difference/mean": 0.012032726779580116, "step": 589 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.0240963648539037e-05, "clip_ratio/low_min": 6.0240963648539037e-05, "clip_ratio/region_mean": 6.0240963648539037e-05, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 744.0, "completions/mean_length": 459.6875, "completions/mean_terminated_length": 427.7930908203125, "completions/min_length": 207.0, "completions/min_terminated_length": 207.0, "entropy": 0.32845132052898407, "epoch": 0.3158458244111349, "frac_reward_zero_std": 0.5, "grad_norm": 0.010784919373691082, "learning_rate": 1e-05, "loss": -0.0052, "num_tokens": 12588333.0, "reward": 0.5, "reward_std": 0.2314550280570984, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5080004930496216, "sampling/importance_sampling_ratio/max": 1.996599793434143, "sampling/importance_sampling_ratio/mean": 0.9994009733200073, "sampling/importance_sampling_ratio/min": 0.6954110264778137, "sampling/sampling_logp_difference/max": 0.6914455890655518, "sampling/sampling_logp_difference/mean": 0.011339749209582806, "step": 590 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.970178997609764e-05, "clip_ratio/low_min": 4.970178997609764e-05, "clip_ratio/region_mean": 4.970178997609764e-05, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 749.0, "completions/mean_length": 483.4375, "completions/mean_terminated_length": 454.0, "completions/min_length": 236.0, "completions/min_terminated_length": 236.0, "entropy": 0.33000001311302185, "epoch": 0.3163811563169165, "frac_reward_zero_std": 0.25, "grad_norm": 0.018891392275691032, "learning_rate": 1e-05, "loss": 0.0613, "num_tokens": 12607691.0, "reward": 0.65625, "reward_std": 0.3061639964580536, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.4541513919830322, "sampling/importance_sampling_ratio/mean": 0.9999512434005737, "sampling/importance_sampling_ratio/min": 0.6845542788505554, "sampling/sampling_logp_difference/max": 0.37898731231689453, "sampling/sampling_logp_difference/mean": 0.011431398801505566, "step": 591 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 642.0, "completions/mean_length": 427.5, "completions/mean_terminated_length": 404.8000183105469, "completions/min_length": 264.0, "completions/min_terminated_length": 264.0, "entropy": 0.3442673273384571, "epoch": 0.3169164882226981, "frac_reward_zero_std": 0.25, "grad_norm": 0.004809168633073568, "learning_rate": 1e-05, "loss": 0.0952, "num_tokens": 12624931.0, "reward": 0.90625, "reward_std": 0.2651650309562683, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.438783884048462, "sampling/importance_sampling_ratio/mean": 1.0002009868621826, "sampling/importance_sampling_ratio/min": 0.7311169505119324, "sampling/sampling_logp_difference/max": 0.3637981414794922, "sampling/sampling_logp_difference/mean": 0.011807357892394066, "step": 592 }, { "clip_ratio/high_max": 9.512937685940415e-05, "clip_ratio/high_mean": 9.512937685940415e-05, "clip_ratio/low_mean": 5.741846689488739e-05, "clip_ratio/low_min": 5.741846689488739e-05, "clip_ratio/region_mean": 0.00015254784375429153, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 593.0, "completions/mean_length": 457.71875, "completions/mean_terminated_length": 413.39288330078125, "completions/min_length": 259.0, "completions/min_terminated_length": 259.0, "entropy": 0.4796469882130623, "epoch": 0.31745182012847967, "frac_reward_zero_std": 0.5, "grad_norm": 0.008637635037302971, "learning_rate": 1e-05, "loss": 0.0625, "num_tokens": 12643434.0, "reward": 0.5, "reward_std": 0.2314550280570984, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5080004930496216, "sampling/importance_sampling_ratio/max": 1.568268060684204, "sampling/importance_sampling_ratio/mean": 0.9996455311775208, "sampling/importance_sampling_ratio/min": 0.6906678676605225, "sampling/sampling_logp_difference/max": 0.44997191429138184, "sampling/sampling_logp_difference/mean": 0.015425682067871094, "step": 593 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00017156328613054939, "clip_ratio/low_min": 0.00017156328613054939, "clip_ratio/region_mean": 0.00017156328613054939, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 720.0, "completions/mean_length": 546.34375, "completions/mean_terminated_length": 472.4583435058594, "completions/min_length": 322.0, "completions/min_terminated_length": 322.0, "entropy": 0.3852994851768017, "epoch": 0.31798715203426126, "frac_reward_zero_std": 0.25, "grad_norm": 0.014427164569497108, "learning_rate": 1e-05, "loss": 0.0107, "num_tokens": 12665341.0, "reward": 0.59375, "reward_std": 0.38816186785697937, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.4005064964294434, "sampling/importance_sampling_ratio/mean": 0.9995067715644836, "sampling/importance_sampling_ratio/min": 0.7024900317192078, "sampling/sampling_logp_difference/max": 0.35312414169311523, "sampling/sampling_logp_difference/mean": 0.012627151794731617, "step": 594 }, { "clip_ratio/high_max": 0.00012202422658447176, "clip_ratio/high_mean": 0.00012202422658447176, "clip_ratio/low_mean": 7.35726862330921e-05, "clip_ratio/low_min": 7.35726862330921e-05, "clip_ratio/region_mean": 0.00019559691281756386, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 737.0, "completions/mean_length": 523.90625, "completions/mean_terminated_length": 478.7037048339844, "completions/min_length": 257.0, "completions/min_terminated_length": 257.0, "entropy": 0.3983924984931946, "epoch": 0.31852248394004284, "frac_reward_zero_std": 0.0, "grad_norm": 0.021224621683359146, "learning_rate": 1e-05, "loss": 0.0517, "num_tokens": 12686002.0, "reward": 0.53125, "reward_std": 0.521792471408844, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.4695851802825928, "sampling/importance_sampling_ratio/mean": 1.0005778074264526, "sampling/importance_sampling_ratio/min": 0.40103861689567566, "sampling/sampling_logp_difference/max": 0.9136976003646851, "sampling/sampling_logp_difference/mean": 0.013541512191295624, "step": 595 }, { "clip_ratio/high_max": 6.256256165215746e-05, "clip_ratio/high_mean": 6.256256165215746e-05, "clip_ratio/low_mean": 0.0002397635907982476, "clip_ratio/low_min": 0.0002397635907982476, "clip_ratio/region_mean": 0.00030232615245040506, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 748.0, "completions/mean_length": 468.4375, "completions/mean_terminated_length": 458.774169921875, "completions/min_length": 232.0, "completions/min_terminated_length": 232.0, "entropy": 0.4168017841875553, "epoch": 0.31905781584582443, "frac_reward_zero_std": 0.0, "grad_norm": 0.02819846011698246, "learning_rate": 1e-05, "loss": 0.0175, "num_tokens": 12705080.0, "reward": 0.59375, "reward_std": 0.4534739851951599, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.4610224962234497, "sampling/importance_sampling_ratio/mean": 0.9996376037597656, "sampling/importance_sampling_ratio/min": 0.6944543719291687, "sampling/sampling_logp_difference/max": 0.3791365623474121, "sampling/sampling_logp_difference/mean": 0.013964653946459293, "step": 596 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 767.0, "completions/mean_length": 422.84375, "completions/mean_terminated_length": 387.137939453125, "completions/min_length": 204.0, "completions/min_terminated_length": 204.0, "entropy": 0.3936327174305916, "epoch": 0.319593147751606, "frac_reward_zero_std": 0.75, "grad_norm": 0.013872303068637848, "learning_rate": 1e-05, "loss": 0.0201, "num_tokens": 12722419.0, "reward": 0.625, "reward_std": 0.13363061845302582, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.5759553909301758, "sampling/importance_sampling_ratio/mean": 0.9993356466293335, "sampling/importance_sampling_ratio/min": 0.7176394462585449, "sampling/sampling_logp_difference/max": 0.4548616409301758, "sampling/sampling_logp_difference/mean": 0.013393946923315525, "step": 597 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 7.062147051328793e-05, "clip_ratio/low_min": 7.062147051328793e-05, "clip_ratio/region_mean": 7.062147051328793e-05, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 761.0, "completions/mean_length": 533.59375, "completions/mean_terminated_length": 500.107177734375, "completions/min_length": 243.0, "completions/min_terminated_length": 243.0, "entropy": 0.4238380491733551, "epoch": 0.32012847965738755, "frac_reward_zero_std": 0.0, "grad_norm": 0.022031327709555626, "learning_rate": 1e-05, "loss": 0.0291, "num_tokens": 12743774.0, "reward": 0.625, "reward_std": 0.48503684997558594, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.6586331129074097, "sampling/importance_sampling_ratio/mean": 0.9996314644813538, "sampling/importance_sampling_ratio/min": 0.39061135053634644, "sampling/sampling_logp_difference/max": 0.9400421977043152, "sampling/sampling_logp_difference/mean": 0.013750353828072548, "step": 598 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00011353315494488925, "clip_ratio/low_min": 0.00011353315494488925, "clip_ratio/region_mean": 0.00011353315494488925, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 756.0, "completions/mean_length": 497.25, "completions/mean_terminated_length": 469.2413635253906, "completions/min_length": 219.0, "completions/min_terminated_length": 219.0, "entropy": 0.4273698627948761, "epoch": 0.32066381156316914, "frac_reward_zero_std": 0.25, "grad_norm": 0.020897885784506798, "learning_rate": 1e-05, "loss": 0.0968, "num_tokens": 12764022.0, "reward": 0.71875, "reward_std": 0.378745436668396, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.283443570137024, "sampling/importance_sampling_ratio/mean": 0.9996061325073242, "sampling/importance_sampling_ratio/min": 0.6696034073829651, "sampling/sampling_logp_difference/max": 0.40106964111328125, "sampling/sampling_logp_difference/mean": 0.01368781179189682, "step": 599 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 630.0, "completions/max_terminated_length": 630.0, "completions/mean_length": 367.5625, "completions/mean_terminated_length": 367.5625, "completions/min_length": 213.0, "completions/min_terminated_length": 213.0, "entropy": 0.3283414952456951, "epoch": 0.32119914346895073, "frac_reward_zero_std": 0.75, "grad_norm": 0.0027471999637782574, "learning_rate": 1e-05, "loss": 0.0416, "num_tokens": 12779328.0, "reward": 0.96875, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.96875, "rewards/accuracy_reward/std": 0.1767766922712326, "sampling/importance_sampling_ratio/max": 1.3942617177963257, "sampling/importance_sampling_ratio/mean": 0.9999634027481079, "sampling/importance_sampling_ratio/min": 0.4934152662754059, "sampling/sampling_logp_difference/max": 0.7064042091369629, "sampling/sampling_logp_difference/mean": 0.011736900545656681, "step": 600 }, { "clip_ratio/high_max": 7.022471982054412e-05, "clip_ratio/high_mean": 7.022471982054412e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 7.022471982054412e-05, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 719.0, "completions/mean_length": 523.40625, "completions/mean_terminated_length": 454.91998291015625, "completions/min_length": 301.0, "completions/min_terminated_length": 301.0, "entropy": 0.42866112291812897, "epoch": 0.3217344753747323, "frac_reward_zero_std": 0.25, "grad_norm": 0.007983977906405926, "learning_rate": 1e-05, "loss": 0.0655, "num_tokens": 12799813.0, "reward": 0.625, "reward_std": 0.3924052119255066, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.2967842817306519, "sampling/importance_sampling_ratio/mean": 0.9998101592063904, "sampling/importance_sampling_ratio/min": 0.6835389137268066, "sampling/sampling_logp_difference/max": 0.38047170639038086, "sampling/sampling_logp_difference/mean": 0.013447530567646027, "step": 601 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.741846689488739e-05, "clip_ratio/low_min": 5.741846689488739e-05, "clip_ratio/region_mean": 5.741846689488739e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 690.0, "completions/max_terminated_length": 690.0, "completions/mean_length": 427.09375, "completions/mean_terminated_length": 427.09375, "completions/min_length": 196.0, "completions/min_terminated_length": 196.0, "entropy": 0.31813596189022064, "epoch": 0.3222698072805139, "frac_reward_zero_std": 0.5, "grad_norm": 0.003531959606334567, "learning_rate": 1e-05, "loss": 0.0475, "num_tokens": 12817360.0, "reward": 0.6875, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.4063985347747803, "sampling/importance_sampling_ratio/mean": 0.9997484087944031, "sampling/importance_sampling_ratio/min": 0.7401788234710693, "sampling/sampling_logp_difference/max": 0.3410322666168213, "sampling/sampling_logp_difference/mean": 0.011469905264675617, "step": 602 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 701.0, "completions/max_terminated_length": 701.0, "completions/mean_length": 459.8125, "completions/mean_terminated_length": 459.8125, "completions/min_length": 227.0, "completions/min_terminated_length": 227.0, "entropy": 0.3275734893977642, "epoch": 0.3228051391862955, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": -0.0353, "num_tokens": 12835994.0, "reward": 0.96875, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.96875, "rewards/accuracy_reward/std": 0.1767766922712326, "sampling/importance_sampling_ratio/max": 1.4133315086364746, "sampling/importance_sampling_ratio/mean": 1.0001341104507446, "sampling/importance_sampling_ratio/min": 0.6654146313667297, "sampling/sampling_logp_difference/max": 0.4073449373245239, "sampling/sampling_logp_difference/mean": 0.011468447744846344, "step": 603 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 768.0, "completions/mean_length": 452.125, "completions/mean_terminated_length": 431.0666809082031, "completions/min_length": 230.0, "completions/min_terminated_length": 230.0, "entropy": 0.4624815322458744, "epoch": 0.3233404710920771, "frac_reward_zero_std": 0.5, "grad_norm": 0.006904337089508772, "learning_rate": 1e-05, "loss": 0.0485, "num_tokens": 12854846.0, "reward": 0.46875, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.46875, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.582192063331604, "sampling/importance_sampling_ratio/mean": 0.9995688199996948, "sampling/importance_sampling_ratio/min": 0.6303673982620239, "sampling/sampling_logp_difference/max": 0.4614524841308594, "sampling/sampling_logp_difference/mean": 0.014672099612653255, "step": 604 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.321413482306525e-05, "clip_ratio/low_min": 5.321413482306525e-05, "clip_ratio/region_mean": 5.321413482306525e-05, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 691.0, "completions/mean_length": 520.65625, "completions/mean_terminated_length": 474.85186767578125, "completions/min_length": 282.0, "completions/min_terminated_length": 282.0, "entropy": 0.3870318531990051, "epoch": 0.3238758029978587, "frac_reward_zero_std": 0.0, "grad_norm": 0.01451573520898819, "learning_rate": 1e-05, "loss": 0.0396, "num_tokens": 12875155.0, "reward": 0.53125, "reward_std": 0.4807935357093811, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.308606505393982, "sampling/importance_sampling_ratio/mean": 0.9997498393058777, "sampling/importance_sampling_ratio/min": 0.721451997756958, "sampling/sampling_logp_difference/max": 0.3264894485473633, "sampling/sampling_logp_difference/mean": 0.012777287513017654, "step": 605 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 679.0, "completions/mean_length": 480.21875, "completions/mean_terminated_length": 450.4482727050781, "completions/min_length": 221.0, "completions/min_terminated_length": 221.0, "entropy": 0.4058426357805729, "epoch": 0.32441113490364026, "frac_reward_zero_std": 0.5, "grad_norm": 0.03000659868121147, "learning_rate": 1e-05, "loss": 0.0309, "num_tokens": 12893898.0, "reward": 0.84375, "reward_std": 0.22201895713806152, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.3476595878601074, "sampling/importance_sampling_ratio/mean": 0.9998955726623535, "sampling/importance_sampling_ratio/min": 0.6948140859603882, "sampling/sampling_logp_difference/max": 0.36411094665527344, "sampling/sampling_logp_difference/mean": 0.012886201031506062, "step": 606 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.538325058296323e-05, "clip_ratio/low_min": 5.538325058296323e-05, "clip_ratio/region_mean": 5.538325058296323e-05, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 635.0, "completions/mean_length": 459.3125, "completions/mean_terminated_length": 438.7333679199219, "completions/min_length": 197.0, "completions/min_terminated_length": 197.0, "entropy": 0.44721322134137154, "epoch": 0.32494646680942185, "frac_reward_zero_std": 0.5, "grad_norm": 0.032096076756715775, "learning_rate": 1e-05, "loss": 0.0456, "num_tokens": 12911964.0, "reward": 0.75, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.4550565481185913, "sampling/importance_sampling_ratio/mean": 0.9998563528060913, "sampling/importance_sampling_ratio/min": 0.7085832953453064, "sampling/sampling_logp_difference/max": 0.3750448226928711, "sampling/sampling_logp_difference/mean": 0.013640429824590683, "step": 607 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00023667131608817726, "clip_ratio/low_min": 0.00023667131608817726, "clip_ratio/region_mean": 0.00023667131608817726, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 746.0, "completions/mean_length": 473.75, "completions/mean_terminated_length": 454.13336181640625, "completions/min_length": 211.0, "completions/min_terminated_length": 211.0, "entropy": 0.4183771573007107, "epoch": 0.32548179871520344, "frac_reward_zero_std": 0.25, "grad_norm": 0.018005147576332092, "learning_rate": 1e-05, "loss": 0.0978, "num_tokens": 12930604.0, "reward": 0.8125, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.5010231733322144, "sampling/importance_sampling_ratio/mean": 0.9998956322669983, "sampling/importance_sampling_ratio/min": 0.5953898429870605, "sampling/sampling_logp_difference/max": 0.5185388326644897, "sampling/sampling_logp_difference/mean": 0.01284195389598608, "step": 608 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.335040623322129e-05, "clip_ratio/low_min": 5.335040623322129e-05, "clip_ratio/region_mean": 5.335040623322129e-05, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 766.0, "completions/mean_length": 515.46875, "completions/mean_terminated_length": 507.32257080078125, "completions/min_length": 321.0, "completions/min_terminated_length": 321.0, "entropy": 0.3799743019044399, "epoch": 0.32601713062098503, "frac_reward_zero_std": 0.5, "grad_norm": 0.006943284999579191, "learning_rate": 1e-05, "loss": 0.0255, "num_tokens": 12950715.0, "reward": 0.5625, "reward_std": 0.249358132481575, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.4181941747665405, "sampling/importance_sampling_ratio/mean": 0.9999389052391052, "sampling/importance_sampling_ratio/min": 0.6583727598190308, "sampling/sampling_logp_difference/max": 0.4179840087890625, "sampling/sampling_logp_difference/mean": 0.012957867234945297, "step": 609 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 8.400537626584992e-05, "clip_ratio/low_min": 8.400537626584992e-05, "clip_ratio/region_mean": 8.400537626584992e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 555.0, "completions/max_terminated_length": 555.0, "completions/mean_length": 385.8125, "completions/mean_terminated_length": 385.8125, "completions/min_length": 240.0, "completions/min_terminated_length": 240.0, "entropy": 0.3592875115573406, "epoch": 0.3265524625267666, "frac_reward_zero_std": 0.5, "grad_norm": 0.0037383507005870342, "learning_rate": 1e-05, "loss": 0.0117, "num_tokens": 12966477.0, "reward": 0.9375, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.9375, "rewards/accuracy_reward/std": 0.24593468010425568, "sampling/importance_sampling_ratio/max": 1.344467043876648, "sampling/importance_sampling_ratio/mean": 0.9999433159828186, "sampling/importance_sampling_ratio/min": 0.5734022259712219, "sampling/sampling_logp_difference/max": 0.5561678409576416, "sampling/sampling_logp_difference/mean": 0.01277139876037836, "step": 610 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 674.0, "completions/mean_length": 461.6875, "completions/mean_terminated_length": 441.2666931152344, "completions/min_length": 252.0, "completions/min_terminated_length": 252.0, "entropy": 0.3126893974840641, "epoch": 0.3270877944325482, "frac_reward_zero_std": 0.5, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0295, "num_tokens": 12984747.0, "reward": 0.78125, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.4169516563415527, "sampling/importance_sampling_ratio/mean": 1.00009024143219, "sampling/importance_sampling_ratio/min": 0.5525392889976501, "sampling/sampling_logp_difference/max": 0.5932307243347168, "sampling/sampling_logp_difference/mean": 0.011285481974482536, "step": 611 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 602.0, "completions/mean_length": 433.5, "completions/mean_terminated_length": 411.20001220703125, "completions/min_length": 137.0, "completions/min_terminated_length": 137.0, "entropy": 0.39331403747200966, "epoch": 0.32762312633832974, "frac_reward_zero_std": 0.0, "grad_norm": 0.012627610936760902, "learning_rate": 1e-05, "loss": -0.1073, "num_tokens": 13002347.0, "reward": 0.5625, "reward_std": 0.4355512857437134, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.3340296745300293, "sampling/importance_sampling_ratio/mean": 0.9993400573730469, "sampling/importance_sampling_ratio/min": 0.6539542078971863, "sampling/sampling_logp_difference/max": 0.42471790313720703, "sampling/sampling_logp_difference/mean": 0.011053561232984066, "step": 612 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.345177826005965e-05, "clip_ratio/low_min": 6.345177826005965e-05, "clip_ratio/region_mean": 6.345177826005965e-05, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 645.0, "completions/mean_length": 395.84375, "completions/mean_terminated_length": 371.0333557128906, "completions/min_length": 206.0, "completions/min_terminated_length": 206.0, "entropy": 0.37275737151503563, "epoch": 0.32815845824411133, "frac_reward_zero_std": 0.25, "grad_norm": 0.005499366670846939, "learning_rate": 1e-05, "loss": 0.034, "num_tokens": 13018510.0, "reward": 0.53125, "reward_std": 0.2651650309562683, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.3023018836975098, "sampling/importance_sampling_ratio/mean": 0.9996027946472168, "sampling/importance_sampling_ratio/min": 0.6578198075294495, "sampling/sampling_logp_difference/max": 0.4188241958618164, "sampling/sampling_logp_difference/mean": 0.012931509874761105, "step": 613 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 640.0, "completions/mean_length": 472.1875, "completions/mean_terminated_length": 429.9285888671875, "completions/min_length": 224.0, "completions/min_terminated_length": 224.0, "entropy": 0.3782094642519951, "epoch": 0.3286937901498929, "frac_reward_zero_std": 0.5, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": -0.0212, "num_tokens": 13037268.0, "reward": 0.5625, "reward_std": 0.249358132481575, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.4559943675994873, "sampling/importance_sampling_ratio/mean": 0.9992681741714478, "sampling/importance_sampling_ratio/min": 0.6216511726379395, "sampling/sampling_logp_difference/max": 0.4753761291503906, "sampling/sampling_logp_difference/mean": 0.01303521916270256, "step": 614 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00013793977996101603, "clip_ratio/low_min": 0.00013793977996101603, "clip_ratio/region_mean": 0.00013793977996101603, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 571.0, "completions/mean_length": 408.21875, "completions/mean_terminated_length": 396.6128845214844, "completions/min_length": 253.0, "completions/min_terminated_length": 253.0, "entropy": 0.35767824575304985, "epoch": 0.3292291220556745, "frac_reward_zero_std": 0.5, "grad_norm": 0.00944463536143303, "learning_rate": 1e-05, "loss": 0.0179, "num_tokens": 13053867.0, "reward": 0.8125, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.6619890928268433, "sampling/importance_sampling_ratio/mean": 0.9997155666351318, "sampling/importance_sampling_ratio/min": 0.7145463228225708, "sampling/sampling_logp_difference/max": 0.5080151557922363, "sampling/sampling_logp_difference/mean": 0.012090201489627361, "step": 615 }, { "clip_ratio/high_max": 6.449948705267161e-05, "clip_ratio/high_mean": 6.449948705267161e-05, "clip_ratio/low_mean": 5.383290408644825e-05, "clip_ratio/low_min": 5.383290408644825e-05, "clip_ratio/region_mean": 0.00011833239113911986, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 645.0, "completions/mean_length": 493.40625, "completions/mean_terminated_length": 454.1785888671875, "completions/min_length": 226.0, "completions/min_terminated_length": 226.0, "entropy": 0.28268441930413246, "epoch": 0.3297644539614561, "frac_reward_zero_std": 0.5, "grad_norm": 0.005956647451967001, "learning_rate": 1e-05, "loss": 0.0877, "num_tokens": 13073520.0, "reward": 0.78125, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.462528109550476, "sampling/importance_sampling_ratio/mean": 0.9998534917831421, "sampling/importance_sampling_ratio/min": 0.6425939798355103, "sampling/sampling_logp_difference/max": 0.4422421455383301, "sampling/sampling_logp_difference/mean": 0.009625250473618507, "step": 616 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 732.0, "completions/mean_length": 476.5, "completions/mean_terminated_length": 467.0967712402344, "completions/min_length": 232.0, "completions/min_terminated_length": 232.0, "entropy": 0.3729204908013344, "epoch": 0.3302997858672377, "frac_reward_zero_std": 0.5, "grad_norm": 0.013258181512355804, "learning_rate": 1e-05, "loss": -0.0302, "num_tokens": 13092888.0, "reward": 0.875, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.000476360321045, "sampling/importance_sampling_ratio/min": 0.696682870388031, "sampling/sampling_logp_difference/max": 0.710841178894043, "sampling/sampling_logp_difference/mean": 0.013029306195676327, "step": 617 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 511.0, "completions/max_terminated_length": 511.0, "completions/mean_length": 372.09375, "completions/mean_terminated_length": 372.09375, "completions/min_length": 163.0, "completions/min_terminated_length": 163.0, "entropy": 0.28513216972351074, "epoch": 0.33083511777301927, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 13108515.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.4207388162612915, "sampling/importance_sampling_ratio/mean": 0.9997943043708801, "sampling/importance_sampling_ratio/min": 0.6902952194213867, "sampling/sampling_logp_difference/max": 0.370635986328125, "sampling/sampling_logp_difference/mean": 0.01070329174399376, "step": 618 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 710.0, "completions/mean_length": 454.90625, "completions/mean_terminated_length": 434.0333557128906, "completions/min_length": 189.0, "completions/min_terminated_length": 189.0, "entropy": 0.27062502689659595, "epoch": 0.33137044967880086, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0277, "num_tokens": 13126352.0, "reward": 0.9375, "reward_std": 0.1157275140285492, "rewards/accuracy_reward/mean": 0.9375, "rewards/accuracy_reward/std": 0.24593468010425568, "sampling/importance_sampling_ratio/max": 1.4356149435043335, "sampling/importance_sampling_ratio/mean": 1.0000866651535034, "sampling/importance_sampling_ratio/min": 0.6938625574111938, "sampling/sampling_logp_difference/max": 0.3654813766479492, "sampling/sampling_logp_difference/mean": 0.009605921804904938, "step": 619 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 745.0, "completions/mean_length": 543.625, "completions/mean_terminated_length": 536.3870849609375, "completions/min_length": 318.0, "completions/min_terminated_length": 318.0, "entropy": 0.394099947065115, "epoch": 0.33190578158458245, "frac_reward_zero_std": 0.5, "grad_norm": 0.007365651894360781, "learning_rate": 1e-05, "loss": 0.0206, "num_tokens": 13147708.0, "reward": 0.59375, "reward_std": 0.22201895713806152, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.404664397239685, "sampling/importance_sampling_ratio/mean": 1.0003150701522827, "sampling/importance_sampling_ratio/min": 0.5437097549438477, "sampling/sampling_logp_difference/max": 0.609339714050293, "sampling/sampling_logp_difference/mean": 0.011778902262449265, "step": 620 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00013234963989816606, "clip_ratio/low_min": 0.00013234963989816606, "clip_ratio/region_mean": 0.00013234963989816606, "completions/clipped_ratio": 0.0, "completions/max_length": 645.0, "completions/max_terminated_length": 645.0, "completions/mean_length": 450.15625, "completions/mean_terminated_length": 450.15625, "completions/min_length": 280.0, "completions/min_terminated_length": 280.0, "entropy": 0.35715028643608093, "epoch": 0.33244111349036404, "frac_reward_zero_std": 0.5, "grad_norm": 0.006816867273300886, "learning_rate": 1e-05, "loss": -0.002, "num_tokens": 13165897.0, "reward": 0.625, "reward_std": 0.2314550280570984, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.676544189453125, "sampling/importance_sampling_ratio/mean": 1.000089168548584, "sampling/importance_sampling_ratio/min": 0.7021817564964294, "sampling/sampling_logp_difference/max": 0.5167346000671387, "sampling/sampling_logp_difference/mean": 0.012682432308793068, "step": 621 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.9073725424241275e-05, "clip_ratio/low_min": 5.9073725424241275e-05, "clip_ratio/region_mean": 5.9073725424241275e-05, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 624.0, "completions/mean_length": 431.6875, "completions/mean_terminated_length": 420.83868408203125, "completions/min_length": 263.0, "completions/min_terminated_length": 263.0, "entropy": 0.359247550368309, "epoch": 0.3329764453961456, "frac_reward_zero_std": 0.25, "grad_norm": 0.02203557826578617, "learning_rate": 1e-05, "loss": 0.0327, "num_tokens": 13183423.0, "reward": 0.65625, "reward_std": 0.3966485261917114, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.4204281568527222, "sampling/importance_sampling_ratio/mean": 1.000117301940918, "sampling/importance_sampling_ratio/min": 0.6145386099815369, "sampling/sampling_logp_difference/max": 0.4868835210800171, "sampling/sampling_logp_difference/mean": 0.012450214475393295, "step": 622 }, { "clip_ratio/high_max": 7.827176159480587e-05, "clip_ratio/high_mean": 7.827176159480587e-05, "clip_ratio/low_mean": 7.827176159480587e-05, "clip_ratio/low_min": 7.827176159480587e-05, "clip_ratio/region_mean": 0.00015654352318961173, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 493.0, "completions/mean_length": 336.8125, "completions/mean_terminated_length": 322.9032287597656, "completions/min_length": 114.0, "completions/min_terminated_length": 114.0, "entropy": 0.3818258121609688, "epoch": 0.3335117773019272, "frac_reward_zero_std": 0.5, "grad_norm": 0.02574858069419861, "learning_rate": 1e-05, "loss": -0.0117, "num_tokens": 13197585.0, "reward": 0.9375, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.9375, "rewards/accuracy_reward/std": 0.24593468010425568, "sampling/importance_sampling_ratio/max": 1.4402650594711304, "sampling/importance_sampling_ratio/mean": 1.0006004571914673, "sampling/importance_sampling_ratio/min": 0.6467270255088806, "sampling/sampling_logp_difference/max": 0.43583106994628906, "sampling/sampling_logp_difference/mean": 0.013068610802292824, "step": 623 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 7.951653969939798e-05, "clip_ratio/low_min": 7.951653969939798e-05, "clip_ratio/region_mean": 7.951653969939798e-05, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 736.0, "completions/mean_length": 482.5, "completions/mean_terminated_length": 429.629638671875, "completions/min_length": 253.0, "completions/min_terminated_length": 253.0, "entropy": 0.3787815682590008, "epoch": 0.3340471092077088, "frac_reward_zero_std": 0.5, "grad_norm": 0.009864883497357368, "learning_rate": 1e-05, "loss": 0.023, "num_tokens": 13216705.0, "reward": 0.8125, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.4867147207260132, "sampling/importance_sampling_ratio/mean": 0.9999117255210876, "sampling/importance_sampling_ratio/min": 0.633115291595459, "sampling/sampling_logp_difference/max": 0.45710277557373047, "sampling/sampling_logp_difference/mean": 0.011775805614888668, "step": 624 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 720.0, "completions/mean_length": 541.90625, "completions/mean_terminated_length": 518.5172119140625, "completions/min_length": 270.0, "completions/min_terminated_length": 270.0, "entropy": 0.31354223750531673, "epoch": 0.33458244111349034, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 13238454.0, "reward": 0.75, "reward_std": 0.0, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.41706383228302, "sampling/importance_sampling_ratio/mean": 0.9999799728393555, "sampling/importance_sampling_ratio/min": 0.6495473980903625, "sampling/sampling_logp_difference/max": 0.43147945404052734, "sampling/sampling_logp_difference/mean": 0.011186165735125542, "step": 625 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 618.0, "completions/max_terminated_length": 618.0, "completions/mean_length": 411.5, "completions/mean_terminated_length": 411.5, "completions/min_length": 216.0, "completions/min_terminated_length": 216.0, "entropy": 0.38047345727682114, "epoch": 0.3351177730192719, "frac_reward_zero_std": 0.25, "grad_norm": 0.007136835716664791, "learning_rate": 1e-05, "loss": -0.0182, "num_tokens": 13254950.0, "reward": 0.625, "reward_std": 0.3104073107242584, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.4254165887832642, "sampling/importance_sampling_ratio/mean": 1.000501275062561, "sampling/importance_sampling_ratio/min": 0.6391082406044006, "sampling/sampling_logp_difference/max": 0.4476814270019531, "sampling/sampling_logp_difference/mean": 0.01349923387169838, "step": 626 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00011325611194479279, "clip_ratio/low_min": 0.00011325611194479279, "clip_ratio/region_mean": 0.00011325611194479279, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 685.0, "completions/mean_length": 489.25, "completions/mean_terminated_length": 437.629638671875, "completions/min_length": 276.0, "completions/min_terminated_length": 276.0, "entropy": 0.4465014263987541, "epoch": 0.3356531049250535, "frac_reward_zero_std": 0.5, "grad_norm": 0.009988157078623772, "learning_rate": 1e-05, "loss": 0.0301, "num_tokens": 13274390.0, "reward": 0.75, "reward_std": 0.2314550280570984, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.4333781003952026, "sampling/importance_sampling_ratio/mean": 1.0000110864639282, "sampling/importance_sampling_ratio/min": 0.6930885910987854, "sampling/sampling_logp_difference/max": 0.36659741401672363, "sampling/sampling_logp_difference/mean": 0.013853187672793865, "step": 627 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 745.0, "completions/mean_length": 497.3125, "completions/mean_terminated_length": 488.58062744140625, "completions/min_length": 324.0, "completions/min_terminated_length": 324.0, "entropy": 0.3636351116001606, "epoch": 0.3361884368308351, "frac_reward_zero_std": 0.25, "grad_norm": 0.01138480193912983, "learning_rate": 1e-05, "loss": 0.0442, "num_tokens": 13293672.0, "reward": 0.84375, "reward_std": 0.3061639666557312, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.7428381443023682, "sampling/importance_sampling_ratio/mean": 1.0001919269561768, "sampling/importance_sampling_ratio/min": 0.6661534905433655, "sampling/sampling_logp_difference/max": 0.555514931678772, "sampling/sampling_logp_difference/mean": 0.01124931126832962, "step": 628 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.256518124951981e-05, "clip_ratio/low_min": 5.256518124951981e-05, "clip_ratio/region_mean": 5.256518124951981e-05, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 765.0, "completions/mean_length": 545.46875, "completions/mean_terminated_length": 504.25927734375, "completions/min_length": 259.0, "completions/min_terminated_length": 259.0, "entropy": 0.29527443274855614, "epoch": 0.3367237687366167, "frac_reward_zero_std": 0.5, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0045, "num_tokens": 13314863.0, "reward": 0.84375, "reward_std": 0.22201895713806152, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.4749572277069092, "sampling/importance_sampling_ratio/mean": 0.9999758005142212, "sampling/importance_sampling_ratio/min": 0.6951217651367188, "sampling/sampling_logp_difference/max": 0.3886289596557617, "sampling/sampling_logp_difference/mean": 0.009654700756072998, "step": 629 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 669.0, "completions/mean_length": 438.25, "completions/mean_terminated_length": 427.6128845214844, "completions/min_length": 258.0, "completions/min_terminated_length": 258.0, "entropy": 0.2346278317272663, "epoch": 0.3372591006423983, "frac_reward_zero_std": 0.75, "grad_norm": 0.008940054103732109, "learning_rate": 1e-05, "loss": 0.0038, "num_tokens": 13332535.0, "reward": 0.84375, "reward_std": 0.1293872892856598, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.4977093935012817, "sampling/importance_sampling_ratio/mean": 1.0003480911254883, "sampling/importance_sampling_ratio/min": 0.6041476130485535, "sampling/sampling_logp_difference/max": 0.503936767578125, "sampling/sampling_logp_difference/mean": 0.008601022884249687, "step": 630 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 598.0, "completions/max_terminated_length": 598.0, "completions/mean_length": 367.90625, "completions/mean_terminated_length": 367.90625, "completions/min_length": 176.0, "completions/min_terminated_length": 176.0, "entropy": 0.33554455637931824, "epoch": 0.33779443254817987, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 13348028.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.4243865013122559, "sampling/importance_sampling_ratio/mean": 1.0000324249267578, "sampling/importance_sampling_ratio/min": 0.6829863786697388, "sampling/sampling_logp_difference/max": 0.38128042221069336, "sampling/sampling_logp_difference/mean": 0.011215053498744965, "step": 631 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00013873439456801862, "clip_ratio/low_min": 0.00013873439456801862, "clip_ratio/region_mean": 0.00013873439456801862, "completions/clipped_ratio": 0.0, "completions/max_length": 748.0, "completions/max_terminated_length": 748.0, "completions/mean_length": 448.03125, "completions/mean_terminated_length": 448.03125, "completions/min_length": 286.0, "completions/min_terminated_length": 286.0, "entropy": 0.2793141473084688, "epoch": 0.33832976445396146, "frac_reward_zero_std": 0.75, "grad_norm": 0.007991788908839226, "learning_rate": 1e-05, "loss": -0.0154, "num_tokens": 13365965.0, "reward": 0.90625, "reward_std": 0.1293872892856598, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.4368860721588135, "sampling/importance_sampling_ratio/mean": 0.9999039769172668, "sampling/importance_sampling_ratio/min": 0.6882292032241821, "sampling/sampling_logp_difference/max": 0.37363338470458984, "sampling/sampling_logp_difference/mean": 0.009869934059679508, "step": 632 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 709.0, "completions/mean_length": 499.9375, "completions/mean_terminated_length": 450.2962951660156, "completions/min_length": 209.0, "completions/min_terminated_length": 209.0, "entropy": 0.5051672868430614, "epoch": 0.33886509635974305, "frac_reward_zero_std": 0.25, "grad_norm": 0.009050918743014336, "learning_rate": 1e-05, "loss": 0.0673, "num_tokens": 13386219.0, "reward": 0.59375, "reward_std": 0.3061639964580536, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.551378846168518, "sampling/importance_sampling_ratio/mean": 1.0000944137573242, "sampling/importance_sampling_ratio/min": 0.7343372106552124, "sampling/sampling_logp_difference/max": 0.4391441345214844, "sampling/sampling_logp_difference/mean": 0.014612581580877304, "step": 633 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 749.0, "completions/mean_length": 390.96875, "completions/mean_terminated_length": 378.8064270019531, "completions/min_length": 126.0, "completions/min_terminated_length": 126.0, "entropy": 0.32369521260261536, "epoch": 0.33940042826552463, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0352, "num_tokens": 13402442.0, "reward": 0.90625, "reward_std": 0.1293872892856598, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.4072140455245972, "sampling/importance_sampling_ratio/mean": 0.9999368190765381, "sampling/importance_sampling_ratio/min": 0.6991485357284546, "sampling/sampling_logp_difference/max": 0.3578920364379883, "sampling/sampling_logp_difference/mean": 0.011111059226095676, "step": 634 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.28125, "completions/max_length": 768.0, "completions/max_terminated_length": 631.0, "completions/mean_length": 533.21875, "completions/mean_terminated_length": 441.34783935546875, "completions/min_length": 255.0, "completions/min_terminated_length": 255.0, "entropy": 0.41296880692243576, "epoch": 0.3399357601713062, "frac_reward_zero_std": 0.25, "grad_norm": 0.03107323870062828, "learning_rate": 1e-05, "loss": 0.0451, "num_tokens": 13423177.0, "reward": 0.71875, "reward_std": 0.2651650309562683, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.446143388748169, "sampling/importance_sampling_ratio/mean": 1.0002474784851074, "sampling/importance_sampling_ratio/min": 0.6319281458854675, "sampling/sampling_logp_difference/max": 0.45897960662841797, "sampling/sampling_logp_difference/mean": 0.012981615029275417, "step": 635 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 615.0, "completions/max_terminated_length": 615.0, "completions/mean_length": 403.3125, "completions/mean_terminated_length": 403.3125, "completions/min_length": 117.0, "completions/min_terminated_length": 117.0, "entropy": 0.3445026371628046, "epoch": 0.3404710920770878, "frac_reward_zero_std": 0.75, "grad_norm": 0.0035737641155719757, "learning_rate": 1e-05, "loss": -0.0121, "num_tokens": 13440179.0, "reward": 0.96875, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.96875, "rewards/accuracy_reward/std": 0.1767766922712326, "sampling/importance_sampling_ratio/max": 1.4905427694320679, "sampling/importance_sampling_ratio/mean": 0.9997856616973877, "sampling/importance_sampling_ratio/min": 0.7409813404083252, "sampling/sampling_logp_difference/max": 0.3991403579711914, "sampling/sampling_logp_difference/mean": 0.011083485558629036, "step": 636 }, { "clip_ratio/high_max": 7.126567652449012e-05, "clip_ratio/high_mean": 7.126567652449012e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 7.126567652449012e-05, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 658.0, "completions/mean_length": 484.96875, "completions/mean_terminated_length": 455.6896667480469, "completions/min_length": 220.0, "completions/min_terminated_length": 220.0, "entropy": 0.36797764524817467, "epoch": 0.3410064239828694, "frac_reward_zero_std": 0.25, "grad_norm": 0.008711599744856358, "learning_rate": 1e-05, "loss": -0.0203, "num_tokens": 13460290.0, "reward": 0.59375, "reward_std": 0.3377464711666107, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.9879310131072998, "sampling/importance_sampling_ratio/mean": 0.9996102452278137, "sampling/importance_sampling_ratio/min": 0.6669403910636902, "sampling/sampling_logp_difference/max": 0.6870944499969482, "sampling/sampling_logp_difference/mean": 0.012271219864487648, "step": 637 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00011636956332949921, "clip_ratio/low_min": 0.00011636956332949921, "clip_ratio/region_mean": 0.00011636956332949921, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 759.0, "completions/mean_length": 497.75, "completions/mean_terminated_length": 447.7037048339844, "completions/min_length": 251.0, "completions/min_terminated_length": 251.0, "entropy": 0.5101986601948738, "epoch": 0.341541755888651, "frac_reward_zero_std": 0.25, "grad_norm": 0.01628776825964451, "learning_rate": 1e-05, "loss": 0.0033, "num_tokens": 13480426.0, "reward": 0.46875, "reward_std": 0.3198433816432953, "rewards/accuracy_reward/mean": 0.46875, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0003156661987305, "sampling/importance_sampling_ratio/min": 0.33501747250556946, "sampling/sampling_logp_difference/max": 1.0935726165771484, "sampling/sampling_logp_difference/mean": 0.015251288190484047, "step": 638 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 604.0, "completions/max_terminated_length": 604.0, "completions/mean_length": 438.625, "completions/mean_terminated_length": 438.625, "completions/min_length": 188.0, "completions/min_terminated_length": 188.0, "entropy": 0.2699894309043884, "epoch": 0.3420770877944325, "frac_reward_zero_std": 0.5, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": -0.0404, "num_tokens": 13497926.0, "reward": 0.46875, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.46875, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.383937954902649, "sampling/importance_sampling_ratio/mean": 0.9999804496765137, "sampling/importance_sampling_ratio/min": 0.6456522345542908, "sampling/sampling_logp_difference/max": 0.43749427795410156, "sampling/sampling_logp_difference/mean": 0.010212450288236141, "step": 639 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00022541262296726927, "clip_ratio/low_min": 0.00022541262296726927, "clip_ratio/region_mean": 0.00022541262296726927, "completions/clipped_ratio": 0.0, "completions/max_length": 671.0, "completions/max_terminated_length": 671.0, "completions/mean_length": 421.90625, "completions/mean_terminated_length": 421.90625, "completions/min_length": 97.0, "completions/min_terminated_length": 97.0, "entropy": 0.3346049040555954, "epoch": 0.3426124197002141, "frac_reward_zero_std": 0.25, "grad_norm": 0.02313557080924511, "learning_rate": 1e-05, "loss": -0.0252, "num_tokens": 13515011.0, "reward": 0.84375, "reward_std": 0.3198433816432953, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.348913311958313, "sampling/importance_sampling_ratio/mean": 0.9998977184295654, "sampling/importance_sampling_ratio/min": 0.7032831311225891, "sampling/sampling_logp_difference/max": 0.35199570655822754, "sampling/sampling_logp_difference/mean": 0.011293580755591393, "step": 640 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.0386471886886284e-05, "clip_ratio/low_min": 6.0386471886886284e-05, "clip_ratio/region_mean": 6.0386471886886284e-05, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 642.0, "completions/mean_length": 468.84375, "completions/mean_terminated_length": 459.19354248046875, "completions/min_length": 230.0, "completions/min_terminated_length": 230.0, "entropy": 0.3149367831647396, "epoch": 0.3431477516059957, "frac_reward_zero_std": 0.25, "grad_norm": 0.020712561905384064, "learning_rate": 1e-05, "loss": 0.0052, "num_tokens": 13533518.0, "reward": 0.84375, "reward_std": 0.3198433816432953, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.4185200929641724, "sampling/importance_sampling_ratio/mean": 1.000146508216858, "sampling/importance_sampling_ratio/min": 0.6921027302742004, "sampling/sampling_logp_difference/max": 0.3680208921432495, "sampling/sampling_logp_difference/mean": 0.011046596802771091, "step": 641 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.385609620134346e-05, "clip_ratio/low_min": 5.385609620134346e-05, "clip_ratio/region_mean": 5.385609620134346e-05, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 762.0, "completions/mean_length": 433.90625, "completions/mean_terminated_length": 423.1290283203125, "completions/min_length": 270.0, "completions/min_terminated_length": 270.0, "entropy": 0.39057696238160133, "epoch": 0.3436830835117773, "frac_reward_zero_std": 0.25, "grad_norm": 0.010424681007862091, "learning_rate": 1e-05, "loss": 0.0984, "num_tokens": 13551467.0, "reward": 0.78125, "reward_std": 0.3061639964580536, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.8698256015777588, "sampling/importance_sampling_ratio/mean": 0.999830961227417, "sampling/importance_sampling_ratio/min": 0.266834557056427, "sampling/sampling_logp_difference/max": 1.3211264610290527, "sampling/sampling_logp_difference/mean": 0.013005973771214485, "step": 642 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.28125, "completions/max_length": 768.0, "completions/max_terminated_length": 767.0, "completions/mean_length": 529.125, "completions/mean_terminated_length": 435.6521911621094, "completions/min_length": 251.0, "completions/min_terminated_length": 251.0, "entropy": 0.48676617071032524, "epoch": 0.3442184154175589, "frac_reward_zero_std": 0.5, "grad_norm": 0.008242892101407051, "learning_rate": 1e-05, "loss": -0.0169, "num_tokens": 13572543.0, "reward": 0.625, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.5422800779342651, "sampling/importance_sampling_ratio/mean": 1.000182032585144, "sampling/importance_sampling_ratio/min": 0.6356048583984375, "sampling/sampling_logp_difference/max": 0.45317816734313965, "sampling/sampling_logp_difference/mean": 0.015141142532229424, "step": 643 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 8.457374497083947e-05, "clip_ratio/low_min": 8.457374497083947e-05, "clip_ratio/region_mean": 8.457374497083947e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 703.0, "completions/max_terminated_length": 703.0, "completions/mean_length": 396.0625, "completions/mean_terminated_length": 396.0625, "completions/min_length": 193.0, "completions/min_terminated_length": 193.0, "entropy": 0.356972835958004, "epoch": 0.34475374732334046, "frac_reward_zero_std": 0.0, "grad_norm": 0.015076568350195885, "learning_rate": 1e-05, "loss": -0.0654, "num_tokens": 13588673.0, "reward": 0.84375, "reward_std": 0.3808925747871399, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.4283466339111328, "sampling/importance_sampling_ratio/mean": 1.000098705291748, "sampling/importance_sampling_ratio/min": 0.629047691822052, "sampling/sampling_logp_difference/max": 0.4635481834411621, "sampling/sampling_logp_difference/mean": 0.012696065939962864, "step": 644 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 630.0, "completions/mean_length": 506.125, "completions/mean_terminated_length": 418.8333435058594, "completions/min_length": 228.0, "completions/min_terminated_length": 228.0, "entropy": 0.3814406506717205, "epoch": 0.34528907922912205, "frac_reward_zero_std": 0.5, "grad_norm": 0.020508045330643654, "learning_rate": 1e-05, "loss": 0.021, "num_tokens": 13608421.0, "reward": 0.625, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.390692114830017, "sampling/importance_sampling_ratio/mean": 0.9999074339866638, "sampling/importance_sampling_ratio/min": 0.6654835343360901, "sampling/sampling_logp_difference/max": 0.4072413444519043, "sampling/sampling_logp_difference/mean": 0.012382500804960728, "step": 645 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0001863876823335886, "clip_ratio/low_min": 0.0001863876823335886, "clip_ratio/region_mean": 0.0001863876823335886, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 724.0, "completions/mean_length": 486.625, "completions/mean_terminated_length": 467.86669921875, "completions/min_length": 247.0, "completions/min_terminated_length": 247.0, "entropy": 0.3633422404527664, "epoch": 0.34582441113490364, "frac_reward_zero_std": 0.5, "grad_norm": 0.01649761013686657, "learning_rate": 1e-05, "loss": -0.0065, "num_tokens": 13627873.0, "reward": 0.75, "reward_std": 0.26726123690605164, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.5981051921844482, "sampling/importance_sampling_ratio/mean": 1.0002291202545166, "sampling/importance_sampling_ratio/min": 0.6511904001235962, "sampling/sampling_logp_difference/max": 0.46881866455078125, "sampling/sampling_logp_difference/mean": 0.012197589501738548, "step": 646 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00015401547716464847, "clip_ratio/low_min": 0.00015401547716464847, "clip_ratio/region_mean": 0.00015401547716464847, "completions/clipped_ratio": 0.3125, "completions/max_length": 768.0, "completions/max_terminated_length": 682.0, "completions/mean_length": 561.9375, "completions/mean_terminated_length": 468.2727355957031, "completions/min_length": 265.0, "completions/min_terminated_length": 265.0, "entropy": 0.6201138310134411, "epoch": 0.34635974304068523, "frac_reward_zero_std": 0.0, "grad_norm": 0.012537403963506222, "learning_rate": 1e-05, "loss": 0.0283, "num_tokens": 13650159.0, "reward": 0.34375, "reward_std": 0.4218915104866028, "rewards/accuracy_reward/mean": 0.34375, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.541121482849121, "sampling/importance_sampling_ratio/mean": 0.9999062418937683, "sampling/importance_sampling_ratio/min": 0.44555389881134033, "sampling/sampling_logp_difference/max": 0.8084371089935303, "sampling/sampling_logp_difference/mean": 0.016672369092702866, "step": 647 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00011061752593377605, "clip_ratio/low_min": 0.00011061752593377605, "clip_ratio/region_mean": 0.00011061752593377605, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 709.0, "completions/mean_length": 551.46875, "completions/mean_terminated_length": 479.29168701171875, "completions/min_length": 244.0, "completions/min_terminated_length": 244.0, "entropy": 0.33008667454123497, "epoch": 0.3468950749464668, "frac_reward_zero_std": 0.25, "grad_norm": 0.007998436689376831, "learning_rate": 1e-05, "loss": 0.0697, "num_tokens": 13671574.0, "reward": 0.625, "reward_std": 0.3745020925998688, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.4150848388671875, "sampling/importance_sampling_ratio/mean": 0.9996934533119202, "sampling/importance_sampling_ratio/min": 0.6198036670684814, "sampling/sampling_logp_difference/max": 0.47835254669189453, "sampling/sampling_logp_difference/mean": 0.01092028059065342, "step": 648 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 679.0, "completions/mean_length": 504.71875, "completions/mean_terminated_length": 487.16668701171875, "completions/min_length": 185.0, "completions/min_terminated_length": 185.0, "entropy": 0.2784304544329643, "epoch": 0.3474304068522484, "frac_reward_zero_std": 0.5, "grad_norm": 0.013612071052193642, "learning_rate": 1e-05, "loss": 0.0396, "num_tokens": 13690949.0, "reward": 0.78125, "reward_std": 0.2630178928375244, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.4658979177474976, "sampling/importance_sampling_ratio/mean": 0.9999603629112244, "sampling/importance_sampling_ratio/min": 0.6817265152931213, "sampling/sampling_logp_difference/max": 0.38312673568725586, "sampling/sampling_logp_difference/mean": 0.009534653276205063, "step": 649 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 572.0, "completions/mean_length": 410.5625, "completions/mean_terminated_length": 373.5862121582031, "completions/min_length": 254.0, "completions/min_terminated_length": 254.0, "entropy": 0.4071606919169426, "epoch": 0.34796573875803, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0115, "num_tokens": 13708263.0, "reward": 0.71875, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.4312196969985962, "sampling/importance_sampling_ratio/mean": 1.0000720024108887, "sampling/importance_sampling_ratio/min": 0.6544552445411682, "sampling/sampling_logp_difference/max": 0.4239521026611328, "sampling/sampling_logp_difference/mean": 0.014624012634158134, "step": 650 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0001284026657231152, "clip_ratio/low_min": 0.0001284026657231152, "clip_ratio/region_mean": 0.0001284026657231152, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 757.0, "completions/mean_length": 512.46875, "completions/mean_terminated_length": 427.29168701171875, "completions/min_length": 208.0, "completions/min_terminated_length": 208.0, "entropy": 0.47825566306710243, "epoch": 0.3485010706638116, "frac_reward_zero_std": 0.25, "grad_norm": 0.005921847186982632, "learning_rate": 1e-05, "loss": 0.0701, "num_tokens": 13728846.0, "reward": 0.4375, "reward_std": 0.292504221200943, "rewards/accuracy_reward/mean": 0.4375, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.5161925554275513, "sampling/importance_sampling_ratio/mean": 0.9998018145561218, "sampling/importance_sampling_ratio/min": 0.7247036695480347, "sampling/sampling_logp_difference/max": 0.4162023067474365, "sampling/sampling_logp_difference/mean": 0.014716443605720997, "step": 651 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 621.0, "completions/mean_length": 393.4375, "completions/mean_terminated_length": 381.3548278808594, "completions/min_length": 200.0, "completions/min_terminated_length": 200.0, "entropy": 0.36428897827863693, "epoch": 0.3490364025695932, "frac_reward_zero_std": 0.5, "grad_norm": 0.018470045179128647, "learning_rate": 1e-05, "loss": 0.0203, "num_tokens": 13745156.0, "reward": 0.71875, "reward_std": 0.2630178928375244, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.4548707008361816, "sampling/importance_sampling_ratio/mean": 0.9999221563339233, "sampling/importance_sampling_ratio/min": 0.614672839641571, "sampling/sampling_logp_difference/max": 0.48666512966156006, "sampling/sampling_logp_difference/mean": 0.012761600315570831, "step": 652 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 7.418397581204772e-05, "clip_ratio/low_min": 7.418397581204772e-05, "clip_ratio/region_mean": 7.418397581204772e-05, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 628.0, "completions/mean_length": 480.59375, "completions/mean_terminated_length": 471.32257080078125, "completions/min_length": 297.0, "completions/min_terminated_length": 297.0, "entropy": 0.3200231231749058, "epoch": 0.3495717344753747, "frac_reward_zero_std": 0.0, "grad_norm": 0.014736565761268139, "learning_rate": 1e-05, "loss": 0.0422, "num_tokens": 13764647.0, "reward": 0.53125, "reward_std": 0.47137707471847534, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.3195416927337646, "sampling/importance_sampling_ratio/mean": 1.000119686126709, "sampling/importance_sampling_ratio/min": 0.7368139624595642, "sampling/sampling_logp_difference/max": 0.305419921875, "sampling/sampling_logp_difference/mean": 0.010633639059960842, "step": 653 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 710.0, "completions/mean_length": 453.21875, "completions/mean_terminated_length": 420.6551818847656, "completions/min_length": 139.0, "completions/min_terminated_length": 139.0, "entropy": 0.4703764244914055, "epoch": 0.3501070663811563, "frac_reward_zero_std": 0.5, "grad_norm": 0.0212630033493042, "learning_rate": 1e-05, "loss": 0.0625, "num_tokens": 13782670.0, "reward": 0.78125, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.7001612186431885, "sampling/importance_sampling_ratio/mean": 1.0002896785736084, "sampling/importance_sampling_ratio/min": 0.6003448367118835, "sampling/sampling_logp_difference/max": 0.5307230949401855, "sampling/sampling_logp_difference/mean": 0.014233189634978771, "step": 654 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 688.0, "completions/mean_length": 465.5, "completions/mean_terminated_length": 445.3333435058594, "completions/min_length": 243.0, "completions/min_terminated_length": 243.0, "entropy": 0.4003981351852417, "epoch": 0.3506423982869379, "frac_reward_zero_std": 0.25, "grad_norm": 0.005528174340724945, "learning_rate": 1e-05, "loss": 0.0315, "num_tokens": 13800846.0, "reward": 0.65625, "reward_std": 0.3061639964580536, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.4268187284469604, "sampling/importance_sampling_ratio/mean": 0.9999117255210876, "sampling/importance_sampling_ratio/min": 0.5273811221122742, "sampling/sampling_logp_difference/max": 0.6398317813873291, "sampling/sampling_logp_difference/mean": 0.013050133362412453, "step": 655 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00023159264674177393, "clip_ratio/low_min": 0.00023159264674177393, "clip_ratio/region_mean": 0.00023159264674177393, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 704.0, "completions/mean_length": 534.375, "completions/mean_terminated_length": 510.2069091796875, "completions/min_length": 282.0, "completions/min_terminated_length": 282.0, "entropy": 0.35114817321300507, "epoch": 0.3511777301927195, "frac_reward_zero_std": 0.25, "grad_norm": 0.014032700099050999, "learning_rate": 1e-05, "loss": 0.0372, "num_tokens": 13821162.0, "reward": 0.6875, "reward_std": 0.3924051821231842, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.4244271516799927, "sampling/importance_sampling_ratio/mean": 0.9999833106994629, "sampling/importance_sampling_ratio/min": 0.5283624529838562, "sampling/sampling_logp_difference/max": 0.6379728317260742, "sampling/sampling_logp_difference/mean": 0.011497631669044495, "step": 656 }, { "clip_ratio/high_max": 8.620689914096147e-05, "clip_ratio/high_mean": 8.620689914096147e-05, "clip_ratio/low_mean": 0.0001526529958937317, "clip_ratio/low_min": 0.0001526529958937317, "clip_ratio/region_mean": 0.00023885989503469318, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 660.0, "completions/mean_length": 398.0625, "completions/mean_terminated_length": 373.4000244140625, "completions/min_length": 163.0, "completions/min_terminated_length": 163.0, "entropy": 0.5512331612408161, "epoch": 0.35171306209850106, "frac_reward_zero_std": 0.25, "grad_norm": 0.023126527667045593, "learning_rate": 1e-05, "loss": 0.0397, "num_tokens": 13838556.0, "reward": 0.71875, "reward_std": 0.3787454068660736, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.4397109746932983, "sampling/importance_sampling_ratio/mean": 0.9997005462646484, "sampling/importance_sampling_ratio/min": 0.40250781178474426, "sampling/sampling_logp_difference/max": 0.9100408554077148, "sampling/sampling_logp_difference/mean": 0.01570245809853077, "step": 657 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 679.0, "completions/mean_length": 440.15625, "completions/mean_terminated_length": 429.58062744140625, "completions/min_length": 79.0, "completions/min_terminated_length": 79.0, "entropy": 0.3326597884297371, "epoch": 0.35224839400428265, "frac_reward_zero_std": 0.25, "grad_norm": 0.0073088291101157665, "learning_rate": 1e-05, "loss": -0.0253, "num_tokens": 13856865.0, "reward": 0.75, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.2980173826217651, "sampling/importance_sampling_ratio/mean": 0.999835193157196, "sampling/importance_sampling_ratio/min": 0.6736611127853394, "sampling/sampling_logp_difference/max": 0.39502811431884766, "sampling/sampling_logp_difference/mean": 0.011770878918468952, "step": 658 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.650994717143476e-05, "clip_ratio/low_min": 5.650994717143476e-05, "clip_ratio/region_mean": 5.650994717143476e-05, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 750.0, "completions/mean_length": 464.4375, "completions/mean_terminated_length": 421.0714416503906, "completions/min_length": 205.0, "completions/min_terminated_length": 205.0, "entropy": 0.48483504354953766, "epoch": 0.35278372591006424, "frac_reward_zero_std": 0.75, "grad_norm": 0.011631009168922901, "learning_rate": 1e-05, "loss": 0.0158, "num_tokens": 13875719.0, "reward": 0.625, "reward_std": 0.13363061845302582, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.5468271970748901, "sampling/importance_sampling_ratio/mean": 0.9996281266212463, "sampling/importance_sampling_ratio/min": 0.7024893164634705, "sampling/sampling_logp_difference/max": 0.4362058639526367, "sampling/sampling_logp_difference/mean": 0.013573600910604, "step": 659 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 642.0, "completions/mean_length": 454.84375, "completions/mean_terminated_length": 433.9666748046875, "completions/min_length": 186.0, "completions/min_terminated_length": 186.0, "entropy": 0.39848157204687595, "epoch": 0.3533190578158458, "frac_reward_zero_std": 0.0, "grad_norm": 0.03598235547542572, "learning_rate": 1e-05, "loss": 0.0276, "num_tokens": 13894562.0, "reward": 0.59375, "reward_std": 0.4534739851951599, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.537682056427002, "sampling/importance_sampling_ratio/mean": 1.0001001358032227, "sampling/importance_sampling_ratio/min": 0.6504964828491211, "sampling/sampling_logp_difference/max": 0.43027615547180176, "sampling/sampling_logp_difference/mean": 0.01200496032834053, "step": 660 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0001919140195241198, "clip_ratio/low_min": 0.0001919140195241198, "clip_ratio/region_mean": 0.0001919140195241198, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 549.0, "completions/mean_length": 429.3125, "completions/mean_terminated_length": 406.7333679199219, "completions/min_length": 295.0, "completions/min_terminated_length": 295.0, "entropy": 0.35921452194452286, "epoch": 0.3538543897216274, "frac_reward_zero_std": 0.25, "grad_norm": 0.010375307872891426, "learning_rate": 1e-05, "loss": 0.035, "num_tokens": 13911956.0, "reward": 0.625, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.4255915880203247, "sampling/importance_sampling_ratio/mean": 0.9996388554573059, "sampling/importance_sampling_ratio/min": 0.6840957999229431, "sampling/sampling_logp_difference/max": 0.3796572685241699, "sampling/sampling_logp_difference/mean": 0.012640676461160183, "step": 661 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00015690759755671024, "clip_ratio/low_min": 0.00015690759755671024, "clip_ratio/region_mean": 0.00015690759755671024, "completions/clipped_ratio": 0.0, "completions/max_length": 699.0, "completions/max_terminated_length": 699.0, "completions/mean_length": 439.84375, "completions/mean_terminated_length": 439.84375, "completions/min_length": 216.0, "completions/min_terminated_length": 216.0, "entropy": 0.29182940907776356, "epoch": 0.354389721627409, "frac_reward_zero_std": 0.5, "grad_norm": 0.008143672719597816, "learning_rate": 1e-05, "loss": -0.0614, "num_tokens": 13929199.0, "reward": 0.875, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 1.4570568799972534, "sampling/importance_sampling_ratio/mean": 1.0003750324249268, "sampling/importance_sampling_ratio/min": 0.6930297613143921, "sampling/sampling_logp_difference/max": 0.3764185905456543, "sampling/sampling_logp_difference/mean": 0.01032546628266573, "step": 662 }, { "clip_ratio/high_max": 5.612932363874279e-05, "clip_ratio/high_mean": 5.612932363874279e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 5.612932363874279e-05, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 719.0, "completions/mean_length": 538.25, "completions/mean_terminated_length": 473.91998291015625, "completions/min_length": 302.0, "completions/min_terminated_length": 302.0, "entropy": 0.5008703619241714, "epoch": 0.3549250535331906, "frac_reward_zero_std": 0.5, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": -0.0193, "num_tokens": 13950527.0, "reward": 0.53125, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.3500277996063232, "sampling/importance_sampling_ratio/mean": 1.0000468492507935, "sampling/importance_sampling_ratio/min": 0.6496276259422302, "sampling/sampling_logp_difference/max": 0.43135595321655273, "sampling/sampling_logp_difference/mean": 0.014918780885636806, "step": 663 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00010224949073744938, "clip_ratio/low_min": 0.00010224949073744938, "clip_ratio/region_mean": 0.00010224949073744938, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 638.0, "completions/mean_length": 476.78125, "completions/mean_terminated_length": 457.36669921875, "completions/min_length": 212.0, "completions/min_terminated_length": 212.0, "entropy": 0.5086811780929565, "epoch": 0.3554603854389722, "frac_reward_zero_std": 0.25, "grad_norm": 0.01617942377924919, "learning_rate": 1e-05, "loss": 0.0788, "num_tokens": 13970232.0, "reward": 0.625, "reward_std": 0.3924052119255066, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.5820857286453247, "sampling/importance_sampling_ratio/mean": 0.9999898076057434, "sampling/importance_sampling_ratio/min": 0.6589893102645874, "sampling/sampling_logp_difference/max": 0.4587440490722656, "sampling/sampling_logp_difference/mean": 0.015467606484889984, "step": 664 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 8.778089977568015e-05, "clip_ratio/low_min": 8.778089977568015e-05, "clip_ratio/region_mean": 8.778089977568015e-05, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 650.0, "completions/mean_length": 448.1875, "completions/mean_terminated_length": 426.86669921875, "completions/min_length": 139.0, "completions/min_terminated_length": 139.0, "entropy": 0.3695897329598665, "epoch": 0.35599571734475377, "frac_reward_zero_std": 0.25, "grad_norm": 0.008000156842172146, "learning_rate": 1e-05, "loss": 0.1095, "num_tokens": 13988006.0, "reward": 0.65625, "reward_std": 0.3198433816432953, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.436724305152893, "sampling/importance_sampling_ratio/mean": 1.0006052255630493, "sampling/importance_sampling_ratio/min": 0.5921891927719116, "sampling/sampling_logp_difference/max": 0.5239291191101074, "sampling/sampling_logp_difference/mean": 0.012634573504328728, "step": 665 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00013520821812562644, "clip_ratio/low_min": 0.00013520821812562644, "clip_ratio/region_mean": 0.00013520821812562644, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 737.0, "completions/mean_length": 502.90625, "completions/mean_terminated_length": 485.2333679199219, "completions/min_length": 244.0, "completions/min_terminated_length": 244.0, "entropy": 0.34534798562526703, "epoch": 0.35653104925053536, "frac_reward_zero_std": 0.5, "grad_norm": 0.016071680933237076, "learning_rate": 1e-05, "loss": -0.0186, "num_tokens": 14007827.0, "reward": 0.78125, "reward_std": 0.2630178928375244, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.5614160299301147, "sampling/importance_sampling_ratio/mean": 0.9999233484268188, "sampling/importance_sampling_ratio/min": 0.62237948179245, "sampling/sampling_logp_difference/max": 0.47420525550842285, "sampling/sampling_logp_difference/mean": 0.011669373139739037, "step": 666 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.677350756945089e-05, "clip_ratio/low_min": 6.677350756945089e-05, "clip_ratio/region_mean": 6.677350756945089e-05, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 653.0, "completions/mean_length": 435.375, "completions/mean_terminated_length": 400.96551513671875, "completions/min_length": 200.0, "completions/min_terminated_length": 200.0, "entropy": 0.5260384231805801, "epoch": 0.3570663811563169, "frac_reward_zero_std": 0.25, "grad_norm": 0.018121743574738503, "learning_rate": 1e-05, "loss": 0.0219, "num_tokens": 14025255.0, "reward": 0.5625, "reward_std": 0.3514062464237213, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.5001248121261597, "sampling/importance_sampling_ratio/mean": 0.9995136857032776, "sampling/importance_sampling_ratio/min": 0.5823060870170593, "sampling/sampling_logp_difference/max": 0.5407590866088867, "sampling/sampling_logp_difference/mean": 0.01620151288807392, "step": 667 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00012657707702601328, "clip_ratio/low_min": 0.00012657707702601328, "clip_ratio/region_mean": 0.00012657707702601328, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 690.0, "completions/mean_length": 476.125, "completions/mean_terminated_length": 434.4285888671875, "completions/min_length": 204.0, "completions/min_terminated_length": 204.0, "entropy": 0.31281290389597416, "epoch": 0.3576017130620985, "frac_reward_zero_std": 0.0, "grad_norm": 0.02866942621767521, "learning_rate": 1e-05, "loss": 0.0745, "num_tokens": 14043707.0, "reward": 0.6875, "reward_std": 0.4671337604522705, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.4010796546936035, "sampling/importance_sampling_ratio/mean": 0.9998789429664612, "sampling/importance_sampling_ratio/min": 0.6633136868476868, "sampling/sampling_logp_difference/max": 0.4105072021484375, "sampling/sampling_logp_difference/mean": 0.011560408398509026, "step": 668 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 708.0, "completions/mean_length": 437.6875, "completions/mean_terminated_length": 403.5172424316406, "completions/min_length": 244.0, "completions/min_terminated_length": 244.0, "entropy": 0.5037291385233402, "epoch": 0.35813704496788007, "frac_reward_zero_std": 0.25, "grad_norm": 0.004610450007021427, "learning_rate": 1e-05, "loss": 0.1006, "num_tokens": 14061801.0, "reward": 0.78125, "reward_std": 0.3377464711666107, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.6962294578552246, "sampling/importance_sampling_ratio/mean": 0.9997002482414246, "sampling/importance_sampling_ratio/min": 0.5090535283088684, "sampling/sampling_logp_difference/max": 0.6752021312713623, "sampling/sampling_logp_difference/mean": 0.014454632066190243, "step": 669 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.018064985051751e-05, "clip_ratio/low_min": 5.018064985051751e-05, "clip_ratio/region_mean": 5.018064985051751e-05, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 700.0, "completions/mean_length": 504.5, "completions/mean_terminated_length": 430.7200012207031, "completions/min_length": 285.0, "completions/min_terminated_length": 285.0, "entropy": 0.4414107985794544, "epoch": 0.35867237687366166, "frac_reward_zero_std": 0.5, "grad_norm": 0.01271494198590517, "learning_rate": 1e-05, "loss": 0.0736, "num_tokens": 14082105.0, "reward": 0.625, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.6699410676956177, "sampling/importance_sampling_ratio/mean": 1.0002871751785278, "sampling/importance_sampling_ratio/min": 0.6470206379890442, "sampling/sampling_logp_difference/max": 0.5127882957458496, "sampling/sampling_logp_difference/mean": 0.01434128824621439, "step": 670 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 759.0, "completions/mean_length": 491.125, "completions/mean_terminated_length": 472.66668701171875, "completions/min_length": 233.0, "completions/min_terminated_length": 233.0, "entropy": 0.2826128099113703, "epoch": 0.35920770877944325, "frac_reward_zero_std": 0.25, "grad_norm": 0.006041218061000109, "learning_rate": 1e-05, "loss": 0.0748, "num_tokens": 14101333.0, "reward": 0.78125, "reward_std": 0.3377464711666107, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.2799896001815796, "sampling/importance_sampling_ratio/mean": 1.0001157522201538, "sampling/importance_sampling_ratio/min": 0.642038106918335, "sampling/sampling_logp_difference/max": 0.44310760498046875, "sampling/sampling_logp_difference/mean": 0.01015233900398016, "step": 671 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.256256165215746e-05, "clip_ratio/low_min": 6.256256165215746e-05, "clip_ratio/region_mean": 6.256256165215746e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 621.0, "completions/max_terminated_length": 621.0, "completions/mean_length": 453.625, "completions/mean_terminated_length": 453.625, "completions/min_length": 263.0, "completions/min_terminated_length": 263.0, "entropy": 0.38719524815678596, "epoch": 0.35974304068522484, "frac_reward_zero_std": 0.25, "grad_norm": 0.022147536277770996, "learning_rate": 1e-05, "loss": 0.002, "num_tokens": 14119873.0, "reward": 0.78125, "reward_std": 0.3608423173427582, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.5876615047454834, "sampling/importance_sampling_ratio/mean": 0.9994799494743347, "sampling/importance_sampling_ratio/min": 0.5754033327102661, "sampling/sampling_logp_difference/max": 0.5526840686798096, "sampling/sampling_logp_difference/mean": 0.013436345383524895, "step": 672 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 691.0, "completions/max_terminated_length": 691.0, "completions/mean_length": 430.71875, "completions/mean_terminated_length": 430.71875, "completions/min_length": 153.0, "completions/min_terminated_length": 153.0, "entropy": 0.319727785885334, "epoch": 0.3602783725910064, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": -0.0142, "num_tokens": 14137200.0, "reward": 0.96875, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.96875, "rewards/accuracy_reward/std": 0.1767766922712326, "sampling/importance_sampling_ratio/max": 1.4579832553863525, "sampling/importance_sampling_ratio/mean": 1.0001784563064575, "sampling/importance_sampling_ratio/min": 0.709961473941803, "sampling/sampling_logp_difference/max": 0.37705421447753906, "sampling/sampling_logp_difference/mean": 0.012015475891530514, "step": 673 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.341958214761689e-05, "clip_ratio/low_min": 6.341958214761689e-05, "clip_ratio/region_mean": 6.341958214761689e-05, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 721.0, "completions/mean_length": 520.625, "completions/mean_terminated_length": 495.03448486328125, "completions/min_length": 294.0, "completions/min_terminated_length": 294.0, "entropy": 0.4333329312503338, "epoch": 0.360813704496788, "frac_reward_zero_std": 0.25, "grad_norm": 0.009912843815982342, "learning_rate": 1e-05, "loss": 0.0571, "num_tokens": 14157988.0, "reward": 0.65625, "reward_std": 0.3377465009689331, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.4718139171600342, "sampling/importance_sampling_ratio/mean": 1.00019371509552, "sampling/importance_sampling_ratio/min": 0.6560470461845398, "sampling/sampling_logp_difference/max": 0.42152273654937744, "sampling/sampling_logp_difference/mean": 0.014145986177027225, "step": 674 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 648.0, "completions/mean_length": 419.625, "completions/mean_terminated_length": 383.5862121582031, "completions/min_length": 199.0, "completions/min_terminated_length": 199.0, "entropy": 0.39590614289045334, "epoch": 0.3613490364025696, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": -0.041, "num_tokens": 14175240.0, "reward": 0.90625, "reward_std": 0.1293872892856598, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.4585431814193726, "sampling/importance_sampling_ratio/mean": 0.9998171329498291, "sampling/importance_sampling_ratio/min": 0.5023759007453918, "sampling/sampling_logp_difference/max": 0.6884067058563232, "sampling/sampling_logp_difference/mean": 0.014292944222688675, "step": 675 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 728.0, "completions/max_terminated_length": 728.0, "completions/mean_length": 363.53125, "completions/mean_terminated_length": 363.53125, "completions/min_length": 169.0, "completions/min_terminated_length": 169.0, "entropy": 0.4874013550579548, "epoch": 0.3618843683083512, "frac_reward_zero_std": 0.75, "grad_norm": 0.014141431078314781, "learning_rate": 1e-05, "loss": -0.0292, "num_tokens": 14190841.0, "reward": 0.875, "reward_std": 0.13363061845302582, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 1.411044955253601, "sampling/importance_sampling_ratio/mean": 1.0006593465805054, "sampling/importance_sampling_ratio/min": 0.6551950573921204, "sampling/sampling_logp_difference/max": 0.42282235622406006, "sampling/sampling_logp_difference/mean": 0.015140259638428688, "step": 676 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00011285867003607564, "clip_ratio/low_min": 0.00011285867003607564, "clip_ratio/region_mean": 0.00011285867003607564, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 705.0, "completions/mean_length": 480.15625, "completions/mean_terminated_length": 450.3793029785156, "completions/min_length": 257.0, "completions/min_terminated_length": 257.0, "entropy": 0.4115572366863489, "epoch": 0.3624197002141328, "frac_reward_zero_std": 0.25, "grad_norm": 0.00806187279522419, "learning_rate": 1e-05, "loss": -0.0317, "num_tokens": 14210054.0, "reward": 0.65625, "reward_std": 0.3198433816432953, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.4364715814590454, "sampling/importance_sampling_ratio/mean": 0.9999154806137085, "sampling/importance_sampling_ratio/min": 0.6824835538864136, "sampling/sampling_logp_difference/max": 0.3820168972015381, "sampling/sampling_logp_difference/mean": 0.013969811610877514, "step": 677 }, { "clip_ratio/high_max": 5.225752465776168e-05, "clip_ratio/high_mean": 5.225752465776168e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 5.225752465776168e-05, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 659.0, "completions/mean_length": 440.71875, "completions/mean_terminated_length": 380.1111145019531, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 0.3401738256216049, "epoch": 0.36295503211991437, "frac_reward_zero_std": 0.0, "grad_norm": 0.012882783077657223, "learning_rate": 1e-05, "loss": 0.0524, "num_tokens": 14227341.0, "reward": 0.6875, "reward_std": 0.44403791427612305, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.4207862615585327, "sampling/importance_sampling_ratio/mean": 0.9995315670967102, "sampling/importance_sampling_ratio/min": 0.5772851705551147, "sampling/sampling_logp_difference/max": 0.5494189262390137, "sampling/sampling_logp_difference/mean": 0.01124914363026619, "step": 678 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 634.0, "completions/max_terminated_length": 634.0, "completions/mean_length": 449.125, "completions/mean_terminated_length": 449.125, "completions/min_length": 270.0, "completions/min_terminated_length": 270.0, "entropy": 0.3078611623495817, "epoch": 0.36349036402569596, "frac_reward_zero_std": 0.5, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": -0.0006, "num_tokens": 14245297.0, "reward": 0.71875, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.520932912826538, "sampling/importance_sampling_ratio/mean": 1.0000300407409668, "sampling/importance_sampling_ratio/min": 0.6541504263877869, "sampling/sampling_logp_difference/max": 0.42441797256469727, "sampling/sampling_logp_difference/mean": 0.011252114549279213, "step": 679 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 617.0, "completions/mean_length": 435.5, "completions/mean_terminated_length": 424.774169921875, "completions/min_length": 241.0, "completions/min_terminated_length": 241.0, "entropy": 0.3654438741505146, "epoch": 0.3640256959314775, "frac_reward_zero_std": 0.0, "grad_norm": 0.010212710127234459, "learning_rate": 1e-05, "loss": 0.0872, "num_tokens": 14262729.0, "reward": 0.71875, "reward_std": 0.4218915104866028, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.4961756467819214, "sampling/importance_sampling_ratio/mean": 0.9998728036880493, "sampling/importance_sampling_ratio/min": 0.6332477331161499, "sampling/sampling_logp_difference/max": 0.4568936824798584, "sampling/sampling_logp_difference/mean": 0.01297544315457344, "step": 680 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0001949239886016585, "clip_ratio/low_min": 0.0001949239886016585, "clip_ratio/region_mean": 0.0001949239886016585, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 765.0, "completions/mean_length": 500.1875, "completions/mean_terminated_length": 482.3333435058594, "completions/min_length": 288.0, "completions/min_terminated_length": 288.0, "entropy": 0.42423025891184807, "epoch": 0.3645610278372591, "frac_reward_zero_std": 0.25, "grad_norm": 0.006797696463763714, "learning_rate": 1e-05, "loss": 0.0049, "num_tokens": 14282447.0, "reward": 0.625, "reward_std": 0.292504221200943, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.5203732252120972, "sampling/importance_sampling_ratio/mean": 1.0001957416534424, "sampling/importance_sampling_ratio/min": 0.48946166038513184, "sampling/sampling_logp_difference/max": 0.7144491672515869, "sampling/sampling_logp_difference/mean": 0.014462722465395927, "step": 681 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00015481705850106664, "clip_ratio/low_min": 0.00015481705850106664, "clip_ratio/region_mean": 0.00015481705850106664, "completions/clipped_ratio": 0.0, "completions/max_length": 760.0, "completions/max_terminated_length": 760.0, "completions/mean_length": 462.15625, "completions/mean_terminated_length": 462.15625, "completions/min_length": 285.0, "completions/min_terminated_length": 285.0, "entropy": 0.31800391152501106, "epoch": 0.36509635974304067, "frac_reward_zero_std": 0.0, "grad_norm": 0.013765319250524044, "learning_rate": 1e-05, "loss": 0.0097, "num_tokens": 14301348.0, "reward": 0.59375, "reward_std": 0.4944729208946228, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.3633944988250732, "sampling/importance_sampling_ratio/mean": 0.9994536638259888, "sampling/importance_sampling_ratio/min": 0.6891199350357056, "sampling/sampling_logp_difference/max": 0.37233996391296387, "sampling/sampling_logp_difference/mean": 0.010963515378534794, "step": 682 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 8.256275032181293e-05, "clip_ratio/low_min": 8.256275032181293e-05, "clip_ratio/region_mean": 8.256275032181293e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 606.0, "completions/max_terminated_length": 606.0, "completions/mean_length": 368.71875, "completions/mean_terminated_length": 368.71875, "completions/min_length": 181.0, "completions/min_terminated_length": 181.0, "entropy": 0.36436543986201286, "epoch": 0.36563169164882225, "frac_reward_zero_std": 0.5, "grad_norm": 0.006122628226876259, "learning_rate": 1e-05, "loss": 0.0228, "num_tokens": 14317011.0, "reward": 0.5625, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.7335211038589478, "sampling/importance_sampling_ratio/mean": 1.0002427101135254, "sampling/importance_sampling_ratio/min": 0.4999772608280182, "sampling/sampling_logp_difference/max": 0.693192720413208, "sampling/sampling_logp_difference/mean": 0.013767261058092117, "step": 683 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.49688154226169e-05, "clip_ratio/low_min": 6.49688154226169e-05, "clip_ratio/region_mean": 6.49688154226169e-05, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 739.0, "completions/mean_length": 430.5625, "completions/mean_terminated_length": 419.6773986816406, "completions/min_length": 174.0, "completions/min_terminated_length": 174.0, "entropy": 0.4069690480828285, "epoch": 0.36616702355460384, "frac_reward_zero_std": 0.5, "grad_norm": 0.010354407131671906, "learning_rate": 1e-05, "loss": -0.0044, "num_tokens": 14334461.0, "reward": 0.78125, "reward_std": 0.2630178928375244, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.4599215984344482, "sampling/importance_sampling_ratio/mean": 1.0004851818084717, "sampling/importance_sampling_ratio/min": 0.6137949824333191, "sampling/sampling_logp_difference/max": 0.4880943298339844, "sampling/sampling_logp_difference/mean": 0.013524693436920643, "step": 684 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 747.0, "completions/mean_length": 547.90625, "completions/mean_terminated_length": 486.2799987792969, "completions/min_length": 268.0, "completions/min_terminated_length": 268.0, "entropy": 0.41413573175668716, "epoch": 0.36670235546038543, "frac_reward_zero_std": 0.5, "grad_norm": 0.00839283224195242, "learning_rate": 1e-05, "loss": 0.0102, "num_tokens": 14355978.0, "reward": 0.59375, "reward_std": 0.22201895713806152, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.999715268611908, "sampling/importance_sampling_ratio/min": 0.5842621326446533, "sampling/sampling_logp_difference/max": 0.7234721183776855, "sampling/sampling_logp_difference/mean": 0.014377523213624954, "step": 685 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 733.0, "completions/mean_length": 459.6875, "completions/mean_terminated_length": 439.13336181640625, "completions/min_length": 222.0, "completions/min_terminated_length": 222.0, "entropy": 0.4223700761795044, "epoch": 0.367237687366167, "frac_reward_zero_std": 0.5, "grad_norm": 0.027567055076360703, "learning_rate": 1e-05, "loss": 0.0006, "num_tokens": 14374288.0, "reward": 0.625, "reward_std": 0.2314550280570984, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.46743905544281, "sampling/importance_sampling_ratio/mean": 0.999767541885376, "sampling/importance_sampling_ratio/min": 0.5879977345466614, "sampling/sampling_logp_difference/max": 0.5310320854187012, "sampling/sampling_logp_difference/mean": 0.01458754763007164, "step": 686 }, { "clip_ratio/high_max": 8.148630877258256e-05, "clip_ratio/high_mean": 8.148630877258256e-05, "clip_ratio/low_mean": 0.00031715090153738856, "clip_ratio/low_min": 0.00031715090153738856, "clip_ratio/region_mean": 0.0003986372103099711, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 566.0, "completions/mean_length": 425.0625, "completions/mean_terminated_length": 402.20001220703125, "completions/min_length": 260.0, "completions/min_terminated_length": 260.0, "entropy": 0.39186953753232956, "epoch": 0.3677730192719486, "frac_reward_zero_std": 0.25, "grad_norm": 0.005817718803882599, "learning_rate": 1e-05, "loss": 0.0304, "num_tokens": 14390914.0, "reward": 0.65625, "reward_std": 0.3198433816432953, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.4666051864624023, "sampling/importance_sampling_ratio/mean": 0.9998550415039062, "sampling/importance_sampling_ratio/min": 0.5507513880729675, "sampling/sampling_logp_difference/max": 0.5964717864990234, "sampling/sampling_logp_difference/mean": 0.013279074802994728, "step": 687 }, { "clip_ratio/high_max": 6.826870230725035e-05, "clip_ratio/high_mean": 6.826870230725035e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 6.826870230725035e-05, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 708.0, "completions/mean_length": 462.0, "completions/mean_terminated_length": 418.2857360839844, "completions/min_length": 210.0, "completions/min_terminated_length": 210.0, "entropy": 0.4630816914141178, "epoch": 0.3683083511777302, "frac_reward_zero_std": 0.25, "grad_norm": 0.016125718131661415, "learning_rate": 1e-05, "loss": 0.1038, "num_tokens": 14409274.0, "reward": 0.53125, "reward_std": 0.3608423173427582, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.5871626138687134, "sampling/importance_sampling_ratio/mean": 1.000115156173706, "sampling/importance_sampling_ratio/min": 0.53107750415802, "sampling/sampling_logp_difference/max": 0.6328473091125488, "sampling/sampling_logp_difference/mean": 0.0158088356256485, "step": 688 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00016200369282159954, "clip_ratio/low_min": 0.00016200369282159954, "clip_ratio/region_mean": 0.00016200369282159954, "completions/clipped_ratio": 0.28125, "completions/max_length": 768.0, "completions/max_terminated_length": 743.0, "completions/mean_length": 565.78125, "completions/mean_terminated_length": 486.6521911621094, "completions/min_length": 260.0, "completions/min_terminated_length": 260.0, "entropy": 0.5368899889290333, "epoch": 0.3688436830835118, "frac_reward_zero_std": 0.5, "grad_norm": 0.003974041901528835, "learning_rate": 1e-05, "loss": 0.0156, "num_tokens": 14431915.0, "reward": 0.5625, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.4156824350357056, "sampling/importance_sampling_ratio/mean": 1.0004881620407104, "sampling/importance_sampling_ratio/min": 0.6396004557609558, "sampling/sampling_logp_difference/max": 0.4469115734100342, "sampling/sampling_logp_difference/mean": 0.015277111902832985, "step": 689 }, { "clip_ratio/high_max": 6.92520770826377e-05, "clip_ratio/high_mean": 6.92520770826377e-05, "clip_ratio/low_mean": 5.018064985051751e-05, "clip_ratio/low_min": 5.018064985051751e-05, "clip_ratio/region_mean": 0.00011943272693315521, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 721.0, "completions/mean_length": 451.21875, "completions/mean_terminated_length": 405.96429443359375, "completions/min_length": 245.0, "completions/min_terminated_length": 245.0, "entropy": 0.37817422300577164, "epoch": 0.3693790149892934, "frac_reward_zero_std": 0.25, "grad_norm": 0.012605220079421997, "learning_rate": 1e-05, "loss": 0.0207, "num_tokens": 14450106.0, "reward": 0.71875, "reward_std": 0.35564959049224854, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.4299546480178833, "sampling/importance_sampling_ratio/mean": 1.000266671180725, "sampling/importance_sampling_ratio/min": 0.712934672832489, "sampling/sampling_logp_difference/max": 0.35764265060424805, "sampling/sampling_logp_difference/mean": 0.012577303685247898, "step": 690 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.374032843974419e-05, "clip_ratio/low_min": 5.374032843974419e-05, "clip_ratio/region_mean": 5.374032843974419e-05, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 720.0, "completions/mean_length": 491.53125, "completions/mean_terminated_length": 440.3333435058594, "completions/min_length": 244.0, "completions/min_terminated_length": 244.0, "entropy": 0.5191456601023674, "epoch": 0.36991434689507496, "frac_reward_zero_std": 0.5, "grad_norm": 0.00929737277328968, "learning_rate": 1e-05, "loss": 0.0538, "num_tokens": 14469531.0, "reward": 0.46875, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.46875, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.4154235124588013, "sampling/importance_sampling_ratio/mean": 1.0003197193145752, "sampling/importance_sampling_ratio/min": 0.7000487446784973, "sampling/sampling_logp_difference/max": 0.35660529136657715, "sampling/sampling_logp_difference/mean": 0.01629146933555603, "step": 691 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00012185192463221028, "clip_ratio/low_min": 0.00012185192463221028, "clip_ratio/region_mean": 0.00012185192463221028, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 634.0, "completions/mean_length": 493.3125, "completions/mean_terminated_length": 475.0000305175781, "completions/min_length": 324.0, "completions/min_terminated_length": 324.0, "entropy": 0.25759415701031685, "epoch": 0.37044967880085655, "frac_reward_zero_std": 0.0, "grad_norm": 0.007354686036705971, "learning_rate": 1e-05, "loss": 0.0234, "num_tokens": 14489277.0, "reward": 0.78125, "reward_std": 0.4355708956718445, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.463361144065857, "sampling/importance_sampling_ratio/mean": 0.9997757077217102, "sampling/importance_sampling_ratio/min": 0.5534732937812805, "sampling/sampling_logp_difference/max": 0.5915417671203613, "sampling/sampling_logp_difference/mean": 0.010354440659284592, "step": 692 }, { "clip_ratio/high_max": 5.615453846985474e-05, "clip_ratio/high_mean": 5.615453846985474e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 5.615453846985474e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 695.0, "completions/max_terminated_length": 695.0, "completions/mean_length": 398.25, "completions/mean_terminated_length": 398.25, "completions/min_length": 198.0, "completions/min_terminated_length": 198.0, "entropy": 0.3556918017566204, "epoch": 0.37098501070663814, "frac_reward_zero_std": 0.0, "grad_norm": 0.012502416037023067, "learning_rate": 1e-05, "loss": 0.0473, "num_tokens": 14505637.0, "reward": 0.75, "reward_std": 0.4261348247528076, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.6686698198318481, "sampling/importance_sampling_ratio/mean": 0.9999684691429138, "sampling/importance_sampling_ratio/min": 0.4408150315284729, "sampling/sampling_logp_difference/max": 0.8191299438476562, "sampling/sampling_logp_difference/mean": 0.012879938818514347, "step": 693 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00011354028174537234, "clip_ratio/low_min": 0.00011354028174537234, "clip_ratio/region_mean": 0.00011354028174537234, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 757.0, "completions/mean_length": 562.28125, "completions/mean_terminated_length": 504.67999267578125, "completions/min_length": 332.0, "completions/min_terminated_length": 332.0, "entropy": 0.3640853948891163, "epoch": 0.3715203426124197, "frac_reward_zero_std": 0.5, "grad_norm": 0.003080160589888692, "learning_rate": 1e-05, "loss": 0.0164, "num_tokens": 14527262.0, "reward": 0.75, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.6592509746551514, "sampling/importance_sampling_ratio/mean": 1.000012993812561, "sampling/importance_sampling_ratio/min": 0.6692094802856445, "sampling/sampling_logp_difference/max": 0.5063662528991699, "sampling/sampling_logp_difference/mean": 0.012055626139044762, "step": 694 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 706.0, "completions/mean_length": 487.40625, "completions/mean_terminated_length": 478.3548278808594, "completions/min_length": 254.0, "completions/min_terminated_length": 254.0, "entropy": 0.3632141798734665, "epoch": 0.37205567451820126, "frac_reward_zero_std": 0.25, "grad_norm": 0.013796296902000904, "learning_rate": 1e-05, "loss": -0.0126, "num_tokens": 14546179.0, "reward": 0.6875, "reward_std": 0.3514062762260437, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.7792364358901978, "sampling/importance_sampling_ratio/mean": 1.000428557395935, "sampling/importance_sampling_ratio/min": 0.5009722709655762, "sampling/sampling_logp_difference/max": 0.6912045478820801, "sampling/sampling_logp_difference/mean": 0.012903879396617413, "step": 695 }, { "clip_ratio/high_max": 5.18242122780066e-05, "clip_ratio/high_mean": 5.18242122780066e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 5.18242122780066e-05, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 662.0, "completions/mean_length": 491.25, "completions/mean_terminated_length": 462.6206970214844, "completions/min_length": 185.0, "completions/min_terminated_length": 185.0, "entropy": 0.3077428266406059, "epoch": 0.37259100642398285, "frac_reward_zero_std": 0.0, "grad_norm": 0.006799117662012577, "learning_rate": 1e-05, "loss": 0.0882, "num_tokens": 14565803.0, "reward": 0.84375, "reward_std": 0.3808925747871399, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.6996805667877197, "sampling/importance_sampling_ratio/mean": 0.9997897744178772, "sampling/importance_sampling_ratio/min": 0.6941378712654114, "sampling/sampling_logp_difference/max": 0.5304403305053711, "sampling/sampling_logp_difference/mean": 0.011125020682811737, "step": 696 }, { "clip_ratio/high_max": 7.77363165980205e-05, "clip_ratio/high_mean": 7.77363165980205e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 7.77363165980205e-05, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 568.0, "completions/mean_length": 436.90625, "completions/mean_terminated_length": 389.6071472167969, "completions/min_length": 185.0, "completions/min_terminated_length": 185.0, "entropy": 0.43571585789322853, "epoch": 0.37312633832976444, "frac_reward_zero_std": 0.0, "grad_norm": 0.025787562131881714, "learning_rate": 1e-05, "loss": 0.085, "num_tokens": 14583352.0, "reward": 0.65625, "reward_std": 0.4218915104866028, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.4428902864456177, "sampling/importance_sampling_ratio/mean": 1.0001848936080933, "sampling/importance_sampling_ratio/min": 0.6767501831054688, "sampling/sampling_logp_difference/max": 0.3904531002044678, "sampling/sampling_logp_difference/mean": 0.014643594622612, "step": 697 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0001140519707405474, "clip_ratio/low_min": 0.0001140519707405474, "clip_ratio/region_mean": 0.0001140519707405474, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 651.0, "completions/mean_length": 506.09375, "completions/mean_terminated_length": 479.0, "completions/min_length": 258.0, "completions/min_terminated_length": 258.0, "entropy": 0.3874698430299759, "epoch": 0.37366167023554603, "frac_reward_zero_std": 0.25, "grad_norm": 0.01362792868167162, "learning_rate": 1e-05, "loss": -0.0161, "num_tokens": 14603547.0, "reward": 0.65625, "reward_std": 0.38816186785697937, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.3679933547973633, "sampling/importance_sampling_ratio/mean": 0.9998596906661987, "sampling/importance_sampling_ratio/min": 0.5134339332580566, "sampling/sampling_logp_difference/max": 0.6666338443756104, "sampling/sampling_logp_difference/mean": 0.013550950214266777, "step": 698 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 590.0, "completions/mean_length": 403.90625, "completions/mean_terminated_length": 392.1612854003906, "completions/min_length": 204.0, "completions/min_terminated_length": 204.0, "entropy": 0.2696386091411114, "epoch": 0.3741970021413276, "frac_reward_zero_std": 0.75, "grad_norm": 0.0030069833155721426, "learning_rate": 1e-05, "loss": 0.086, "num_tokens": 14619824.0, "reward": 0.96875, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.96875, "rewards/accuracy_reward/std": 0.1767766922712326, "sampling/importance_sampling_ratio/max": 1.4245977401733398, "sampling/importance_sampling_ratio/mean": 0.9999731183052063, "sampling/importance_sampling_ratio/min": 0.6120657920837402, "sampling/sampling_logp_difference/max": 0.49091553688049316, "sampling/sampling_logp_difference/mean": 0.010597498156130314, "step": 699 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 461.0, "completions/max_terminated_length": 461.0, "completions/mean_length": 309.6875, "completions/mean_terminated_length": 309.6875, "completions/min_length": 186.0, "completions/min_terminated_length": 186.0, "entropy": 0.42275048419833183, "epoch": 0.3747323340471092, "frac_reward_zero_std": 0.5, "grad_norm": 0.03343091160058975, "learning_rate": 1e-05, "loss": 0.0205, "num_tokens": 14632942.0, "reward": 0.8125, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.4157845973968506, "sampling/importance_sampling_ratio/mean": 0.9995958209037781, "sampling/importance_sampling_ratio/min": 0.6339309215545654, "sampling/sampling_logp_difference/max": 0.45581531524658203, "sampling/sampling_logp_difference/mean": 0.013705016113817692, "step": 700 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 689.0, "completions/max_terminated_length": 689.0, "completions/mean_length": 453.71875, "completions/mean_terminated_length": 453.71875, "completions/min_length": 152.0, "completions/min_terminated_length": 152.0, "entropy": 0.3207152709364891, "epoch": 0.3752676659528908, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": -0.0306, "num_tokens": 14651053.0, "reward": 0.71875, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.4452935457229614, "sampling/importance_sampling_ratio/mean": 1.0002785921096802, "sampling/importance_sampling_ratio/min": 0.6024765372276306, "sampling/sampling_logp_difference/max": 0.5067065954208374, "sampling/sampling_logp_difference/mean": 0.01190304383635521, "step": 701 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 9.356287773698568e-05, "clip_ratio/low_min": 9.356287773698568e-05, "clip_ratio/region_mean": 9.356287773698568e-05, "completions/clipped_ratio": 0.28125, "completions/max_length": 768.0, "completions/max_terminated_length": 636.0, "completions/mean_length": 474.65625, "completions/mean_terminated_length": 359.86956787109375, "completions/min_length": 185.0, "completions/min_terminated_length": 185.0, "entropy": 0.5178338028490543, "epoch": 0.3758029978586724, "frac_reward_zero_std": 0.5, "grad_norm": 0.004330488853156567, "learning_rate": 1e-05, "loss": 0.117, "num_tokens": 14670354.0, "reward": 0.625, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.5016812086105347, "sampling/importance_sampling_ratio/mean": 0.9999881386756897, "sampling/importance_sampling_ratio/min": 0.5800591111183167, "sampling/sampling_logp_difference/max": 0.5446252822875977, "sampling/sampling_logp_difference/mean": 0.01547443401068449, "step": 702 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 585.0, "completions/mean_length": 357.375, "completions/mean_terminated_length": 344.1290283203125, "completions/min_length": 147.0, "completions/min_terminated_length": 147.0, "entropy": 0.4904642216861248, "epoch": 0.37633832976445397, "frac_reward_zero_std": 0.5, "grad_norm": 0.012364492751657963, "learning_rate": 1e-05, "loss": -0.011, "num_tokens": 14685302.0, "reward": 0.875, "reward_std": 0.2314550280570984, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 1.433635950088501, "sampling/importance_sampling_ratio/mean": 0.9998182058334351, "sampling/importance_sampling_ratio/min": 0.6609511375427246, "sampling/sampling_logp_difference/max": 0.4140753746032715, "sampling/sampling_logp_difference/mean": 0.01529625616967678, "step": 703 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.375, "completions/max_length": 768.0, "completions/max_terminated_length": 704.0, "completions/mean_length": 599.59375, "completions/mean_terminated_length": 498.5500183105469, "completions/min_length": 341.0, "completions/min_terminated_length": 341.0, "entropy": 0.6024130024015903, "epoch": 0.37687366167023556, "frac_reward_zero_std": 0.5, "grad_norm": 0.0067597646266222, "learning_rate": 1e-05, "loss": -0.0056, "num_tokens": 14708481.0, "reward": 0.375, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.375, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.4328670501708984, "sampling/importance_sampling_ratio/mean": 0.999868631362915, "sampling/importance_sampling_ratio/min": 0.6925622224807739, "sampling/sampling_logp_difference/max": 0.3673572540283203, "sampling/sampling_logp_difference/mean": 0.016798943281173706, "step": 704 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.771006544819102e-05, "clip_ratio/low_min": 5.771006544819102e-05, "clip_ratio/region_mean": 5.771006544819102e-05, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 759.0, "completions/mean_length": 460.1875, "completions/mean_terminated_length": 450.258056640625, "completions/min_length": 284.0, "completions/min_terminated_length": 284.0, "entropy": 0.40252557396888733, "epoch": 0.37740899357601715, "frac_reward_zero_std": 0.5, "grad_norm": 0.011444158852100372, "learning_rate": 1e-05, "loss": -0.0376, "num_tokens": 14727135.0, "reward": 0.84375, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.410793662071228, "sampling/importance_sampling_ratio/mean": 0.999970555305481, "sampling/importance_sampling_ratio/min": 0.608476996421814, "sampling/sampling_logp_difference/max": 0.49679625034332275, "sampling/sampling_logp_difference/mean": 0.01365983672440052, "step": 705 }, { "clip_ratio/high_max": 0.0001688751290203072, "clip_ratio/high_mean": 0.0001688751290203072, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0001688751290203072, "completions/clipped_ratio": 0.0, "completions/max_length": 731.0, "completions/max_terminated_length": 731.0, "completions/mean_length": 393.40625, "completions/mean_terminated_length": 393.40625, "completions/min_length": 185.0, "completions/min_terminated_length": 185.0, "entropy": 0.37903517857193947, "epoch": 0.37794432548179874, "frac_reward_zero_std": 0.25, "grad_norm": 0.009346889331936836, "learning_rate": 1e-05, "loss": -0.0052, "num_tokens": 14743468.0, "reward": 0.6875, "reward_std": 0.2925041913986206, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.410951852798462, "sampling/importance_sampling_ratio/mean": 0.9999088048934937, "sampling/importance_sampling_ratio/min": 0.5254480838775635, "sampling/sampling_logp_difference/max": 0.6435039043426514, "sampling/sampling_logp_difference/mean": 0.012299933470785618, "step": 706 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00022300455020740628, "clip_ratio/low_min": 0.00022300455020740628, "clip_ratio/region_mean": 0.00022300455020740628, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 630.0, "completions/mean_length": 506.0, "completions/mean_terminated_length": 445.5384826660156, "completions/min_length": 300.0, "completions/min_terminated_length": 300.0, "entropy": 0.4007762297987938, "epoch": 0.3784796573875803, "frac_reward_zero_std": 0.25, "grad_norm": 0.00942362379282713, "learning_rate": 1e-05, "loss": 0.1047, "num_tokens": 14763260.0, "reward": 0.6875, "reward_std": 0.3514062464237213, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.436396598815918, "sampling/importance_sampling_ratio/mean": 0.9998412728309631, "sampling/importance_sampling_ratio/min": 0.5992105603218079, "sampling/sampling_logp_difference/max": 0.5121423006057739, "sampling/sampling_logp_difference/mean": 0.01251544151455164, "step": 707 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 648.0, "completions/mean_length": 399.71875, "completions/mean_terminated_length": 387.83868408203125, "completions/min_length": 213.0, "completions/min_terminated_length": 213.0, "entropy": 0.29389157705008984, "epoch": 0.37901498929336186, "frac_reward_zero_std": 0.25, "grad_norm": 0.005059751681983471, "learning_rate": 1e-05, "loss": -0.0461, "num_tokens": 14779451.0, "reward": 0.90625, "reward_std": 0.2651650309562683, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.4614378213882446, "sampling/importance_sampling_ratio/mean": 1.0001176595687866, "sampling/importance_sampling_ratio/min": 0.629279613494873, "sampling/sampling_logp_difference/max": 0.4631795883178711, "sampling/sampling_logp_difference/mean": 0.011236912570893764, "step": 708 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.361323175951838e-05, "clip_ratio/low_min": 6.361323175951838e-05, "clip_ratio/region_mean": 6.361323175951838e-05, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 521.0, "completions/mean_length": 418.375, "completions/mean_terminated_length": 368.4285888671875, "completions/min_length": 219.0, "completions/min_terminated_length": 219.0, "entropy": 0.5839998461306095, "epoch": 0.37955032119914345, "frac_reward_zero_std": 0.0, "grad_norm": 0.022702999413013458, "learning_rate": 1e-05, "loss": 0.191, "num_tokens": 14796911.0, "reward": 0.6875, "reward_std": 0.49022960662841797, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.4120562076568604, "sampling/importance_sampling_ratio/mean": 0.9997467994689941, "sampling/importance_sampling_ratio/min": 0.5374916791915894, "sampling/sampling_logp_difference/max": 0.6208419799804688, "sampling/sampling_logp_difference/mean": 0.01453771535307169, "step": 709 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00015049399371491745, "clip_ratio/low_min": 0.00015049399371491745, "clip_ratio/region_mean": 0.00015049399371491745, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 764.0, "completions/mean_length": 524.25, "completions/mean_terminated_length": 468.0000305175781, "completions/min_length": 187.0, "completions/min_terminated_length": 187.0, "entropy": 0.4507652670145035, "epoch": 0.38008565310492504, "frac_reward_zero_std": 0.25, "grad_norm": 0.017059093341231346, "learning_rate": 1e-05, "loss": 0.0521, "num_tokens": 14818367.0, "reward": 0.625, "reward_std": 0.3650856614112854, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.6400338411331177, "sampling/importance_sampling_ratio/mean": 1.0003429651260376, "sampling/importance_sampling_ratio/min": 0.6167190670967102, "sampling/sampling_logp_difference/max": 0.4947168827056885, "sampling/sampling_logp_difference/mean": 0.015008334070444107, "step": 710 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00010794473200803623, "clip_ratio/low_min": 0.00010794473200803623, "clip_ratio/region_mean": 0.00010794473200803623, "completions/clipped_ratio": 0.0, "completions/max_length": 698.0, "completions/max_terminated_length": 698.0, "completions/mean_length": 348.3125, "completions/mean_terminated_length": 348.3125, "completions/min_length": 166.0, "completions/min_terminated_length": 166.0, "entropy": 0.35897016897797585, "epoch": 0.3806209850107066, "frac_reward_zero_std": 0.5, "grad_norm": 0.008120217360556126, "learning_rate": 1e-05, "loss": -0.0577, "num_tokens": 14832921.0, "reward": 0.84375, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.4298087358474731, "sampling/importance_sampling_ratio/mean": 0.9998185634613037, "sampling/importance_sampling_ratio/min": 0.6612955331802368, "sampling/sampling_logp_difference/max": 0.41355443000793457, "sampling/sampling_logp_difference/mean": 0.012855300679802895, "step": 711 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 7.896398892626166e-05, "clip_ratio/low_min": 7.896398892626166e-05, "clip_ratio/region_mean": 7.896398892626166e-05, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 687.0, "completions/mean_length": 448.53125, "completions/mean_terminated_length": 415.4827575683594, "completions/min_length": 225.0, "completions/min_terminated_length": 225.0, "entropy": 0.5796360373497009, "epoch": 0.3811563169164882, "frac_reward_zero_std": 0.25, "grad_norm": 0.026035917922854424, "learning_rate": 1e-05, "loss": 0.1085, "num_tokens": 14851474.0, "reward": 0.6875, "reward_std": 0.3514062464237213, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.4607065916061401, "sampling/importance_sampling_ratio/mean": 0.9997735619544983, "sampling/importance_sampling_ratio/min": 0.6867302656173706, "sampling/sampling_logp_difference/max": 0.378920316696167, "sampling/sampling_logp_difference/mean": 0.015060271136462688, "step": 712 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00015512864774791524, "clip_ratio/low_min": 0.00015512864774791524, "clip_ratio/region_mean": 0.00015512864774791524, "completions/clipped_ratio": 0.0, "completions/max_length": 703.0, "completions/max_terminated_length": 703.0, "completions/mean_length": 402.34375, "completions/mean_terminated_length": 402.34375, "completions/min_length": 168.0, "completions/min_terminated_length": 168.0, "entropy": 0.3063546046614647, "epoch": 0.3816916488222698, "frac_reward_zero_std": 0.25, "grad_norm": 0.020387133583426476, "learning_rate": 1e-05, "loss": -0.0582, "num_tokens": 14867613.0, "reward": 0.5625, "reward_std": 0.3745020925998688, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.3914622068405151, "sampling/importance_sampling_ratio/mean": 0.9998612403869629, "sampling/importance_sampling_ratio/min": 0.5678632855415344, "sampling/sampling_logp_difference/max": 0.5658745765686035, "sampling/sampling_logp_difference/mean": 0.011173569597303867, "step": 713 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 695.0, "completions/mean_length": 509.40625, "completions/mean_terminated_length": 482.6551818847656, "completions/min_length": 283.0, "completions/min_terminated_length": 283.0, "entropy": 0.3332985360175371, "epoch": 0.3822269807280514, "frac_reward_zero_std": 0.5, "grad_norm": 0.019818589091300964, "learning_rate": 1e-05, "loss": -0.0187, "num_tokens": 14887714.0, "reward": 0.75, "reward_std": 0.2314550280570984, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.618808627128601, "sampling/importance_sampling_ratio/mean": 1.000177264213562, "sampling/importance_sampling_ratio/min": 0.5244354605674744, "sampling/sampling_logp_difference/max": 0.6454329490661621, "sampling/sampling_logp_difference/mean": 0.011827697046101093, "step": 714 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0003420163739065174, "clip_ratio/low_min": 0.0003420163739065174, "clip_ratio/region_mean": 0.0003420163739065174, "completions/clipped_ratio": 0.28125, "completions/max_length": 768.0, "completions/max_terminated_length": 736.0, "completions/mean_length": 549.59375, "completions/mean_terminated_length": 464.13043212890625, "completions/min_length": 299.0, "completions/min_terminated_length": 299.0, "entropy": 0.48910192772746086, "epoch": 0.382762312633833, "frac_reward_zero_std": 0.25, "grad_norm": 0.014854305423796177, "learning_rate": 1e-05, "loss": 0.0358, "num_tokens": 14909349.0, "reward": 0.625, "reward_std": 0.3104073107242584, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.5460256338119507, "sampling/importance_sampling_ratio/mean": 1.0004106760025024, "sampling/importance_sampling_ratio/min": 0.6284617185592651, "sampling/sampling_logp_difference/max": 0.4644801616668701, "sampling/sampling_logp_difference/mean": 0.014490020461380482, "step": 715 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 731.0, "completions/mean_length": 454.3125, "completions/mean_terminated_length": 444.19354248046875, "completions/min_length": 265.0, "completions/min_terminated_length": 265.0, "entropy": 0.2621281649917364, "epoch": 0.38329764453961457, "frac_reward_zero_std": 0.25, "grad_norm": 0.015931304544210434, "learning_rate": 1e-05, "loss": -0.0098, "num_tokens": 14927527.0, "reward": 0.75, "reward_std": 0.3745020925998688, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.542803406715393, "sampling/importance_sampling_ratio/mean": 1.0000957250595093, "sampling/importance_sampling_ratio/min": 0.6055469512939453, "sampling/sampling_logp_difference/max": 0.5016231536865234, "sampling/sampling_logp_difference/mean": 0.010087703354656696, "step": 716 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 635.0, "completions/mean_length": 437.40625, "completions/mean_terminated_length": 415.36669921875, "completions/min_length": 261.0, "completions/min_terminated_length": 261.0, "entropy": 0.2905181087553501, "epoch": 0.38383297644539616, "frac_reward_zero_std": 0.5, "grad_norm": 0.0051934183575212955, "learning_rate": 1e-05, "loss": 0.0278, "num_tokens": 14945580.0, "reward": 0.8125, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.2746304273605347, "sampling/importance_sampling_ratio/mean": 0.9997221827507019, "sampling/importance_sampling_ratio/min": 0.4813854694366455, "sampling/sampling_logp_difference/max": 0.7310869693756104, "sampling/sampling_logp_difference/mean": 0.010493318550288677, "step": 717 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.131362922838889e-05, "clip_ratio/low_min": 5.131362922838889e-05, "clip_ratio/region_mean": 5.131362922838889e-05, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 698.0, "completions/mean_length": 528.875, "completions/mean_terminated_length": 473.69232177734375, "completions/min_length": 290.0, "completions/min_terminated_length": 290.0, "entropy": 0.319519704207778, "epoch": 0.38436830835117775, "frac_reward_zero_std": 0.75, "grad_norm": 0.00915689580142498, "learning_rate": 1e-05, "loss": 0.0297, "num_tokens": 14966128.0, "reward": 0.84375, "reward_std": 0.1293872892856598, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.6557259559631348, "sampling/importance_sampling_ratio/mean": 0.999800443649292, "sampling/importance_sampling_ratio/min": 0.4600535035133362, "sampling/sampling_logp_difference/max": 0.7764124870300293, "sampling/sampling_logp_difference/mean": 0.011192606762051582, "step": 718 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 7.720815483480692e-05, "clip_ratio/low_min": 7.720815483480692e-05, "clip_ratio/region_mean": 7.720815483480692e-05, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 633.0, "completions/mean_length": 457.875, "completions/mean_terminated_length": 447.8709411621094, "completions/min_length": 217.0, "completions/min_terminated_length": 217.0, "entropy": 0.3259826861321926, "epoch": 0.38490364025695933, "frac_reward_zero_std": 0.25, "grad_norm": 0.025625785812735558, "learning_rate": 1e-05, "loss": 0.0263, "num_tokens": 14984572.0, "reward": 0.78125, "reward_std": 0.3377464711666107, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.4572973251342773, "sampling/importance_sampling_ratio/mean": 0.9995626211166382, "sampling/importance_sampling_ratio/min": 0.6916748881340027, "sampling/sampling_logp_difference/max": 0.3765835762023926, "sampling/sampling_logp_difference/mean": 0.01140644121915102, "step": 719 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 620.0, "completions/max_terminated_length": 620.0, "completions/mean_length": 372.46875, "completions/mean_terminated_length": 372.46875, "completions/min_length": 149.0, "completions/min_terminated_length": 149.0, "entropy": 0.33895523846149445, "epoch": 0.3854389721627409, "frac_reward_zero_std": 0.25, "grad_norm": 0.021499278023838997, "learning_rate": 1e-05, "loss": 0.0171, "num_tokens": 14999691.0, "reward": 0.8125, "reward_std": 0.3104073107242584, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.4709117412567139, "sampling/importance_sampling_ratio/mean": 0.9995214939117432, "sampling/importance_sampling_ratio/min": 0.4620845317840576, "sampling/sampling_logp_difference/max": 0.7720074653625488, "sampling/sampling_logp_difference/mean": 0.011521605774760246, "step": 720 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00011531365453265607, "clip_ratio/low_min": 0.00011531365453265607, "clip_ratio/region_mean": 0.00011531365453265607, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 566.0, "completions/mean_length": 370.46875, "completions/mean_terminated_length": 357.6451416015625, "completions/min_length": 117.0, "completions/min_terminated_length": 117.0, "entropy": 0.3961496278643608, "epoch": 0.3859743040685225, "frac_reward_zero_std": 0.25, "grad_norm": 0.014711255207657814, "learning_rate": 1e-05, "loss": -0.0293, "num_tokens": 15015146.0, "reward": 0.75, "reward_std": 0.3514062464237213, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.4367763996124268, "sampling/importance_sampling_ratio/mean": 0.9994955658912659, "sampling/importance_sampling_ratio/min": 0.6811802387237549, "sampling/sampling_logp_difference/max": 0.3839282989501953, "sampling/sampling_logp_difference/mean": 0.013433506712317467, "step": 721 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 7.309941429411992e-05, "clip_ratio/low_min": 7.309941429411992e-05, "clip_ratio/region_mean": 7.309941429411992e-05, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 606.0, "completions/mean_length": 411.15625, "completions/mean_terminated_length": 399.6451416015625, "completions/min_length": 180.0, "completions/min_terminated_length": 180.0, "entropy": 0.3904821388423443, "epoch": 0.38650963597430404, "frac_reward_zero_std": 0.25, "grad_norm": 0.00928311888128519, "learning_rate": 1e-05, "loss": 0.1394, "num_tokens": 15032263.0, "reward": 0.625, "reward_std": 0.292504221200943, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.4140853881835938, "sampling/importance_sampling_ratio/mean": 0.9999125599861145, "sampling/importance_sampling_ratio/min": 0.6743881702423096, "sampling/sampling_logp_difference/max": 0.3939495086669922, "sampling/sampling_logp_difference/mean": 0.01285515632480383, "step": 722 }, { "clip_ratio/high_max": 5.451373726828024e-05, "clip_ratio/high_mean": 5.451373726828024e-05, "clip_ratio/low_mean": 5.451373726828024e-05, "clip_ratio/low_min": 5.451373726828024e-05, "clip_ratio/region_mean": 0.00010902747453656048, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 748.0, "completions/mean_length": 456.40625, "completions/mean_terminated_length": 435.63336181640625, "completions/min_length": 211.0, "completions/min_terminated_length": 211.0, "entropy": 0.4317764453589916, "epoch": 0.38704496788008563, "frac_reward_zero_std": 0.5, "grad_norm": 0.007342847995460033, "learning_rate": 1e-05, "loss": 0.0119, "num_tokens": 15050956.0, "reward": 0.90625, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.624866247177124, "sampling/importance_sampling_ratio/mean": 0.9999297261238098, "sampling/importance_sampling_ratio/min": 0.27815917134284973, "sampling/sampling_logp_difference/max": 1.2795617580413818, "sampling/sampling_logp_difference/mean": 0.013796807266771793, "step": 723 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00036409543827176094, "clip_ratio/low_min": 0.00036409543827176094, "clip_ratio/region_mean": 0.00036409543827176094, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 561.0, "completions/mean_length": 437.28125, "completions/mean_terminated_length": 403.0689697265625, "completions/min_length": 144.0, "completions/min_terminated_length": 144.0, "entropy": 0.4697720594704151, "epoch": 0.3875802997858672, "frac_reward_zero_std": 0.25, "grad_norm": 0.018577823415398598, "learning_rate": 1e-05, "loss": -0.0288, "num_tokens": 15068581.0, "reward": 0.53125, "reward_std": 0.3471629321575165, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.4698411226272583, "sampling/importance_sampling_ratio/mean": 0.9999327659606934, "sampling/importance_sampling_ratio/min": 0.6675266623497009, "sampling/sampling_logp_difference/max": 0.4041759967803955, "sampling/sampling_logp_difference/mean": 0.014572784304618835, "step": 724 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 664.0, "completions/max_terminated_length": 664.0, "completions/mean_length": 388.15625, "completions/mean_terminated_length": 388.15625, "completions/min_length": 242.0, "completions/min_terminated_length": 242.0, "entropy": 0.3312032427638769, "epoch": 0.3881156316916488, "frac_reward_zero_std": 0.75, "grad_norm": 0.003970189485698938, "learning_rate": 1e-05, "loss": -0.0488, "num_tokens": 15084394.0, "reward": 0.96875, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.96875, "rewards/accuracy_reward/std": 0.1767766922712326, "sampling/importance_sampling_ratio/max": 1.610827922821045, "sampling/importance_sampling_ratio/mean": 1.0001180171966553, "sampling/importance_sampling_ratio/min": 0.484560489654541, "sampling/sampling_logp_difference/max": 0.724513053894043, "sampling/sampling_logp_difference/mean": 0.011727024801075459, "step": 725 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 649.0, "completions/mean_length": 364.125, "completions/mean_terminated_length": 351.0967712402344, "completions/min_length": 188.0, "completions/min_terminated_length": 188.0, "entropy": 0.36562174931168556, "epoch": 0.3886509635974304, "frac_reward_zero_std": 0.5, "grad_norm": 0.016024533659219742, "learning_rate": 1e-05, "loss": 0.0509, "num_tokens": 15099534.0, "reward": 0.8125, "reward_std": 0.249358132481575, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.591576099395752, "sampling/importance_sampling_ratio/mean": 1.0002087354660034, "sampling/importance_sampling_ratio/min": 0.6780989170074463, "sampling/sampling_logp_difference/max": 0.4647247791290283, "sampling/sampling_logp_difference/mean": 0.013160875998437405, "step": 726 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 7.566586282337084e-05, "clip_ratio/low_min": 7.566586282337084e-05, "clip_ratio/region_mean": 7.566586282337084e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 641.0, "completions/max_terminated_length": 641.0, "completions/mean_length": 385.84375, "completions/mean_terminated_length": 385.84375, "completions/min_length": 187.0, "completions/min_terminated_length": 187.0, "entropy": 0.38310887292027473, "epoch": 0.389186295503212, "frac_reward_zero_std": 0.5, "grad_norm": 0.017197776585817337, "learning_rate": 1e-05, "loss": 0.0102, "num_tokens": 15115641.0, "reward": 0.78125, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.641198992729187, "sampling/importance_sampling_ratio/mean": 0.9994255304336548, "sampling/importance_sampling_ratio/min": 0.6325100064277649, "sampling/sampling_logp_difference/max": 0.49542713165283203, "sampling/sampling_logp_difference/mean": 0.013108815997838974, "step": 727 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00014766686945222318, "clip_ratio/low_min": 0.00014766686945222318, "clip_ratio/region_mean": 0.00014766686945222318, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 664.0, "completions/mean_length": 474.6875, "completions/mean_terminated_length": 465.2257995605469, "completions/min_length": 270.0, "completions/min_terminated_length": 270.0, "entropy": 0.30199407041072845, "epoch": 0.3897216274089936, "frac_reward_zero_std": 0.75, "grad_norm": 0.009411138482391834, "learning_rate": 1e-05, "loss": -0.013, "num_tokens": 15134671.0, "reward": 0.875, "reward_std": 0.13363061845302582, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 1.4857972860336304, "sampling/importance_sampling_ratio/mean": 0.9998932480812073, "sampling/importance_sampling_ratio/min": 0.6065194010734558, "sampling/sampling_logp_difference/max": 0.5000185966491699, "sampling/sampling_logp_difference/mean": 0.010988927446305752, "step": 728 }, { "clip_ratio/high_max": 5.658669033437036e-05, "clip_ratio/high_mean": 5.658669033437036e-05, "clip_ratio/low_mean": 0.00033626535878283903, "clip_ratio/low_min": 0.00033626535878283903, "clip_ratio/region_mean": 0.0003928520491172094, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 743.0, "completions/mean_length": 549.3125, "completions/mean_terminated_length": 488.0799865722656, "completions/min_length": 239.0, "completions/min_terminated_length": 239.0, "entropy": 0.4881167598068714, "epoch": 0.39025695931477516, "frac_reward_zero_std": 0.0, "grad_norm": 0.010733065195381641, "learning_rate": 1e-05, "loss": 0.0184, "num_tokens": 15156265.0, "reward": 0.4375, "reward_std": 0.48503684997558594, "rewards/accuracy_reward/mean": 0.4375, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.5695544481277466, "sampling/importance_sampling_ratio/mean": 0.9994789958000183, "sampling/importance_sampling_ratio/min": 0.599615216255188, "sampling/sampling_logp_difference/max": 0.5114671587944031, "sampling/sampling_logp_difference/mean": 0.01568576507270336, "step": 729 }, { "clip_ratio/high_max": 8.620689914096147e-05, "clip_ratio/high_mean": 8.620689914096147e-05, "clip_ratio/low_mean": 7.122506940504536e-05, "clip_ratio/low_min": 7.122506940504536e-05, "clip_ratio/region_mean": 0.00015743196854600683, "completions/clipped_ratio": 0.0, "completions/max_length": 712.0, "completions/max_terminated_length": 712.0, "completions/mean_length": 388.40625, "completions/mean_terminated_length": 388.40625, "completions/min_length": 224.0, "completions/min_terminated_length": 224.0, "entropy": 0.3228759430348873, "epoch": 0.39079229122055675, "frac_reward_zero_std": 0.75, "grad_norm": 0.005625385325402021, "learning_rate": 1e-05, "loss": 0.0241, "num_tokens": 15172470.0, "reward": 0.9375, "reward_std": 0.1157275140285492, "rewards/accuracy_reward/mean": 0.9375, "rewards/accuracy_reward/std": 0.24593468010425568, "sampling/importance_sampling_ratio/max": 1.4138686656951904, "sampling/importance_sampling_ratio/mean": 1.0001715421676636, "sampling/importance_sampling_ratio/min": 0.5930461287498474, "sampling/sampling_logp_difference/max": 0.5224831104278564, "sampling/sampling_logp_difference/mean": 0.011669726110994816, "step": 730 }, { "clip_ratio/high_max": 8.890469325706363e-05, "clip_ratio/high_mean": 8.890469325706363e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 8.890469325706363e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 596.0, "completions/max_terminated_length": 596.0, "completions/mean_length": 396.4375, "completions/mean_terminated_length": 396.4375, "completions/min_length": 209.0, "completions/min_terminated_length": 209.0, "entropy": 0.3301063999533653, "epoch": 0.39132762312633834, "frac_reward_zero_std": 0.75, "grad_norm": 0.0034774274099618196, "learning_rate": 1e-05, "loss": 0.0083, "num_tokens": 15188868.0, "reward": 0.96875, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.96875, "rewards/accuracy_reward/std": 0.1767766922712326, "sampling/importance_sampling_ratio/max": 1.457810878753662, "sampling/importance_sampling_ratio/mean": 1.000235915184021, "sampling/importance_sampling_ratio/min": 0.5868493318557739, "sampling/sampling_logp_difference/max": 0.532987117767334, "sampling/sampling_logp_difference/mean": 0.012261013500392437, "step": 731 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00015230859571602196, "clip_ratio/low_min": 0.00015230859571602196, "clip_ratio/region_mean": 0.00015230859571602196, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 736.0, "completions/mean_length": 473.09375, "completions/mean_terminated_length": 430.96429443359375, "completions/min_length": 199.0, "completions/min_terminated_length": 199.0, "entropy": 0.45796767994761467, "epoch": 0.39186295503211993, "frac_reward_zero_std": 0.5, "grad_norm": 0.014501573517918587, "learning_rate": 1e-05, "loss": 0.0022, "num_tokens": 15207639.0, "reward": 0.53125, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.4648370742797852, "sampling/importance_sampling_ratio/mean": 1.0001122951507568, "sampling/importance_sampling_ratio/min": 0.6425051689147949, "sampling/sampling_logp_difference/max": 0.442380428314209, "sampling/sampling_logp_difference/mean": 0.014262106269598007, "step": 732 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 596.0, "completions/max_terminated_length": 596.0, "completions/mean_length": 386.9375, "completions/mean_terminated_length": 386.9375, "completions/min_length": 135.0, "completions/min_terminated_length": 135.0, "entropy": 0.37069207057356834, "epoch": 0.3923982869379015, "frac_reward_zero_std": 0.25, "grad_norm": 0.01740621030330658, "learning_rate": 1e-05, "loss": 0.0277, "num_tokens": 15223837.0, "reward": 0.59375, "reward_std": 0.38816186785697937, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.5206844806671143, "sampling/importance_sampling_ratio/mean": 1.0001450777053833, "sampling/importance_sampling_ratio/min": 0.5678588151931763, "sampling/sampling_logp_difference/max": 0.5658824443817139, "sampling/sampling_logp_difference/mean": 0.013012561947107315, "step": 733 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 644.0, "completions/max_terminated_length": 644.0, "completions/mean_length": 410.90625, "completions/mean_terminated_length": 410.90625, "completions/min_length": 152.0, "completions/min_terminated_length": 152.0, "entropy": 0.3397655766457319, "epoch": 0.3929336188436831, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": -0.0291, "num_tokens": 15240210.0, "reward": 0.875, "reward_std": 0.13363061845302582, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 1.3749804496765137, "sampling/importance_sampling_ratio/mean": 1.0000972747802734, "sampling/importance_sampling_ratio/min": 0.6986644268035889, "sampling/sampling_logp_difference/max": 0.3585846424102783, "sampling/sampling_logp_difference/mean": 0.011467776261270046, "step": 734 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 7.114399340935051e-05, "clip_ratio/low_min": 7.114399340935051e-05, "clip_ratio/region_mean": 7.114399340935051e-05, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 751.0, "completions/mean_length": 493.875, "completions/mean_terminated_length": 465.5172424316406, "completions/min_length": 222.0, "completions/min_terminated_length": 222.0, "entropy": 0.4104369692504406, "epoch": 0.39346895074946464, "frac_reward_zero_std": 0.5, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": -0.0472, "num_tokens": 15260166.0, "reward": 0.71875, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.417318344116211, "sampling/importance_sampling_ratio/mean": 0.9997690916061401, "sampling/importance_sampling_ratio/min": 0.578517496585846, "sampling/sampling_logp_difference/max": 0.5472865104675293, "sampling/sampling_logp_difference/mean": 0.013015126809477806, "step": 735 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00014819206262473017, "clip_ratio/low_min": 0.00014819206262473017, "clip_ratio/region_mean": 0.00014819206262473017, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 665.0, "completions/mean_length": 425.15625, "completions/mean_terminated_length": 414.0967712402344, "completions/min_length": 214.0, "completions/min_terminated_length": 214.0, "entropy": 0.3317549005150795, "epoch": 0.39400428265524623, "frac_reward_zero_std": 0.25, "grad_norm": 0.013737526722252369, "learning_rate": 1e-05, "loss": -0.0222, "num_tokens": 15277451.0, "reward": 0.53125, "reward_std": 0.35564959049224854, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.5341882705688477, "sampling/importance_sampling_ratio/mean": 1.0003247261047363, "sampling/importance_sampling_ratio/min": 0.7361739277839661, "sampling/sampling_logp_difference/max": 0.42800140380859375, "sampling/sampling_logp_difference/mean": 0.011700326576828957, "step": 736 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 7.552870374638587e-05, "clip_ratio/low_min": 7.552870374638587e-05, "clip_ratio/region_mean": 7.552870374638587e-05, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 736.0, "completions/mean_length": 417.53125, "completions/mean_terminated_length": 406.2257995605469, "completions/min_length": 181.0, "completions/min_terminated_length": 181.0, "entropy": 0.4715849831700325, "epoch": 0.3945396145610278, "frac_reward_zero_std": 0.5, "grad_norm": 0.02280639484524727, "learning_rate": 1e-05, "loss": 0.0249, "num_tokens": 15294276.0, "reward": 0.75, "reward_std": 0.26726123690605164, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.6859614849090576, "sampling/importance_sampling_ratio/mean": 1.000662922859192, "sampling/importance_sampling_ratio/min": 0.619315505027771, "sampling/sampling_logp_difference/max": 0.5223360061645508, "sampling/sampling_logp_difference/mean": 0.014331185258924961, "step": 737 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.819421832915395e-05, "clip_ratio/low_min": 6.819421832915395e-05, "clip_ratio/region_mean": 6.819421832915395e-05, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 759.0, "completions/mean_length": 418.625, "completions/mean_terminated_length": 395.3333435058594, "completions/min_length": 168.0, "completions/min_terminated_length": 168.0, "entropy": 0.3739110343158245, "epoch": 0.3950749464668094, "frac_reward_zero_std": 0.75, "grad_norm": 0.00491142924875021, "learning_rate": 1e-05, "loss": 0.0652, "num_tokens": 15311112.0, "reward": 0.71875, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.4075877666473389, "sampling/importance_sampling_ratio/mean": 0.9998874068260193, "sampling/importance_sampling_ratio/min": 0.2507037818431854, "sampling/sampling_logp_difference/max": 1.3834831714630127, "sampling/sampling_logp_difference/mean": 0.01229048240929842, "step": 738 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 563.0, "completions/max_terminated_length": 563.0, "completions/mean_length": 374.0, "completions/mean_terminated_length": 374.0, "completions/min_length": 236.0, "completions/min_terminated_length": 236.0, "entropy": 0.33726467192173004, "epoch": 0.395610278372591, "frac_reward_zero_std": 0.5, "grad_norm": 0.017529767006635666, "learning_rate": 1e-05, "loss": -0.006, "num_tokens": 15327032.0, "reward": 0.90625, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.628007173538208, "sampling/importance_sampling_ratio/mean": 1.0002076625823975, "sampling/importance_sampling_ratio/min": 0.4969369173049927, "sampling/sampling_logp_difference/max": 0.6992921829223633, "sampling/sampling_logp_difference/mean": 0.012618626467883587, "step": 739 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.577866977546364e-05, "clip_ratio/low_min": 5.577866977546364e-05, "clip_ratio/region_mean": 5.577866977546364e-05, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 722.0, "completions/mean_length": 496.8125, "completions/mean_terminated_length": 468.75860595703125, "completions/min_length": 251.0, "completions/min_terminated_length": 251.0, "entropy": 0.40716972202062607, "epoch": 0.3961456102783726, "frac_reward_zero_std": 0.25, "grad_norm": 0.031049173325300217, "learning_rate": 1e-05, "loss": 0.0438, "num_tokens": 15346810.0, "reward": 0.625, "reward_std": 0.292504221200943, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.3598735332489014, "sampling/importance_sampling_ratio/mean": 1.0001327991485596, "sampling/importance_sampling_ratio/min": 0.6353383660316467, "sampling/sampling_logp_difference/max": 0.4535975456237793, "sampling/sampling_logp_difference/mean": 0.013288475573062897, "step": 740 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00011844850814668462, "clip_ratio/low_min": 0.00011844850814668462, "clip_ratio/region_mean": 0.00011844850814668462, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 711.0, "completions/mean_length": 464.21875, "completions/mean_terminated_length": 432.7930908203125, "completions/min_length": 234.0, "completions/min_terminated_length": 234.0, "entropy": 0.4071118049323559, "epoch": 0.3966809421841542, "frac_reward_zero_std": 0.25, "grad_norm": 0.03432130441069603, "learning_rate": 1e-05, "loss": 0.0626, "num_tokens": 15366033.0, "reward": 0.625, "reward_std": 0.3104073107242584, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.4833122491836548, "sampling/importance_sampling_ratio/mean": 1.000255823135376, "sampling/importance_sampling_ratio/min": 0.6084573268890381, "sampling/sampling_logp_difference/max": 0.496828556060791, "sampling/sampling_logp_difference/mean": 0.013487053103744984, "step": 741 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.545696694753133e-05, "clip_ratio/low_min": 5.545696694753133e-05, "clip_ratio/region_mean": 5.545696694753133e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 703.0, "completions/max_terminated_length": 703.0, "completions/mean_length": 465.34375, "completions/mean_terminated_length": 465.34375, "completions/min_length": 183.0, "completions/min_terminated_length": 183.0, "entropy": 0.2904929928481579, "epoch": 0.39721627408993576, "frac_reward_zero_std": 0.5, "grad_norm": 0.036213818937540054, "learning_rate": 1e-05, "loss": 0.0622, "num_tokens": 15384164.0, "reward": 0.875, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 1.4013344049453735, "sampling/importance_sampling_ratio/mean": 1.000303030014038, "sampling/importance_sampling_ratio/min": 0.6378161907196045, "sampling/sampling_logp_difference/max": 0.4497051239013672, "sampling/sampling_logp_difference/mean": 0.010160043835639954, "step": 742 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 674.0, "completions/max_terminated_length": 674.0, "completions/mean_length": 421.6875, "completions/mean_terminated_length": 421.6875, "completions/min_length": 237.0, "completions/min_terminated_length": 237.0, "entropy": 0.2593152653425932, "epoch": 0.39775160599571735, "frac_reward_zero_std": 0.5, "grad_norm": 0.007283906452357769, "learning_rate": 1e-05, "loss": -0.0511, "num_tokens": 15401266.0, "reward": 0.78125, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.4111536741256714, "sampling/importance_sampling_ratio/mean": 0.9999212622642517, "sampling/importance_sampling_ratio/min": 0.5152935981750488, "sampling/sampling_logp_difference/max": 0.6630184650421143, "sampling/sampling_logp_difference/mean": 0.009853643365204334, "step": 743 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 692.0, "completions/max_terminated_length": 692.0, "completions/mean_length": 370.90625, "completions/mean_terminated_length": 370.90625, "completions/min_length": 171.0, "completions/min_terminated_length": 171.0, "entropy": 0.3156023509800434, "epoch": 0.39828693790149894, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": -0.018, "num_tokens": 15416503.0, "reward": 0.9375, "reward_std": 0.1157275140285492, "rewards/accuracy_reward/mean": 0.9375, "rewards/accuracy_reward/std": 0.24593468010425568, "sampling/importance_sampling_ratio/max": 1.3144745826721191, "sampling/importance_sampling_ratio/mean": 1.000388503074646, "sampling/importance_sampling_ratio/min": 0.6407133936882019, "sampling/sampling_logp_difference/max": 0.4451730251312256, "sampling/sampling_logp_difference/mean": 0.011980942450463772, "step": 744 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00015106660430319607, "clip_ratio/low_min": 0.00015106660430319607, "clip_ratio/region_mean": 0.00015106660430319607, "completions/clipped_ratio": 0.0, "completions/max_length": 568.0, "completions/max_terminated_length": 568.0, "completions/mean_length": 384.375, "completions/mean_terminated_length": 384.375, "completions/min_length": 198.0, "completions/min_terminated_length": 198.0, "entropy": 0.28047935478389263, "epoch": 0.3988222698072805, "frac_reward_zero_std": 0.25, "grad_norm": 0.023195985704660416, "learning_rate": 1e-05, "loss": 0.0368, "num_tokens": 15432419.0, "reward": 0.75, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.4278185367584229, "sampling/importance_sampling_ratio/mean": 1.0004290342330933, "sampling/importance_sampling_ratio/min": 0.638532280921936, "sampling/sampling_logp_difference/max": 0.44858312606811523, "sampling/sampling_logp_difference/mean": 0.010663620196282864, "step": 745 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 586.0, "completions/max_terminated_length": 586.0, "completions/mean_length": 361.5625, "completions/mean_terminated_length": 361.5625, "completions/min_length": 169.0, "completions/min_terminated_length": 169.0, "entropy": 0.331597950309515, "epoch": 0.3993576017130621, "frac_reward_zero_std": 0.5, "grad_norm": 0.00515330582857132, "learning_rate": 1e-05, "loss": 0.0095, "num_tokens": 15447477.0, "reward": 0.9375, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.9375, "rewards/accuracy_reward/std": 0.24593468010425568, "sampling/importance_sampling_ratio/max": 1.3726600408554077, "sampling/importance_sampling_ratio/mean": 1.000138282775879, "sampling/importance_sampling_ratio/min": 0.7044076323509216, "sampling/sampling_logp_difference/max": 0.35039806365966797, "sampling/sampling_logp_difference/mean": 0.011656593531370163, "step": 746 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 8.191350207198411e-05, "clip_ratio/low_min": 8.191350207198411e-05, "clip_ratio/region_mean": 8.191350207198411e-05, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 673.0, "completions/mean_length": 418.5625, "completions/mean_terminated_length": 382.4137878417969, "completions/min_length": 184.0, "completions/min_terminated_length": 184.0, "entropy": 0.4204455576837063, "epoch": 0.3998929336188437, "frac_reward_zero_std": 0.5, "grad_norm": 0.010350142605602741, "learning_rate": 1e-05, "loss": 0.0967, "num_tokens": 15464631.0, "reward": 0.84375, "reward_std": 0.22201895713806152, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.7982512712478638, "sampling/importance_sampling_ratio/mean": 1.0005135536193848, "sampling/importance_sampling_ratio/min": 0.3905912935733795, "sampling/sampling_logp_difference/max": 0.9400935173034668, "sampling/sampling_logp_difference/mean": 0.013929876498878002, "step": 747 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.823144212830812e-05, "clip_ratio/low_min": 6.823144212830812e-05, "clip_ratio/region_mean": 6.823144212830812e-05, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 563.0, "completions/mean_length": 395.3125, "completions/mean_terminated_length": 383.2903137207031, "completions/min_length": 248.0, "completions/min_terminated_length": 248.0, "entropy": 0.30783896148204803, "epoch": 0.4004282655246253, "frac_reward_zero_std": 0.5, "grad_norm": 0.016324976459145546, "learning_rate": 1e-05, "loss": 0.0313, "num_tokens": 15480793.0, "reward": 0.8125, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.6437350511550903, "sampling/importance_sampling_ratio/mean": 0.9998080730438232, "sampling/importance_sampling_ratio/min": 0.6735740900039673, "sampling/sampling_logp_difference/max": 0.49697113037109375, "sampling/sampling_logp_difference/mean": 0.011754120700061321, "step": 748 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 694.0, "completions/mean_length": 501.875, "completions/mean_terminated_length": 463.857177734375, "completions/min_length": 192.0, "completions/min_terminated_length": 192.0, "entropy": 0.41706426069140434, "epoch": 0.4009635974304068, "frac_reward_zero_std": 0.5, "grad_norm": 0.006824329495429993, "learning_rate": 1e-05, "loss": 0.0041, "num_tokens": 15501141.0, "reward": 0.65625, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.5268117189407349, "sampling/importance_sampling_ratio/mean": 1.0005855560302734, "sampling/importance_sampling_ratio/min": 0.6713352799415588, "sampling/sampling_logp_difference/max": 0.42318177223205566, "sampling/sampling_logp_difference/mean": 0.013412107713520527, "step": 749 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 7.01065655448474e-05, "clip_ratio/low_min": 7.01065655448474e-05, "clip_ratio/region_mean": 7.01065655448474e-05, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 685.0, "completions/mean_length": 457.4375, "completions/mean_terminated_length": 413.0714416503906, "completions/min_length": 223.0, "completions/min_terminated_length": 223.0, "entropy": 0.29431847855448723, "epoch": 0.4014989293361884, "frac_reward_zero_std": 0.5, "grad_norm": 0.009221713058650494, "learning_rate": 1e-05, "loss": 0.0008, "num_tokens": 15519403.0, "reward": 0.59375, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.5546016693115234, "sampling/importance_sampling_ratio/mean": 1.0002026557922363, "sampling/importance_sampling_ratio/min": 0.6902912259101868, "sampling/sampling_logp_difference/max": 0.4412193298339844, "sampling/sampling_logp_difference/mean": 0.010672741569578648, "step": 750 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 8.555784006603062e-05, "clip_ratio/low_min": 8.555784006603062e-05, "clip_ratio/region_mean": 8.555784006603062e-05, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 742.0, "completions/mean_length": 422.40625, "completions/mean_terminated_length": 411.258056640625, "completions/min_length": 284.0, "completions/min_terminated_length": 284.0, "entropy": 0.3825416825711727, "epoch": 0.40203426124197, "frac_reward_zero_std": 0.25, "grad_norm": 0.020340463146567345, "learning_rate": 1e-05, "loss": 0.0204, "num_tokens": 15536352.0, "reward": 0.625, "reward_std": 0.3104073107242584, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.5766780376434326, "sampling/importance_sampling_ratio/mean": 1.0000587701797485, "sampling/importance_sampling_ratio/min": 0.6515544652938843, "sampling/sampling_logp_difference/max": 0.4553201198577881, "sampling/sampling_logp_difference/mean": 0.014280795119702816, "step": 751 }, { "clip_ratio/high_max": 7.427213131450117e-05, "clip_ratio/high_mean": 7.427213131450117e-05, "clip_ratio/low_mean": 0.0002810747319017537, "clip_ratio/low_min": 0.0002810747319017537, "clip_ratio/region_mean": 0.00035534686321625486, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 710.0, "completions/mean_length": 427.21875, "completions/mean_terminated_length": 416.2257995605469, "completions/min_length": 160.0, "completions/min_terminated_length": 160.0, "entropy": 0.34784119576215744, "epoch": 0.4025695931477516, "frac_reward_zero_std": 0.25, "grad_norm": 0.016556397080421448, "learning_rate": 1e-05, "loss": -0.0522, "num_tokens": 15553831.0, "reward": 0.65625, "reward_std": 0.3377465009689331, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.4274108409881592, "sampling/importance_sampling_ratio/mean": 1.0001895427703857, "sampling/importance_sampling_ratio/min": 0.2745887339115143, "sampling/sampling_logp_difference/max": 1.292480707168579, "sampling/sampling_logp_difference/mean": 0.012321370653808117, "step": 752 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 653.0, "completions/mean_length": 450.40625, "completions/mean_terminated_length": 417.5517272949219, "completions/min_length": 211.0, "completions/min_terminated_length": 211.0, "entropy": 0.34333814308047295, "epoch": 0.4031049250535332, "frac_reward_zero_std": 0.5, "grad_norm": 0.019724832847714424, "learning_rate": 1e-05, "loss": 0.0748, "num_tokens": 15571956.0, "reward": 0.8125, "reward_std": 0.249358132481575, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.4537216424942017, "sampling/importance_sampling_ratio/mean": 1.000090479850769, "sampling/importance_sampling_ratio/min": 0.4299718141555786, "sampling/sampling_logp_difference/max": 0.8440356254577637, "sampling/sampling_logp_difference/mean": 0.01199343428015709, "step": 753 }, { "clip_ratio/high_max": 7.822277984814718e-05, "clip_ratio/high_mean": 7.822277984814718e-05, "clip_ratio/low_mean": 0.0003144714064546861, "clip_ratio/low_min": 0.0003144714064546861, "clip_ratio/region_mean": 0.00039269418630283326, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 761.0, "completions/mean_length": 513.8125, "completions/mean_terminated_length": 477.5000305175781, "completions/min_length": 250.0, "completions/min_terminated_length": 250.0, "entropy": 0.4728994816541672, "epoch": 0.40364025695931477, "frac_reward_zero_std": 0.0, "grad_norm": 0.014718258753418922, "learning_rate": 1e-05, "loss": 0.0584, "num_tokens": 15592542.0, "reward": 0.5, "reward_std": 0.4355512857437134, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5080004930496216, "sampling/importance_sampling_ratio/max": 1.545873761177063, "sampling/importance_sampling_ratio/mean": 0.9999228119850159, "sampling/importance_sampling_ratio/min": 0.6337277889251709, "sampling/sampling_logp_difference/max": 0.45613574981689453, "sampling/sampling_logp_difference/mean": 0.015861382707953453, "step": 754 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00018514944531489164, "clip_ratio/low_min": 0.00018514944531489164, "clip_ratio/region_mean": 0.00018514944531489164, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 468.0, "completions/mean_length": 436.0625, "completions/mean_terminated_length": 325.41668701171875, "completions/min_length": 158.0, "completions/min_terminated_length": 158.0, "entropy": 0.4311661459505558, "epoch": 0.40417558886509636, "frac_reward_zero_std": 0.5, "grad_norm": 0.01695352792739868, "learning_rate": 1e-05, "loss": 0.0771, "num_tokens": 15609712.0, "reward": 0.40625, "reward_std": 0.22201895713806152, "rewards/accuracy_reward/mean": 0.40625, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.3483442068099976, "sampling/importance_sampling_ratio/mean": 0.9998624324798584, "sampling/importance_sampling_ratio/min": 0.7367224097251892, "sampling/sampling_logp_difference/max": 0.3055441379547119, "sampling/sampling_logp_difference/mean": 0.013261335901916027, "step": 755 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 702.0, "completions/mean_length": 471.9375, "completions/mean_terminated_length": 452.20001220703125, "completions/min_length": 294.0, "completions/min_terminated_length": 294.0, "entropy": 0.42721887305378914, "epoch": 0.40471092077087795, "frac_reward_zero_std": 0.5, "grad_norm": 0.008616846054792404, "learning_rate": 1e-05, "loss": 0.0055, "num_tokens": 15629094.0, "reward": 0.625, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0001200437545776, "sampling/importance_sampling_ratio/min": 0.6894100308418274, "sampling/sampling_logp_difference/max": 0.7046043872833252, "sampling/sampling_logp_difference/mean": 0.013750944286584854, "step": 756 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 603.0, "completions/max_terminated_length": 603.0, "completions/mean_length": 404.65625, "completions/mean_terminated_length": 404.65625, "completions/min_length": 242.0, "completions/min_terminated_length": 242.0, "entropy": 0.30308121256530285, "epoch": 0.40524625267665954, "frac_reward_zero_std": 0.5, "grad_norm": 0.0057063233107328415, "learning_rate": 1e-05, "loss": -0.0013, "num_tokens": 15645803.0, "reward": 0.90625, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.39963960647583, "sampling/importance_sampling_ratio/mean": 1.0008162260055542, "sampling/importance_sampling_ratio/min": 0.7349751591682434, "sampling/sampling_logp_difference/max": 0.3362147808074951, "sampling/sampling_logp_difference/mean": 0.011067013256251812, "step": 757 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.558236782439053e-05, "clip_ratio/low_min": 6.558236782439053e-05, "clip_ratio/region_mean": 6.558236782439053e-05, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 579.0, "completions/mean_length": 386.3125, "completions/mean_terminated_length": 374.0, "completions/min_length": 197.0, "completions/min_terminated_length": 197.0, "entropy": 0.3252818398177624, "epoch": 0.4057815845824411, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0137, "num_tokens": 15661629.0, "reward": 0.6875, "reward_std": 0.1157275140285492, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.3640577793121338, "sampling/importance_sampling_ratio/mean": 1.0000487565994263, "sampling/importance_sampling_ratio/min": 0.7221677303314209, "sampling/sampling_logp_difference/max": 0.3254978656768799, "sampling/sampling_logp_difference/mean": 0.011510277166962624, "step": 758 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 740.0, "completions/mean_length": 478.8125, "completions/mean_terminated_length": 448.89654541015625, "completions/min_length": 243.0, "completions/min_terminated_length": 243.0, "entropy": 0.3173256739974022, "epoch": 0.4063169164882227, "frac_reward_zero_std": 0.25, "grad_norm": 0.021319890394806862, "learning_rate": 1e-05, "loss": -0.0141, "num_tokens": 15681375.0, "reward": 0.6875, "reward_std": 0.3745020925998688, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.4550646543502808, "sampling/importance_sampling_ratio/mean": 0.9998688697814941, "sampling/importance_sampling_ratio/min": 0.6559000611305237, "sampling/sampling_logp_difference/max": 0.4217468500137329, "sampling/sampling_logp_difference/mean": 0.011187671683728695, "step": 759 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.627784023294225e-05, "clip_ratio/low_min": 6.627784023294225e-05, "clip_ratio/region_mean": 6.627784023294225e-05, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 765.0, "completions/mean_length": 542.59375, "completions/mean_terminated_length": 510.39288330078125, "completions/min_length": 304.0, "completions/min_terminated_length": 304.0, "entropy": 0.3270022738724947, "epoch": 0.4068522483940043, "frac_reward_zero_std": 0.25, "grad_norm": 0.00968860648572445, "learning_rate": 1e-05, "loss": 0.043, "num_tokens": 15702282.0, "reward": 0.53125, "reward_std": 0.35564959049224854, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.40469229221344, "sampling/importance_sampling_ratio/mean": 1.00003981590271, "sampling/importance_sampling_ratio/min": 0.5530411601066589, "sampling/sampling_logp_difference/max": 0.592322826385498, "sampling/sampling_logp_difference/mean": 0.01104754488915205, "step": 760 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 617.0, "completions/mean_length": 456.75, "completions/mean_terminated_length": 436.0000305175781, "completions/min_length": 110.0, "completions/min_terminated_length": 110.0, "entropy": 0.30460382997989655, "epoch": 0.4073875802997859, "frac_reward_zero_std": 0.5, "grad_norm": 0.026091277599334717, "learning_rate": 1e-05, "loss": 0.0078, "num_tokens": 15720786.0, "reward": 0.8125, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.7048051357269287, "sampling/importance_sampling_ratio/mean": 1.0001603364944458, "sampling/importance_sampling_ratio/min": 0.5779150724411011, "sampling/sampling_logp_difference/max": 0.5483283996582031, "sampling/sampling_logp_difference/mean": 0.011420255526900291, "step": 761 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 735.0, "completions/mean_length": 535.78125, "completions/mean_terminated_length": 492.77777099609375, "completions/min_length": 175.0, "completions/min_terminated_length": 175.0, "entropy": 0.32452690601348877, "epoch": 0.4079229122055675, "frac_reward_zero_std": 0.25, "grad_norm": 0.0213970635086298, "learning_rate": 1e-05, "loss": 0.0421, "num_tokens": 15741219.0, "reward": 0.6875, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.9757503271102905, "sampling/importance_sampling_ratio/mean": 1.0006074905395508, "sampling/importance_sampling_ratio/min": 0.42033618688583374, "sampling/sampling_logp_difference/max": 0.8667004108428955, "sampling/sampling_logp_difference/mean": 0.011018093675374985, "step": 762 }, { "clip_ratio/high_max": 6.449948705267161e-05, "clip_ratio/high_mean": 6.449948705267161e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 6.449948705267161e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 647.0, "completions/max_terminated_length": 647.0, "completions/mean_length": 412.3125, "completions/mean_terminated_length": 412.3125, "completions/min_length": 191.0, "completions/min_terminated_length": 191.0, "entropy": 0.3106103427708149, "epoch": 0.408458244111349, "frac_reward_zero_std": 0.5, "grad_norm": 0.00918335746973753, "learning_rate": 1e-05, "loss": 0.0212, "num_tokens": 15758221.0, "reward": 0.75, "reward_std": 0.2314550280570984, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.4989961385726929, "sampling/importance_sampling_ratio/mean": 0.999900758266449, "sampling/importance_sampling_ratio/min": 0.6642783284187317, "sampling/sampling_logp_difference/max": 0.4090540409088135, "sampling/sampling_logp_difference/mean": 0.011544481851160526, "step": 763 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 654.0, "completions/mean_length": 480.0, "completions/mean_terminated_length": 426.6666564941406, "completions/min_length": 247.0, "completions/min_terminated_length": 247.0, "entropy": 0.39441378600895405, "epoch": 0.4089935760171306, "frac_reward_zero_std": 0.5, "grad_norm": 0.02085895650088787, "learning_rate": 1e-05, "loss": 0.0797, "num_tokens": 15777245.0, "reward": 0.75, "reward_std": 0.2587745785713196, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.474548578262329, "sampling/importance_sampling_ratio/mean": 0.9995726346969604, "sampling/importance_sampling_ratio/min": 0.6380072236061096, "sampling/sampling_logp_difference/max": 0.4494056701660156, "sampling/sampling_logp_difference/mean": 0.013304685242474079, "step": 764 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 693.0, "completions/mean_length": 376.96875, "completions/mean_terminated_length": 364.3548278808594, "completions/min_length": 195.0, "completions/min_terminated_length": 195.0, "entropy": 0.33709727600216866, "epoch": 0.4095289079229122, "frac_reward_zero_std": 0.75, "grad_norm": 0.014469758607447147, "learning_rate": 1e-05, "loss": 0.0022, "num_tokens": 15793364.0, "reward": 0.875, "reward_std": 0.13363061845302582, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 1.4618092775344849, "sampling/importance_sampling_ratio/mean": 1.000153660774231, "sampling/importance_sampling_ratio/min": 0.6918150782585144, "sampling/sampling_logp_difference/max": 0.37967491149902344, "sampling/sampling_logp_difference/mean": 0.011879919096827507, "step": 765 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 9.697439963929355e-05, "clip_ratio/low_min": 9.697439963929355e-05, "clip_ratio/region_mean": 9.697439963929355e-05, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 694.0, "completions/mean_length": 531.96875, "completions/mean_terminated_length": 488.25927734375, "completions/min_length": 285.0, "completions/min_terminated_length": 285.0, "entropy": 0.305579349398613, "epoch": 0.4100642398286938, "frac_reward_zero_std": 0.25, "grad_norm": 0.011317568831145763, "learning_rate": 1e-05, "loss": 0.07, "num_tokens": 15815179.0, "reward": 0.78125, "reward_std": 0.3471629321575165, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.4919722080230713, "sampling/importance_sampling_ratio/mean": 1.0002607107162476, "sampling/importance_sampling_ratio/min": 0.6147410869598389, "sampling/sampling_logp_difference/max": 0.4865540862083435, "sampling/sampling_logp_difference/mean": 0.011284776963293552, "step": 766 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.217028228798881e-05, "clip_ratio/low_min": 5.217028228798881e-05, "clip_ratio/region_mean": 5.217028228798881e-05, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 654.0, "completions/mean_length": 491.625, "completions/mean_terminated_length": 473.20001220703125, "completions/min_length": 188.0, "completions/min_terminated_length": 188.0, "entropy": 0.3676033690571785, "epoch": 0.41059957173447537, "frac_reward_zero_std": 0.25, "grad_norm": 0.011424537748098373, "learning_rate": 1e-05, "loss": 0.0137, "num_tokens": 15834815.0, "reward": 0.6875, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.4128429889678955, "sampling/importance_sampling_ratio/mean": 1.0006253719329834, "sampling/importance_sampling_ratio/min": 0.6142329573631287, "sampling/sampling_logp_difference/max": 0.4873809814453125, "sampling/sampling_logp_difference/mean": 0.01319599524140358, "step": 767 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 707.0, "completions/max_terminated_length": 707.0, "completions/mean_length": 376.75, "completions/mean_terminated_length": 376.75, "completions/min_length": 140.0, "completions/min_terminated_length": 140.0, "entropy": 0.3216324485838413, "epoch": 0.41113490364025695, "frac_reward_zero_std": 0.75, "grad_norm": 0.00625705998390913, "learning_rate": 1e-05, "loss": -0.0106, "num_tokens": 15850623.0, "reward": 0.96875, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.96875, "rewards/accuracy_reward/std": 0.1767766922712326, "sampling/importance_sampling_ratio/max": 1.4294602870941162, "sampling/importance_sampling_ratio/mean": 0.9997236132621765, "sampling/importance_sampling_ratio/min": 0.6545900702476501, "sampling/sampling_logp_difference/max": 0.42374610900878906, "sampling/sampling_logp_difference/mean": 0.011525521986186504, "step": 768 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 671.0, "completions/mean_length": 400.28125, "completions/mean_terminated_length": 388.4193420410156, "completions/min_length": 157.0, "completions/min_terminated_length": 157.0, "entropy": 0.32398680597543716, "epoch": 0.41167023554603854, "frac_reward_zero_std": 0.5, "grad_norm": 0.02308107353746891, "learning_rate": 1e-05, "loss": 0.0472, "num_tokens": 15866608.0, "reward": 0.78125, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.541273593902588, "sampling/importance_sampling_ratio/mean": 1.0000206232070923, "sampling/importance_sampling_ratio/min": 0.43122372031211853, "sampling/sampling_logp_difference/max": 0.8411282300949097, "sampling/sampling_logp_difference/mean": 0.011534423567354679, "step": 769 }, { "clip_ratio/high_max": 5.9326055634301156e-05, "clip_ratio/high_mean": 5.9326055634301156e-05, "clip_ratio/low_mean": 0.00020713766207336448, "clip_ratio/low_min": 0.00020713766207336448, "clip_ratio/region_mean": 0.00026646371770766564, "completions/clipped_ratio": 0.0, "completions/max_length": 766.0, "completions/max_terminated_length": 766.0, "completions/mean_length": 438.59375, "completions/mean_terminated_length": 438.59375, "completions/min_length": 224.0, "completions/min_terminated_length": 224.0, "entropy": 0.4114176370203495, "epoch": 0.41220556745182013, "frac_reward_zero_std": 0.25, "grad_norm": 0.03290877863764763, "learning_rate": 1e-05, "loss": -0.0332, "num_tokens": 15884699.0, "reward": 0.65625, "reward_std": 0.3198433816432953, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.539549708366394, "sampling/importance_sampling_ratio/mean": 0.9995622634887695, "sampling/importance_sampling_ratio/min": 0.38092902302742004, "sampling/sampling_logp_difference/max": 0.9651422500610352, "sampling/sampling_logp_difference/mean": 0.013242307119071484, "step": 770 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 753.0, "completions/mean_length": 437.09375, "completions/mean_terminated_length": 415.0333557128906, "completions/min_length": 186.0, "completions/min_terminated_length": 186.0, "entropy": 0.3268381766974926, "epoch": 0.4127408993576017, "frac_reward_zero_std": 0.5, "grad_norm": 0.010581866838037968, "learning_rate": 1e-05, "loss": -0.0477, "num_tokens": 15903150.0, "reward": 0.5625, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.5415682792663574, "sampling/importance_sampling_ratio/mean": 0.9997550845146179, "sampling/importance_sampling_ratio/min": 0.7361999154090881, "sampling/sampling_logp_difference/max": 0.43280029296875, "sampling/sampling_logp_difference/mean": 0.01146602351218462, "step": 771 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00017565976304467767, "clip_ratio/low_min": 0.00017565976304467767, "clip_ratio/region_mean": 0.00017565976304467767, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 765.0, "completions/mean_length": 414.25, "completions/mean_terminated_length": 402.83868408203125, "completions/min_length": 151.0, "completions/min_terminated_length": 151.0, "entropy": 0.29830528050661087, "epoch": 0.4132762312633833, "frac_reward_zero_std": 0.5, "grad_norm": 0.0038797277957201004, "learning_rate": 1e-05, "loss": -0.027, "num_tokens": 15920742.0, "reward": 0.65625, "reward_std": 0.22201895713806152, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.3394511938095093, "sampling/importance_sampling_ratio/mean": 1.000575065612793, "sampling/importance_sampling_ratio/min": 0.6225912570953369, "sampling/sampling_logp_difference/max": 0.4738650321960449, "sampling/sampling_logp_difference/mean": 0.011488824151456356, "step": 772 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 665.0, "completions/max_terminated_length": 665.0, "completions/mean_length": 365.375, "completions/mean_terminated_length": 365.375, "completions/min_length": 161.0, "completions/min_terminated_length": 161.0, "entropy": 0.34343548864126205, "epoch": 0.4138115631691649, "frac_reward_zero_std": 0.75, "grad_norm": 0.004457968287169933, "learning_rate": 1e-05, "loss": 0.0274, "num_tokens": 15936346.0, "reward": 0.9375, "reward_std": 0.1157275140285492, "rewards/accuracy_reward/mean": 0.9375, "rewards/accuracy_reward/std": 0.24593468010425568, "sampling/importance_sampling_ratio/max": 1.5066066980361938, "sampling/importance_sampling_ratio/mean": 0.9996554255485535, "sampling/importance_sampling_ratio/min": 0.6448359489440918, "sampling/sampling_logp_difference/max": 0.43875932693481445, "sampling/sampling_logp_difference/mean": 0.012474888935685158, "step": 773 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 8.538251131540164e-05, "clip_ratio/low_min": 8.538251131540164e-05, "clip_ratio/region_mean": 8.538251131540164e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 650.0, "completions/max_terminated_length": 650.0, "completions/mean_length": 365.375, "completions/mean_terminated_length": 365.375, "completions/min_length": 165.0, "completions/min_terminated_length": 165.0, "entropy": 0.3918217346072197, "epoch": 0.4143468950749465, "frac_reward_zero_std": 0.5, "grad_norm": 0.01897948421537876, "learning_rate": 1e-05, "loss": -0.0121, "num_tokens": 15951646.0, "reward": 0.6875, "reward_std": 0.2587745785713196, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.5322632789611816, "sampling/importance_sampling_ratio/mean": 0.9997677206993103, "sampling/importance_sampling_ratio/min": 0.4341105818748474, "sampling/sampling_logp_difference/max": 0.8344559669494629, "sampling/sampling_logp_difference/mean": 0.014131310395896435, "step": 774 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 642.0, "completions/mean_length": 433.3125, "completions/mean_terminated_length": 398.6896667480469, "completions/min_length": 142.0, "completions/min_terminated_length": 142.0, "entropy": 0.4856613911688328, "epoch": 0.4148822269807281, "frac_reward_zero_std": 0.25, "grad_norm": 0.0216378141194582, "learning_rate": 1e-05, "loss": -0.0469, "num_tokens": 15969920.0, "reward": 0.53125, "reward_std": 0.3471629321575165, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.3805956840515137, "sampling/importance_sampling_ratio/mean": 1.0000503063201904, "sampling/importance_sampling_ratio/min": 0.6940709948539734, "sampling/sampling_logp_difference/max": 0.36518096923828125, "sampling/sampling_logp_difference/mean": 0.015132423490285873, "step": 775 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.975143358227797e-05, "clip_ratio/low_min": 5.975143358227797e-05, "clip_ratio/region_mean": 5.975143358227797e-05, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 733.0, "completions/mean_length": 530.5, "completions/mean_terminated_length": 475.69232177734375, "completions/min_length": 256.0, "completions/min_terminated_length": 256.0, "entropy": 0.3959058076143265, "epoch": 0.41541755888650966, "frac_reward_zero_std": 0.25, "grad_norm": 0.017261290922760963, "learning_rate": 1e-05, "loss": 0.0329, "num_tokens": 15990808.0, "reward": 0.59375, "reward_std": 0.3966485261917114, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.4253036975860596, "sampling/importance_sampling_ratio/mean": 1.00001859664917, "sampling/importance_sampling_ratio/min": 0.6032301783561707, "sampling/sampling_logp_difference/max": 0.5054564476013184, "sampling/sampling_logp_difference/mean": 0.013239888474345207, "step": 776 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 492.0, "completions/max_terminated_length": 492.0, "completions/mean_length": 375.65625, "completions/mean_terminated_length": 375.65625, "completions/min_length": 215.0, "completions/min_terminated_length": 215.0, "entropy": 0.37662187218666077, "epoch": 0.4159528907922912, "frac_reward_zero_std": 0.25, "grad_norm": 0.02308535948395729, "learning_rate": 1e-05, "loss": 0.0741, "num_tokens": 16006133.0, "reward": 0.625, "reward_std": 0.3924051821231842, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.4557102918624878, "sampling/importance_sampling_ratio/mean": 0.999885618686676, "sampling/importance_sampling_ratio/min": 0.5420427322387695, "sampling/sampling_logp_difference/max": 0.6124105453491211, "sampling/sampling_logp_difference/mean": 0.013395894318819046, "step": 777 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00026709352459874935, "clip_ratio/low_min": 0.00026709352459874935, "clip_ratio/region_mean": 0.00026709352459874935, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 742.0, "completions/mean_length": 515.875, "completions/mean_terminated_length": 469.1851806640625, "completions/min_length": 191.0, "completions/min_terminated_length": 191.0, "entropy": 0.4640541672706604, "epoch": 0.4164882226980728, "frac_reward_zero_std": 0.25, "grad_norm": 0.021424466744065285, "learning_rate": 1e-05, "loss": 0.0808, "num_tokens": 16026473.0, "reward": 0.40625, "reward_std": 0.3198433816432953, "rewards/accuracy_reward/mean": 0.40625, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.5008653402328491, "sampling/importance_sampling_ratio/mean": 1.0001349449157715, "sampling/importance_sampling_ratio/min": 0.6110365390777588, "sampling/sampling_logp_difference/max": 0.4925985336303711, "sampling/sampling_logp_difference/mean": 0.014934550039470196, "step": 778 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 9.834775846684352e-05, "clip_ratio/low_min": 9.834775846684352e-05, "clip_ratio/region_mean": 9.834775846684352e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 581.0, "completions/max_terminated_length": 581.0, "completions/mean_length": 351.78125, "completions/mean_terminated_length": 351.78125, "completions/min_length": 228.0, "completions/min_terminated_length": 228.0, "entropy": 0.4013065788894892, "epoch": 0.4170235546038544, "frac_reward_zero_std": 0.0, "grad_norm": 0.011998678557574749, "learning_rate": 1e-05, "loss": 0.0307, "num_tokens": 16041354.0, "reward": 0.71875, "reward_std": 0.4218915104866028, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.5025750398635864, "sampling/importance_sampling_ratio/mean": 1.0004608631134033, "sampling/importance_sampling_ratio/min": 0.6503225564956665, "sampling/sampling_logp_difference/max": 0.4302867650985718, "sampling/sampling_logp_difference/mean": 0.013495108112692833, "step": 779 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 765.0, "completions/max_terminated_length": 765.0, "completions/mean_length": 380.625, "completions/mean_terminated_length": 380.625, "completions/min_length": 205.0, "completions/min_terminated_length": 205.0, "entropy": 0.2916193753480911, "epoch": 0.41755888650963596, "frac_reward_zero_std": 0.75, "grad_norm": 0.021915575489401817, "learning_rate": 1e-05, "loss": 0.0358, "num_tokens": 16056734.0, "reward": 0.78125, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.4232560396194458, "sampling/importance_sampling_ratio/mean": 1.0000630617141724, "sampling/importance_sampling_ratio/min": 0.7022632360458374, "sampling/sampling_logp_difference/max": 0.35344696044921875, "sampling/sampling_logp_difference/mean": 0.010835547931492329, "step": 780 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 720.0, "completions/mean_length": 458.15625, "completions/mean_terminated_length": 426.10345458984375, "completions/min_length": 205.0, "completions/min_terminated_length": 205.0, "entropy": 0.3535945191979408, "epoch": 0.41809421841541755, "frac_reward_zero_std": 0.5, "grad_norm": 0.0052179982885718346, "learning_rate": 1e-05, "loss": -0.0112, "num_tokens": 16075339.0, "reward": 0.6875, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.6308897733688354, "sampling/importance_sampling_ratio/mean": 0.9999247193336487, "sampling/importance_sampling_ratio/min": 0.716689944267273, "sampling/sampling_logp_difference/max": 0.48912572860717773, "sampling/sampling_logp_difference/mean": 0.012360227294266224, "step": 781 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 748.0, "completions/mean_length": 460.53125, "completions/mean_terminated_length": 428.72412109375, "completions/min_length": 213.0, "completions/min_terminated_length": 213.0, "entropy": 0.3926909938454628, "epoch": 0.41862955032119914, "frac_reward_zero_std": 0.25, "grad_norm": 0.022419588640332222, "learning_rate": 1e-05, "loss": 0.0481, "num_tokens": 16093644.0, "reward": 0.6875, "reward_std": 0.3745020925998688, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.4974701404571533, "sampling/importance_sampling_ratio/mean": 0.9997333288192749, "sampling/importance_sampling_ratio/min": 0.6168071627616882, "sampling/sampling_logp_difference/max": 0.483198881149292, "sampling/sampling_logp_difference/mean": 0.013082112185657024, "step": 782 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 705.0, "completions/mean_length": 450.125, "completions/mean_terminated_length": 439.8709411621094, "completions/min_length": 128.0, "completions/min_terminated_length": 128.0, "entropy": 0.3309701308608055, "epoch": 0.41916488222698073, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0183, "num_tokens": 16112088.0, "reward": 0.9375, "reward_std": 0.1157275140285492, "rewards/accuracy_reward/mean": 0.9375, "rewards/accuracy_reward/std": 0.24593468010425568, "sampling/importance_sampling_ratio/max": 1.4328806400299072, "sampling/importance_sampling_ratio/mean": 1.0003955364227295, "sampling/importance_sampling_ratio/min": 0.6102308630943298, "sampling/sampling_logp_difference/max": 0.49391794204711914, "sampling/sampling_logp_difference/mean": 0.012100967578589916, "step": 783 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00014480544632533565, "clip_ratio/low_min": 0.00014480544632533565, "clip_ratio/region_mean": 0.00014480544632533565, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 634.0, "completions/mean_length": 432.03125, "completions/mean_terminated_length": 354.5, "completions/min_length": 227.0, "completions/min_terminated_length": 227.0, "entropy": 0.4955846704542637, "epoch": 0.4197002141327623, "frac_reward_zero_std": 0.0, "grad_norm": 0.03532204404473305, "learning_rate": 1e-05, "loss": 0.1908, "num_tokens": 16130065.0, "reward": 0.71875, "reward_std": 0.4218915104866028, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.7675895690917969, "sampling/importance_sampling_ratio/mean": 1.0000088214874268, "sampling/importance_sampling_ratio/min": 0.6349325180053711, "sampling/sampling_logp_difference/max": 0.5696167945861816, "sampling/sampling_logp_difference/mean": 0.01592966914176941, "step": 784 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00015654352318961173, "clip_ratio/low_min": 0.00015654352318961173, "clip_ratio/region_mean": 0.00015654352318961173, "completions/clipped_ratio": 0.0, "completions/max_length": 520.0, "completions/max_terminated_length": 520.0, "completions/mean_length": 363.3125, "completions/mean_terminated_length": 363.3125, "completions/min_length": 166.0, "completions/min_terminated_length": 166.0, "entropy": 0.33013381995260715, "epoch": 0.4202355460385439, "frac_reward_zero_std": 0.75, "grad_norm": 0.008598454296588898, "learning_rate": 1e-05, "loss": -0.0179, "num_tokens": 16145811.0, "reward": 0.9375, "reward_std": 0.1157275140285492, "rewards/accuracy_reward/mean": 0.9375, "rewards/accuracy_reward/std": 0.24593468010425568, "sampling/importance_sampling_ratio/max": 1.4315712451934814, "sampling/importance_sampling_ratio/mean": 1.0002683401107788, "sampling/importance_sampling_ratio/min": 0.7052102088928223, "sampling/sampling_logp_difference/max": 0.35877251625061035, "sampling/sampling_logp_difference/mean": 0.012170505709946156, "step": 785 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 609.0, "completions/max_terminated_length": 609.0, "completions/mean_length": 397.5625, "completions/mean_terminated_length": 397.5625, "completions/min_length": 248.0, "completions/min_terminated_length": 248.0, "entropy": 0.3210936710238457, "epoch": 0.4207708779443255, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 16162293.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.35883629322052, "sampling/importance_sampling_ratio/mean": 0.999962568283081, "sampling/importance_sampling_ratio/min": 0.7317671179771423, "sampling/sampling_logp_difference/max": 0.3122929334640503, "sampling/sampling_logp_difference/mean": 0.011883299797773361, "step": 786 }, { "clip_ratio/high_max": 8.934953802963719e-05, "clip_ratio/high_mean": 8.934953802963719e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 8.934953802963719e-05, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 730.0, "completions/mean_length": 370.5625, "completions/mean_terminated_length": 357.7419128417969, "completions/min_length": 209.0, "completions/min_terminated_length": 209.0, "entropy": 0.3573397099971771, "epoch": 0.4213062098501071, "frac_reward_zero_std": 0.5, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0943, "num_tokens": 16177567.0, "reward": 0.84375, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.4566199779510498, "sampling/importance_sampling_ratio/mean": 0.9997912049293518, "sampling/importance_sampling_ratio/min": 0.7044780254364014, "sampling/sampling_logp_difference/max": 0.37611865997314453, "sampling/sampling_logp_difference/mean": 0.012392064556479454, "step": 787 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 557.0, "completions/mean_length": 381.15625, "completions/mean_terminated_length": 368.6773986816406, "completions/min_length": 174.0, "completions/min_terminated_length": 174.0, "entropy": 0.34688152372837067, "epoch": 0.42184154175588867, "frac_reward_zero_std": 0.25, "grad_norm": 0.01136648841202259, "learning_rate": 1e-05, "loss": 0.0767, "num_tokens": 16193516.0, "reward": 0.84375, "reward_std": 0.3198433816432953, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.5200353860855103, "sampling/importance_sampling_ratio/mean": 0.9998480677604675, "sampling/importance_sampling_ratio/min": 0.7074972987174988, "sampling/sampling_logp_difference/max": 0.4187335968017578, "sampling/sampling_logp_difference/mean": 0.012763068079948425, "step": 788 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0002685058643692173, "clip_ratio/low_min": 0.0002685058643692173, "clip_ratio/region_mean": 0.0002685058643692173, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 607.0, "completions/mean_length": 432.40625, "completions/mean_terminated_length": 397.6896667480469, "completions/min_length": 173.0, "completions/min_terminated_length": 173.0, "entropy": 0.4548214189708233, "epoch": 0.42237687366167026, "frac_reward_zero_std": 0.25, "grad_norm": 0.006262308452278376, "learning_rate": 1e-05, "loss": 0.0085, "num_tokens": 16211633.0, "reward": 0.375, "reward_std": 0.2925041913986206, "rewards/accuracy_reward/mean": 0.375, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.5168640613555908, "sampling/importance_sampling_ratio/mean": 1.0004241466522217, "sampling/importance_sampling_ratio/min": 0.6472005248069763, "sampling/sampling_logp_difference/max": 0.43509912490844727, "sampling/sampling_logp_difference/mean": 0.015050098299980164, "step": 789 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.160670454846695e-05, "clip_ratio/low_min": 6.160670454846695e-05, "clip_ratio/region_mean": 6.160670454846695e-05, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 752.0, "completions/mean_length": 422.3125, "completions/mean_terminated_length": 358.2962951660156, "completions/min_length": 143.0, "completions/min_terminated_length": 143.0, "entropy": 0.41861243173480034, "epoch": 0.4229122055674518, "frac_reward_zero_std": 0.5, "grad_norm": 0.03162233531475067, "learning_rate": 1e-05, "loss": 0.0262, "num_tokens": 16228923.0, "reward": 0.75, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.6621532440185547, "sampling/importance_sampling_ratio/mean": 1.0001317262649536, "sampling/importance_sampling_ratio/min": 0.692443311214447, "sampling/sampling_logp_difference/max": 0.5081138610839844, "sampling/sampling_logp_difference/mean": 0.014441372826695442, "step": 790 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.849321314599365e-05, "clip_ratio/low_min": 5.849321314599365e-05, "clip_ratio/region_mean": 5.849321314599365e-05, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 684.0, "completions/mean_length": 479.625, "completions/mean_terminated_length": 449.7930908203125, "completions/min_length": 197.0, "completions/min_terminated_length": 197.0, "entropy": 0.41815035790205, "epoch": 0.4234475374732334, "frac_reward_zero_std": 0.5, "grad_norm": 0.01708553172647953, "learning_rate": 1e-05, "loss": -0.0637, "num_tokens": 16248471.0, "reward": 0.71875, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.3802968263626099, "sampling/importance_sampling_ratio/mean": 1.0000295639038086, "sampling/importance_sampling_ratio/min": 0.6753427386283875, "sampling/sampling_logp_difference/max": 0.3925349712371826, "sampling/sampling_logp_difference/mean": 0.013590806163847446, "step": 791 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00011505786824272946, "clip_ratio/low_min": 0.00011505786824272946, "clip_ratio/region_mean": 0.00011505786824272946, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 640.0, "completions/mean_length": 432.9375, "completions/mean_terminated_length": 410.60003662109375, "completions/min_length": 254.0, "completions/min_terminated_length": 254.0, "entropy": 0.407636433839798, "epoch": 0.42398286937901497, "frac_reward_zero_std": 0.25, "grad_norm": 0.007025112397968769, "learning_rate": 1e-05, "loss": 0.0684, "num_tokens": 16265901.0, "reward": 0.78125, "reward_std": 0.3377465009689331, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.4147555828094482, "sampling/importance_sampling_ratio/mean": 1.0002249479293823, "sampling/importance_sampling_ratio/min": 0.7398054003715515, "sampling/sampling_logp_difference/max": 0.346956729888916, "sampling/sampling_logp_difference/mean": 0.013874690048396587, "step": 792 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 584.0, "completions/max_terminated_length": 584.0, "completions/mean_length": 344.4375, "completions/mean_terminated_length": 344.4375, "completions/min_length": 182.0, "completions/min_terminated_length": 182.0, "entropy": 0.26546439342200756, "epoch": 0.42451820128479656, "frac_reward_zero_std": 0.75, "grad_norm": 0.004844346549361944, "learning_rate": 1e-05, "loss": -0.0328, "num_tokens": 16280411.0, "reward": 0.96875, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.96875, "rewards/accuracy_reward/std": 0.1767766922712326, "sampling/importance_sampling_ratio/max": 1.4307457208633423, "sampling/importance_sampling_ratio/mean": 1.0005249977111816, "sampling/importance_sampling_ratio/min": 0.6472405195236206, "sampling/sampling_logp_difference/max": 0.43503737449645996, "sampling/sampling_logp_difference/mean": 0.010227811522781849, "step": 793 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 445.0, "completions/max_terminated_length": 445.0, "completions/mean_length": 296.84375, "completions/mean_terminated_length": 296.84375, "completions/min_length": 192.0, "completions/min_terminated_length": 192.0, "entropy": 0.29051942750811577, "epoch": 0.42505353319057815, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": -0.0149, "num_tokens": 16293462.0, "reward": 0.96875, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.96875, "rewards/accuracy_reward/std": 0.1767766922712326, "sampling/importance_sampling_ratio/max": 1.3314151763916016, "sampling/importance_sampling_ratio/mean": 0.9995257258415222, "sampling/importance_sampling_ratio/min": 0.6889463067054749, "sampling/sampling_logp_difference/max": 0.3725919723510742, "sampling/sampling_logp_difference/mean": 0.010967799462378025, "step": 794 }, { "clip_ratio/high_max": 7.126567652449012e-05, "clip_ratio/high_mean": 7.126567652449012e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 7.126567652449012e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 561.0, "completions/max_terminated_length": 561.0, "completions/mean_length": 359.6875, "completions/mean_terminated_length": 359.6875, "completions/min_length": 238.0, "completions/min_terminated_length": 238.0, "entropy": 0.3822439946234226, "epoch": 0.42558886509635974, "frac_reward_zero_std": 0.0, "grad_norm": 0.013289808295667171, "learning_rate": 1e-05, "loss": -0.0491, "num_tokens": 16308764.0, "reward": 0.65625, "reward_std": 0.4355708956718445, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.758047342300415, "sampling/importance_sampling_ratio/mean": 1.0002474784851074, "sampling/importance_sampling_ratio/min": 0.6286794543266296, "sampling/sampling_logp_difference/max": 0.5642037391662598, "sampling/sampling_logp_difference/mean": 0.014009960927069187, "step": 795 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 665.0, "completions/mean_length": 399.9375, "completions/mean_terminated_length": 388.06451416015625, "completions/min_length": 118.0, "completions/min_terminated_length": 118.0, "entropy": 0.33113632909953594, "epoch": 0.4261241970021413, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": -0.0067, "num_tokens": 16325202.0, "reward": 0.8125, "reward_std": 0.1157275140285492, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.5197477340698242, "sampling/importance_sampling_ratio/mean": 0.9992085099220276, "sampling/importance_sampling_ratio/min": 0.7125191688537598, "sampling/sampling_logp_difference/max": 0.4185444116592407, "sampling/sampling_logp_difference/mean": 0.01204575877636671, "step": 796 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 610.0, "completions/mean_length": 353.65625, "completions/mean_terminated_length": 340.2903137207031, "completions/min_length": 115.0, "completions/min_terminated_length": 115.0, "entropy": 0.370099201798439, "epoch": 0.4266595289079229, "frac_reward_zero_std": 0.5, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0091, "num_tokens": 16340007.0, "reward": 0.84375, "reward_std": 0.22201895713806152, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.5763697624206543, "sampling/importance_sampling_ratio/mean": 0.9999203085899353, "sampling/importance_sampling_ratio/min": 0.7019596099853516, "sampling/sampling_logp_difference/max": 0.4551246166229248, "sampling/sampling_logp_difference/mean": 0.012925157323479652, "step": 797 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00025131575239356607, "clip_ratio/low_min": 0.00025131575239356607, "clip_ratio/region_mean": 0.00025131575239356607, "completions/clipped_ratio": 0.0, "completions/max_length": 600.0, "completions/max_terminated_length": 600.0, "completions/mean_length": 384.28125, "completions/mean_terminated_length": 384.28125, "completions/min_length": 187.0, "completions/min_terminated_length": 187.0, "entropy": 0.37455303221940994, "epoch": 0.4271948608137045, "frac_reward_zero_std": 0.0, "grad_norm": 0.008700243197381496, "learning_rate": 1e-05, "loss": -0.0144, "num_tokens": 16356024.0, "reward": 0.65625, "reward_std": 0.3808925747871399, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.3682764768600464, "sampling/importance_sampling_ratio/mean": 0.9998393058776855, "sampling/importance_sampling_ratio/min": 0.7417562007904053, "sampling/sampling_logp_difference/max": 0.3135519027709961, "sampling/sampling_logp_difference/mean": 0.012510780245065689, "step": 798 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0001096491250791587, "clip_ratio/low_min": 0.0001096491250791587, "clip_ratio/region_mean": 0.0001096491250791587, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 693.0, "completions/mean_length": 490.6875, "completions/mean_terminated_length": 472.20001220703125, "completions/min_length": 259.0, "completions/min_terminated_length": 259.0, "entropy": 0.33674395829439163, "epoch": 0.4277301927194861, "frac_reward_zero_std": 0.5, "grad_norm": 0.015756376087665558, "learning_rate": 1e-05, "loss": 0.0313, "num_tokens": 16375702.0, "reward": 0.84375, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.4444522857666016, "sampling/importance_sampling_ratio/mean": 0.9997134804725647, "sampling/importance_sampling_ratio/min": 0.6950921416282654, "sampling/sampling_logp_difference/max": 0.36773014068603516, "sampling/sampling_logp_difference/mean": 0.012204481288790703, "step": 799 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 547.0, "completions/mean_length": 375.78125, "completions/mean_terminated_length": 363.1290283203125, "completions/min_length": 240.0, "completions/min_terminated_length": 240.0, "entropy": 0.3607005886733532, "epoch": 0.4282655246252677, "frac_reward_zero_std": 0.0, "grad_norm": 0.02751448005437851, "learning_rate": 1e-05, "loss": 0.0044, "num_tokens": 16391791.0, "reward": 0.71875, "reward_std": 0.4397946000099182, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.455745816230774, "sampling/importance_sampling_ratio/mean": 0.999874472618103, "sampling/importance_sampling_ratio/min": 0.6866875290870667, "sampling/sampling_logp_difference/max": 0.37587594985961914, "sampling/sampling_logp_difference/mean": 0.012193486094474792, "step": 800 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 654.0, "completions/mean_length": 423.78125, "completions/mean_terminated_length": 412.6773986816406, "completions/min_length": 213.0, "completions/min_terminated_length": 213.0, "entropy": 0.3118172138929367, "epoch": 0.42880085653104927, "frac_reward_zero_std": 0.25, "grad_norm": 0.007535708136856556, "learning_rate": 1e-05, "loss": 0.0731, "num_tokens": 16408984.0, "reward": 0.71875, "reward_std": 0.2651650309562683, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.418203592300415, "sampling/importance_sampling_ratio/mean": 1.000199317932129, "sampling/importance_sampling_ratio/min": 0.6029441952705383, "sampling/sampling_logp_difference/max": 0.5059306621551514, "sampling/sampling_logp_difference/mean": 0.011605561710894108, "step": 801 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00014550624473486096, "clip_ratio/low_min": 0.00014550624473486096, "clip_ratio/region_mean": 0.00014550624473486096, "completions/clipped_ratio": 0.0, "completions/max_length": 703.0, "completions/max_terminated_length": 703.0, "completions/mean_length": 442.1875, "completions/mean_terminated_length": 442.1875, "completions/min_length": 195.0, "completions/min_terminated_length": 195.0, "entropy": 0.36222807317972183, "epoch": 0.42933618843683086, "frac_reward_zero_std": 0.25, "grad_norm": 0.005875537637621164, "learning_rate": 1e-05, "loss": 0.067, "num_tokens": 16427414.0, "reward": 0.625, "reward_std": 0.3104073107242584, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.5121146440505981, "sampling/importance_sampling_ratio/mean": 1.000623106956482, "sampling/importance_sampling_ratio/min": 0.6826663017272949, "sampling/sampling_logp_difference/max": 0.4135091304779053, "sampling/sampling_logp_difference/mean": 0.012216384522616863, "step": 802 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 711.0, "completions/mean_length": 437.34375, "completions/mean_terminated_length": 403.137939453125, "completions/min_length": 212.0, "completions/min_terminated_length": 212.0, "entropy": 0.33751334995031357, "epoch": 0.42987152034261245, "frac_reward_zero_std": 0.5, "grad_norm": 0.012162872590124607, "learning_rate": 1e-05, "loss": 0.069, "num_tokens": 16445105.0, "reward": 0.8125, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.4444760084152222, "sampling/importance_sampling_ratio/mean": 1.0000667572021484, "sampling/importance_sampling_ratio/min": 0.7367410063743591, "sampling/sampling_logp_difference/max": 0.36774659156799316, "sampling/sampling_logp_difference/mean": 0.011439654976129532, "step": 803 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.171700468054041e-05, "clip_ratio/low_min": 5.171700468054041e-05, "clip_ratio/region_mean": 5.171700468054041e-05, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 759.0, "completions/mean_length": 425.1875, "completions/mean_terminated_length": 346.0769348144531, "completions/min_length": 227.0, "completions/min_terminated_length": 227.0, "entropy": 0.3881307542324066, "epoch": 0.430406852248394, "frac_reward_zero_std": 0.5, "grad_norm": 0.007201123051345348, "learning_rate": 1e-05, "loss": 0.0101, "num_tokens": 16462727.0, "reward": 0.75, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.395493507385254, "sampling/importance_sampling_ratio/mean": 1.0002069473266602, "sampling/importance_sampling_ratio/min": 0.7024174332618713, "sampling/sampling_logp_difference/max": 0.3532273769378662, "sampling/sampling_logp_difference/mean": 0.013050492852926254, "step": 804 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 596.0, "completions/mean_length": 374.90625, "completions/mean_terminated_length": 362.2257995605469, "completions/min_length": 213.0, "completions/min_terminated_length": 213.0, "entropy": 0.3300849013030529, "epoch": 0.43094218415417557, "frac_reward_zero_std": 0.25, "grad_norm": 0.029466472566127777, "learning_rate": 1e-05, "loss": 0.0395, "num_tokens": 16478060.0, "reward": 0.8125, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.3755929470062256, "sampling/importance_sampling_ratio/mean": 0.9999516606330872, "sampling/importance_sampling_ratio/min": 0.7007484436035156, "sampling/sampling_logp_difference/max": 0.3556063175201416, "sampling/sampling_logp_difference/mean": 0.01132247131317854, "step": 805 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 668.0, "completions/max_terminated_length": 668.0, "completions/mean_length": 334.40625, "completions/mean_terminated_length": 334.40625, "completions/min_length": 203.0, "completions/min_terminated_length": 203.0, "entropy": 0.36720867455005646, "epoch": 0.43147751605995716, "frac_reward_zero_std": 0.5, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0017, "num_tokens": 16492113.0, "reward": 0.90625, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.440938949584961, "sampling/importance_sampling_ratio/mean": 0.9999064207077026, "sampling/importance_sampling_ratio/min": 0.6023098230361938, "sampling/sampling_logp_difference/max": 0.5069832801818848, "sampling/sampling_logp_difference/mean": 0.012224563397467136, "step": 806 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 764.0, "completions/max_terminated_length": 764.0, "completions/mean_length": 396.8125, "completions/mean_terminated_length": 396.8125, "completions/min_length": 128.0, "completions/min_terminated_length": 128.0, "entropy": 0.3561983220279217, "epoch": 0.43201284796573874, "frac_reward_zero_std": 0.75, "grad_norm": 0.003809213638305664, "learning_rate": 1e-05, "loss": -0.0303, "num_tokens": 16508411.0, "reward": 0.71875, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.6199498176574707, "sampling/importance_sampling_ratio/mean": 0.9994025826454163, "sampling/importance_sampling_ratio/min": 0.009181640110909939, "sampling/sampling_logp_difference/max": 4.690549373626709, "sampling/sampling_logp_difference/mean": 0.013144487515091896, "step": 807 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 8.526603050995618e-05, "clip_ratio/low_min": 8.526603050995618e-05, "clip_ratio/region_mean": 8.526603050995618e-05, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 503.0, "completions/mean_length": 360.1875, "completions/mean_terminated_length": 333.0000305175781, "completions/min_length": 137.0, "completions/min_terminated_length": 137.0, "entropy": 0.4667294919490814, "epoch": 0.43254817987152033, "frac_reward_zero_std": 0.25, "grad_norm": 0.006308328825980425, "learning_rate": 1e-05, "loss": 0.0873, "num_tokens": 16523657.0, "reward": 0.84375, "reward_std": 0.3061639666557312, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.435367226600647, "sampling/importance_sampling_ratio/mean": 0.999539315700531, "sampling/importance_sampling_ratio/min": 0.6807668805122375, "sampling/sampling_logp_difference/max": 0.3845353126525879, "sampling/sampling_logp_difference/mean": 0.015322824008762836, "step": 808 }, { "clip_ratio/high_max": 7.851758709875867e-05, "clip_ratio/high_mean": 7.851758709875867e-05, "clip_ratio/low_mean": 5.8823530707741156e-05, "clip_ratio/low_min": 5.8823530707741156e-05, "clip_ratio/region_mean": 0.00013734111780649982, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 713.0, "completions/mean_length": 459.125, "completions/mean_terminated_length": 438.5333557128906, "completions/min_length": 254.0, "completions/min_terminated_length": 254.0, "entropy": 0.34118691086769104, "epoch": 0.4330835117773019, "frac_reward_zero_std": 0.0, "grad_norm": 0.013867797330021858, "learning_rate": 1e-05, "loss": 0.0408, "num_tokens": 16542181.0, "reward": 0.75, "reward_std": 0.4492306709289551, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.4446160793304443, "sampling/importance_sampling_ratio/mean": 0.9995635747909546, "sampling/importance_sampling_ratio/min": 0.6446837782859802, "sampling/sampling_logp_difference/max": 0.438995361328125, "sampling/sampling_logp_difference/mean": 0.011937168426811695, "step": 809 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 727.0, "completions/mean_length": 403.96875, "completions/mean_terminated_length": 392.2257995605469, "completions/min_length": 214.0, "completions/min_terminated_length": 214.0, "entropy": 0.328909769654274, "epoch": 0.4336188436830835, "frac_reward_zero_std": 0.25, "grad_norm": 0.03904910385608673, "learning_rate": 1e-05, "loss": -0.0073, "num_tokens": 16558692.0, "reward": 0.84375, "reward_std": 0.3198433816432953, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.417783498764038, "sampling/importance_sampling_ratio/mean": 0.9995311498641968, "sampling/importance_sampling_ratio/min": 0.4941488802433014, "sampling/sampling_logp_difference/max": 0.704918384552002, "sampling/sampling_logp_difference/mean": 0.011775069870054722, "step": 810 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 728.0, "completions/mean_length": 439.34375, "completions/mean_terminated_length": 417.433349609375, "completions/min_length": 186.0, "completions/min_terminated_length": 186.0, "entropy": 0.3284086771309376, "epoch": 0.4341541755888651, "frac_reward_zero_std": 0.5, "grad_norm": 0.006778714247047901, "learning_rate": 1e-05, "loss": 0.0118, "num_tokens": 16576383.0, "reward": 0.71875, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.3113287687301636, "sampling/importance_sampling_ratio/mean": 1.0002131462097168, "sampling/importance_sampling_ratio/min": 0.7099736928939819, "sampling/sampling_logp_difference/max": 0.3425273895263672, "sampling/sampling_logp_difference/mean": 0.010776509530842304, "step": 811 }, { "clip_ratio/high_max": 6.634819874307141e-05, "clip_ratio/high_mean": 6.634819874307141e-05, "clip_ratio/low_mean": 6.456611299654469e-05, "clip_ratio/low_min": 6.456611299654469e-05, "clip_ratio/region_mean": 0.0001309143117396161, "completions/clipped_ratio": 0.0, "completions/max_length": 658.0, "completions/max_terminated_length": 658.0, "completions/mean_length": 458.8125, "completions/mean_terminated_length": 458.8125, "completions/min_length": 244.0, "completions/min_terminated_length": 244.0, "entropy": 0.2822544202208519, "epoch": 0.4346895074946467, "frac_reward_zero_std": 0.5, "grad_norm": 0.016039883717894554, "learning_rate": 1e-05, "loss": -0.0285, "num_tokens": 16594857.0, "reward": 0.75, "reward_std": 0.26726123690605164, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.3284289836883545, "sampling/importance_sampling_ratio/mean": 0.9999332427978516, "sampling/importance_sampling_ratio/min": 0.6991372108459473, "sampling/sampling_logp_difference/max": 0.3579082489013672, "sampling/sampling_logp_difference/mean": 0.01015300303697586, "step": 812 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00036391491448739544, "clip_ratio/low_min": 0.00036391491448739544, "clip_ratio/region_mean": 0.00036391491448739544, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 706.0, "completions/mean_length": 462.71875, "completions/mean_terminated_length": 406.1851806640625, "completions/min_length": 113.0, "completions/min_terminated_length": 113.0, "entropy": 0.3695691302418709, "epoch": 0.4352248394004283, "frac_reward_zero_std": 0.25, "grad_norm": 0.018846383318305016, "learning_rate": 1e-05, "loss": 0.0614, "num_tokens": 16613288.0, "reward": 0.53125, "reward_std": 0.3787454068660736, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.444919228553772, "sampling/importance_sampling_ratio/mean": 1.0001083612442017, "sampling/importance_sampling_ratio/min": 0.7263509631156921, "sampling/sampling_logp_difference/max": 0.3680534362792969, "sampling/sampling_logp_difference/mean": 0.01269935630261898, "step": 813 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 558.0, "completions/mean_length": 386.5625, "completions/mean_terminated_length": 374.258056640625, "completions/min_length": 227.0, "completions/min_terminated_length": 227.0, "entropy": 0.33550575375556946, "epoch": 0.43576017130620986, "frac_reward_zero_std": 0.0, "grad_norm": 0.023693492636084557, "learning_rate": 1e-05, "loss": 0.0422, "num_tokens": 16629266.0, "reward": 0.71875, "reward_std": 0.4534739851951599, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.420576572418213, "sampling/importance_sampling_ratio/mean": 0.9998122453689575, "sampling/importance_sampling_ratio/min": 0.5436431169509888, "sampling/sampling_logp_difference/max": 0.6094622611999512, "sampling/sampling_logp_difference/mean": 0.01205992978066206, "step": 814 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 8.229097875300795e-05, "clip_ratio/low_min": 8.229097875300795e-05, "clip_ratio/region_mean": 8.229097875300795e-05, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 757.0, "completions/mean_length": 436.09375, "completions/mean_terminated_length": 425.3870849609375, "completions/min_length": 166.0, "completions/min_terminated_length": 166.0, "entropy": 0.3688822388648987, "epoch": 0.43629550321199145, "frac_reward_zero_std": 0.5, "grad_norm": 0.017541784793138504, "learning_rate": 1e-05, "loss": 0.0057, "num_tokens": 16646789.0, "reward": 0.75, "reward_std": 0.26726123690605164, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.5963969230651855, "sampling/importance_sampling_ratio/mean": 0.9998249411582947, "sampling/importance_sampling_ratio/min": 0.706638753414154, "sampling/sampling_logp_difference/max": 0.46774911880493164, "sampling/sampling_logp_difference/mean": 0.011717635206878185, "step": 815 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0001592984117451124, "clip_ratio/low_min": 0.0001592984117451124, "clip_ratio/region_mean": 0.0001592984117451124, "completions/clipped_ratio": 0.0, "completions/max_length": 542.0, "completions/max_terminated_length": 542.0, "completions/mean_length": 379.1875, "completions/mean_terminated_length": 379.1875, "completions/min_length": 202.0, "completions/min_terminated_length": 202.0, "entropy": 0.3667549714446068, "epoch": 0.43683083511777304, "frac_reward_zero_std": 0.5, "grad_norm": 0.012002070434391499, "learning_rate": 1e-05, "loss": -0.0052, "num_tokens": 16662811.0, "reward": 0.84375, "reward_std": 0.22201895713806152, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.4121633768081665, "sampling/importance_sampling_ratio/mean": 0.9996801614761353, "sampling/importance_sampling_ratio/min": 0.6350990533828735, "sampling/sampling_logp_difference/max": 0.45397424697875977, "sampling/sampling_logp_difference/mean": 0.012512888759374619, "step": 816 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 654.0, "completions/max_terminated_length": 654.0, "completions/mean_length": 353.90625, "completions/mean_terminated_length": 353.90625, "completions/min_length": 181.0, "completions/min_terminated_length": 181.0, "entropy": 0.3619252145290375, "epoch": 0.43736616702355463, "frac_reward_zero_std": 0.5, "grad_norm": 0.017897985875606537, "learning_rate": 1e-05, "loss": -0.0003, "num_tokens": 16678032.0, "reward": 0.875, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 1.5810739994049072, "sampling/importance_sampling_ratio/mean": 0.9999575614929199, "sampling/importance_sampling_ratio/min": 0.5207005143165588, "sampling/sampling_logp_difference/max": 0.6525802612304688, "sampling/sampling_logp_difference/mean": 0.012551712803542614, "step": 817 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 673.0, "completions/mean_length": 479.03125, "completions/mean_terminated_length": 469.70965576171875, "completions/min_length": 285.0, "completions/min_terminated_length": 285.0, "entropy": 0.45719290524721146, "epoch": 0.43790149892933616, "frac_reward_zero_std": 0.5, "grad_norm": 0.011253081262111664, "learning_rate": 1e-05, "loss": -0.0216, "num_tokens": 16697849.0, "reward": 0.78125, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.4122263193130493, "sampling/importance_sampling_ratio/mean": 1.0000659227371216, "sampling/importance_sampling_ratio/min": 0.4903346002101898, "sampling/sampling_logp_difference/max": 0.7126672863960266, "sampling/sampling_logp_difference/mean": 0.014308209531009197, "step": 818 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00017018811195157468, "clip_ratio/low_min": 0.00017018811195157468, "clip_ratio/region_mean": 0.00017018811195157468, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 712.0, "completions/mean_length": 488.15625, "completions/mean_terminated_length": 436.3333435058594, "completions/min_length": 145.0, "completions/min_terminated_length": 145.0, "entropy": 0.3782646246254444, "epoch": 0.43843683083511775, "frac_reward_zero_std": 0.0, "grad_norm": 0.008200446143746376, "learning_rate": 1e-05, "loss": 0.0629, "num_tokens": 16717374.0, "reward": 0.65625, "reward_std": 0.3808925747871399, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.5117660760879517, "sampling/importance_sampling_ratio/mean": 1.000573992729187, "sampling/importance_sampling_ratio/min": 0.6699599027633667, "sampling/sampling_logp_difference/max": 0.41327857971191406, "sampling/sampling_logp_difference/mean": 0.012531363405287266, "step": 819 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 612.0, "completions/mean_length": 411.8125, "completions/mean_terminated_length": 400.32257080078125, "completions/min_length": 211.0, "completions/min_terminated_length": 211.0, "entropy": 0.33313389122486115, "epoch": 0.43897216274089934, "frac_reward_zero_std": 0.5, "grad_norm": 0.009693180210888386, "learning_rate": 1e-05, "loss": 0.0022, "num_tokens": 16734384.0, "reward": 0.65625, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.4284695386886597, "sampling/importance_sampling_ratio/mean": 0.9999052882194519, "sampling/importance_sampling_ratio/min": 0.7230563163757324, "sampling/sampling_logp_difference/max": 0.35660362243652344, "sampling/sampling_logp_difference/mean": 0.011355124413967133, "step": 820 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00025044032736332156, "clip_ratio/low_min": 0.00025044032736332156, "clip_ratio/region_mean": 0.00025044032736332156, "completions/clipped_ratio": 0.34375, "completions/max_length": 768.0, "completions/max_terminated_length": 695.0, "completions/mean_length": 586.25, "completions/mean_terminated_length": 491.0476379394531, "completions/min_length": 302.0, "completions/min_terminated_length": 302.0, "entropy": 0.5317547507584095, "epoch": 0.43950749464668093, "frac_reward_zero_std": 0.25, "grad_norm": 0.019287187606096268, "learning_rate": 1e-05, "loss": 0.1012, "num_tokens": 16758704.0, "reward": 0.21875, "reward_std": 0.3608423173427582, "rewards/accuracy_reward/mean": 0.21875, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.7616195678710938, "sampling/importance_sampling_ratio/mean": 1.0005301237106323, "sampling/importance_sampling_ratio/min": 0.6851746439933777, "sampling/sampling_logp_difference/max": 0.5662336349487305, "sampling/sampling_logp_difference/mean": 0.016038743779063225, "step": 821 }, { "clip_ratio/high_max": 0.00014140272105578333, "clip_ratio/high_mean": 0.00014140272105578333, "clip_ratio/low_mean": 6.819421832915395e-05, "clip_ratio/low_min": 6.819421832915395e-05, "clip_ratio/region_mean": 0.00020959693938493729, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 578.0, "completions/mean_length": 403.5625, "completions/mean_terminated_length": 379.2666931152344, "completions/min_length": 145.0, "completions/min_terminated_length": 145.0, "entropy": 0.3709445372223854, "epoch": 0.4400428265524625, "frac_reward_zero_std": 0.75, "grad_norm": 0.022648319602012634, "learning_rate": 1e-05, "loss": 0.0545, "num_tokens": 16775290.0, "reward": 0.90625, "reward_std": 0.1293872892856598, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.5541414022445679, "sampling/importance_sampling_ratio/mean": 0.9995556473731995, "sampling/importance_sampling_ratio/min": 0.6441525220870972, "sampling/sampling_logp_difference/max": 0.44092321395874023, "sampling/sampling_logp_difference/mean": 0.012571304105222225, "step": 822 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.000172176311025396, "clip_ratio/low_min": 0.000172176311025396, "clip_ratio/region_mean": 0.000172176311025396, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 716.0, "completions/mean_length": 482.125, "completions/mean_terminated_length": 429.1851806640625, "completions/min_length": 248.0, "completions/min_terminated_length": 248.0, "entropy": 0.4144217111170292, "epoch": 0.4405781584582441, "frac_reward_zero_std": 0.75, "grad_norm": 0.009128664620220661, "learning_rate": 1e-05, "loss": -0.0164, "num_tokens": 16794414.0, "reward": 0.625, "reward_std": 0.13363061845302582, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.3344744443893433, "sampling/importance_sampling_ratio/mean": 1.0005537271499634, "sampling/importance_sampling_ratio/min": 0.6125915050506592, "sampling/sampling_logp_difference/max": 0.49005699157714844, "sampling/sampling_logp_difference/mean": 0.01385396160185337, "step": 823 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 673.0, "completions/max_terminated_length": 673.0, "completions/mean_length": 412.28125, "completions/mean_terminated_length": 412.28125, "completions/min_length": 248.0, "completions/min_terminated_length": 248.0, "entropy": 0.2766641639173031, "epoch": 0.4411134903640257, "frac_reward_zero_std": 0.5, "grad_norm": 0.0059517971239984035, "learning_rate": 1e-05, "loss": -0.0004, "num_tokens": 16810903.0, "reward": 0.90625, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.5597145557403564, "sampling/importance_sampling_ratio/mean": 1.0004196166992188, "sampling/importance_sampling_ratio/min": 0.6959481239318848, "sampling/sampling_logp_difference/max": 0.4445028305053711, "sampling/sampling_logp_difference/mean": 0.009880322962999344, "step": 824 }, { "clip_ratio/high_max": 8.111615898087621e-05, "clip_ratio/high_mean": 8.111615898087621e-05, "clip_ratio/low_mean": 0.00012761488687829114, "clip_ratio/low_min": 0.00012761488687829114, "clip_ratio/region_mean": 0.00020873104585916735, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 639.0, "completions/mean_length": 518.15625, "completions/mean_terminated_length": 471.8888854980469, "completions/min_length": 235.0, "completions/min_terminated_length": 235.0, "entropy": 0.3968745395541191, "epoch": 0.4416488222698073, "frac_reward_zero_std": 0.5, "grad_norm": 0.004003781825304031, "learning_rate": 1e-05, "loss": 0.0379, "num_tokens": 16831468.0, "reward": 0.71875, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.4546730518341064, "sampling/importance_sampling_ratio/mean": 1.0002210140228271, "sampling/importance_sampling_ratio/min": 0.5168008804321289, "sampling/sampling_logp_difference/max": 0.660097599029541, "sampling/sampling_logp_difference/mean": 0.01302696205675602, "step": 825 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 8.272667037090287e-05, "clip_ratio/low_min": 8.272667037090287e-05, "clip_ratio/region_mean": 8.272667037090287e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 556.0, "completions/max_terminated_length": 556.0, "completions/mean_length": 347.375, "completions/mean_terminated_length": 347.375, "completions/min_length": 196.0, "completions/min_terminated_length": 196.0, "entropy": 0.3648577034473419, "epoch": 0.4421841541755889, "frac_reward_zero_std": 0.25, "grad_norm": 0.010631679557263851, "learning_rate": 1e-05, "loss": 0.0125, "num_tokens": 16845952.0, "reward": 0.71875, "reward_std": 0.3198433816432953, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.6777235269546509, "sampling/importance_sampling_ratio/mean": 1.000125765800476, "sampling/importance_sampling_ratio/min": 0.5925402641296387, "sampling/sampling_logp_difference/max": 0.5233364105224609, "sampling/sampling_logp_difference/mean": 0.013822091743350029, "step": 826 }, { "clip_ratio/high_max": 6.613756704609841e-05, "clip_ratio/high_mean": 6.613756704609841e-05, "clip_ratio/low_mean": 7.155122875701636e-05, "clip_ratio/low_min": 7.155122875701636e-05, "clip_ratio/region_mean": 0.00013768879580311477, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 703.0, "completions/mean_length": 429.5625, "completions/mean_terminated_length": 366.8888854980469, "completions/min_length": 92.0, "completions/min_terminated_length": 92.0, "entropy": 0.48558948189020157, "epoch": 0.44271948608137046, "frac_reward_zero_std": 0.5, "grad_norm": 0.017857933416962624, "learning_rate": 1e-05, "loss": 0.0762, "num_tokens": 16863378.0, "reward": 0.6875, "reward_std": 0.2587745785713196, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.4482274055480957, "sampling/importance_sampling_ratio/mean": 0.9998704791069031, "sampling/importance_sampling_ratio/min": 0.6519090533256531, "sampling/sampling_logp_difference/max": 0.42785024642944336, "sampling/sampling_logp_difference/mean": 0.015063859522342682, "step": 827 }, { "clip_ratio/high_max": 8.922198321670294e-05, "clip_ratio/high_mean": 8.922198321670294e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 8.922198321670294e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 660.0, "completions/max_terminated_length": 660.0, "completions/mean_length": 381.71875, "completions/mean_terminated_length": 381.71875, "completions/min_length": 139.0, "completions/min_terminated_length": 139.0, "entropy": 0.3979560546576977, "epoch": 0.44325481798715205, "frac_reward_zero_std": 0.5, "grad_norm": 0.009561480954289436, "learning_rate": 1e-05, "loss": -0.0238, "num_tokens": 16879217.0, "reward": 0.75, "reward_std": 0.2314550280570984, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.5563980340957642, "sampling/importance_sampling_ratio/mean": 0.9998828768730164, "sampling/importance_sampling_ratio/min": 0.7124536037445068, "sampling/sampling_logp_difference/max": 0.44237422943115234, "sampling/sampling_logp_difference/mean": 0.012851578183472157, "step": 828 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 742.0, "completions/mean_length": 470.25, "completions/mean_terminated_length": 460.6451416015625, "completions/min_length": 189.0, "completions/min_terminated_length": 189.0, "entropy": 0.29934935458004475, "epoch": 0.44379014989293364, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": -0.0037, "num_tokens": 16898049.0, "reward": 0.71875, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.2824440002441406, "sampling/importance_sampling_ratio/mean": 1.000228762626648, "sampling/importance_sampling_ratio/min": 0.6554047465324402, "sampling/sampling_logp_difference/max": 0.4225022792816162, "sampling/sampling_logp_difference/mean": 0.010391879826784134, "step": 829 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 7.086167897796258e-05, "clip_ratio/low_min": 7.086167897796258e-05, "clip_ratio/region_mean": 7.086167897796258e-05, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 712.0, "completions/mean_length": 469.09375, "completions/mean_terminated_length": 400.1153869628906, "completions/min_length": 134.0, "completions/min_terminated_length": 134.0, "entropy": 0.5062973909080029, "epoch": 0.4443254817987152, "frac_reward_zero_std": 0.25, "grad_norm": 0.033503368496894836, "learning_rate": 1e-05, "loss": 0.0941, "num_tokens": 16917748.0, "reward": 0.5625, "reward_std": 0.3514062762260437, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.4417634010314941, "sampling/importance_sampling_ratio/mean": 0.999588131904602, "sampling/importance_sampling_ratio/min": 0.5480656027793884, "sampling/sampling_logp_difference/max": 0.6013603210449219, "sampling/sampling_logp_difference/mean": 0.016545923426747322, "step": 830 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 672.0, "completions/mean_length": 484.34375, "completions/mean_terminated_length": 431.8148193359375, "completions/min_length": 165.0, "completions/min_terminated_length": 165.0, "entropy": 0.3425117898732424, "epoch": 0.44486081370449676, "frac_reward_zero_std": 0.75, "grad_norm": 0.008114475756883621, "learning_rate": 1e-05, "loss": 0.0317, "num_tokens": 16937335.0, "reward": 0.65625, "reward_std": 0.1293872892856598, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.527876853942871, "sampling/importance_sampling_ratio/mean": 0.999904453754425, "sampling/importance_sampling_ratio/min": 0.4257214367389679, "sampling/sampling_logp_difference/max": 0.8539700508117676, "sampling/sampling_logp_difference/mean": 0.011804877780377865, "step": 831 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 679.0, "completions/mean_length": 434.09375, "completions/mean_terminated_length": 411.8333435058594, "completions/min_length": 149.0, "completions/min_terminated_length": 149.0, "entropy": 0.3797290250658989, "epoch": 0.44539614561027835, "frac_reward_zero_std": 0.0, "grad_norm": 0.01946469023823738, "learning_rate": 1e-05, "loss": 0.0321, "num_tokens": 16954522.0, "reward": 0.71875, "reward_std": 0.4397946000099182, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.339029312133789, "sampling/importance_sampling_ratio/mean": 1.0002355575561523, "sampling/importance_sampling_ratio/min": 0.7105914950370789, "sampling/sampling_logp_difference/max": 0.3416576385498047, "sampling/sampling_logp_difference/mean": 0.012946255505084991, "step": 832 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 750.0, "completions/mean_length": 416.875, "completions/mean_terminated_length": 405.5483703613281, "completions/min_length": 162.0, "completions/min_terminated_length": 162.0, "entropy": 0.2781375162303448, "epoch": 0.44593147751605994, "frac_reward_zero_std": 0.25, "grad_norm": 0.007427854463458061, "learning_rate": 1e-05, "loss": 0.0927, "num_tokens": 16971846.0, "reward": 0.90625, "reward_std": 0.2651650309562683, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.438772201538086, "sampling/importance_sampling_ratio/mean": 0.9997425079345703, "sampling/importance_sampling_ratio/min": 0.6445795893669128, "sampling/sampling_logp_difference/max": 0.43915700912475586, "sampling/sampling_logp_difference/mean": 0.00972855743020773, "step": 833 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00013874637807020918, "clip_ratio/low_min": 0.00013874637807020918, "clip_ratio/region_mean": 0.00013874637807020918, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 649.0, "completions/mean_length": 421.3125, "completions/mean_terminated_length": 385.4482727050781, "completions/min_length": 182.0, "completions/min_terminated_length": 182.0, "entropy": 0.49338012747466564, "epoch": 0.4464668094218415, "frac_reward_zero_std": 0.0, "grad_norm": 0.03788989409804344, "learning_rate": 1e-05, "loss": -0.0179, "num_tokens": 16989064.0, "reward": 0.5625, "reward_std": 0.47655022144317627, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0000516176223755, "sampling/importance_sampling_ratio/min": 0.5171704292297363, "sampling/sampling_logp_difference/max": 1.6423954963684082, "sampling/sampling_logp_difference/mean": 0.015298415906727314, "step": 834 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 732.0, "completions/mean_length": 462.46875, "completions/mean_terminated_length": 376.91998291015625, "completions/min_length": 203.0, "completions/min_terminated_length": 203.0, "entropy": 0.29654703475534916, "epoch": 0.4470021413276231, "frac_reward_zero_std": 0.25, "grad_norm": 0.00800703652203083, "learning_rate": 1e-05, "loss": -0.0005, "num_tokens": 17007711.0, "reward": 0.625, "reward_std": 0.3104073107242584, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.6471343040466309, "sampling/importance_sampling_ratio/mean": 0.9995182752609253, "sampling/importance_sampling_ratio/min": 0.7529647350311279, "sampling/sampling_logp_difference/max": 0.4990370273590088, "sampling/sampling_logp_difference/mean": 0.010023349896073341, "step": 835 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 654.0, "completions/max_terminated_length": 654.0, "completions/mean_length": 363.71875, "completions/mean_terminated_length": 363.71875, "completions/min_length": 245.0, "completions/min_terminated_length": 245.0, "entropy": 0.30910630337893963, "epoch": 0.4475374732334047, "frac_reward_zero_std": 0.75, "grad_norm": 0.004031539894640446, "learning_rate": 1e-05, "loss": -0.0254, "num_tokens": 17022638.0, "reward": 0.96875, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.96875, "rewards/accuracy_reward/std": 0.1767766922712326, "sampling/importance_sampling_ratio/max": 1.4584299325942993, "sampling/importance_sampling_ratio/mean": 0.9999199509620667, "sampling/importance_sampling_ratio/min": 0.6215628981590271, "sampling/sampling_logp_difference/max": 0.47551822662353516, "sampling/sampling_logp_difference/mean": 0.010782316327095032, "step": 836 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 766.0, "completions/mean_length": 483.46875, "completions/mean_terminated_length": 464.5000305175781, "completions/min_length": 254.0, "completions/min_terminated_length": 254.0, "entropy": 0.4010370597243309, "epoch": 0.4480728051391863, "frac_reward_zero_std": 0.5, "grad_norm": 0.007395565509796143, "learning_rate": 1e-05, "loss": 0.0427, "num_tokens": 17042021.0, "reward": 0.5625, "reward_std": 0.249358132481575, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.3822921514511108, "sampling/importance_sampling_ratio/mean": 0.9999129772186279, "sampling/importance_sampling_ratio/min": 0.5554530024528503, "sampling/sampling_logp_difference/max": 0.5879712104797363, "sampling/sampling_logp_difference/mean": 0.013415156863629818, "step": 837 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 706.0, "completions/mean_length": 478.28125, "completions/mean_terminated_length": 468.9354553222656, "completions/min_length": 335.0, "completions/min_terminated_length": 335.0, "entropy": 0.33436031080782413, "epoch": 0.4486081370449679, "frac_reward_zero_std": 0.25, "grad_norm": 0.020302664488554, "learning_rate": 1e-05, "loss": 0.014, "num_tokens": 17061190.0, "reward": 0.84375, "reward_std": 0.3061639666557312, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.3103904724121094, "sampling/importance_sampling_ratio/mean": 1.0000519752502441, "sampling/importance_sampling_ratio/min": 0.7050955295562744, "sampling/sampling_logp_difference/max": 0.34942197799682617, "sampling/sampling_logp_difference/mean": 0.012042642571032047, "step": 838 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.767731247236952e-05, "clip_ratio/low_min": 6.767731247236952e-05, "clip_ratio/region_mean": 6.767731247236952e-05, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 550.0, "completions/mean_length": 381.34375, "completions/mean_terminated_length": 341.3448181152344, "completions/min_length": 147.0, "completions/min_terminated_length": 147.0, "entropy": 0.45109712332487106, "epoch": 0.44914346895074947, "frac_reward_zero_std": 0.0, "grad_norm": 0.0383065864443779, "learning_rate": 1e-05, "loss": 0.1497, "num_tokens": 17077465.0, "reward": 0.84375, "reward_std": 0.3808925747871399, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.380422830581665, "sampling/importance_sampling_ratio/mean": 1.0000584125518799, "sampling/importance_sampling_ratio/min": 0.7162234783172607, "sampling/sampling_logp_difference/max": 0.33376312255859375, "sampling/sampling_logp_difference/mean": 0.014321033842861652, "step": 839 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.841817230451852e-05, "clip_ratio/low_min": 6.841817230451852e-05, "clip_ratio/region_mean": 6.841817230451852e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 768.0, "completions/max_terminated_length": 768.0, "completions/mean_length": 438.4375, "completions/mean_terminated_length": 438.4375, "completions/min_length": 118.0, "completions/min_terminated_length": 118.0, "entropy": 0.25611102394759655, "epoch": 0.44967880085653106, "frac_reward_zero_std": 0.25, "grad_norm": 0.02322283573448658, "learning_rate": 1e-05, "loss": -0.0443, "num_tokens": 17095527.0, "reward": 0.8125, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.3614468574523926, "sampling/importance_sampling_ratio/mean": 0.9999040365219116, "sampling/importance_sampling_ratio/min": 0.738928496837616, "sampling/sampling_logp_difference/max": 0.3085479736328125, "sampling/sampling_logp_difference/mean": 0.009763798676431179, "step": 840 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00012222748046042398, "clip_ratio/low_min": 0.00012222748046042398, "clip_ratio/region_mean": 0.00012222748046042398, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 732.0, "completions/mean_length": 545.65625, "completions/mean_terminated_length": 471.54168701171875, "completions/min_length": 207.0, "completions/min_terminated_length": 207.0, "entropy": 0.4472469873726368, "epoch": 0.45021413276231265, "frac_reward_zero_std": 0.25, "grad_norm": 0.007340646348893642, "learning_rate": 1e-05, "loss": 0.0369, "num_tokens": 17117588.0, "reward": 0.53125, "reward_std": 0.2651650309562683, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.6117594242095947, "sampling/importance_sampling_ratio/mean": 0.9999203681945801, "sampling/importance_sampling_ratio/min": 0.7105607390403748, "sampling/sampling_logp_difference/max": 0.4773263931274414, "sampling/sampling_logp_difference/mean": 0.014217482879757881, "step": 841 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 663.0, "completions/max_terminated_length": 663.0, "completions/mean_length": 387.90625, "completions/mean_terminated_length": 387.90625, "completions/min_length": 209.0, "completions/min_terminated_length": 209.0, "entropy": 0.3326467014849186, "epoch": 0.45074946466809424, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": -0.0131, "num_tokens": 17133601.0, "reward": 0.71875, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.4478331804275513, "sampling/importance_sampling_ratio/mean": 1.0000418424606323, "sampling/importance_sampling_ratio/min": 0.672549307346344, "sampling/sampling_logp_difference/max": 0.3966798782348633, "sampling/sampling_logp_difference/mean": 0.01194804161787033, "step": 842 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 655.0, "completions/max_terminated_length": 655.0, "completions/mean_length": 366.78125, "completions/mean_terminated_length": 366.78125, "completions/min_length": 224.0, "completions/min_terminated_length": 224.0, "entropy": 0.3126358576118946, "epoch": 0.4512847965738758, "frac_reward_zero_std": 0.75, "grad_norm": 0.005991787184029818, "learning_rate": 1e-05, "loss": 0.0164, "num_tokens": 17149074.0, "reward": 0.84375, "reward_std": 0.1293872892856598, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.4585145711898804, "sampling/importance_sampling_ratio/mean": 0.9998881816864014, "sampling/importance_sampling_ratio/min": 0.7193760871887207, "sampling/sampling_logp_difference/max": 0.37741851806640625, "sampling/sampling_logp_difference/mean": 0.010992232710123062, "step": 843 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0003193222510162741, "clip_ratio/low_min": 0.0003193222510162741, "clip_ratio/region_mean": 0.0003193222510162741, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 647.0, "completions/mean_length": 406.1875, "completions/mean_terminated_length": 339.1851806640625, "completions/min_length": 153.0, "completions/min_terminated_length": 153.0, "entropy": 0.6021257527172565, "epoch": 0.4518201284796574, "frac_reward_zero_std": 0.5, "grad_norm": 0.010459443554282188, "learning_rate": 1e-05, "loss": 0.0969, "num_tokens": 17165568.0, "reward": 0.625, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.4254722595214844, "sampling/importance_sampling_ratio/mean": 1.000404715538025, "sampling/importance_sampling_ratio/min": 0.7119803428649902, "sampling/sampling_logp_difference/max": 0.35450315475463867, "sampling/sampling_logp_difference/mean": 0.01792016252875328, "step": 844 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00013062438665656373, "clip_ratio/low_min": 0.00013062438665656373, "clip_ratio/region_mean": 0.00013062438665656373, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 588.0, "completions/mean_length": 453.34375, "completions/mean_terminated_length": 432.36669921875, "completions/min_length": 196.0, "completions/min_terminated_length": 196.0, "entropy": 0.35442223213613033, "epoch": 0.45235546038543895, "frac_reward_zero_std": 0.75, "grad_norm": 0.0104680759832263, "learning_rate": 1e-05, "loss": 0.0372, "num_tokens": 17184403.0, "reward": 0.84375, "reward_std": 0.1293872892856598, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.352452278137207, "sampling/importance_sampling_ratio/mean": 0.9995409250259399, "sampling/importance_sampling_ratio/min": 0.7116348147392273, "sampling/sampling_logp_difference/max": 0.34019041061401367, "sampling/sampling_logp_difference/mean": 0.011902401223778725, "step": 845 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.380806735251099e-05, "clip_ratio/low_min": 6.380806735251099e-05, "clip_ratio/region_mean": 6.380806735251099e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 722.0, "completions/max_terminated_length": 722.0, "completions/mean_length": 457.25, "completions/mean_terminated_length": 457.25, "completions/min_length": 262.0, "completions/min_terminated_length": 262.0, "entropy": 0.3293653018772602, "epoch": 0.45289079229122053, "frac_reward_zero_std": 0.75, "grad_norm": 0.011456511914730072, "learning_rate": 1e-05, "loss": -0.0053, "num_tokens": 17202603.0, "reward": 0.65625, "reward_std": 0.1293872892856598, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.4396300315856934, "sampling/importance_sampling_ratio/mean": 1.000016212463379, "sampling/importance_sampling_ratio/min": 0.6384013295173645, "sampling/sampling_logp_difference/max": 0.4487881660461426, "sampling/sampling_logp_difference/mean": 0.01195397786796093, "step": 846 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 540.0, "completions/mean_length": 396.75, "completions/mean_terminated_length": 372.0000305175781, "completions/min_length": 191.0, "completions/min_terminated_length": 191.0, "entropy": 0.3784095495939255, "epoch": 0.4534261241970021, "frac_reward_zero_std": 0.5, "grad_norm": 0.013537553139030933, "learning_rate": 1e-05, "loss": 0.0311, "num_tokens": 17219283.0, "reward": 0.6875, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.360169529914856, "sampling/importance_sampling_ratio/mean": 1.0003794431686401, "sampling/importance_sampling_ratio/min": 0.6856853365898132, "sampling/sampling_logp_difference/max": 0.3773365020751953, "sampling/sampling_logp_difference/mean": 0.012845630757510662, "step": 847 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0001144701091106981, "clip_ratio/low_min": 0.0001144701091106981, "clip_ratio/region_mean": 0.0001144701091106981, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 752.0, "completions/mean_length": 554.28125, "completions/mean_terminated_length": 514.7037353515625, "completions/min_length": 282.0, "completions/min_terminated_length": 282.0, "entropy": 0.40647515282034874, "epoch": 0.4539614561027837, "frac_reward_zero_std": 0.25, "grad_norm": 0.010650386102497578, "learning_rate": 1e-05, "loss": 0.0133, "num_tokens": 17241532.0, "reward": 0.75, "reward_std": 0.3514062762260437, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.452175259590149, "sampling/importance_sampling_ratio/mean": 0.9999576807022095, "sampling/importance_sampling_ratio/min": 0.47834256291389465, "sampling/sampling_logp_difference/max": 0.7374281883239746, "sampling/sampling_logp_difference/mean": 0.013392100110650063, "step": 848 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 714.0, "completions/mean_length": 422.03125, "completions/mean_terminated_length": 410.8709411621094, "completions/min_length": 206.0, "completions/min_terminated_length": 206.0, "entropy": 0.39763497561216354, "epoch": 0.4544967880085653, "frac_reward_zero_std": 0.0, "grad_norm": 0.027767548337578773, "learning_rate": 1e-05, "loss": 0.0641, "num_tokens": 17258725.0, "reward": 0.84375, "reward_std": 0.3808925747871399, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.908989429473877, "sampling/importance_sampling_ratio/mean": 1.0007014274597168, "sampling/importance_sampling_ratio/min": 0.6914438605308533, "sampling/sampling_logp_difference/max": 0.6465740203857422, "sampling/sampling_logp_difference/mean": 0.0119412150233984, "step": 849 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.121449405327439e-05, "clip_ratio/low_min": 6.121449405327439e-05, "clip_ratio/region_mean": 6.121449405327439e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 669.0, "completions/max_terminated_length": 669.0, "completions/mean_length": 413.5625, "completions/mean_terminated_length": 413.5625, "completions/min_length": 95.0, "completions/min_terminated_length": 95.0, "entropy": 0.22440005093812943, "epoch": 0.4550321199143469, "frac_reward_zero_std": 0.5, "grad_norm": 0.002890060655772686, "learning_rate": 1e-05, "loss": 0.0081, "num_tokens": 17275559.0, "reward": 0.9375, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.9375, "rewards/accuracy_reward/std": 0.24593468010425568, "sampling/importance_sampling_ratio/max": 1.4582141637802124, "sampling/importance_sampling_ratio/mean": 1.000386357307434, "sampling/importance_sampling_ratio/min": 0.7644913196563721, "sampling/sampling_logp_difference/max": 0.3772125244140625, "sampling/sampling_logp_difference/mean": 0.008429231122136116, "step": 850 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 7.058159098960459e-05, "clip_ratio/low_min": 7.058159098960459e-05, "clip_ratio/region_mean": 7.058159098960459e-05, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 758.0, "completions/mean_length": 508.34375, "completions/mean_terminated_length": 471.2500305175781, "completions/min_length": 213.0, "completions/min_terminated_length": 213.0, "entropy": 0.4065246470272541, "epoch": 0.4555674518201285, "frac_reward_zero_std": 0.25, "grad_norm": 0.005690491758286953, "learning_rate": 1e-05, "loss": 0.0381, "num_tokens": 17295666.0, "reward": 0.8125, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.3454594612121582, "sampling/importance_sampling_ratio/mean": 0.9997237324714661, "sampling/importance_sampling_ratio/min": 0.7095920443534851, "sampling/sampling_logp_difference/max": 0.3430650234222412, "sampling/sampling_logp_difference/mean": 0.01417690608650446, "step": 851 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 642.0, "completions/mean_length": 425.9375, "completions/mean_terminated_length": 403.13336181640625, "completions/min_length": 220.0, "completions/min_terminated_length": 220.0, "entropy": 0.389895211905241, "epoch": 0.45610278372591007, "frac_reward_zero_std": 0.25, "grad_norm": 0.01650960184633732, "learning_rate": 1e-05, "loss": 0.0086, "num_tokens": 17313104.0, "reward": 0.59375, "reward_std": 0.378745436668396, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.4076075553894043, "sampling/importance_sampling_ratio/mean": 1.000275731086731, "sampling/importance_sampling_ratio/min": 0.7483003735542297, "sampling/sampling_logp_difference/max": 0.3418915271759033, "sampling/sampling_logp_difference/mean": 0.012765714898705482, "step": 852 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 605.0, "completions/mean_length": 397.84375, "completions/mean_terminated_length": 373.16668701171875, "completions/min_length": 243.0, "completions/min_terminated_length": 243.0, "entropy": 0.35864674486219883, "epoch": 0.45663811563169165, "frac_reward_zero_std": 0.5, "grad_norm": 0.004349736031144857, "learning_rate": 1e-05, "loss": 0.0152, "num_tokens": 17329195.0, "reward": 0.6875, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.2833842039108276, "sampling/importance_sampling_ratio/mean": 1.0001198053359985, "sampling/importance_sampling_ratio/min": 0.6706969141960144, "sampling/sampling_logp_difference/max": 0.3994380235671997, "sampling/sampling_logp_difference/mean": 0.012003584764897823, "step": 853 }, { "clip_ratio/high_max": 0.00011682243348332122, "clip_ratio/high_mean": 0.00011682243348332122, "clip_ratio/low_mean": 0.00013551987285609357, "clip_ratio/low_min": 0.00013551987285609357, "clip_ratio/region_mean": 0.0002523423063394148, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 611.0, "completions/mean_length": 399.40625, "completions/mean_terminated_length": 361.2758483886719, "completions/min_length": 183.0, "completions/min_terminated_length": 183.0, "entropy": 0.4235917590558529, "epoch": 0.45717344753747324, "frac_reward_zero_std": 0.25, "grad_norm": 0.01636618562042713, "learning_rate": 1e-05, "loss": 0.0596, "num_tokens": 17345904.0, "reward": 0.625, "reward_std": 0.3745020925998688, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.4649571180343628, "sampling/importance_sampling_ratio/mean": 0.9999454021453857, "sampling/importance_sampling_ratio/min": 0.7032767534255981, "sampling/sampling_logp_difference/max": 0.38182592391967773, "sampling/sampling_logp_difference/mean": 0.014212645590305328, "step": 854 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.846585554536432e-05, "clip_ratio/low_min": 5.846585554536432e-05, "clip_ratio/region_mean": 5.846585554536432e-05, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 743.0, "completions/mean_length": 395.34375, "completions/mean_terminated_length": 383.32257080078125, "completions/min_length": 167.0, "completions/min_terminated_length": 167.0, "entropy": 0.3481990545988083, "epoch": 0.45770877944325483, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0059, "num_tokens": 17362179.0, "reward": 0.90625, "reward_std": 0.1293872892856598, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.43292236328125, "sampling/importance_sampling_ratio/mean": 1.000043272972107, "sampling/importance_sampling_ratio/min": 0.5405396819114685, "sampling/sampling_logp_difference/max": 0.6151871681213379, "sampling/sampling_logp_difference/mean": 0.012787885963916779, "step": 855 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 579.0, "completions/mean_length": 416.21875, "completions/mean_terminated_length": 404.8709411621094, "completions/min_length": 271.0, "completions/min_terminated_length": 271.0, "entropy": 0.29986532032489777, "epoch": 0.4582441113490364, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": -0.0105, "num_tokens": 17378866.0, "reward": 0.9375, "reward_std": 0.1157275140285492, "rewards/accuracy_reward/mean": 0.9375, "rewards/accuracy_reward/std": 0.24593468010425568, "sampling/importance_sampling_ratio/max": 1.420054316520691, "sampling/importance_sampling_ratio/mean": 1.0008354187011719, "sampling/importance_sampling_ratio/min": 0.6028842329978943, "sampling/sampling_logp_difference/max": 0.5060300827026367, "sampling/sampling_logp_difference/mean": 0.011007579043507576, "step": 856 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00012850145139964297, "clip_ratio/low_min": 0.00012850145139964297, "clip_ratio/region_mean": 0.00012850145139964297, "completions/clipped_ratio": 0.0, "completions/max_length": 705.0, "completions/max_terminated_length": 705.0, "completions/mean_length": 388.875, "completions/mean_terminated_length": 388.875, "completions/min_length": 169.0, "completions/min_terminated_length": 169.0, "entropy": 0.34492479264736176, "epoch": 0.458779443254818, "frac_reward_zero_std": 0.5, "grad_norm": 0.025023143738508224, "learning_rate": 1e-05, "loss": 0.0539, "num_tokens": 17394622.0, "reward": 0.78125, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.4472174644470215, "sampling/importance_sampling_ratio/mean": 0.9998632669448853, "sampling/importance_sampling_ratio/min": 0.6163671612739563, "sampling/sampling_logp_difference/max": 0.48391246795654297, "sampling/sampling_logp_difference/mean": 0.011631453409790993, "step": 857 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 551.0, "completions/mean_length": 450.875, "completions/mean_terminated_length": 377.69232177734375, "completions/min_length": 228.0, "completions/min_terminated_length": 228.0, "entropy": 0.3829144723713398, "epoch": 0.4593147751605996, "frac_reward_zero_std": 0.25, "grad_norm": 0.007320623379200697, "learning_rate": 1e-05, "loss": 0.1101, "num_tokens": 17413202.0, "reward": 0.71875, "reward_std": 0.3198433816432953, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.4191597700119019, "sampling/importance_sampling_ratio/mean": 1.0002152919769287, "sampling/importance_sampling_ratio/min": 0.7294127345085144, "sampling/sampling_logp_difference/max": 0.3500649929046631, "sampling/sampling_logp_difference/mean": 0.012862490490078926, "step": 858 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 749.0, "completions/max_terminated_length": 749.0, "completions/mean_length": 426.6875, "completions/mean_terminated_length": 426.6875, "completions/min_length": 216.0, "completions/min_terminated_length": 216.0, "entropy": 0.3332531042397022, "epoch": 0.45985010706638113, "frac_reward_zero_std": 0.25, "grad_norm": 0.00850115716457367, "learning_rate": 1e-05, "loss": -0.0237, "num_tokens": 17430104.0, "reward": 0.8125, "reward_std": 0.3471825420856476, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.4328937530517578, "sampling/importance_sampling_ratio/mean": 0.999755322933197, "sampling/importance_sampling_ratio/min": 0.6934176683425903, "sampling/sampling_logp_difference/max": 0.3661227226257324, "sampling/sampling_logp_difference/mean": 0.011567413806915283, "step": 859 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0001035625537042506, "clip_ratio/low_min": 0.0001035625537042506, "clip_ratio/region_mean": 0.0001035625537042506, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 701.0, "completions/mean_length": 387.65625, "completions/mean_terminated_length": 348.3103332519531, "completions/min_length": 168.0, "completions/min_terminated_length": 168.0, "entropy": 0.4081963486969471, "epoch": 0.4603854389721627, "frac_reward_zero_std": 0.5, "grad_norm": 0.018700972199440002, "learning_rate": 1e-05, "loss": 0.0589, "num_tokens": 17445869.0, "reward": 0.65625, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.4094253778457642, "sampling/importance_sampling_ratio/mean": 0.9998759031295776, "sampling/importance_sampling_ratio/min": 0.6700186133384705, "sampling/sampling_logp_difference/max": 0.4004497528076172, "sampling/sampling_logp_difference/mean": 0.01435065921396017, "step": 860 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.662439459934831e-05, "clip_ratio/low_min": 4.662439459934831e-05, "clip_ratio/region_mean": 4.662439459934831e-05, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 741.0, "completions/mean_length": 498.03125, "completions/mean_terminated_length": 448.03704833984375, "completions/min_length": 314.0, "completions/min_terminated_length": 314.0, "entropy": 0.3133188635110855, "epoch": 0.4609207708779443, "frac_reward_zero_std": 0.5, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0012, "num_tokens": 17465102.0, "reward": 0.75, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.3276058435440063, "sampling/importance_sampling_ratio/mean": 1.000036358833313, "sampling/importance_sampling_ratio/min": 0.6978370547294617, "sampling/sampling_logp_difference/max": 0.3597695827484131, "sampling/sampling_logp_difference/mean": 0.010118676349520683, "step": 861 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 602.0, "completions/mean_length": 457.0, "completions/mean_terminated_length": 446.9677429199219, "completions/min_length": 331.0, "completions/min_terminated_length": 331.0, "entropy": 0.3007192686200142, "epoch": 0.4614561027837259, "frac_reward_zero_std": 0.5, "grad_norm": 0.02058938704431057, "learning_rate": 1e-05, "loss": -0.0113, "num_tokens": 17483334.0, "reward": 0.90625, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.5101215839385986, "sampling/importance_sampling_ratio/mean": 1.0001330375671387, "sampling/importance_sampling_ratio/min": 0.46541279554367065, "sampling/sampling_logp_difference/max": 0.7648305892944336, "sampling/sampling_logp_difference/mean": 0.01065827812999487, "step": 862 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00023827057157177478, "clip_ratio/low_min": 0.00023827057157177478, "clip_ratio/region_mean": 0.00023827057157177478, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 757.0, "completions/mean_length": 450.625, "completions/mean_terminated_length": 440.3870849609375, "completions/min_length": 179.0, "completions/min_terminated_length": 179.0, "entropy": 0.37086210399866104, "epoch": 0.4619914346895075, "frac_reward_zero_std": 0.0, "grad_norm": 0.019463149830698967, "learning_rate": 1e-05, "loss": 0.0812, "num_tokens": 17501522.0, "reward": 0.65625, "reward_std": 0.5038893818855286, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.4115018844604492, "sampling/importance_sampling_ratio/mean": 0.9998630285263062, "sampling/importance_sampling_ratio/min": 0.6237742304801941, "sampling/sampling_logp_difference/max": 0.4719667434692383, "sampling/sampling_logp_difference/mean": 0.012444799765944481, "step": 863 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 623.0, "completions/max_terminated_length": 623.0, "completions/mean_length": 343.5, "completions/mean_terminated_length": 343.5, "completions/min_length": 217.0, "completions/min_terminated_length": 217.0, "entropy": 0.36748653277754784, "epoch": 0.4625267665952891, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": -0.0061, "num_tokens": 17515770.0, "reward": 0.90625, "reward_std": 0.1293872892856598, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.3397990465164185, "sampling/importance_sampling_ratio/mean": 1.0000654458999634, "sampling/importance_sampling_ratio/min": 0.6854734420776367, "sampling/sampling_logp_difference/max": 0.37764549255371094, "sampling/sampling_logp_difference/mean": 0.012020249851047993, "step": 864 }, { "clip_ratio/high_max": 5.4656757129123434e-05, "clip_ratio/high_mean": 5.4656757129123434e-05, "clip_ratio/low_mean": 0.0002011603064602241, "clip_ratio/low_min": 0.0002011603064602241, "clip_ratio/region_mean": 0.0002558170635893475, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 629.0, "completions/mean_length": 446.65625, "completions/mean_terminated_length": 387.1481628417969, "completions/min_length": 218.0, "completions/min_terminated_length": 218.0, "entropy": 0.49589866027235985, "epoch": 0.46306209850107066, "frac_reward_zero_std": 0.0, "grad_norm": 0.029117299243807793, "learning_rate": 1e-05, "loss": 0.0569, "num_tokens": 17534079.0, "reward": 0.59375, "reward_std": 0.4534739851951599, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.4676965475082397, "sampling/importance_sampling_ratio/mean": 1.0003687143325806, "sampling/importance_sampling_ratio/min": 0.7300150394439697, "sampling/sampling_logp_difference/max": 0.3836941719055176, "sampling/sampling_logp_difference/mean": 0.0150558827444911, "step": 865 }, { "clip_ratio/high_max": 0.00015352282935054973, "clip_ratio/high_mean": 0.00015352282935054973, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00015352282935054973, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 606.0, "completions/mean_length": 413.5, "completions/mean_terminated_length": 389.86669921875, "completions/min_length": 147.0, "completions/min_terminated_length": 147.0, "entropy": 0.2860215548425913, "epoch": 0.46359743040685225, "frac_reward_zero_std": 0.25, "grad_norm": 0.010892197489738464, "learning_rate": 1e-05, "loss": 0.0288, "num_tokens": 17550839.0, "reward": 0.84375, "reward_std": 0.3061639964580536, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.294333815574646, "sampling/importance_sampling_ratio/mean": 1.0000544786453247, "sampling/importance_sampling_ratio/min": 0.638538658618927, "sampling/sampling_logp_difference/max": 0.44857311248779297, "sampling/sampling_logp_difference/mean": 0.010317453183233738, "step": 866 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.838074477855116e-05, "clip_ratio/low_min": 6.838074477855116e-05, "clip_ratio/region_mean": 6.838074477855116e-05, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 633.0, "completions/mean_length": 458.5625, "completions/mean_terminated_length": 437.933349609375, "completions/min_length": 225.0, "completions/min_terminated_length": 225.0, "entropy": 0.3842085562646389, "epoch": 0.46413276231263384, "frac_reward_zero_std": 0.5, "grad_norm": 0.019818734377622604, "learning_rate": 1e-05, "loss": 0.0632, "num_tokens": 17569249.0, "reward": 0.875, "reward_std": 0.2314550280570984, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 1.4064277410507202, "sampling/importance_sampling_ratio/mean": 1.0000783205032349, "sampling/importance_sampling_ratio/min": 0.5575892329216003, "sampling/sampling_logp_difference/max": 0.5841326713562012, "sampling/sampling_logp_difference/mean": 0.01301675196737051, "step": 867 }, { "clip_ratio/high_max": 6.811989442212507e-05, "clip_ratio/high_mean": 6.811989442212507e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 6.811989442212507e-05, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 646.0, "completions/mean_length": 447.75, "completions/mean_terminated_length": 414.6206970214844, "completions/min_length": 78.0, "completions/min_terminated_length": 78.0, "entropy": 0.38831511698663235, "epoch": 0.46466809421841543, "frac_reward_zero_std": 0.5, "grad_norm": 0.007741203065961599, "learning_rate": 1e-05, "loss": 0.0503, "num_tokens": 17587409.0, "reward": 0.625, "reward_std": 0.2314550280570984, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.6087449789047241, "sampling/importance_sampling_ratio/mean": 1.0000803470611572, "sampling/importance_sampling_ratio/min": 0.7040154337882996, "sampling/sampling_logp_difference/max": 0.47545433044433594, "sampling/sampling_logp_difference/mean": 0.012356244027614594, "step": 868 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 754.0, "completions/mean_length": 414.65625, "completions/mean_terminated_length": 403.258056640625, "completions/min_length": 173.0, "completions/min_terminated_length": 173.0, "entropy": 0.3263944126665592, "epoch": 0.465203426124197, "frac_reward_zero_std": 0.5, "grad_norm": 0.007845853455364704, "learning_rate": 1e-05, "loss": -0.0206, "num_tokens": 17604606.0, "reward": 0.90625, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.3820526599884033, "sampling/importance_sampling_ratio/mean": 0.999838650226593, "sampling/importance_sampling_ratio/min": 0.6143204569816589, "sampling/sampling_logp_difference/max": 0.48723864555358887, "sampling/sampling_logp_difference/mean": 0.011147018522024155, "step": 869 }, { "clip_ratio/high_max": 5.989458441035822e-05, "clip_ratio/high_mean": 5.989458441035822e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 5.989458441035822e-05, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 725.0, "completions/mean_length": 508.375, "completions/mean_terminated_length": 491.0666809082031, "completions/min_length": 250.0, "completions/min_terminated_length": 250.0, "entropy": 0.3418238162994385, "epoch": 0.4657387580299786, "frac_reward_zero_std": 0.25, "grad_norm": 0.039786387234926224, "learning_rate": 1e-05, "loss": 0.0471, "num_tokens": 17625138.0, "reward": 0.84375, "reward_std": 0.3061639964580536, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9999599456787109, "sampling/importance_sampling_ratio/min": 0.5652012825012207, "sampling/sampling_logp_difference/max": 0.7570457458496094, "sampling/sampling_logp_difference/mean": 0.011174959130585194, "step": 870 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 729.0, "completions/mean_length": 456.78125, "completions/mean_terminated_length": 424.5862121582031, "completions/min_length": 251.0, "completions/min_terminated_length": 251.0, "entropy": 0.388157669454813, "epoch": 0.4662740899357602, "frac_reward_zero_std": 0.25, "grad_norm": 0.010254666209220886, "learning_rate": 1e-05, "loss": 0.0579, "num_tokens": 17643979.0, "reward": 0.5625, "reward_std": 0.3104073107242584, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.4111849069595337, "sampling/importance_sampling_ratio/mean": 1.0003020763397217, "sampling/importance_sampling_ratio/min": 0.7047721743583679, "sampling/sampling_logp_difference/max": 0.3498806953430176, "sampling/sampling_logp_difference/mean": 0.013184558600187302, "step": 871 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 701.0, "completions/max_terminated_length": 701.0, "completions/mean_length": 463.9375, "completions/mean_terminated_length": 463.9375, "completions/min_length": 238.0, "completions/min_terminated_length": 238.0, "entropy": 0.27669578045606613, "epoch": 0.4668094218415418, "frac_reward_zero_std": 0.5, "grad_norm": 0.007386947050690651, "learning_rate": 1e-05, "loss": -0.0044, "num_tokens": 17662769.0, "reward": 0.875, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 1.3984256982803345, "sampling/importance_sampling_ratio/mean": 0.999940812587738, "sampling/importance_sampling_ratio/min": 0.5103549361228943, "sampling/sampling_logp_difference/max": 0.6726489067077637, "sampling/sampling_logp_difference/mean": 0.010137831792235374, "step": 872 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 651.0, "completions/mean_length": 450.5625, "completions/mean_terminated_length": 417.7241516113281, "completions/min_length": 239.0, "completions/min_terminated_length": 239.0, "entropy": 0.4348072949796915, "epoch": 0.4673447537473233, "frac_reward_zero_std": 0.5, "grad_norm": 0.014925865456461906, "learning_rate": 1e-05, "loss": 0.03, "num_tokens": 17680675.0, "reward": 0.75, "reward_std": 0.2587745785713196, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.3342413902282715, "sampling/importance_sampling_ratio/mean": 0.9997825622558594, "sampling/importance_sampling_ratio/min": 0.7196564078330994, "sampling/sampling_logp_difference/max": 0.3289813995361328, "sampling/sampling_logp_difference/mean": 0.013186180964112282, "step": 873 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 518.0, "completions/max_terminated_length": 518.0, "completions/mean_length": 351.1875, "completions/mean_terminated_length": 351.1875, "completions/min_length": 215.0, "completions/min_terminated_length": 215.0, "entropy": 0.28410062938928604, "epoch": 0.4678800856531049, "frac_reward_zero_std": 0.5, "grad_norm": 0.013831309974193573, "learning_rate": 1e-05, "loss": 0.0254, "num_tokens": 17695313.0, "reward": 0.875, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 1.350860357284546, "sampling/importance_sampling_ratio/mean": 0.9998568892478943, "sampling/importance_sampling_ratio/min": 0.7161739468574524, "sampling/sampling_logp_difference/max": 0.3338322639465332, "sampling/sampling_logp_difference/mean": 0.01030992902815342, "step": 874 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00013835086429025978, "clip_ratio/low_min": 0.00013835086429025978, "clip_ratio/region_mean": 0.00013835086429025978, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 759.0, "completions/mean_length": 529.125, "completions/mean_terminated_length": 462.239990234375, "completions/min_length": 133.0, "completions/min_terminated_length": 133.0, "entropy": 0.376115333288908, "epoch": 0.4684154175588865, "frac_reward_zero_std": 0.0, "grad_norm": 0.021969245746731758, "learning_rate": 1e-05, "loss": -0.0103, "num_tokens": 17715909.0, "reward": 0.5, "reward_std": 0.49871626496315, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5080004930496216, "sampling/importance_sampling_ratio/max": 1.5781770944595337, "sampling/importance_sampling_ratio/mean": 0.9997391104698181, "sampling/importance_sampling_ratio/min": 0.6547223925590515, "sampling/sampling_logp_difference/max": 0.4562704563140869, "sampling/sampling_logp_difference/mean": 0.012313427403569221, "step": 875 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0001707992414594628, "clip_ratio/low_min": 0.0001707992414594628, "clip_ratio/region_mean": 0.0001707992414594628, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 633.0, "completions/mean_length": 398.65625, "completions/mean_terminated_length": 386.7419128417969, "completions/min_length": 120.0, "completions/min_terminated_length": 120.0, "entropy": 0.34080640971660614, "epoch": 0.4689507494646681, "frac_reward_zero_std": 0.25, "grad_norm": 0.005396557506173849, "learning_rate": 1e-05, "loss": -0.0499, "num_tokens": 17732170.0, "reward": 0.6875, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.3275684118270874, "sampling/importance_sampling_ratio/mean": 1.0001745223999023, "sampling/importance_sampling_ratio/min": 0.6981101036071777, "sampling/sampling_logp_difference/max": 0.359378457069397, "sampling/sampling_logp_difference/mean": 0.01163567416369915, "step": 876 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 698.0, "completions/mean_length": 451.5, "completions/mean_terminated_length": 418.75860595703125, "completions/min_length": 212.0, "completions/min_terminated_length": 212.0, "entropy": 0.3072942551225424, "epoch": 0.46948608137044967, "frac_reward_zero_std": 0.25, "grad_norm": 0.012186689302325249, "learning_rate": 1e-05, "loss": 0.0212, "num_tokens": 17750562.0, "reward": 0.78125, "reward_std": 0.3471629321575165, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.5272384881973267, "sampling/importance_sampling_ratio/mean": 0.9996962547302246, "sampling/importance_sampling_ratio/min": 0.5193907618522644, "sampling/sampling_logp_difference/max": 0.6550987958908081, "sampling/sampling_logp_difference/mean": 0.010424166917800903, "step": 877 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00019841270113829523, "clip_ratio/low_min": 0.00019841270113829523, "clip_ratio/region_mean": 0.00019841270113829523, "completions/clipped_ratio": 0.0, "completions/max_length": 537.0, "completions/max_terminated_length": 537.0, "completions/mean_length": 354.09375, "completions/mean_terminated_length": 354.09375, "completions/min_length": 153.0, "completions/min_terminated_length": 153.0, "entropy": 0.3449370674788952, "epoch": 0.47002141327623126, "frac_reward_zero_std": 0.5, "grad_norm": 0.010278141126036644, "learning_rate": 1e-05, "loss": -0.0316, "num_tokens": 17765437.0, "reward": 0.875, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 1.73139226436615, "sampling/importance_sampling_ratio/mean": 0.9997285604476929, "sampling/importance_sampling_ratio/min": 0.6772655248641968, "sampling/sampling_logp_difference/max": 0.5489258766174316, "sampling/sampling_logp_difference/mean": 0.011303069069981575, "step": 878 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00015254774189088494, "clip_ratio/low_min": 0.00015254774189088494, "clip_ratio/region_mean": 0.00015254774189088494, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 741.0, "completions/mean_length": 472.6875, "completions/mean_terminated_length": 418.0, "completions/min_length": 157.0, "completions/min_terminated_length": 157.0, "entropy": 0.5267374888062477, "epoch": 0.47055674518201285, "frac_reward_zero_std": 0.25, "grad_norm": 0.03600321710109711, "learning_rate": 1e-05, "loss": 0.0225, "num_tokens": 17784539.0, "reward": 0.625, "reward_std": 0.3104073107242584, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.3712495565414429, "sampling/importance_sampling_ratio/mean": 0.9998849630355835, "sampling/importance_sampling_ratio/min": 0.6972718238830566, "sampling/sampling_logp_difference/max": 0.3605799674987793, "sampling/sampling_logp_difference/mean": 0.015775874257087708, "step": 879 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.026135841035284e-05, "clip_ratio/low_min": 5.026135841035284e-05, "clip_ratio/region_mean": 5.026135841035284e-05, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 764.0, "completions/mean_length": 502.40625, "completions/mean_terminated_length": 474.9310302734375, "completions/min_length": 260.0, "completions/min_terminated_length": 260.0, "entropy": 0.4410126060247421, "epoch": 0.47109207708779444, "frac_reward_zero_std": 0.25, "grad_norm": 0.021375300362706184, "learning_rate": 1e-05, "loss": 0.036, "num_tokens": 17804896.0, "reward": 0.625, "reward_std": 0.3650856614112854, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.6762452125549316, "sampling/importance_sampling_ratio/mean": 0.9997879266738892, "sampling/importance_sampling_ratio/min": 0.7033208608627319, "sampling/sampling_logp_difference/max": 0.5165562629699707, "sampling/sampling_logp_difference/mean": 0.01409416738897562, "step": 880 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 648.0, "completions/max_terminated_length": 648.0, "completions/mean_length": 404.03125, "completions/mean_terminated_length": 404.03125, "completions/min_length": 226.0, "completions/min_terminated_length": 226.0, "entropy": 0.3333063628524542, "epoch": 0.471627408993576, "frac_reward_zero_std": 0.5, "grad_norm": 0.0034336980897933245, "learning_rate": 1e-05, "loss": 0.0139, "num_tokens": 17821465.0, "reward": 0.9375, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.9375, "rewards/accuracy_reward/std": 0.24593468010425568, "sampling/importance_sampling_ratio/max": 1.3419228792190552, "sampling/importance_sampling_ratio/mean": 0.9997738003730774, "sampling/importance_sampling_ratio/min": 0.694087564945221, "sampling/sampling_logp_difference/max": 0.3651571273803711, "sampling/sampling_logp_difference/mean": 0.012087167240679264, "step": 881 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 736.0, "completions/mean_length": 504.65625, "completions/mean_terminated_length": 477.4137878417969, "completions/min_length": 277.0, "completions/min_terminated_length": 277.0, "entropy": 0.36456388235092163, "epoch": 0.4721627408993576, "frac_reward_zero_std": 0.75, "grad_norm": 0.007763247471302748, "learning_rate": 1e-05, "loss": 0.0129, "num_tokens": 17841726.0, "reward": 0.59375, "reward_std": 0.1293872892856598, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.4377187490463257, "sampling/importance_sampling_ratio/mean": 1.0000646114349365, "sampling/importance_sampling_ratio/min": 0.7447316646575928, "sampling/sampling_logp_difference/max": 0.36305761337280273, "sampling/sampling_logp_difference/mean": 0.012421871535480022, "step": 882 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 555.0, "completions/mean_length": 334.46875, "completions/mean_terminated_length": 320.4838562011719, "completions/min_length": 135.0, "completions/min_terminated_length": 135.0, "entropy": 0.39696090295910835, "epoch": 0.4726980728051392, "frac_reward_zero_std": 0.25, "grad_norm": 0.017396535724401474, "learning_rate": 1e-05, "loss": -0.0025, "num_tokens": 17856061.0, "reward": 0.65625, "reward_std": 0.3377465009689331, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.3676894903182983, "sampling/importance_sampling_ratio/mean": 1.0004018545150757, "sampling/importance_sampling_ratio/min": 0.6673457026481628, "sampling/sampling_logp_difference/max": 0.404447078704834, "sampling/sampling_logp_difference/mean": 0.01251395232975483, "step": 883 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00011150757927680388, "clip_ratio/low_min": 0.00011150757927680388, "clip_ratio/region_mean": 0.00011150757927680388, "completions/clipped_ratio": 0.0, "completions/max_length": 434.0, "completions/max_terminated_length": 434.0, "completions/mean_length": 297.53125, "completions/mean_terminated_length": 297.53125, "completions/min_length": 164.0, "completions/min_terminated_length": 164.0, "entropy": 0.34722578153014183, "epoch": 0.4732334047109208, "frac_reward_zero_std": 0.75, "grad_norm": 0.005463318433612585, "learning_rate": 1e-05, "loss": -0.0168, "num_tokens": 17868838.0, "reward": 0.96875, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.96875, "rewards/accuracy_reward/std": 0.1767766922712326, "sampling/importance_sampling_ratio/max": 1.5137042999267578, "sampling/importance_sampling_ratio/mean": 1.0004953145980835, "sampling/importance_sampling_ratio/min": 0.6887400150299072, "sampling/sampling_logp_difference/max": 0.41455984115600586, "sampling/sampling_logp_difference/mean": 0.013013110496103764, "step": 884 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 621.0, "completions/mean_length": 427.03125, "completions/mean_terminated_length": 404.3000183105469, "completions/min_length": 284.0, "completions/min_terminated_length": 284.0, "entropy": 0.43913501501083374, "epoch": 0.4737687366167024, "frac_reward_zero_std": 0.5, "grad_norm": 0.006735073868185282, "learning_rate": 1e-05, "loss": 0.0015, "num_tokens": 17885999.0, "reward": 0.6875, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.5701171159744263, "sampling/importance_sampling_ratio/mean": 1.0001567602157593, "sampling/importance_sampling_ratio/min": 0.7139126658439636, "sampling/sampling_logp_difference/max": 0.45115017890930176, "sampling/sampling_logp_difference/mean": 0.013849540613591671, "step": 885 }, { "clip_ratio/high_max": 5.67923671042081e-05, "clip_ratio/high_mean": 5.67923671042081e-05, "clip_ratio/low_mean": 0.00017409470456186682, "clip_ratio/low_min": 0.00017409470456186682, "clip_ratio/region_mean": 0.00023088707166607492, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 747.0, "completions/mean_length": 477.9375, "completions/mean_terminated_length": 447.9310302734375, "completions/min_length": 258.0, "completions/min_terminated_length": 258.0, "entropy": 0.42922772839665413, "epoch": 0.4743040685224839, "frac_reward_zero_std": 0.25, "grad_norm": 0.015612448565661907, "learning_rate": 1e-05, "loss": 0.0822, "num_tokens": 17905693.0, "reward": 0.6875, "reward_std": 0.3514062464237213, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.5738472938537598, "sampling/importance_sampling_ratio/mean": 1.0004065036773682, "sampling/importance_sampling_ratio/min": 0.541984498500824, "sampling/sampling_logp_difference/max": 0.6125178337097168, "sampling/sampling_logp_difference/mean": 0.014382733963429928, "step": 886 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00020096462685614824, "clip_ratio/low_min": 0.00020096462685614824, "clip_ratio/region_mean": 0.00020096462685614824, "completions/clipped_ratio": 0.0, "completions/max_length": 671.0, "completions/max_terminated_length": 671.0, "completions/mean_length": 381.0625, "completions/mean_terminated_length": 381.0625, "completions/min_length": 209.0, "completions/min_terminated_length": 209.0, "entropy": 0.3459123782813549, "epoch": 0.4748394004282655, "frac_reward_zero_std": 0.5, "grad_norm": 0.007520102430135012, "learning_rate": 1e-05, "loss": -0.0128, "num_tokens": 17921447.0, "reward": 0.78125, "reward_std": 0.2630178928375244, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.2851744890213013, "sampling/importance_sampling_ratio/mean": 0.9995664358139038, "sampling/importance_sampling_ratio/min": 0.6430121064186096, "sampling/sampling_logp_difference/max": 0.441591739654541, "sampling/sampling_logp_difference/mean": 0.011842616833746433, "step": 887 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 7.759156142128631e-05, "clip_ratio/low_min": 7.759156142128631e-05, "clip_ratio/region_mean": 7.759156142128631e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 525.0, "completions/max_terminated_length": 525.0, "completions/mean_length": 373.34375, "completions/mean_terminated_length": 373.34375, "completions/min_length": 228.0, "completions/min_terminated_length": 228.0, "entropy": 0.2709067929536104, "epoch": 0.4753747323340471, "frac_reward_zero_std": 0.25, "grad_norm": 0.020747454836964607, "learning_rate": 1e-05, "loss": 0.0305, "num_tokens": 17937346.0, "reward": 0.71875, "reward_std": 0.2651650309562683, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.3925729990005493, "sampling/importance_sampling_ratio/mean": 0.9998383522033691, "sampling/importance_sampling_ratio/min": 0.6875101923942566, "sampling/sampling_logp_difference/max": 0.3746786117553711, "sampling/sampling_logp_difference/mean": 0.009892567992210388, "step": 888 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.413545634131879e-05, "clip_ratio/low_min": 6.413545634131879e-05, "clip_ratio/region_mean": 6.413545634131879e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 681.0, "completions/max_terminated_length": 681.0, "completions/mean_length": 484.4375, "completions/mean_terminated_length": 484.4375, "completions/min_length": 268.0, "completions/min_terminated_length": 268.0, "entropy": 0.3678322583436966, "epoch": 0.4759100642398287, "frac_reward_zero_std": 0.0, "grad_norm": 0.02119183912873268, "learning_rate": 1e-05, "loss": -0.0034, "num_tokens": 17956944.0, "reward": 0.5625, "reward_std": 0.4355512857437134, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 0.9998264908790588, "sampling/importance_sampling_ratio/min": 0.654708206653595, "sampling/sampling_logp_difference/max": 0.8237423896789551, "sampling/sampling_logp_difference/mean": 0.012639732100069523, "step": 889 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 608.0, "completions/mean_length": 449.71875, "completions/mean_terminated_length": 439.45159912109375, "completions/min_length": 196.0, "completions/min_terminated_length": 196.0, "entropy": 0.3518533445894718, "epoch": 0.47644539614561027, "frac_reward_zero_std": 0.0, "grad_norm": 0.037341851741075516, "learning_rate": 1e-05, "loss": -0.0234, "num_tokens": 17975487.0, "reward": 0.8125, "reward_std": 0.3945523500442505, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.4925684928894043, "sampling/importance_sampling_ratio/mean": 1.0000295639038086, "sampling/importance_sampling_ratio/min": 0.6937601566314697, "sampling/sampling_logp_difference/max": 0.4004983901977539, "sampling/sampling_logp_difference/mean": 0.012080343440175056, "step": 890 }, { "clip_ratio/high_max": 9.197939652949572e-05, "clip_ratio/high_mean": 9.197939652949572e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 9.197939652949572e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 600.0, "completions/max_terminated_length": 600.0, "completions/mean_length": 358.65625, "completions/mean_terminated_length": 358.65625, "completions/min_length": 170.0, "completions/min_terminated_length": 170.0, "entropy": 0.4045837000012398, "epoch": 0.47698072805139186, "frac_reward_zero_std": 0.25, "grad_norm": 0.010673610493540764, "learning_rate": 1e-05, "loss": 0.0327, "num_tokens": 17990628.0, "reward": 0.875, "reward_std": 0.2925041913986206, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 1.5819710493087769, "sampling/importance_sampling_ratio/mean": 0.9999442100524902, "sampling/importance_sampling_ratio/min": 0.689908504486084, "sampling/sampling_logp_difference/max": 0.45867156982421875, "sampling/sampling_logp_difference/mean": 0.01309248898178339, "step": 891 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00013506213144864887, "clip_ratio/low_min": 0.00013506213144864887, "clip_ratio/region_mean": 0.00013506213144864887, "completions/clipped_ratio": 0.0, "completions/max_length": 651.0, "completions/max_terminated_length": 651.0, "completions/mean_length": 425.21875, "completions/mean_terminated_length": 425.21875, "completions/min_length": 202.0, "completions/min_terminated_length": 202.0, "entropy": 0.30039442516863346, "epoch": 0.47751605995717344, "frac_reward_zero_std": 0.25, "grad_norm": 0.01618858054280281, "learning_rate": 1e-05, "loss": 0.0064, "num_tokens": 18007995.0, "reward": 0.65625, "reward_std": 0.3787454068660736, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.4497257471084595, "sampling/importance_sampling_ratio/mean": 1.0000301599502563, "sampling/importance_sampling_ratio/min": 0.6121593713760376, "sampling/sampling_logp_difference/max": 0.49076271057128906, "sampling/sampling_logp_difference/mean": 0.01163009274750948, "step": 892 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.830223744851537e-05, "clip_ratio/low_min": 5.830223744851537e-05, "clip_ratio/region_mean": 5.830223744851537e-05, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 757.0, "completions/mean_length": 419.84375, "completions/mean_terminated_length": 396.63336181640625, "completions/min_length": 239.0, "completions/min_terminated_length": 239.0, "entropy": 0.34264643490314484, "epoch": 0.47805139186295503, "frac_reward_zero_std": 0.5, "grad_norm": 0.01804599165916443, "learning_rate": 1e-05, "loss": 0.0074, "num_tokens": 18025510.0, "reward": 0.84375, "reward_std": 0.22201895713806152, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.401878833770752, "sampling/importance_sampling_ratio/mean": 0.9999934434890747, "sampling/importance_sampling_ratio/min": 0.7447401285171509, "sampling/sampling_logp_difference/max": 0.3378133773803711, "sampling/sampling_logp_difference/mean": 0.01168726198375225, "step": 893 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 634.0, "completions/max_terminated_length": 634.0, "completions/mean_length": 408.53125, "completions/mean_terminated_length": 408.53125, "completions/min_length": 239.0, "completions/min_terminated_length": 239.0, "entropy": 0.4389772601425648, "epoch": 0.4785867237687366, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0302, "num_tokens": 18042199.0, "reward": 0.59375, "reward_std": 0.1293872892856598, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.3717992305755615, "sampling/importance_sampling_ratio/mean": 1.0001963376998901, "sampling/importance_sampling_ratio/min": 0.7095062732696533, "sampling/sampling_logp_difference/max": 0.3431859016418457, "sampling/sampling_logp_difference/mean": 0.012897140346467495, "step": 894 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 9.15080527192913e-05, "clip_ratio/low_min": 9.15080527192913e-05, "clip_ratio/region_mean": 9.15080527192913e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 585.0, "completions/max_terminated_length": 585.0, "completions/mean_length": 355.3125, "completions/mean_terminated_length": 355.3125, "completions/min_length": 132.0, "completions/min_terminated_length": 132.0, "entropy": 0.3668481893837452, "epoch": 0.4791220556745182, "frac_reward_zero_std": 0.5, "grad_norm": 0.010617706924676895, "learning_rate": 1e-05, "loss": 0.0338, "num_tokens": 18057177.0, "reward": 0.6875, "reward_std": 0.249358132481575, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.5940654277801514, "sampling/importance_sampling_ratio/mean": 1.0004090070724487, "sampling/importance_sampling_ratio/min": 0.5578004121780396, "sampling/sampling_logp_difference/max": 0.5837540626525879, "sampling/sampling_logp_difference/mean": 0.01301537174731493, "step": 895 }, { "clip_ratio/high_max": 8.973438525572419e-05, "clip_ratio/high_mean": 8.973438525572419e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 8.973438525572419e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 724.0, "completions/max_terminated_length": 724.0, "completions/mean_length": 389.53125, "completions/mean_terminated_length": 389.53125, "completions/min_length": 125.0, "completions/min_terminated_length": 125.0, "entropy": 0.3034164123237133, "epoch": 0.4796573875802998, "frac_reward_zero_std": 0.5, "grad_norm": 0.0055317021906375885, "learning_rate": 1e-05, "loss": 0.0266, "num_tokens": 18073554.0, "reward": 0.9375, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.9375, "rewards/accuracy_reward/std": 0.24593468010425568, "sampling/importance_sampling_ratio/max": 1.3771259784698486, "sampling/importance_sampling_ratio/mean": 0.9995330572128296, "sampling/importance_sampling_ratio/min": 0.676887571811676, "sampling/sampling_logp_difference/max": 0.3902500867843628, "sampling/sampling_logp_difference/mean": 0.011133765801787376, "step": 896 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 694.0, "completions/max_terminated_length": 694.0, "completions/mean_length": 382.5625, "completions/mean_terminated_length": 382.5625, "completions/min_length": 213.0, "completions/min_terminated_length": 213.0, "entropy": 0.3819986581802368, "epoch": 0.4801927194860814, "frac_reward_zero_std": 0.5, "grad_norm": 0.0030285643879324198, "learning_rate": 1e-05, "loss": 0.0614, "num_tokens": 18089500.0, "reward": 0.9375, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.9375, "rewards/accuracy_reward/std": 0.24593468010425568, "sampling/importance_sampling_ratio/max": 1.466621994972229, "sampling/importance_sampling_ratio/mean": 0.9995065927505493, "sampling/importance_sampling_ratio/min": 0.6455044746398926, "sampling/sampling_logp_difference/max": 0.43772315979003906, "sampling/sampling_logp_difference/mean": 0.01318583358079195, "step": 897 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 664.0, "completions/max_terminated_length": 664.0, "completions/mean_length": 359.84375, "completions/mean_terminated_length": 359.84375, "completions/min_length": 182.0, "completions/min_terminated_length": 182.0, "entropy": 0.42644477635622025, "epoch": 0.480728051391863, "frac_reward_zero_std": 0.5, "grad_norm": 0.0070777940563857555, "learning_rate": 1e-05, "loss": 0.0103, "num_tokens": 18104695.0, "reward": 0.75, "reward_std": 0.2314550280570984, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.3902513980865479, "sampling/importance_sampling_ratio/mean": 0.9994193315505981, "sampling/importance_sampling_ratio/min": 0.7062434554100037, "sampling/sampling_logp_difference/max": 0.3477952480316162, "sampling/sampling_logp_difference/mean": 0.014963710680603981, "step": 898 }, { "clip_ratio/high_max": 5.995203900965862e-05, "clip_ratio/high_mean": 5.995203900965862e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 5.995203900965862e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 754.0, "completions/max_terminated_length": 754.0, "completions/mean_length": 480.28125, "completions/mean_terminated_length": 480.28125, "completions/min_length": 235.0, "completions/min_terminated_length": 235.0, "entropy": 0.32665304839611053, "epoch": 0.48126338329764456, "frac_reward_zero_std": 0.5, "grad_norm": 0.0037798136472702026, "learning_rate": 1e-05, "loss": 0.014, "num_tokens": 18123992.0, "reward": 0.90625, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.432084560394287, "sampling/importance_sampling_ratio/mean": 1.000036358833313, "sampling/importance_sampling_ratio/min": 0.69344562292099, "sampling/sampling_logp_difference/max": 0.36608242988586426, "sampling/sampling_logp_difference/mean": 0.01185980997979641, "step": 899 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0001020408162730746, "clip_ratio/low_min": 0.0001020408162730746, "clip_ratio/region_mean": 0.0001020408162730746, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 681.0, "completions/mean_length": 369.28125, "completions/mean_terminated_length": 356.4193420410156, "completions/min_length": 189.0, "completions/min_terminated_length": 189.0, "entropy": 0.39549417793750763, "epoch": 0.4817987152034261, "frac_reward_zero_std": 0.0, "grad_norm": 0.016363950446248055, "learning_rate": 1e-05, "loss": 0.0588, "num_tokens": 18139761.0, "reward": 0.71875, "reward_std": 0.4765698313713074, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.5296053886413574, "sampling/importance_sampling_ratio/mean": 1.000364899635315, "sampling/importance_sampling_ratio/min": 0.6586078405380249, "sampling/sampling_logp_difference/max": 0.42500972747802734, "sampling/sampling_logp_difference/mean": 0.01368680689483881, "step": 900 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.073858094168827e-05, "clip_ratio/low_min": 6.073858094168827e-05, "clip_ratio/region_mean": 6.073858094168827e-05, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 632.0, "completions/mean_length": 419.21875, "completions/mean_terminated_length": 407.9677429199219, "completions/min_length": 178.0, "completions/min_terminated_length": 178.0, "entropy": 0.3452339880168438, "epoch": 0.4823340471092077, "frac_reward_zero_std": 0.5, "grad_norm": 0.021957147866487503, "learning_rate": 1e-05, "loss": 0.0221, "num_tokens": 18157088.0, "reward": 0.875, "reward_std": 0.2314550280570984, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 1.3405760526657104, "sampling/importance_sampling_ratio/mean": 0.9995335340499878, "sampling/importance_sampling_ratio/min": 0.3529183864593506, "sampling/sampling_logp_difference/max": 1.0415184497833252, "sampling/sampling_logp_difference/mean": 0.012348595075309277, "step": 901 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 754.0, "completions/mean_length": 522.5, "completions/mean_terminated_length": 487.4285888671875, "completions/min_length": 268.0, "completions/min_terminated_length": 268.0, "entropy": 0.46427471563220024, "epoch": 0.4828693790149893, "frac_reward_zero_std": 0.25, "grad_norm": 0.032786671072244644, "learning_rate": 1e-05, "loss": -0.0236, "num_tokens": 18177576.0, "reward": 0.3125, "reward_std": 0.3514062762260437, "rewards/accuracy_reward/mean": 0.3125, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.4217407703399658, "sampling/importance_sampling_ratio/mean": 0.9997016191482544, "sampling/importance_sampling_ratio/min": 0.6393582224845886, "sampling/sampling_logp_difference/max": 0.44729042053222656, "sampling/sampling_logp_difference/mean": 0.015123426914215088, "step": 902 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.2279381634434685e-05, "clip_ratio/low_min": 5.2279381634434685e-05, "clip_ratio/region_mean": 5.2279381634434685e-05, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 753.0, "completions/mean_length": 465.78125, "completions/mean_terminated_length": 445.63336181640625, "completions/min_length": 167.0, "completions/min_terminated_length": 167.0, "entropy": 0.29401976987719536, "epoch": 0.48340471092077086, "frac_reward_zero_std": 0.25, "grad_norm": 0.022586828097701073, "learning_rate": 1e-05, "loss": 0.0886, "num_tokens": 18197137.0, "reward": 0.8125, "reward_std": 0.3471825420856476, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.3233128786087036, "sampling/importance_sampling_ratio/mean": 0.9994017481803894, "sampling/importance_sampling_ratio/min": 0.7277961373329163, "sampling/sampling_logp_difference/max": 0.3177342414855957, "sampling/sampling_logp_difference/mean": 0.010220268741250038, "step": 903 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 559.0, "completions/mean_length": 379.78125, "completions/mean_terminated_length": 367.258056640625, "completions/min_length": 165.0, "completions/min_terminated_length": 165.0, "entropy": 0.2967627942562103, "epoch": 0.48394004282655245, "frac_reward_zero_std": 0.75, "grad_norm": 0.014837553724646568, "learning_rate": 1e-05, "loss": 0.0585, "num_tokens": 18212674.0, "reward": 0.90625, "reward_std": 0.1293872892856598, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.5250288248062134, "sampling/importance_sampling_ratio/mean": 1.0000722408294678, "sampling/importance_sampling_ratio/min": 0.7006058096885681, "sampling/sampling_logp_difference/max": 0.4220132827758789, "sampling/sampling_logp_difference/mean": 0.010664481669664383, "step": 904 }, { "clip_ratio/high_max": 8.73515018611215e-05, "clip_ratio/high_mean": 8.73515018611215e-05, "clip_ratio/low_mean": 0.00019531226280378178, "clip_ratio/low_min": 0.00019531226280378178, "clip_ratio/region_mean": 0.0002826637646649033, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 768.0, "completions/mean_length": 425.28125, "completions/mean_terminated_length": 414.2257995605469, "completions/min_length": 185.0, "completions/min_terminated_length": 185.0, "entropy": 0.5105507746338844, "epoch": 0.48447537473233404, "frac_reward_zero_std": 0.25, "grad_norm": 0.01044401153922081, "learning_rate": 1e-05, "loss": 0.0212, "num_tokens": 18230419.0, "reward": 0.5625, "reward_std": 0.2925041913986206, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.6065908670425415, "sampling/importance_sampling_ratio/mean": 0.9995835423469543, "sampling/importance_sampling_ratio/min": 0.6064783334732056, "sampling/sampling_logp_difference/max": 0.5000863075256348, "sampling/sampling_logp_difference/mean": 0.015154949389398098, "step": 905 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 726.0, "completions/mean_length": 429.125, "completions/mean_terminated_length": 418.19354248046875, "completions/min_length": 240.0, "completions/min_terminated_length": 240.0, "entropy": 0.3160056099295616, "epoch": 0.48501070663811563, "frac_reward_zero_std": 0.75, "grad_norm": 0.006075732409954071, "learning_rate": 1e-05, "loss": 0.0006, "num_tokens": 18247967.0, "reward": 0.78125, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.457342505455017, "sampling/importance_sampling_ratio/mean": 1.0002059936523438, "sampling/importance_sampling_ratio/min": 0.6344391703605652, "sampling/sampling_logp_difference/max": 0.45501387119293213, "sampling/sampling_logp_difference/mean": 0.012042401358485222, "step": 906 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.486767233582214e-05, "clip_ratio/low_min": 6.486767233582214e-05, "clip_ratio/region_mean": 6.486767233582214e-05, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 701.0, "completions/mean_length": 445.09375, "completions/mean_terminated_length": 434.6773986816406, "completions/min_length": 294.0, "completions/min_terminated_length": 294.0, "entropy": 0.26794357411563396, "epoch": 0.4855460385438972, "frac_reward_zero_std": 0.5, "grad_norm": 0.029239822179079056, "learning_rate": 1e-05, "loss": 0.05, "num_tokens": 18265538.0, "reward": 0.75, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.4175670146942139, "sampling/importance_sampling_ratio/mean": 1.0000230073928833, "sampling/importance_sampling_ratio/min": 0.6305237412452698, "sampling/sampling_logp_difference/max": 0.46120452880859375, "sampling/sampling_logp_difference/mean": 0.010698436759412289, "step": 907 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00022639386588707566, "clip_ratio/low_min": 0.00022639386588707566, "clip_ratio/region_mean": 0.00022639386588707566, "completions/clipped_ratio": 0.0, "completions/max_length": 665.0, "completions/max_terminated_length": 665.0, "completions/mean_length": 426.875, "completions/mean_terminated_length": 426.875, "completions/min_length": 205.0, "completions/min_terminated_length": 205.0, "entropy": 0.34501947835087776, "epoch": 0.4860813704496788, "frac_reward_zero_std": 0.25, "grad_norm": 0.01808975823223591, "learning_rate": 1e-05, "loss": 0.0413, "num_tokens": 18282846.0, "reward": 0.40625, "reward_std": 0.38816186785697937, "rewards/accuracy_reward/mean": 0.40625, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.534271001815796, "sampling/importance_sampling_ratio/mean": 0.9991494417190552, "sampling/importance_sampling_ratio/min": 0.7002196907997131, "sampling/sampling_logp_difference/max": 0.4280552864074707, "sampling/sampling_logp_difference/mean": 0.01250333059579134, "step": 908 }, { "clip_ratio/high_max": 0.00013555483019445091, "clip_ratio/high_mean": 0.00013555483019445091, "clip_ratio/low_mean": 6.702412792947143e-05, "clip_ratio/low_min": 6.702412792947143e-05, "clip_ratio/region_mean": 0.00020257895812392235, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 712.0, "completions/mean_length": 453.9375, "completions/mean_terminated_length": 421.4482727050781, "completions/min_length": 179.0, "completions/min_terminated_length": 179.0, "entropy": 0.4775955379009247, "epoch": 0.4866167023554604, "frac_reward_zero_std": 0.0, "grad_norm": 0.01463873777538538, "learning_rate": 1e-05, "loss": -0.0133, "num_tokens": 18302212.0, "reward": 0.59375, "reward_std": 0.3987956643104553, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.7850691080093384, "sampling/importance_sampling_ratio/mean": 1.0003458261489868, "sampling/importance_sampling_ratio/min": 0.6452078223228455, "sampling/sampling_logp_difference/max": 0.5794570446014404, "sampling/sampling_logp_difference/mean": 0.014868216589093208, "step": 909 }, { "clip_ratio/high_max": 7.237985118990764e-05, "clip_ratio/high_mean": 7.237985118990764e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 7.237985118990764e-05, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 660.0, "completions/mean_length": 444.15625, "completions/mean_terminated_length": 433.70965576171875, "completions/min_length": 277.0, "completions/min_terminated_length": 277.0, "entropy": 0.33012066781520844, "epoch": 0.487152034261242, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0347, "num_tokens": 18320297.0, "reward": 0.9375, "reward_std": 0.1157275140285492, "rewards/accuracy_reward/mean": 0.9375, "rewards/accuracy_reward/std": 0.24593468010425568, "sampling/importance_sampling_ratio/max": 1.5443007946014404, "sampling/importance_sampling_ratio/mean": 0.9999580383300781, "sampling/importance_sampling_ratio/min": 0.6941600441932678, "sampling/sampling_logp_difference/max": 0.4345712661743164, "sampling/sampling_logp_difference/mean": 0.011883266270160675, "step": 910 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 7.841907063266262e-05, "clip_ratio/low_min": 7.841907063266262e-05, "clip_ratio/region_mean": 7.841907063266262e-05, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 620.0, "completions/mean_length": 448.53125, "completions/mean_terminated_length": 402.89288330078125, "completions/min_length": 149.0, "completions/min_terminated_length": 149.0, "entropy": 0.3132179379463196, "epoch": 0.4876873661670236, "frac_reward_zero_std": 0.5, "grad_norm": 0.01716768555343151, "learning_rate": 1e-05, "loss": 0.0517, "num_tokens": 18338282.0, "reward": 0.8125, "reward_std": 0.249358132481575, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.6091407537460327, "sampling/importance_sampling_ratio/mean": 1.0003584623336792, "sampling/importance_sampling_ratio/min": 0.327725887298584, "sampling/sampling_logp_difference/max": 1.1155776977539062, "sampling/sampling_logp_difference/mean": 0.011485504917800426, "step": 911 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 658.0, "completions/mean_length": 485.4375, "completions/mean_terminated_length": 456.2069091796875, "completions/min_length": 299.0, "completions/min_terminated_length": 299.0, "entropy": 0.39809851720929146, "epoch": 0.48822269807280516, "frac_reward_zero_std": 0.25, "grad_norm": 0.005597486160695553, "learning_rate": 1e-05, "loss": 0.0598, "num_tokens": 18357880.0, "reward": 0.34375, "reward_std": 0.3471629321575165, "rewards/accuracy_reward/mean": 0.34375, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.859739899635315, "sampling/importance_sampling_ratio/mean": 1.0005167722702026, "sampling/importance_sampling_ratio/min": 0.5668060779571533, "sampling/sampling_logp_difference/max": 0.6204366683959961, "sampling/sampling_logp_difference/mean": 0.013395301066339016, "step": 912 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 726.0, "completions/max_terminated_length": 726.0, "completions/mean_length": 417.0625, "completions/mean_terminated_length": 417.0625, "completions/min_length": 189.0, "completions/min_terminated_length": 189.0, "entropy": 0.37044214457273483, "epoch": 0.48875802997858675, "frac_reward_zero_std": 0.25, "grad_norm": 0.026986071839928627, "learning_rate": 1e-05, "loss": 0.0587, "num_tokens": 18375458.0, "reward": 0.84375, "reward_std": 0.3061639964580536, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.3652336597442627, "sampling/importance_sampling_ratio/mean": 0.999951183795929, "sampling/importance_sampling_ratio/min": 0.7097204923629761, "sampling/sampling_logp_difference/max": 0.3428840637207031, "sampling/sampling_logp_difference/mean": 0.012635950930416584, "step": 913 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.688068242510781e-05, "clip_ratio/low_min": 6.688068242510781e-05, "clip_ratio/region_mean": 6.688068242510781e-05, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 667.0, "completions/mean_length": 429.625, "completions/mean_terminated_length": 407.0666809082031, "completions/min_length": 208.0, "completions/min_terminated_length": 208.0, "entropy": 0.35477233305573463, "epoch": 0.4892933618843683, "frac_reward_zero_std": 0.5, "grad_norm": 0.005654192063957453, "learning_rate": 1e-05, "loss": -0.0307, "num_tokens": 18392870.0, "reward": 0.75, "reward_std": 0.2314550280570984, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.4293205738067627, "sampling/importance_sampling_ratio/mean": 0.9999894499778748, "sampling/importance_sampling_ratio/min": 0.7082498669624329, "sampling/sampling_logp_difference/max": 0.35719919204711914, "sampling/sampling_logp_difference/mean": 0.012324618175625801, "step": 914 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 607.0, "completions/mean_length": 419.875, "completions/mean_terminated_length": 396.66668701171875, "completions/min_length": 196.0, "completions/min_terminated_length": 196.0, "entropy": 0.41905029118061066, "epoch": 0.48982869379014987, "frac_reward_zero_std": 0.5, "grad_norm": 0.021844206377863884, "learning_rate": 1e-05, "loss": 0.031, "num_tokens": 18410490.0, "reward": 0.625, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.3654179573059082, "sampling/importance_sampling_ratio/mean": 1.0001132488250732, "sampling/importance_sampling_ratio/min": 0.6708257794380188, "sampling/sampling_logp_difference/max": 0.39924585819244385, "sampling/sampling_logp_difference/mean": 0.014455149881541729, "step": 915 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 8.218277798732743e-05, "clip_ratio/low_min": 8.218277798732743e-05, "clip_ratio/region_mean": 8.218277798732743e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 643.0, "completions/max_terminated_length": 643.0, "completions/mean_length": 384.9375, "completions/mean_terminated_length": 384.9375, "completions/min_length": 132.0, "completions/min_terminated_length": 132.0, "entropy": 0.387206245213747, "epoch": 0.49036402569593146, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0493, "num_tokens": 18426696.0, "reward": 0.90625, "reward_std": 0.1293872892856598, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.6574963331222534, "sampling/importance_sampling_ratio/mean": 0.9998041987419128, "sampling/importance_sampling_ratio/min": 0.7262021899223328, "sampling/sampling_logp_difference/max": 0.5053081512451172, "sampling/sampling_logp_difference/mean": 0.013358903117477894, "step": 916 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 742.0, "completions/mean_length": 512.6875, "completions/mean_terminated_length": 486.2758483886719, "completions/min_length": 230.0, "completions/min_terminated_length": 230.0, "entropy": 0.35392509400844574, "epoch": 0.49089935760171305, "frac_reward_zero_std": 0.25, "grad_norm": 0.02172613888978958, "learning_rate": 1e-05, "loss": 0.0728, "num_tokens": 18446902.0, "reward": 0.71875, "reward_std": 0.3608423173427582, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.3223521709442139, "sampling/importance_sampling_ratio/mean": 0.9995226263999939, "sampling/importance_sampling_ratio/min": 0.6217734813690186, "sampling/sampling_logp_difference/max": 0.47517943382263184, "sampling/sampling_logp_difference/mean": 0.012370355427265167, "step": 917 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 8.729050023248419e-05, "clip_ratio/low_min": 8.729050023248419e-05, "clip_ratio/region_mean": 8.729050023248419e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 597.0, "completions/max_terminated_length": 597.0, "completions/mean_length": 381.5, "completions/mean_terminated_length": 381.5, "completions/min_length": 234.0, "completions/min_terminated_length": 234.0, "entropy": 0.3457867503166199, "epoch": 0.49143468950749464, "frac_reward_zero_std": 0.25, "grad_norm": 0.005011693574488163, "learning_rate": 1e-05, "loss": -0.0139, "num_tokens": 18462854.0, "reward": 0.84375, "reward_std": 0.3061639964580536, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.459420919418335, "sampling/importance_sampling_ratio/mean": 1.000162959098816, "sampling/importance_sampling_ratio/min": 0.6616084575653076, "sampling/sampling_logp_difference/max": 0.41308140754699707, "sampling/sampling_logp_difference/mean": 0.012941376306116581, "step": 918 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 9.321401739725843e-05, "clip_ratio/low_min": 9.321401739725843e-05, "clip_ratio/region_mean": 9.321401739725843e-05, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 748.0, "completions/mean_length": 406.90625, "completions/mean_terminated_length": 395.258056640625, "completions/min_length": 140.0, "completions/min_terminated_length": 140.0, "entropy": 0.3992702253162861, "epoch": 0.4919700214132762, "frac_reward_zero_std": 0.5, "grad_norm": 0.004719160031527281, "learning_rate": 1e-05, "loss": 0.0304, "num_tokens": 18479811.0, "reward": 0.46875, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.46875, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.4867140054702759, "sampling/importance_sampling_ratio/mean": 0.9998989701271057, "sampling/importance_sampling_ratio/min": 0.655383288860321, "sampling/sampling_logp_difference/max": 0.4225350618362427, "sampling/sampling_logp_difference/mean": 0.013475977815687656, "step": 919 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00023113489805837162, "clip_ratio/low_min": 0.00023113489805837162, "clip_ratio/region_mean": 0.00023113489805837162, "completions/clipped_ratio": 0.375, "completions/max_length": 768.0, "completions/max_terminated_length": 731.0, "completions/mean_length": 586.28125, "completions/mean_terminated_length": 477.25, "completions/min_length": 295.0, "completions/min_terminated_length": 295.0, "entropy": 0.5839615426957607, "epoch": 0.4925053533190578, "frac_reward_zero_std": 0.25, "grad_norm": 0.013810857199132442, "learning_rate": 1e-05, "loss": 0.0409, "num_tokens": 18502924.0, "reward": 0.3125, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.3125, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.4311981201171875, "sampling/importance_sampling_ratio/mean": 0.9998413324356079, "sampling/importance_sampling_ratio/min": 0.447336882352829, "sampling/sampling_logp_difference/max": 0.804443359375, "sampling/sampling_logp_difference/mean": 0.01793341338634491, "step": 920 }, { "clip_ratio/high_max": 6.145526276668534e-05, "clip_ratio/high_mean": 6.145526276668534e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 6.145526276668534e-05, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 762.0, "completions/mean_length": 450.8125, "completions/mean_terminated_length": 440.58062744140625, "completions/min_length": 276.0, "completions/min_terminated_length": 276.0, "entropy": 0.31450655683875084, "epoch": 0.4930406852248394, "frac_reward_zero_std": 0.25, "grad_norm": 0.0160969328135252, "learning_rate": 1e-05, "loss": 0.0518, "num_tokens": 18521374.0, "reward": 0.875, "reward_std": 0.2925041913986206, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 1.6108704805374146, "sampling/importance_sampling_ratio/mean": 0.9998055696487427, "sampling/importance_sampling_ratio/min": 0.6242378354072571, "sampling/sampling_logp_difference/max": 0.4767746925354004, "sampling/sampling_logp_difference/mean": 0.011314880102872849, "step": 921 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00016263027646346018, "clip_ratio/low_min": 0.00016263027646346018, "clip_ratio/region_mean": 0.00016263027646346018, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 658.0, "completions/mean_length": 511.03125, "completions/mean_terminated_length": 439.0799865722656, "completions/min_length": 227.0, "completions/min_terminated_length": 227.0, "entropy": 0.46680184081196785, "epoch": 0.493576017130621, "frac_reward_zero_std": 0.0, "grad_norm": 0.01299319788813591, "learning_rate": 1e-05, "loss": 0.0957, "num_tokens": 18541751.0, "reward": 0.46875, "reward_std": 0.4765698313713074, "rewards/accuracy_reward/mean": 0.46875, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.4235397577285767, "sampling/importance_sampling_ratio/mean": 0.9999666810035706, "sampling/importance_sampling_ratio/min": 0.6164647340774536, "sampling/sampling_logp_difference/max": 0.48375415802001953, "sampling/sampling_logp_difference/mean": 0.014782430604100227, "step": 922 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0001957513886736706, "clip_ratio/low_min": 0.0001957513886736706, "clip_ratio/region_mean": 0.0001957513886736706, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 714.0, "completions/mean_length": 489.53125, "completions/mean_terminated_length": 411.55999755859375, "completions/min_length": 174.0, "completions/min_terminated_length": 174.0, "entropy": 0.4491739198565483, "epoch": 0.4941113490364026, "frac_reward_zero_std": 0.25, "grad_norm": 0.008904200047254562, "learning_rate": 1e-05, "loss": 0.0112, "num_tokens": 18561216.0, "reward": 0.65625, "reward_std": 0.3198433816432953, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.4237003326416016, "sampling/importance_sampling_ratio/mean": 1.0003609657287598, "sampling/importance_sampling_ratio/min": 0.5650072693824768, "sampling/sampling_logp_difference/max": 0.5709166526794434, "sampling/sampling_logp_difference/mean": 0.014157862402498722, "step": 923 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.49350622552447e-05, "clip_ratio/low_min": 6.49350622552447e-05, "clip_ratio/region_mean": 6.49350622552447e-05, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 647.0, "completions/mean_length": 441.90625, "completions/mean_terminated_length": 408.17242431640625, "completions/min_length": 211.0, "completions/min_terminated_length": 211.0, "entropy": 0.435975544154644, "epoch": 0.49464668094218417, "frac_reward_zero_std": 0.0, "grad_norm": 0.01936471462249756, "learning_rate": 1e-05, "loss": 0.1249, "num_tokens": 18579189.0, "reward": 0.59375, "reward_std": 0.4807935357093811, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.4452024698257446, "sampling/importance_sampling_ratio/mean": 0.9998061656951904, "sampling/importance_sampling_ratio/min": 0.6416178941726685, "sampling/sampling_logp_difference/max": 0.44376230239868164, "sampling/sampling_logp_difference/mean": 0.014069242402911186, "step": 924 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 8.300133049488068e-05, "clip_ratio/low_min": 8.300133049488068e-05, "clip_ratio/region_mean": 8.300133049488068e-05, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 545.0, "completions/mean_length": 454.53125, "completions/mean_terminated_length": 396.4814758300781, "completions/min_length": 296.0, "completions/min_terminated_length": 296.0, "entropy": 0.42174603790044785, "epoch": 0.49518201284796576, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0071, "num_tokens": 18597758.0, "reward": 0.71875, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.4341439008712769, "sampling/importance_sampling_ratio/mean": 1.0000791549682617, "sampling/importance_sampling_ratio/min": 0.6485762000083923, "sampling/sampling_logp_difference/max": 0.43297576904296875, "sampling/sampling_logp_difference/mean": 0.013600548729300499, "step": 925 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 723.0, "completions/mean_length": 417.53125, "completions/mean_terminated_length": 406.2257995605469, "completions/min_length": 195.0, "completions/min_terminated_length": 195.0, "entropy": 0.3950592279434204, "epoch": 0.49571734475374735, "frac_reward_zero_std": 0.25, "grad_norm": 0.011150640435516834, "learning_rate": 1e-05, "loss": -0.0002, "num_tokens": 18615231.0, "reward": 0.75, "reward_std": 0.3514062464237213, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.535329818725586, "sampling/importance_sampling_ratio/mean": 1.0002329349517822, "sampling/importance_sampling_ratio/min": 0.6307322382926941, "sampling/sampling_logp_difference/max": 0.4608738422393799, "sampling/sampling_logp_difference/mean": 0.013399777002632618, "step": 926 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 626.0, "completions/max_terminated_length": 626.0, "completions/mean_length": 387.0625, "completions/mean_terminated_length": 387.0625, "completions/min_length": 167.0, "completions/min_terminated_length": 167.0, "entropy": 0.48306476324796677, "epoch": 0.49625267665952894, "frac_reward_zero_std": 0.25, "grad_norm": 0.008646552450954914, "learning_rate": 1e-05, "loss": 0.0408, "num_tokens": 18630673.0, "reward": 0.71875, "reward_std": 0.3198433816432953, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.5809540748596191, "sampling/importance_sampling_ratio/mean": 1.0004370212554932, "sampling/importance_sampling_ratio/min": 0.4879007339477539, "sampling/sampling_logp_difference/max": 0.7176432609558105, "sampling/sampling_logp_difference/mean": 0.014795218594372272, "step": 927 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.6957175072748214e-05, "clip_ratio/low_min": 4.6957175072748214e-05, "clip_ratio/region_mean": 4.6957175072748214e-05, "completions/clipped_ratio": 0.25, "completions/max_length": 768.0, "completions/max_terminated_length": 758.0, "completions/mean_length": 527.6875, "completions/mean_terminated_length": 447.5833435058594, "completions/min_length": 280.0, "completions/min_terminated_length": 280.0, "entropy": 0.5890818536281586, "epoch": 0.49678800856531047, "frac_reward_zero_std": 0.5, "grad_norm": 0.005541226826608181, "learning_rate": 1e-05, "loss": 0.0658, "num_tokens": 18651943.0, "reward": 0.5, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5080004930496216, "sampling/importance_sampling_ratio/max": 1.7834889888763428, "sampling/importance_sampling_ratio/mean": 1.0003581047058105, "sampling/importance_sampling_ratio/min": 0.710831880569458, "sampling/sampling_logp_difference/max": 0.5785715579986572, "sampling/sampling_logp_difference/mean": 0.01759391278028488, "step": 928 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.1461505790939555e-05, "clip_ratio/low_min": 5.1461505790939555e-05, "clip_ratio/region_mean": 5.1461505790939555e-05, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 765.0, "completions/mean_length": 521.625, "completions/mean_terminated_length": 505.20001220703125, "completions/min_length": 236.0, "completions/min_terminated_length": 236.0, "entropy": 0.4014683738350868, "epoch": 0.49732334047109206, "frac_reward_zero_std": 0.25, "grad_norm": 0.02768176980316639, "learning_rate": 1e-05, "loss": 0.0296, "num_tokens": 18672435.0, "reward": 0.78125, "reward_std": 0.3377465009689331, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.4099376201629639, "sampling/importance_sampling_ratio/mean": 0.9998350143432617, "sampling/importance_sampling_ratio/min": 0.7214834690093994, "sampling/sampling_logp_difference/max": 0.34354543685913086, "sampling/sampling_logp_difference/mean": 0.012690262869000435, "step": 929 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 577.0, "completions/mean_length": 399.84375, "completions/mean_terminated_length": 387.9677429199219, "completions/min_length": 253.0, "completions/min_terminated_length": 253.0, "entropy": 0.2748953439295292, "epoch": 0.49785867237687365, "frac_reward_zero_std": 0.5, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0179, "num_tokens": 18688526.0, "reward": 0.625, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.3876456022262573, "sampling/importance_sampling_ratio/mean": 0.9994591474533081, "sampling/importance_sampling_ratio/min": 0.7178421020507812, "sampling/sampling_logp_difference/max": 0.33150577545166016, "sampling/sampling_logp_difference/mean": 0.01044301874935627, "step": 930 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00015519846783718094, "clip_ratio/low_min": 0.00015519846783718094, "clip_ratio/region_mean": 0.00015519846783718094, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 727.0, "completions/mean_length": 492.09375, "completions/mean_terminated_length": 441.0, "completions/min_length": 262.0, "completions/min_terminated_length": 262.0, "entropy": 0.4753256067633629, "epoch": 0.49839400428265523, "frac_reward_zero_std": 0.0, "grad_norm": 0.01695760153234005, "learning_rate": 1e-05, "loss": 0.0353, "num_tokens": 18708081.0, "reward": 0.3125, "reward_std": 0.4082317352294922, "rewards/accuracy_reward/mean": 0.3125, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.3967517614364624, "sampling/importance_sampling_ratio/mean": 1.0000271797180176, "sampling/importance_sampling_ratio/min": 0.6274396181106567, "sampling/sampling_logp_difference/max": 0.4661078453063965, "sampling/sampling_logp_difference/mean": 0.014627913944423199, "step": 931 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 4.770992381963879e-05, "clip_ratio/low_min": 4.770992381963879e-05, "clip_ratio/region_mean": 4.770992381963879e-05, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 550.0, "completions/mean_length": 410.3125, "completions/mean_terminated_length": 344.0740661621094, "completions/min_length": 130.0, "completions/min_terminated_length": 130.0, "entropy": 0.48293643072247505, "epoch": 0.4989293361884368, "frac_reward_zero_std": 0.25, "grad_norm": 0.02340005896985531, "learning_rate": 1e-05, "loss": 0.0345, "num_tokens": 18725459.0, "reward": 0.65625, "reward_std": 0.3377465009689331, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.626429796218872, "sampling/importance_sampling_ratio/mean": 1.0000079870224, "sampling/importance_sampling_ratio/min": 0.6924667954444885, "sampling/sampling_logp_difference/max": 0.4863872528076172, "sampling/sampling_logp_difference/mean": 0.015874313190579414, "step": 932 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 564.0, "completions/max_terminated_length": 564.0, "completions/mean_length": 433.375, "completions/mean_terminated_length": 433.375, "completions/min_length": 199.0, "completions/min_terminated_length": 199.0, "entropy": 0.31241924688220024, "epoch": 0.4994646680942184, "frac_reward_zero_std": 0.5, "grad_norm": 0.008866383694112301, "learning_rate": 1e-05, "loss": -0.007, "num_tokens": 18742935.0, "reward": 0.8125, "reward_std": 0.2587745785713196, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.4002013206481934, "sampling/importance_sampling_ratio/mean": 1.0002180337905884, "sampling/importance_sampling_ratio/min": 0.6917651295661926, "sampling/sampling_logp_difference/max": 0.36850881576538086, "sampling/sampling_logp_difference/mean": 0.011351009830832481, "step": 933 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 7.042253855615854e-05, "clip_ratio/low_min": 7.042253855615854e-05, "clip_ratio/region_mean": 7.042253855615854e-05, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 559.0, "completions/mean_length": 416.46875, "completions/mean_terminated_length": 393.0333557128906, "completions/min_length": 218.0, "completions/min_terminated_length": 218.0, "entropy": 0.36351118236780167, "epoch": 0.5, "frac_reward_zero_std": 0.25, "grad_norm": 0.015173356048762798, "learning_rate": 1e-05, "loss": 0.0831, "num_tokens": 18759926.0, "reward": 0.65625, "reward_std": 0.3061639666557312, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.472735047340393, "sampling/importance_sampling_ratio/mean": 0.9995389580726624, "sampling/importance_sampling_ratio/min": 0.6942189931869507, "sampling/sampling_logp_difference/max": 0.38712120056152344, "sampling/sampling_logp_difference/mean": 0.012699334882199764, "step": 934 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.681818220182322e-05, "clip_ratio/low_min": 5.681818220182322e-05, "clip_ratio/region_mean": 5.681818220182322e-05, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 763.0, "completions/mean_length": 471.3125, "completions/mean_terminated_length": 440.6206970214844, "completions/min_length": 219.0, "completions/min_terminated_length": 219.0, "entropy": 0.3409218303859234, "epoch": 0.5005353319057816, "frac_reward_zero_std": 0.5, "grad_norm": 0.009462235495448112, "learning_rate": 1e-05, "loss": -0.0104, "num_tokens": 18778928.0, "reward": 0.875, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 1.3341530561447144, "sampling/importance_sampling_ratio/mean": 1.0001732110977173, "sampling/importance_sampling_ratio/min": 0.5364710688591003, "sampling/sampling_logp_difference/max": 0.6227426528930664, "sampling/sampling_logp_difference/mean": 0.0118362195789814, "step": 935 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00015170325787039474, "clip_ratio/low_min": 0.00015170325787039474, "clip_ratio/region_mean": 0.00015170325787039474, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 640.0, "completions/mean_length": 393.53125, "completions/mean_terminated_length": 381.45159912109375, "completions/min_length": 263.0, "completions/min_terminated_length": 263.0, "entropy": 0.3642778657376766, "epoch": 0.5010706638115632, "frac_reward_zero_std": 0.0, "grad_norm": 0.021121865138411522, "learning_rate": 1e-05, "loss": -0.0228, "num_tokens": 18794825.0, "reward": 0.65625, "reward_std": 0.4397946000099182, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.3397612571716309, "sampling/importance_sampling_ratio/mean": 0.999901533126831, "sampling/importance_sampling_ratio/min": 0.6939085721969604, "sampling/sampling_logp_difference/max": 0.365415096282959, "sampling/sampling_logp_difference/mean": 0.012214125134050846, "step": 936 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 642.0, "completions/max_terminated_length": 642.0, "completions/mean_length": 406.25, "completions/mean_terminated_length": 406.25, "completions/min_length": 176.0, "completions/min_terminated_length": 176.0, "entropy": 0.31350530311465263, "epoch": 0.5016059957173448, "frac_reward_zero_std": 0.25, "grad_norm": 0.008388281799852848, "learning_rate": 1e-05, "loss": -0.0677, "num_tokens": 18811657.0, "reward": 0.4375, "reward_std": 0.2925041913986206, "rewards/accuracy_reward/mean": 0.4375, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.435948371887207, "sampling/importance_sampling_ratio/mean": 0.9997534155845642, "sampling/importance_sampling_ratio/min": 0.7478278875350952, "sampling/sampling_logp_difference/max": 0.36182546615600586, "sampling/sampling_logp_difference/mean": 0.011519297026097775, "step": 937 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 8.896797226043418e-05, "clip_ratio/low_min": 8.896797226043418e-05, "clip_ratio/region_mean": 8.896797226043418e-05, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 551.0, "completions/mean_length": 313.15625, "completions/mean_terminated_length": 298.4838562011719, "completions/min_length": 150.0, "completions/min_terminated_length": 150.0, "entropy": 0.4786602444946766, "epoch": 0.5021413276231264, "frac_reward_zero_std": 0.5, "grad_norm": 0.004355534911155701, "learning_rate": 1e-05, "loss": 0.1188, "num_tokens": 18825790.0, "reward": 0.9375, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.9375, "rewards/accuracy_reward/std": 0.24593468010425568, "sampling/importance_sampling_ratio/max": 1.4010833501815796, "sampling/importance_sampling_ratio/mean": 0.9997631907463074, "sampling/importance_sampling_ratio/min": 0.6279602646827698, "sampling/sampling_logp_difference/max": 0.46527838706970215, "sampling/sampling_logp_difference/mean": 0.015440299175679684, "step": 938 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 582.0, "completions/mean_length": 386.4375, "completions/mean_terminated_length": 374.1290283203125, "completions/min_length": 213.0, "completions/min_terminated_length": 213.0, "entropy": 0.3480219542980194, "epoch": 0.5026766595289079, "frac_reward_zero_std": 0.25, "grad_norm": 0.032420963048934937, "learning_rate": 1e-05, "loss": 0.1361, "num_tokens": 18841556.0, "reward": 0.875, "reward_std": 0.292504221200943, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 1.4075968265533447, "sampling/importance_sampling_ratio/mean": 1.0001126527786255, "sampling/importance_sampling_ratio/min": 0.7367861866950989, "sampling/sampling_logp_difference/max": 0.34188389778137207, "sampling/sampling_logp_difference/mean": 0.011499529704451561, "step": 939 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 694.0, "completions/mean_length": 404.1875, "completions/mean_terminated_length": 392.45159912109375, "completions/min_length": 146.0, "completions/min_terminated_length": 146.0, "entropy": 0.3695135787129402, "epoch": 0.5032119914346895, "frac_reward_zero_std": 0.5, "grad_norm": 0.00645273644477129, "learning_rate": 1e-05, "loss": 0.0247, "num_tokens": 18858218.0, "reward": 0.71875, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.4694135189056396, "sampling/importance_sampling_ratio/mean": 0.9998634457588196, "sampling/importance_sampling_ratio/min": 0.6921114325523376, "sampling/sampling_logp_difference/max": 0.38486337661743164, "sampling/sampling_logp_difference/mean": 0.013259908184409142, "step": 940 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 622.0, "completions/max_terminated_length": 622.0, "completions/mean_length": 413.25, "completions/mean_terminated_length": 413.25, "completions/min_length": 211.0, "completions/min_terminated_length": 211.0, "entropy": 0.3180437833070755, "epoch": 0.5037473233404711, "frac_reward_zero_std": 0.5, "grad_norm": 0.015431229956448078, "learning_rate": 1e-05, "loss": 0.0246, "num_tokens": 18875578.0, "reward": 0.71875, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.5414668321609497, "sampling/importance_sampling_ratio/mean": 0.9999116659164429, "sampling/importance_sampling_ratio/min": 0.5779845118522644, "sampling/sampling_logp_difference/max": 0.5482082366943359, "sampling/sampling_logp_difference/mean": 0.01156378909945488, "step": 941 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 637.0, "completions/mean_length": 395.84375, "completions/mean_terminated_length": 383.83868408203125, "completions/min_length": 206.0, "completions/min_terminated_length": 206.0, "entropy": 0.354518286883831, "epoch": 0.5042826552462527, "frac_reward_zero_std": 0.25, "grad_norm": 0.009431066922843456, "learning_rate": 1e-05, "loss": -0.0146, "num_tokens": 18892029.0, "reward": 0.5625, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0001990795135498, "sampling/importance_sampling_ratio/min": 0.6317819952964783, "sampling/sampling_logp_difference/max": 0.6941823959350586, "sampling/sampling_logp_difference/mean": 0.012293101288378239, "step": 942 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 510.0, "completions/max_terminated_length": 510.0, "completions/mean_length": 369.1875, "completions/mean_terminated_length": 369.1875, "completions/min_length": 216.0, "completions/min_terminated_length": 216.0, "entropy": 0.34562911093235016, "epoch": 0.5048179871520343, "frac_reward_zero_std": 0.5, "grad_norm": 0.005273902323096991, "learning_rate": 1e-05, "loss": -0.0083, "num_tokens": 18907563.0, "reward": 0.9375, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.9375, "rewards/accuracy_reward/std": 0.24593468010425568, "sampling/importance_sampling_ratio/max": 1.4901328086853027, "sampling/importance_sampling_ratio/mean": 1.000502586364746, "sampling/importance_sampling_ratio/min": 0.6604971885681152, "sampling/sampling_logp_difference/max": 0.4147624969482422, "sampling/sampling_logp_difference/mean": 0.013175347819924355, "step": 943 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 561.0, "completions/max_terminated_length": 561.0, "completions/mean_length": 375.15625, "completions/mean_terminated_length": 375.15625, "completions/min_length": 203.0, "completions/min_terminated_length": 203.0, "entropy": 0.30867623165249825, "epoch": 0.5053533190578159, "frac_reward_zero_std": 0.5, "grad_norm": 0.021698860451579094, "learning_rate": 1e-05, "loss": -0.0058, "num_tokens": 18922760.0, "reward": 0.75, "reward_std": 0.2314550280570984, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.4776546955108643, "sampling/importance_sampling_ratio/mean": 1.0001270771026611, "sampling/importance_sampling_ratio/min": 0.7079949378967285, "sampling/sampling_logp_difference/max": 0.3904561996459961, "sampling/sampling_logp_difference/mean": 0.011195884086191654, "step": 944 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 689.0, "completions/mean_length": 422.46875, "completions/mean_terminated_length": 411.32257080078125, "completions/min_length": 201.0, "completions/min_terminated_length": 201.0, "entropy": 0.276916678994894, "epoch": 0.5058886509635975, "frac_reward_zero_std": 0.5, "grad_norm": 0.0067413561046123505, "learning_rate": 1e-05, "loss": 0.0147, "num_tokens": 18939519.0, "reward": 0.90625, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.4725924730300903, "sampling/importance_sampling_ratio/mean": 0.999957263469696, "sampling/importance_sampling_ratio/min": 0.6265785694122314, "sampling/sampling_logp_difference/max": 0.4674811363220215, "sampling/sampling_logp_difference/mean": 0.010225052013993263, "step": 945 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00013108201892464422, "clip_ratio/low_min": 0.00013108201892464422, "clip_ratio/region_mean": 0.00013108201892464422, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 749.0, "completions/mean_length": 482.875, "completions/mean_terminated_length": 453.3793029785156, "completions/min_length": 257.0, "completions/min_terminated_length": 257.0, "entropy": 0.4874328598380089, "epoch": 0.5064239828693791, "frac_reward_zero_std": 0.25, "grad_norm": 0.013364087790250778, "learning_rate": 1e-05, "loss": 0.1019, "num_tokens": 18958635.0, "reward": 0.6875, "reward_std": 0.38298875093460083, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.4269742965698242, "sampling/importance_sampling_ratio/mean": 1.0001400709152222, "sampling/importance_sampling_ratio/min": 0.6578408479690552, "sampling/sampling_logp_difference/max": 0.4187922477722168, "sampling/sampling_logp_difference/mean": 0.015182300470769405, "step": 946 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 517.0, "completions/mean_length": 364.25, "completions/mean_terminated_length": 351.2257995605469, "completions/min_length": 239.0, "completions/min_terminated_length": 239.0, "entropy": 0.36630511470139027, "epoch": 0.5069593147751607, "frac_reward_zero_std": 0.5, "grad_norm": 0.024675855413079262, "learning_rate": 1e-05, "loss": -0.0003, "num_tokens": 18973491.0, "reward": 0.8125, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.4747982025146484, "sampling/importance_sampling_ratio/mean": 0.9998736381530762, "sampling/importance_sampling_ratio/min": 0.6933584809303284, "sampling/sampling_logp_difference/max": 0.3885211944580078, "sampling/sampling_logp_difference/mean": 0.01280051376670599, "step": 947 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 563.0, "completions/max_terminated_length": 563.0, "completions/mean_length": 377.125, "completions/mean_terminated_length": 377.125, "completions/min_length": 153.0, "completions/min_terminated_length": 153.0, "entropy": 0.28161306865513325, "epoch": 0.5074946466809421, "frac_reward_zero_std": 0.75, "grad_norm": 0.006737569812685251, "learning_rate": 1e-05, "loss": -0.0079, "num_tokens": 18989191.0, "reward": 0.90625, "reward_std": 0.1293872892856598, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.3683444261550903, "sampling/importance_sampling_ratio/mean": 1.0001842975616455, "sampling/importance_sampling_ratio/min": 0.6362093091011047, "sampling/sampling_logp_difference/max": 0.4522275924682617, "sampling/sampling_logp_difference/mean": 0.010499055497348309, "step": 948 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 582.0, "completions/max_terminated_length": 582.0, "completions/mean_length": 412.125, "completions/mean_terminated_length": 412.125, "completions/min_length": 251.0, "completions/min_terminated_length": 251.0, "entropy": 0.34446657821536064, "epoch": 0.5080299785867237, "frac_reward_zero_std": 0.5, "grad_norm": 0.013441391289234161, "learning_rate": 1e-05, "loss": 0.0209, "num_tokens": 19006339.0, "reward": 0.6875, "reward_std": 0.249358132481575, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.4331457614898682, "sampling/importance_sampling_ratio/mean": 1.000077486038208, "sampling/importance_sampling_ratio/min": 0.6813332438468933, "sampling/sampling_logp_difference/max": 0.38370370864868164, "sampling/sampling_logp_difference/mean": 0.012086731381714344, "step": 949 }, { "clip_ratio/high_max": 7.725587056484073e-05, "clip_ratio/high_mean": 7.725587056484073e-05, "clip_ratio/low_mean": 7.612667832290754e-05, "clip_ratio/low_min": 7.612667832290754e-05, "clip_ratio/region_mean": 0.00015338254888774827, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 699.0, "completions/mean_length": 425.3125, "completions/mean_terminated_length": 414.258056640625, "completions/min_length": 257.0, "completions/min_terminated_length": 257.0, "entropy": 0.34499289840459824, "epoch": 0.5085653104925053, "frac_reward_zero_std": 0.25, "grad_norm": 0.009643463417887688, "learning_rate": 1e-05, "loss": 0.0195, "num_tokens": 19023717.0, "reward": 0.8125, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.565743327140808, "sampling/importance_sampling_ratio/mean": 1.0002692937850952, "sampling/importance_sampling_ratio/min": 0.711341381072998, "sampling/sampling_logp_difference/max": 0.4483606815338135, "sampling/sampling_logp_difference/mean": 0.01318529061973095, "step": 950 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 485.0, "completions/max_terminated_length": 485.0, "completions/mean_length": 344.4375, "completions/mean_terminated_length": 344.4375, "completions/min_length": 172.0, "completions/min_terminated_length": 172.0, "entropy": 0.37679310515522957, "epoch": 0.5091006423982869, "frac_reward_zero_std": 0.75, "grad_norm": 0.0057759517803788185, "learning_rate": 1e-05, "loss": -0.0297, "num_tokens": 19038259.0, "reward": 0.96875, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.96875, "rewards/accuracy_reward/std": 0.1767766922712326, "sampling/importance_sampling_ratio/max": 1.6021724939346313, "sampling/importance_sampling_ratio/mean": 1.0000699758529663, "sampling/importance_sampling_ratio/min": 0.6835008263587952, "sampling/sampling_logp_difference/max": 0.471360445022583, "sampling/sampling_logp_difference/mean": 0.01331338007003069, "step": 951 }, { "clip_ratio/high_max": 9.600614430382848e-05, "clip_ratio/high_mean": 9.600614430382848e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 9.600614430382848e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 712.0, "completions/max_terminated_length": 712.0, "completions/mean_length": 359.71875, "completions/mean_terminated_length": 359.71875, "completions/min_length": 169.0, "completions/min_terminated_length": 169.0, "entropy": 0.30372188799083233, "epoch": 0.5096359743040685, "frac_reward_zero_std": 0.25, "grad_norm": 0.028122534975409508, "learning_rate": 1e-05, "loss": -0.0696, "num_tokens": 19053226.0, "reward": 0.8125, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.5301743745803833, "sampling/importance_sampling_ratio/mean": 1.0003701448440552, "sampling/importance_sampling_ratio/min": 0.6938403844833374, "sampling/sampling_logp_difference/max": 0.4253816604614258, "sampling/sampling_logp_difference/mean": 0.011475645937025547, "step": 952 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 7.327080675167963e-05, "clip_ratio/low_min": 7.327080675167963e-05, "clip_ratio/region_mean": 7.327080675167963e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 716.0, "completions/max_terminated_length": 716.0, "completions/mean_length": 387.125, "completions/mean_terminated_length": 387.125, "completions/min_length": 158.0, "completions/min_terminated_length": 158.0, "entropy": 0.47781757824122906, "epoch": 0.5101713062098501, "frac_reward_zero_std": 0.5, "grad_norm": 0.0038248978089541197, "learning_rate": 1e-05, "loss": -0.0164, "num_tokens": 19069398.0, "reward": 0.53125, "reward_std": 0.2041158676147461, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.4815354347229004, "sampling/importance_sampling_ratio/mean": 1.0005252361297607, "sampling/importance_sampling_ratio/min": 0.6363780498504639, "sampling/sampling_logp_difference/max": 0.4519624710083008, "sampling/sampling_logp_difference/mean": 0.014442318119108677, "step": 953 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 698.0, "completions/max_terminated_length": 698.0, "completions/mean_length": 378.1875, "completions/mean_terminated_length": 378.1875, "completions/min_length": 203.0, "completions/min_terminated_length": 203.0, "entropy": 0.35734890028834343, "epoch": 0.5107066381156317, "frac_reward_zero_std": 0.5, "grad_norm": 0.0068240100517869, "learning_rate": 1e-05, "loss": 0.0674, "num_tokens": 19085964.0, "reward": 0.8125, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.449040412902832, "sampling/importance_sampling_ratio/mean": 0.999864935874939, "sampling/importance_sampling_ratio/min": 0.703772246837616, "sampling/sampling_logp_difference/max": 0.37090158462524414, "sampling/sampling_logp_difference/mean": 0.012334410101175308, "step": 954 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 746.0, "completions/max_terminated_length": 746.0, "completions/mean_length": 379.40625, "completions/mean_terminated_length": 379.40625, "completions/min_length": 170.0, "completions/min_terminated_length": 170.0, "entropy": 0.35313479602336884, "epoch": 0.5112419700214133, "frac_reward_zero_std": 0.0, "grad_norm": 0.009084006771445274, "learning_rate": 1e-05, "loss": -0.002, "num_tokens": 19101953.0, "reward": 0.8125, "reward_std": 0.4082317352294922, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.4120314121246338, "sampling/importance_sampling_ratio/mean": 1.00031578540802, "sampling/importance_sampling_ratio/min": 0.5684295892715454, "sampling/sampling_logp_difference/max": 0.5648777484893799, "sampling/sampling_logp_difference/mean": 0.012110366486012936, "step": 955 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 7.400829053949565e-05, "clip_ratio/low_min": 7.400829053949565e-05, "clip_ratio/region_mean": 7.400829053949565e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 593.0, "completions/max_terminated_length": 593.0, "completions/mean_length": 349.96875, "completions/mean_terminated_length": 349.96875, "completions/min_length": 183.0, "completions/min_terminated_length": 183.0, "entropy": 0.42891527339816093, "epoch": 0.5117773019271948, "frac_reward_zero_std": 0.25, "grad_norm": 0.010238466784358025, "learning_rate": 1e-05, "loss": 0.0668, "num_tokens": 19116664.0, "reward": 0.75, "reward_std": 0.292504221200943, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.3717490434646606, "sampling/importance_sampling_ratio/mean": 0.9998944997787476, "sampling/importance_sampling_ratio/min": 0.5717089772224426, "sampling/sampling_logp_difference/max": 0.5591251850128174, "sampling/sampling_logp_difference/mean": 0.013886130414903164, "step": 956 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 554.0, "completions/mean_length": 396.375, "completions/mean_terminated_length": 384.3870849609375, "completions/min_length": 172.0, "completions/min_terminated_length": 172.0, "entropy": 0.3211304396390915, "epoch": 0.5123126338329764, "frac_reward_zero_std": 0.5, "grad_norm": 0.007500725798308849, "learning_rate": 1e-05, "loss": 0.001, "num_tokens": 19132716.0, "reward": 0.9375, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.9375, "rewards/accuracy_reward/std": 0.24593468010425568, "sampling/importance_sampling_ratio/max": 1.4600927829742432, "sampling/importance_sampling_ratio/mean": 1.0001349449157715, "sampling/importance_sampling_ratio/min": 0.4625379741191864, "sampling/sampling_logp_difference/max": 0.771026611328125, "sampling/sampling_logp_difference/mean": 0.011396470479667187, "step": 957 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 708.0, "completions/mean_length": 426.875, "completions/mean_terminated_length": 415.8709411621094, "completions/min_length": 259.0, "completions/min_terminated_length": 259.0, "entropy": 0.3517173733562231, "epoch": 0.512847965738758, "frac_reward_zero_std": 0.25, "grad_norm": 0.03817927837371826, "learning_rate": 1e-05, "loss": 0.0745, "num_tokens": 19150328.0, "reward": 0.875, "reward_std": 0.292504221200943, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0000733137130737, "sampling/importance_sampling_ratio/min": 0.700235903263092, "sampling/sampling_logp_difference/max": 3.6121015548706055, "sampling/sampling_logp_difference/mean": 0.012966733425855637, "step": 958 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 752.0, "completions/mean_length": 428.0625, "completions/mean_terminated_length": 417.0967712402344, "completions/min_length": 189.0, "completions/min_terminated_length": 189.0, "entropy": 0.30252283439040184, "epoch": 0.5133832976445396, "frac_reward_zero_std": 0.25, "grad_norm": 0.007528909482061863, "learning_rate": 1e-05, "loss": -0.0173, "num_tokens": 19167682.0, "reward": 0.90625, "reward_std": 0.2651650309562683, "rewards/accuracy_reward/mean": 0.90625, "rewards/accuracy_reward/std": 0.2961445748806, "sampling/importance_sampling_ratio/max": 1.4333529472351074, "sampling/importance_sampling_ratio/mean": 1.0002542734146118, "sampling/importance_sampling_ratio/min": 0.6225548982620239, "sampling/sampling_logp_difference/max": 0.4739234447479248, "sampling/sampling_logp_difference/mean": 0.01132252998650074, "step": 959 }, { "clip_ratio/high_max": 6.234413740457967e-05, "clip_ratio/high_mean": 6.234413740457967e-05, "clip_ratio/low_mean": 6.234413740457967e-05, "clip_ratio/low_min": 6.234413740457967e-05, "clip_ratio/region_mean": 0.00012468827480915934, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 761.0, "completions/mean_length": 477.28125, "completions/mean_terminated_length": 457.9000244140625, "completions/min_length": 172.0, "completions/min_terminated_length": 172.0, "entropy": 0.4286377467215061, "epoch": 0.5139186295503212, "frac_reward_zero_std": 0.25, "grad_norm": 0.03617941960692406, "learning_rate": 1e-05, "loss": -0.0133, "num_tokens": 19186891.0, "reward": 0.28125, "reward_std": 0.3471629321575165, "rewards/accuracy_reward/mean": 0.28125, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.509759545326233, "sampling/importance_sampling_ratio/mean": 1.0001592636108398, "sampling/importance_sampling_ratio/min": 0.4537002742290497, "sampling/sampling_logp_difference/max": 0.790318489074707, "sampling/sampling_logp_difference/mean": 0.013955602422356606, "step": 960 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 545.0, "completions/max_terminated_length": 545.0, "completions/mean_length": 382.0625, "completions/mean_terminated_length": 382.0625, "completions/min_length": 252.0, "completions/min_terminated_length": 252.0, "entropy": 0.32199371233582497, "epoch": 0.5144539614561028, "frac_reward_zero_std": 0.5, "grad_norm": 0.007633667904883623, "learning_rate": 1e-05, "loss": -0.0181, "num_tokens": 19202765.0, "reward": 0.71875, "reward_std": 0.2630178928375244, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.4206713438034058, "sampling/importance_sampling_ratio/mean": 1.000057578086853, "sampling/importance_sampling_ratio/min": 0.7581333518028259, "sampling/sampling_logp_difference/max": 0.35112953186035156, "sampling/sampling_logp_difference/mean": 0.01190586481243372, "step": 961 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0002020564061240293, "clip_ratio/low_min": 0.0002020564061240293, "clip_ratio/region_mean": 0.0002020564061240293, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 603.0, "completions/mean_length": 392.09375, "completions/mean_terminated_length": 353.2069091796875, "completions/min_length": 165.0, "completions/min_terminated_length": 165.0, "entropy": 0.4132481776177883, "epoch": 0.5149892933618844, "frac_reward_zero_std": 0.5, "grad_norm": 0.02047305926680565, "learning_rate": 1e-05, "loss": 0.065, "num_tokens": 19219240.0, "reward": 0.65625, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.4062188863754272, "sampling/importance_sampling_ratio/mean": 0.9996537566184998, "sampling/importance_sampling_ratio/min": 0.5742837190628052, "sampling/sampling_logp_difference/max": 0.5546317100524902, "sampling/sampling_logp_difference/mean": 0.014492764137685299, "step": 962 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 7.225433364510536e-05, "clip_ratio/low_min": 7.225433364510536e-05, "clip_ratio/region_mean": 7.225433364510536e-05, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 636.0, "completions/mean_length": 372.90625, "completions/mean_terminated_length": 360.1612854003906, "completions/min_length": 116.0, "completions/min_terminated_length": 116.0, "entropy": 0.38584131747484207, "epoch": 0.515524625267666, "frac_reward_zero_std": 0.5, "grad_norm": 0.02891843393445015, "learning_rate": 1e-05, "loss": 0.0883, "num_tokens": 19235221.0, "reward": 0.84375, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.6288459300994873, "sampling/importance_sampling_ratio/mean": 0.9996564984321594, "sampling/importance_sampling_ratio/min": 0.6750528216362, "sampling/sampling_logp_difference/max": 0.48787176609039307, "sampling/sampling_logp_difference/mean": 0.013608647510409355, "step": 963 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00010113268945133314, "clip_ratio/low_min": 0.00010113268945133314, "clip_ratio/region_mean": 0.00010113268945133314, "completions/clipped_ratio": 0.0, "completions/max_length": 580.0, "completions/max_terminated_length": 580.0, "completions/mean_length": 354.96875, "completions/mean_terminated_length": 354.96875, "completions/min_length": 208.0, "completions/min_terminated_length": 208.0, "entropy": 0.347682636231184, "epoch": 0.5160599571734475, "frac_reward_zero_std": 0.25, "grad_norm": 0.017356660217046738, "learning_rate": 1e-05, "loss": 0.0517, "num_tokens": 19250188.0, "reward": 0.75, "reward_std": 0.3745020925998688, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.4867526292800903, "sampling/importance_sampling_ratio/mean": 0.9998435974121094, "sampling/importance_sampling_ratio/min": 0.6546799540519714, "sampling/sampling_logp_difference/max": 0.42360877990722656, "sampling/sampling_logp_difference/mean": 0.01204556692391634, "step": 964 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 617.0, "completions/mean_length": 408.1875, "completions/mean_terminated_length": 341.5555725097656, "completions/min_length": 63.0, "completions/min_terminated_length": 63.0, "entropy": 0.3895486332476139, "epoch": 0.5165952890792291, "frac_reward_zero_std": 0.0, "grad_norm": 0.021510498598217964, "learning_rate": 1e-05, "loss": 0.0327, "num_tokens": 19266978.0, "reward": 0.5625, "reward_std": 0.4492306709289551, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.413079023361206, "sampling/importance_sampling_ratio/mean": 0.999768078327179, "sampling/importance_sampling_ratio/min": 0.6693138480186462, "sampling/sampling_logp_difference/max": 0.4015021324157715, "sampling/sampling_logp_difference/mean": 0.012658570893108845, "step": 965 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00028588686109287664, "clip_ratio/low_min": 0.00028588686109287664, "clip_ratio/region_mean": 0.00028588686109287664, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 728.0, "completions/mean_length": 421.5, "completions/mean_terminated_length": 410.32257080078125, "completions/min_length": 222.0, "completions/min_terminated_length": 222.0, "entropy": 0.3282371163368225, "epoch": 0.5171306209850107, "frac_reward_zero_std": 0.5, "grad_norm": 0.02918657474219799, "learning_rate": 1e-05, "loss": 0.0189, "num_tokens": 19284522.0, "reward": 0.75, "reward_std": 0.2314550280570984, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.3269658088684082, "sampling/importance_sampling_ratio/mean": 1.0001468658447266, "sampling/importance_sampling_ratio/min": 0.7058061957359314, "sampling/sampling_logp_difference/max": 0.34841465950012207, "sampling/sampling_logp_difference/mean": 0.01187682244926691, "step": 966 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 9.645061800256371e-05, "clip_ratio/low_min": 9.645061800256371e-05, "clip_ratio/region_mean": 9.645061800256371e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 758.0, "completions/max_terminated_length": 758.0, "completions/mean_length": 383.0625, "completions/mean_terminated_length": 383.0625, "completions/min_length": 158.0, "completions/min_terminated_length": 158.0, "entropy": 0.2954042498022318, "epoch": 0.5176659528907923, "frac_reward_zero_std": 0.75, "grad_norm": 0.014282194897532463, "learning_rate": 1e-05, "loss": -0.019, "num_tokens": 19300828.0, "reward": 0.84375, "reward_std": 0.1293872892856598, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.4239294528961182, "sampling/importance_sampling_ratio/mean": 0.9997754693031311, "sampling/importance_sampling_ratio/min": 0.7234726548194885, "sampling/sampling_logp_difference/max": 0.3534202575683594, "sampling/sampling_logp_difference/mean": 0.01094186119735241, "step": 967 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0003659766080090776, "clip_ratio/low_min": 0.0003659766080090776, "clip_ratio/region_mean": 0.0003659766080090776, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 724.0, "completions/mean_length": 457.84375, "completions/mean_terminated_length": 425.75860595703125, "completions/min_length": 274.0, "completions/min_terminated_length": 274.0, "entropy": 0.37592070922255516, "epoch": 0.5182012847965739, "frac_reward_zero_std": 0.0, "grad_norm": 0.023928089067339897, "learning_rate": 1e-05, "loss": 0.0383, "num_tokens": 19319767.0, "reward": 0.53125, "reward_std": 0.4628904461860657, "rewards/accuracy_reward/mean": 0.53125, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.4722683429718018, "sampling/importance_sampling_ratio/mean": 0.9997448325157166, "sampling/importance_sampling_ratio/min": 0.6514332890510559, "sampling/sampling_logp_difference/max": 0.42858028411865234, "sampling/sampling_logp_difference/mean": 0.013530357740819454, "step": 968 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.3487376135308295e-05, "clip_ratio/low_min": 5.3487376135308295e-05, "clip_ratio/region_mean": 5.3487376135308295e-05, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 710.0, "completions/mean_length": 497.46875, "completions/mean_terminated_length": 447.370361328125, "completions/min_length": 212.0, "completions/min_terminated_length": 212.0, "entropy": 0.46024415642023087, "epoch": 0.5187366167023555, "frac_reward_zero_std": 0.25, "grad_norm": 0.02193511091172695, "learning_rate": 1e-05, "loss": 0.0531, "num_tokens": 19340118.0, "reward": 0.65625, "reward_std": 0.3787454068660736, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.4351786375045776, "sampling/importance_sampling_ratio/mean": 0.9999463558197021, "sampling/importance_sampling_ratio/min": 0.6362149715423584, "sampling/sampling_logp_difference/max": 0.45221877098083496, "sampling/sampling_logp_difference/mean": 0.01444620918482542, "step": 969 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 448.0, "completions/max_terminated_length": 448.0, "completions/mean_length": 307.875, "completions/mean_terminated_length": 307.875, "completions/min_length": 132.0, "completions/min_terminated_length": 132.0, "entropy": 0.29755398631095886, "epoch": 0.5192719486081371, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 19353602.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.4207556247711182, "sampling/importance_sampling_ratio/mean": 0.9998729228973389, "sampling/importance_sampling_ratio/min": 0.6982030272483826, "sampling/sampling_logp_difference/max": 0.35924530029296875, "sampling/sampling_logp_difference/mean": 0.010936449281871319, "step": 970 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 7.365940109593794e-05, "clip_ratio/low_min": 7.365940109593794e-05, "clip_ratio/region_mean": 7.365940109593794e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 591.0, "completions/max_terminated_length": 591.0, "completions/mean_length": 413.9375, "completions/mean_terminated_length": 413.9375, "completions/min_length": 208.0, "completions/min_terminated_length": 208.0, "entropy": 0.3776747062802315, "epoch": 0.5198072805139187, "frac_reward_zero_std": 0.0, "grad_norm": 0.012585047632455826, "learning_rate": 1e-05, "loss": 0.0091, "num_tokens": 19370800.0, "reward": 0.625, "reward_std": 0.4629100561141968, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.4471746683120728, "sampling/importance_sampling_ratio/mean": 1.0008338689804077, "sampling/importance_sampling_ratio/min": 0.6913242340087891, "sampling/sampling_logp_difference/max": 0.3696131706237793, "sampling/sampling_logp_difference/mean": 0.012984559871256351, "step": 971 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00015318627993110567, "clip_ratio/low_min": 0.00015318627993110567, "clip_ratio/region_mean": 0.00015318627993110567, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 682.0, "completions/mean_length": 467.875, "completions/mean_terminated_length": 398.6153869628906, "completions/min_length": 243.0, "completions/min_terminated_length": 243.0, "entropy": 0.42555257491767406, "epoch": 0.5203426124197003, "frac_reward_zero_std": 0.25, "grad_norm": 0.009568733163177967, "learning_rate": 1e-05, "loss": 0.0551, "num_tokens": 19389188.0, "reward": 0.59375, "reward_std": 0.3608423173427582, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.4687670469284058, "sampling/importance_sampling_ratio/mean": 1.00002121925354, "sampling/importance_sampling_ratio/min": 0.713232696056366, "sampling/sampling_logp_difference/max": 0.38442325592041016, "sampling/sampling_logp_difference/mean": 0.013383888639509678, "step": 972 }, { "clip_ratio/high_max": 9.773260535439476e-05, "clip_ratio/high_mean": 9.773260535439476e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 9.773260535439476e-05, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 752.0, "completions/mean_length": 422.84375, "completions/mean_terminated_length": 411.70965576171875, "completions/min_length": 194.0, "completions/min_terminated_length": 194.0, "entropy": 0.4044637493789196, "epoch": 0.5208779443254818, "frac_reward_zero_std": 0.25, "grad_norm": 0.050879448652267456, "learning_rate": 1e-05, "loss": -0.016, "num_tokens": 19406719.0, "reward": 0.8125, "reward_std": 0.3471825420856476, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.5698542594909668, "sampling/importance_sampling_ratio/mean": 0.9999532103538513, "sampling/importance_sampling_ratio/min": 0.32599741220474243, "sampling/sampling_logp_difference/max": 1.120865821838379, "sampling/sampling_logp_difference/mean": 0.013992219232022762, "step": 973 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 602.0, "completions/max_terminated_length": 602.0, "completions/mean_length": 302.0625, "completions/mean_terminated_length": 302.0625, "completions/min_length": 194.0, "completions/min_terminated_length": 194.0, "entropy": 0.33476197347044945, "epoch": 0.5214132762312634, "frac_reward_zero_std": 0.5, "grad_norm": 0.008505664765834808, "learning_rate": 1e-05, "loss": 0.0517, "num_tokens": 19420081.0, "reward": 0.9375, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.9375, "rewards/accuracy_reward/std": 0.24593468010425568, "sampling/importance_sampling_ratio/max": 1.7984052896499634, "sampling/importance_sampling_ratio/mean": 0.9994386434555054, "sampling/importance_sampling_ratio/min": 0.6588560342788696, "sampling/sampling_logp_difference/max": 0.5869003534317017, "sampling/sampling_logp_difference/mean": 0.012204364873468876, "step": 974 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 7.735148392384872e-05, "clip_ratio/low_min": 7.735148392384872e-05, "clip_ratio/region_mean": 7.735148392384872e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 571.0, "completions/max_terminated_length": 571.0, "completions/mean_length": 401.5, "completions/mean_terminated_length": 401.5, "completions/min_length": 192.0, "completions/min_terminated_length": 192.0, "entropy": 0.3273180201649666, "epoch": 0.521948608137045, "frac_reward_zero_std": 0.25, "grad_norm": 0.012674779631197453, "learning_rate": 1e-05, "loss": 0.0221, "num_tokens": 19436961.0, "reward": 0.5, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5080004930496216, "sampling/importance_sampling_ratio/max": 1.4262446165084839, "sampling/importance_sampling_ratio/mean": 0.9996024370193481, "sampling/importance_sampling_ratio/min": 0.5284674763679504, "sampling/sampling_logp_difference/max": 0.6377739906311035, "sampling/sampling_logp_difference/mean": 0.01177900843322277, "step": 975 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00010665528679965064, "clip_ratio/low_min": 0.00010665528679965064, "clip_ratio/region_mean": 0.00010665528679965064, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 622.0, "completions/mean_length": 373.75, "completions/mean_terminated_length": 361.0322570800781, "completions/min_length": 151.0, "completions/min_terminated_length": 151.0, "entropy": 0.3929736278951168, "epoch": 0.5224839400428265, "frac_reward_zero_std": 0.25, "grad_norm": 0.01882655918598175, "learning_rate": 1e-05, "loss": 0.0018, "num_tokens": 19452497.0, "reward": 0.6875, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.5898834466934204, "sampling/importance_sampling_ratio/mean": 0.9998830556869507, "sampling/importance_sampling_ratio/min": 0.6962289214134216, "sampling/sampling_logp_difference/max": 0.46366071701049805, "sampling/sampling_logp_difference/mean": 0.01369271520525217, "step": 976 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 760.0, "completions/mean_length": 613.1875, "completions/mean_terminated_length": 577.4615478515625, "completions/min_length": 397.0, "completions/min_terminated_length": 397.0, "entropy": 0.4036356657743454, "epoch": 0.5230192719486081, "frac_reward_zero_std": 0.0, "grad_norm": 0.00760468328371644, "learning_rate": 1e-05, "loss": 0.0376, "num_tokens": 19476799.0, "reward": 0.46875, "reward_std": 0.3808925747871399, "rewards/accuracy_reward/mean": 0.46875, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.6402665376663208, "sampling/importance_sampling_ratio/mean": 0.999738335609436, "sampling/importance_sampling_ratio/min": 0.5129676461219788, "sampling/sampling_logp_difference/max": 0.6675424575805664, "sampling/sampling_logp_difference/mean": 0.012650548480451107, "step": 977 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 8.729050023248419e-05, "clip_ratio/low_min": 8.729050023248419e-05, "clip_ratio/region_mean": 8.729050023248419e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 608.0, "completions/max_terminated_length": 608.0, "completions/mean_length": 355.15625, "completions/mean_terminated_length": 355.15625, "completions/min_length": 185.0, "completions/min_terminated_length": 185.0, "entropy": 0.30730387568473816, "epoch": 0.5235546038543897, "frac_reward_zero_std": 0.0, "grad_norm": 0.03878138214349747, "learning_rate": 1e-05, "loss": 0.0036, "num_tokens": 19491404.0, "reward": 0.84375, "reward_std": 0.3808925747871399, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.4566916227340698, "sampling/importance_sampling_ratio/mean": 1.000020980834961, "sampling/importance_sampling_ratio/min": 0.7223418354988098, "sampling/sampling_logp_difference/max": 0.37616777420043945, "sampling/sampling_logp_difference/mean": 0.010783746838569641, "step": 978 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0001304120960412547, "clip_ratio/low_min": 0.0001304120960412547, "clip_ratio/region_mean": 0.0001304120960412547, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 718.0, "completions/mean_length": 443.5625, "completions/mean_terminated_length": 421.933349609375, "completions/min_length": 156.0, "completions/min_terminated_length": 156.0, "entropy": 0.3865148574113846, "epoch": 0.5240899357601713, "frac_reward_zero_std": 0.25, "grad_norm": 0.00789199024438858, "learning_rate": 1e-05, "loss": 0.1018, "num_tokens": 19509558.0, "reward": 0.65625, "reward_std": 0.3061639666557312, "rewards/accuracy_reward/mean": 0.65625, "rewards/accuracy_reward/std": 0.4825586974620819, "sampling/importance_sampling_ratio/max": 1.9852715730667114, "sampling/importance_sampling_ratio/mean": 1.0002254247665405, "sampling/importance_sampling_ratio/min": 0.6256375908851624, "sampling/sampling_logp_difference/max": 0.685755729675293, "sampling/sampling_logp_difference/mean": 0.013112067244946957, "step": 979 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 7.237985118990764e-05, "clip_ratio/low_min": 7.237985118990764e-05, "clip_ratio/region_mean": 7.237985118990764e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 527.0, "completions/max_terminated_length": 527.0, "completions/mean_length": 377.8125, "completions/mean_terminated_length": 377.8125, "completions/min_length": 175.0, "completions/min_terminated_length": 175.0, "entropy": 0.36957137659192085, "epoch": 0.5246252676659529, "frac_reward_zero_std": 0.5, "grad_norm": 0.0217699334025383, "learning_rate": 1e-05, "loss": -0.011, "num_tokens": 19525616.0, "reward": 0.71875, "reward_std": 0.2630178928375244, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.4488766193389893, "sampling/importance_sampling_ratio/mean": 0.9999788999557495, "sampling/importance_sampling_ratio/min": 0.22517076134681702, "sampling/sampling_logp_difference/max": 1.490896224975586, "sampling/sampling_logp_difference/mean": 0.012938423082232475, "step": 980 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.262525130296126e-05, "clip_ratio/low_min": 6.262525130296126e-05, "clip_ratio/region_mean": 6.262525130296126e-05, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 623.0, "completions/mean_length": 421.65625, "completions/mean_terminated_length": 398.5666809082031, "completions/min_length": 110.0, "completions/min_terminated_length": 110.0, "entropy": 0.34409458190202713, "epoch": 0.5251605995717344, "frac_reward_zero_std": 0.25, "grad_norm": 0.022093556821346283, "learning_rate": 1e-05, "loss": 0.0636, "num_tokens": 19542645.0, "reward": 0.59375, "reward_std": 0.3198433816432953, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.3639451265335083, "sampling/importance_sampling_ratio/mean": 0.9999120831489563, "sampling/importance_sampling_ratio/min": 0.4882810711860657, "sampling/sampling_logp_difference/max": 0.7168641090393066, "sampling/sampling_logp_difference/mean": 0.012842332944273949, "step": 981 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 516.0, "completions/max_terminated_length": 516.0, "completions/mean_length": 323.96875, "completions/mean_terminated_length": 323.96875, "completions/min_length": 101.0, "completions/min_terminated_length": 101.0, "entropy": 0.381885290145874, "epoch": 0.525695931477516, "frac_reward_zero_std": 0.5, "grad_norm": 0.014097297564148903, "learning_rate": 1e-05, "loss": -0.003, "num_tokens": 19556292.0, "reward": 0.875, "reward_std": 0.2177756428718567, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 1.2731571197509766, "sampling/importance_sampling_ratio/mean": 0.9999310970306396, "sampling/importance_sampling_ratio/min": 0.7041700482368469, "sampling/sampling_logp_difference/max": 0.3507354259490967, "sampling/sampling_logp_difference/mean": 0.012606794945895672, "step": 982 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00014495554205495864, "clip_ratio/low_min": 0.00014495554205495864, "clip_ratio/region_mean": 0.00014495554205495864, "completions/clipped_ratio": 0.21875, "completions/max_length": 768.0, "completions/max_terminated_length": 691.0, "completions/mean_length": 471.34375, "completions/mean_terminated_length": 388.2799987792969, "completions/min_length": 216.0, "completions/min_terminated_length": 216.0, "entropy": 0.41394589468836784, "epoch": 0.5262312633832976, "frac_reward_zero_std": 0.25, "grad_norm": 0.012932811863720417, "learning_rate": 1e-05, "loss": 0.0856, "num_tokens": 19575559.0, "reward": 0.59375, "reward_std": 0.3966485261917114, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.5204910039901733, "sampling/importance_sampling_ratio/mean": 0.9998944401741028, "sampling/importance_sampling_ratio/min": 0.7181884050369263, "sampling/sampling_logp_difference/max": 0.4190332889556885, "sampling/sampling_logp_difference/mean": 0.013816176913678646, "step": 983 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.71037016925402e-05, "clip_ratio/low_min": 5.71037016925402e-05, "clip_ratio/region_mean": 5.71037016925402e-05, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 689.0, "completions/mean_length": 419.71875, "completions/mean_terminated_length": 408.4838562011719, "completions/min_length": 181.0, "completions/min_terminated_length": 181.0, "entropy": 0.4361092373728752, "epoch": 0.5267665952890792, "frac_reward_zero_std": 0.25, "grad_norm": 0.020809335634112358, "learning_rate": 1e-05, "loss": 0.0327, "num_tokens": 19592974.0, "reward": 0.71875, "reward_std": 0.3787454068660736, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.4286140203475952, "sampling/importance_sampling_ratio/mean": 0.999965250492096, "sampling/importance_sampling_ratio/min": 0.7039120197296143, "sampling/sampling_logp_difference/max": 0.3567047119140625, "sampling/sampling_logp_difference/mean": 0.01424399483948946, "step": 984 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 754.0, "completions/mean_length": 379.03125, "completions/mean_terminated_length": 353.1000061035156, "completions/min_length": 154.0, "completions/min_terminated_length": 154.0, "entropy": 0.3644980266690254, "epoch": 0.5273019271948608, "frac_reward_zero_std": 0.75, "grad_norm": 0.0035406677052378654, "learning_rate": 1e-05, "loss": 0.0137, "num_tokens": 19608679.0, "reward": 0.71875, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.3350573778152466, "sampling/importance_sampling_ratio/mean": 0.9997720718383789, "sampling/importance_sampling_ratio/min": 0.6727628111839294, "sampling/sampling_logp_difference/max": 0.3963625431060791, "sampling/sampling_logp_difference/mean": 0.012701061554253101, "step": 985 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 608.0, "completions/max_terminated_length": 608.0, "completions/mean_length": 371.40625, "completions/mean_terminated_length": 371.40625, "completions/min_length": 248.0, "completions/min_terminated_length": 248.0, "entropy": 0.33607086539268494, "epoch": 0.5278372591006424, "frac_reward_zero_std": 0.5, "grad_norm": 0.007951159030199051, "learning_rate": 1e-05, "loss": 0.0582, "num_tokens": 19624900.0, "reward": 0.875, "reward_std": 0.2314550280570984, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 1.6003811359405518, "sampling/importance_sampling_ratio/mean": 1.0000178813934326, "sampling/importance_sampling_ratio/min": 0.6993299126625061, "sampling/sampling_logp_difference/max": 0.4702417850494385, "sampling/sampling_logp_difference/mean": 0.012674926780164242, "step": 986 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 5.58534411538858e-05, "clip_ratio/low_min": 5.58534411538858e-05, "clip_ratio/region_mean": 5.58534411538858e-05, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 758.0, "completions/mean_length": 475.5, "completions/mean_terminated_length": 445.2413635253906, "completions/min_length": 210.0, "completions/min_terminated_length": 210.0, "entropy": 0.4294469840824604, "epoch": 0.528372591006424, "frac_reward_zero_std": 0.25, "grad_norm": 0.006498498376458883, "learning_rate": 1e-05, "loss": -0.0595, "num_tokens": 19644348.0, "reward": 0.6875, "reward_std": 0.3745020925998688, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.4246615171432495, "sampling/importance_sampling_ratio/mean": 0.9997056126594543, "sampling/importance_sampling_ratio/min": 0.6270196437835693, "sampling/sampling_logp_difference/max": 0.4667774438858032, "sampling/sampling_logp_difference/mean": 0.013875314965844154, "step": 987 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 742.0, "completions/mean_length": 421.28125, "completions/mean_terminated_length": 398.16668701171875, "completions/min_length": 230.0, "completions/min_terminated_length": 230.0, "entropy": 0.3003564439713955, "epoch": 0.5289079229122056, "frac_reward_zero_std": 0.5, "grad_norm": 0.008136115036904812, "learning_rate": 1e-05, "loss": 0.041, "num_tokens": 19661509.0, "reward": 0.875, "reward_std": 0.2314550280570984, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 1.4009886980056763, "sampling/importance_sampling_ratio/mean": 0.9999625086784363, "sampling/importance_sampling_ratio/min": 0.634975254535675, "sampling/sampling_logp_difference/max": 0.45416927337646484, "sampling/sampling_logp_difference/mean": 0.011120658367872238, "step": 988 }, { "clip_ratio/high_max": 0.00010416666918899864, "clip_ratio/high_mean": 0.00010416666918899864, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00010416666918899864, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 561.0, "completions/mean_length": 356.9375, "completions/mean_terminated_length": 343.6773986816406, "completions/min_length": 219.0, "completions/min_terminated_length": 219.0, "entropy": 0.30500293523073196, "epoch": 0.5294432548179872, "frac_reward_zero_std": 0.75, "grad_norm": 0.004428379703313112, "learning_rate": 1e-05, "loss": 0.0043, "num_tokens": 19677003.0, "reward": 0.96875, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.96875, "rewards/accuracy_reward/std": 0.1767766922712326, "sampling/importance_sampling_ratio/max": 1.4127655029296875, "sampling/importance_sampling_ratio/mean": 0.9997833371162415, "sampling/importance_sampling_ratio/min": 0.554033100605011, "sampling/sampling_logp_difference/max": 0.5905308723449707, "sampling/sampling_logp_difference/mean": 0.011077864095568657, "step": 989 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 718.0, "completions/max_terminated_length": 718.0, "completions/mean_length": 404.1875, "completions/mean_terminated_length": 404.1875, "completions/min_length": 228.0, "completions/min_terminated_length": 228.0, "entropy": 0.3824571669101715, "epoch": 0.5299785867237687, "frac_reward_zero_std": 0.5, "grad_norm": 0.005398706533014774, "learning_rate": 1e-05, "loss": -0.0397, "num_tokens": 19693737.0, "reward": 0.8125, "reward_std": 0.249358132481575, "rewards/accuracy_reward/mean": 0.8125, "rewards/accuracy_reward/std": 0.3965577781200409, "sampling/importance_sampling_ratio/max": 1.359811782836914, "sampling/importance_sampling_ratio/mean": 1.0002894401550293, "sampling/importance_sampling_ratio/min": 0.682829737663269, "sampling/sampling_logp_difference/max": 0.38150978088378906, "sampling/sampling_logp_difference/mean": 0.01318275835365057, "step": 990 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 722.0, "completions/max_terminated_length": 722.0, "completions/mean_length": 452.5625, "completions/mean_terminated_length": 452.5625, "completions/min_length": 228.0, "completions/min_terminated_length": 228.0, "entropy": 0.3480992913246155, "epoch": 0.5305139186295503, "frac_reward_zero_std": 0.25, "grad_norm": 0.019097017124295235, "learning_rate": 1e-05, "loss": 0.0147, "num_tokens": 19711667.0, "reward": 0.875, "reward_std": 0.292504221200943, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 1.4442964792251587, "sampling/importance_sampling_ratio/mean": 0.9998614192008972, "sampling/importance_sampling_ratio/min": 0.7023054361343384, "sampling/sampling_logp_difference/max": 0.36762237548828125, "sampling/sampling_logp_difference/mean": 0.012362902984023094, "step": 991 }, { "clip_ratio/high_max": 0.0002538930275477469, "clip_ratio/high_mean": 0.0002538930275477469, "clip_ratio/low_mean": 0.0001179245300590992, "clip_ratio/low_min": 0.0001179245300590992, "clip_ratio/region_mean": 0.0003718175576068461, "completions/clipped_ratio": 0.0, "completions/max_length": 510.0, "completions/max_terminated_length": 510.0, "completions/mean_length": 321.4375, "completions/mean_terminated_length": 321.4375, "completions/min_length": 114.0, "completions/min_terminated_length": 114.0, "entropy": 0.39349352568387985, "epoch": 0.5310492505353319, "frac_reward_zero_std": 0.25, "grad_norm": 0.011738387867808342, "learning_rate": 1e-05, "loss": -0.0452, "num_tokens": 19725401.0, "reward": 0.78125, "reward_std": 0.3608423173427582, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.3910943269729614, "sampling/importance_sampling_ratio/mean": 0.9995290040969849, "sampling/importance_sampling_ratio/min": 0.632828414440155, "sampling/sampling_logp_difference/max": 0.45755600929260254, "sampling/sampling_logp_difference/mean": 0.01345069520175457, "step": 992 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 573.0, "completions/mean_length": 384.96875, "completions/mean_terminated_length": 372.6128845214844, "completions/min_length": 204.0, "completions/min_terminated_length": 204.0, "entropy": 0.267300384119153, "epoch": 0.5315845824411135, "frac_reward_zero_std": 0.25, "grad_norm": 0.016314968466758728, "learning_rate": 1e-05, "loss": 0.0205, "num_tokens": 19741256.0, "reward": 0.84375, "reward_std": 0.3198433816432953, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.3768436908721924, "sampling/importance_sampling_ratio/mean": 1.0003135204315186, "sampling/importance_sampling_ratio/min": 0.7016299366950989, "sampling/sampling_logp_difference/max": 0.35434913635253906, "sampling/sampling_logp_difference/mean": 0.009977073408663273, "step": 993 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 604.0, "completions/max_terminated_length": 604.0, "completions/mean_length": 354.375, "completions/mean_terminated_length": 354.375, "completions/min_length": 191.0, "completions/min_terminated_length": 191.0, "entropy": 0.28781773522496223, "epoch": 0.5321199143468951, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 19756044.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.554656982421875, "sampling/importance_sampling_ratio/mean": 0.9998306632041931, "sampling/importance_sampling_ratio/min": 0.5329753756523132, "sampling/sampling_logp_difference/max": 0.6292800903320312, "sampling/sampling_logp_difference/mean": 0.011215814389288425, "step": 994 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 648.0, "completions/max_terminated_length": 648.0, "completions/mean_length": 375.5, "completions/mean_terminated_length": 375.5, "completions/min_length": 151.0, "completions/min_terminated_length": 151.0, "entropy": 0.3446412794291973, "epoch": 0.5326552462526767, "frac_reward_zero_std": 0.25, "grad_norm": 0.009794474579393864, "learning_rate": 1e-05, "loss": 0.0041, "num_tokens": 19771812.0, "reward": 0.84375, "reward_std": 0.3061639964580536, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.4218099117279053, "sampling/importance_sampling_ratio/mean": 0.9994850158691406, "sampling/importance_sampling_ratio/min": 0.6771134734153748, "sampling/sampling_logp_difference/max": 0.38991641998291016, "sampling/sampling_logp_difference/mean": 0.012730089016258717, "step": 995 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 8.514986257068813e-05, "clip_ratio/low_min": 8.514986257068813e-05, "clip_ratio/region_mean": 8.514986257068813e-05, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 639.0, "completions/mean_length": 363.40625, "completions/mean_terminated_length": 336.433349609375, "completions/min_length": 146.0, "completions/min_terminated_length": 146.0, "entropy": 0.4420875646173954, "epoch": 0.5331905781584583, "frac_reward_zero_std": 0.5, "grad_norm": 0.028622398152947426, "learning_rate": 1e-05, "loss": -0.0261, "num_tokens": 19787257.0, "reward": 0.59375, "reward_std": 0.22201895713806152, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.4544509649276733, "sampling/importance_sampling_ratio/mean": 1.0002591609954834, "sampling/importance_sampling_ratio/min": 0.7278399467468262, "sampling/sampling_logp_difference/max": 0.37462854385375977, "sampling/sampling_logp_difference/mean": 0.015078195370733738, "step": 996 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 653.0, "completions/max_terminated_length": 653.0, "completions/mean_length": 469.125, "completions/mean_terminated_length": 469.125, "completions/min_length": 199.0, "completions/min_terminated_length": 199.0, "entropy": 0.26573713682591915, "epoch": 0.5337259100642399, "frac_reward_zero_std": 0.5, "grad_norm": 0.023011846467852592, "learning_rate": 1e-05, "loss": 0.0082, "num_tokens": 19806317.0, "reward": 0.6875, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.5333566665649414, "sampling/importance_sampling_ratio/mean": 0.9998281598091125, "sampling/importance_sampling_ratio/min": 0.6885629892349243, "sampling/sampling_logp_difference/max": 0.4274592399597168, "sampling/sampling_logp_difference/mean": 0.009900758974254131, "step": 997 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 7.92141945566982e-05, "clip_ratio/low_min": 7.92141945566982e-05, "clip_ratio/region_mean": 7.92141945566982e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 633.0, "completions/max_terminated_length": 633.0, "completions/mean_length": 439.3125, "completions/mean_terminated_length": 439.3125, "completions/min_length": 277.0, "completions/min_terminated_length": 277.0, "entropy": 0.252308351919055, "epoch": 0.5342612419700214, "frac_reward_zero_std": 0.75, "grad_norm": 0.005164835602045059, "learning_rate": 1e-05, "loss": 0.006, "num_tokens": 19824175.0, "reward": 0.9375, "reward_std": 0.1157275140285492, "rewards/accuracy_reward/mean": 0.9375, "rewards/accuracy_reward/std": 0.24593468010425568, "sampling/importance_sampling_ratio/max": 1.7274595499038696, "sampling/importance_sampling_ratio/mean": 0.9996723532676697, "sampling/importance_sampling_ratio/min": 0.6095215678215027, "sampling/sampling_logp_difference/max": 0.5466518402099609, "sampling/sampling_logp_difference/mean": 0.010268724523484707, "step": 998 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 447.0, "completions/max_terminated_length": 447.0, "completions/mean_length": 293.71875, "completions/mean_terminated_length": 293.71875, "completions/min_length": 147.0, "completions/min_terminated_length": 147.0, "entropy": 0.3240813873708248, "epoch": 0.534796573875803, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 19836998.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.434338092803955, "sampling/importance_sampling_ratio/mean": 1.000130295753479, "sampling/importance_sampling_ratio/min": 0.7086703181266785, "sampling/sampling_logp_difference/max": 0.3607034683227539, "sampling/sampling_logp_difference/mean": 0.01266037579625845, "step": 999 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00015957191499182954, "clip_ratio/low_min": 0.00015957191499182954, "clip_ratio/region_mean": 0.00015957191499182954, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 767.0, "completions/mean_length": 530.6875, "completions/mean_terminated_length": 496.7857360839844, "completions/min_length": 317.0, "completions/min_terminated_length": 317.0, "entropy": 0.41408251225948334, "epoch": 0.5353319057815846, "frac_reward_zero_std": 0.25, "grad_norm": 0.012072480283677578, "learning_rate": 1e-05, "loss": 0.0581, "num_tokens": 19857604.0, "reward": 0.375, "reward_std": 0.292504221200943, "rewards/accuracy_reward/mean": 0.375, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.4654985666275024, "sampling/importance_sampling_ratio/mean": 1.0008958578109741, "sampling/importance_sampling_ratio/min": 0.6040355563163757, "sampling/sampling_logp_difference/max": 0.504122257232666, "sampling/sampling_logp_difference/mean": 0.013704602606594563, "step": 1000 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 574.0, "completions/max_terminated_length": 574.0, "completions/mean_length": 383.53125, "completions/mean_terminated_length": 383.53125, "completions/min_length": 222.0, "completions/min_terminated_length": 222.0, "entropy": 0.3190803676843643, "epoch": 0.5358672376873662, "frac_reward_zero_std": 0.25, "grad_norm": 0.008184117265045643, "learning_rate": 1e-05, "loss": -0.0074, "num_tokens": 19874157.0, "reward": 0.875, "reward_std": 0.292504221200943, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 1.3911616802215576, "sampling/importance_sampling_ratio/mean": 1.0001699924468994, "sampling/importance_sampling_ratio/min": 0.7666565775871277, "sampling/sampling_logp_difference/max": 0.33013916015625, "sampling/sampling_logp_difference/mean": 0.011891323141753674, "step": 1001 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.613756704609841e-05, "clip_ratio/low_min": 6.613756704609841e-05, "clip_ratio/region_mean": 6.613756704609841e-05, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 688.0, "completions/mean_length": 417.1875, "completions/mean_terminated_length": 405.8709411621094, "completions/min_length": 168.0, "completions/min_terminated_length": 168.0, "entropy": 0.34403831884264946, "epoch": 0.5364025695931478, "frac_reward_zero_std": 0.5, "grad_norm": 0.022531097754836082, "learning_rate": 1e-05, "loss": 0.0501, "num_tokens": 19891467.0, "reward": 0.78125, "reward_std": 0.2630178928375244, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.3868879079818726, "sampling/importance_sampling_ratio/mean": 0.9997280836105347, "sampling/importance_sampling_ratio/min": 0.7225915789604187, "sampling/sampling_logp_difference/max": 0.32706236839294434, "sampling/sampling_logp_difference/mean": 0.012148319743573666, "step": 1002 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 466.0, "completions/max_terminated_length": 466.0, "completions/mean_length": 342.46875, "completions/mean_terminated_length": 342.46875, "completions/min_length": 201.0, "completions/min_terminated_length": 201.0, "entropy": 0.35112348571419716, "epoch": 0.5369379014989293, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 19906346.0, "reward": 1.0, "reward_std": 0.0, "rewards/accuracy_reward/mean": 1.0, "rewards/accuracy_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 1.2754554748535156, "sampling/importance_sampling_ratio/mean": 1.0004018545150757, "sampling/importance_sampling_ratio/min": 0.7040057182312012, "sampling/sampling_logp_difference/max": 0.3509688377380371, "sampling/sampling_logp_difference/mean": 0.012432698160409927, "step": 1003 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0002144115642295219, "clip_ratio/low_min": 0.0002144115642295219, "clip_ratio/region_mean": 0.0002144115642295219, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 760.0, "completions/mean_length": 455.1875, "completions/mean_terminated_length": 422.82757568359375, "completions/min_length": 156.0, "completions/min_terminated_length": 156.0, "entropy": 0.4676204025745392, "epoch": 0.5374732334047109, "frac_reward_zero_std": 0.25, "grad_norm": 0.020795581862330437, "learning_rate": 1e-05, "loss": -0.1373, "num_tokens": 19924656.0, "reward": 0.5625, "reward_std": 0.3514062464237213, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0006451606750488, "sampling/importance_sampling_ratio/min": 0.6748799681663513, "sampling/sampling_logp_difference/max": 1.088576316833496, "sampling/sampling_logp_difference/mean": 0.0150221586227417, "step": 1004 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 649.0, "completions/max_terminated_length": 649.0, "completions/mean_length": 303.1875, "completions/mean_terminated_length": 303.1875, "completions/min_length": 130.0, "completions/min_terminated_length": 130.0, "entropy": 0.35607171431183815, "epoch": 0.5380085653104925, "frac_reward_zero_std": 0.75, "grad_norm": 0.014990466646850109, "learning_rate": 1e-05, "loss": 0.0291, "num_tokens": 19937366.0, "reward": 0.84375, "reward_std": 0.1293872892856598, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.4300074577331543, "sampling/importance_sampling_ratio/mean": 1.0001391172409058, "sampling/importance_sampling_ratio/min": 0.6600378155708313, "sampling/sampling_logp_difference/max": 0.41545820236206055, "sampling/sampling_logp_difference/mean": 0.012568687088787556, "step": 1005 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.125, "completions/max_length": 768.0, "completions/max_terminated_length": 607.0, "completions/mean_length": 427.1875, "completions/mean_terminated_length": 378.5000305175781, "completions/min_length": 195.0, "completions/min_terminated_length": 195.0, "entropy": 0.4223974570631981, "epoch": 0.538543897216274, "frac_reward_zero_std": 0.25, "grad_norm": 0.01761556603014469, "learning_rate": 1e-05, "loss": 0.1399, "num_tokens": 19954628.0, "reward": 0.75, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.4593396186828613, "sampling/importance_sampling_ratio/mean": 0.9995014071464539, "sampling/importance_sampling_ratio/min": 0.4806101322174072, "sampling/sampling_logp_difference/max": 0.732698917388916, "sampling/sampling_logp_difference/mean": 0.014494835399091244, "step": 1006 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 6.551362457685173e-05, "clip_ratio/low_min": 6.551362457685173e-05, "clip_ratio/region_mean": 6.551362457685173e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 687.0, "completions/max_terminated_length": 687.0, "completions/mean_length": 422.0, "completions/mean_terminated_length": 422.0, "completions/min_length": 212.0, "completions/min_terminated_length": 212.0, "entropy": 0.29704665020108223, "epoch": 0.5390792291220556, "frac_reward_zero_std": 0.25, "grad_norm": 0.01521917525678873, "learning_rate": 1e-05, "loss": 0.0653, "num_tokens": 19972316.0, "reward": 0.625, "reward_std": 0.3924052119255066, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.4309535026550293, "sampling/importance_sampling_ratio/mean": 1.000120759010315, "sampling/importance_sampling_ratio/min": 0.5780975222587585, "sampling/sampling_logp_difference/max": 0.5480127334594727, "sampling/sampling_logp_difference/mean": 0.012352030724287033, "step": 1007 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 628.0, "completions/max_terminated_length": 628.0, "completions/mean_length": 393.78125, "completions/mean_terminated_length": 393.78125, "completions/min_length": 150.0, "completions/min_terminated_length": 150.0, "entropy": 0.35848720744252205, "epoch": 0.5396145610278372, "frac_reward_zero_std": 0.5, "grad_norm": 0.004797272849828005, "learning_rate": 1e-05, "loss": 0.0366, "num_tokens": 19988485.0, "reward": 0.875, "reward_std": 0.2314550280570984, "rewards/accuracy_reward/mean": 0.875, "rewards/accuracy_reward/std": 0.33601075410842896, "sampling/importance_sampling_ratio/max": 1.4358552694320679, "sampling/importance_sampling_ratio/mean": 0.9997005462646484, "sampling/importance_sampling_ratio/min": 0.6208333373069763, "sampling/sampling_logp_difference/max": 0.47669267654418945, "sampling/sampling_logp_difference/mean": 0.014138579368591309, "step": 1008 }, { "clip_ratio/high_max": 0.00011111111234640703, "clip_ratio/high_mean": 0.00011111111234640703, "clip_ratio/low_mean": 7.301401637960225e-05, "clip_ratio/low_min": 7.301401637960225e-05, "clip_ratio/region_mean": 0.00018412512872600928, "completions/clipped_ratio": 0.0625, "completions/max_length": 768.0, "completions/max_terminated_length": 669.0, "completions/mean_length": 399.5, "completions/mean_terminated_length": 374.933349609375, "completions/min_length": 186.0, "completions/min_terminated_length": 186.0, "entropy": 0.37881822884082794, "epoch": 0.5401498929336188, "frac_reward_zero_std": 0.25, "grad_norm": 0.02062937803566456, "learning_rate": 1e-05, "loss": 0.0705, "num_tokens": 20005445.0, "reward": 0.78125, "reward_std": 0.3377465009689331, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.3408777713775635, "sampling/importance_sampling_ratio/mean": 0.9996195435523987, "sampling/importance_sampling_ratio/min": 0.6806493997573853, "sampling/sampling_logp_difference/max": 0.3847079277038574, "sampling/sampling_logp_difference/mean": 0.013024517334997654, "step": 1009 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 8.294625149574131e-05, "clip_ratio/low_min": 8.294625149574131e-05, "clip_ratio/region_mean": 8.294625149574131e-05, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 720.0, "completions/mean_length": 473.375, "completions/mean_terminated_length": 405.3846435546875, "completions/min_length": 253.0, "completions/min_terminated_length": 253.0, "entropy": 0.4926433861255646, "epoch": 0.5406852248394004, "frac_reward_zero_std": 0.5, "grad_norm": 0.007573904003947973, "learning_rate": 1e-05, "loss": 0.0146, "num_tokens": 20024281.0, "reward": 0.25, "reward_std": 0.2314550280570984, "rewards/accuracy_reward/mean": 0.25, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.4932169914245605, "sampling/importance_sampling_ratio/mean": 0.9992649555206299, "sampling/importance_sampling_ratio/min": 0.6486780047416687, "sampling/sampling_logp_difference/max": 0.4328188896179199, "sampling/sampling_logp_difference/mean": 0.016582582145929337, "step": 1010 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 517.0, "completions/max_terminated_length": 517.0, "completions/mean_length": 326.6875, "completions/mean_terminated_length": 326.6875, "completions/min_length": 166.0, "completions/min_terminated_length": 166.0, "entropy": 0.3384322002530098, "epoch": 0.541220556745182, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": 0.0217, "num_tokens": 20038207.0, "reward": 0.96875, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.96875, "rewards/accuracy_reward/std": 0.1767766922712326, "sampling/importance_sampling_ratio/max": 1.4876166582107544, "sampling/importance_sampling_ratio/mean": 1.0000590085983276, "sampling/importance_sampling_ratio/min": 0.6396904587745667, "sampling/sampling_logp_difference/max": 0.44677090644836426, "sampling/sampling_logp_difference/mean": 0.012286476790904999, "step": 1011 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00029437870398396626, "clip_ratio/low_min": 0.00029437870398396626, "clip_ratio/region_mean": 0.00029437870398396626, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 760.0, "completions/mean_length": 489.25, "completions/mean_terminated_length": 424.923095703125, "completions/min_length": 180.0, "completions/min_terminated_length": 180.0, "entropy": 0.4187079556286335, "epoch": 0.5417558886509636, "frac_reward_zero_std": 0.5, "grad_norm": 0.011295115575194359, "learning_rate": 1e-05, "loss": -0.023, "num_tokens": 20058255.0, "reward": 0.625, "reward_std": 0.2314550280570984, "rewards/accuracy_reward/mean": 0.625, "rewards/accuracy_reward/std": 0.49186936020851135, "sampling/importance_sampling_ratio/max": 1.4581892490386963, "sampling/importance_sampling_ratio/mean": 0.9999808073043823, "sampling/importance_sampling_ratio/min": 0.6816900968551636, "sampling/sampling_logp_difference/max": 0.3831801414489746, "sampling/sampling_logp_difference/mean": 0.01373599935323, "step": 1012 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 409.0, "completions/mean_length": 313.3125, "completions/mean_terminated_length": 298.6451416015625, "completions/min_length": 168.0, "completions/min_terminated_length": 168.0, "entropy": 0.31616272777318954, "epoch": 0.5422912205567452, "frac_reward_zero_std": 0.25, "grad_norm": 0.009547683410346508, "learning_rate": 1e-05, "loss": 0.036, "num_tokens": 20072105.0, "reward": 0.84375, "reward_std": 0.3198433816432953, "rewards/accuracy_reward/mean": 0.84375, "rewards/accuracy_reward/std": 0.3689020276069641, "sampling/importance_sampling_ratio/max": 1.4409582614898682, "sampling/importance_sampling_ratio/mean": 0.9999350309371948, "sampling/importance_sampling_ratio/min": 0.7607950568199158, "sampling/sampling_logp_difference/max": 0.3653082847595215, "sampling/sampling_logp_difference/mean": 0.011710098944604397, "step": 1013 }, { "clip_ratio/high_max": 8.234519191319123e-05, "clip_ratio/high_mean": 8.234519191319123e-05, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 8.234519191319123e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 499.0, "completions/max_terminated_length": 499.0, "completions/mean_length": 356.25, "completions/mean_terminated_length": 356.25, "completions/min_length": 217.0, "completions/min_terminated_length": 217.0, "entropy": 0.2891187146306038, "epoch": 0.5428265524625268, "frac_reward_zero_std": 0.75, "grad_norm": 0.0034570067655295134, "learning_rate": 1e-05, "loss": 0.0138, "num_tokens": 20087553.0, "reward": 0.96875, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.96875, "rewards/accuracy_reward/std": 0.1767766922712326, "sampling/importance_sampling_ratio/max": 1.445860743522644, "sampling/importance_sampling_ratio/mean": 0.9999263882637024, "sampling/importance_sampling_ratio/min": 0.7230806946754456, "sampling/sampling_logp_difference/max": 0.36870479583740234, "sampling/sampling_logp_difference/mean": 0.011481990106403828, "step": 1014 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00016371542733395472, "clip_ratio/low_min": 0.00016371542733395472, "clip_ratio/region_mean": 0.00016371542733395472, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 752.0, "completions/mean_length": 387.40625, "completions/mean_terminated_length": 375.1290283203125, "completions/min_length": 175.0, "completions/min_terminated_length": 175.0, "entropy": 0.34031448513269424, "epoch": 0.5433618843683083, "frac_reward_zero_std": 0.25, "grad_norm": 0.0314343087375164, "learning_rate": 1e-05, "loss": 0.1007, "num_tokens": 20103438.0, "reward": 0.71875, "reward_std": 0.3198433816432953, "rewards/accuracy_reward/mean": 0.71875, "rewards/accuracy_reward/std": 0.45680341124534607, "sampling/importance_sampling_ratio/max": 1.4403650760650635, "sampling/importance_sampling_ratio/mean": 0.9999372959136963, "sampling/importance_sampling_ratio/min": 0.5992985963821411, "sampling/sampling_logp_difference/max": 0.5119953155517578, "sampling/sampling_logp_difference/mean": 0.012536661699414253, "step": 1015 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.15625, "completions/max_length": 768.0, "completions/max_terminated_length": 686.0, "completions/mean_length": 427.9375, "completions/mean_terminated_length": 364.96295166015625, "completions/min_length": 138.0, "completions/min_terminated_length": 138.0, "entropy": 0.41648540273308754, "epoch": 0.5438972162740899, "frac_reward_zero_std": 0.75, "grad_norm": 0.012236598879098892, "learning_rate": 1e-05, "loss": -0.0493, "num_tokens": 20120636.0, "reward": 0.6875, "reward_std": 0.1157275140285492, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.4134716987609863, "sampling/importance_sampling_ratio/mean": 0.9997365474700928, "sampling/importance_sampling_ratio/min": 0.6459512114524841, "sampling/sampling_logp_difference/max": 0.43703126907348633, "sampling/sampling_logp_difference/mean": 0.013590492308139801, "step": 1016 }, { "clip_ratio/high_max": 9.904913167702034e-05, "clip_ratio/high_mean": 9.904913167702034e-05, "clip_ratio/low_mean": 0.00010154346091439947, "clip_ratio/low_min": 0.00010154346091439947, "clip_ratio/region_mean": 0.00020059259259141982, "completions/clipped_ratio": 0.0, "completions/max_length": 676.0, "completions/max_terminated_length": 676.0, "completions/mean_length": 336.53125, "completions/mean_terminated_length": 336.53125, "completions/min_length": 133.0, "completions/min_terminated_length": 133.0, "entropy": 0.3056458942592144, "epoch": 0.5444325481798715, "frac_reward_zero_std": 0.5, "grad_norm": 0.0060928938910365105, "learning_rate": 1e-05, "loss": -0.0084, "num_tokens": 20135173.0, "reward": 0.75, "reward_std": 0.1767766922712326, "rewards/accuracy_reward/mean": 0.75, "rewards/accuracy_reward/std": 0.4399413466453552, "sampling/importance_sampling_ratio/max": 1.4335640668869019, "sampling/importance_sampling_ratio/mean": 1.0000736713409424, "sampling/importance_sampling_ratio/min": 0.5954843759536743, "sampling/sampling_logp_difference/max": 0.5183801651000977, "sampling/sampling_logp_difference/mean": 0.011896425858139992, "step": 1017 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00023194549066829495, "clip_ratio/low_min": 0.00023194549066829495, "clip_ratio/region_mean": 0.00023194549066829495, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 704.0, "completions/mean_length": 427.84375, "completions/mean_terminated_length": 416.8709411621094, "completions/min_length": 206.0, "completions/min_terminated_length": 206.0, "entropy": 0.3733966462314129, "epoch": 0.5449678800856531, "frac_reward_zero_std": 0.25, "grad_norm": 0.023733748123049736, "learning_rate": 1e-05, "loss": -0.0237, "num_tokens": 20152880.0, "reward": 0.5, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.5, "rewards/accuracy_reward/std": 0.5080004930496216, "sampling/importance_sampling_ratio/max": 1.5804710388183594, "sampling/importance_sampling_ratio/mean": 0.9996747374534607, "sampling/importance_sampling_ratio/min": 0.5733012557029724, "sampling/sampling_logp_difference/max": 0.5563439130783081, "sampling/sampling_logp_difference/mean": 0.01292388141155243, "step": 1018 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0001584179772180505, "clip_ratio/low_min": 0.0001584179772180505, "clip_ratio/region_mean": 0.0001584179772180505, "completions/clipped_ratio": 0.1875, "completions/max_length": 768.0, "completions/max_terminated_length": 693.0, "completions/mean_length": 447.375, "completions/mean_terminated_length": 373.3846435546875, "completions/min_length": 213.0, "completions/min_terminated_length": 213.0, "entropy": 0.3781808651983738, "epoch": 0.5455032119914347, "frac_reward_zero_std": 0.25, "grad_norm": 0.013881339691579342, "learning_rate": 1e-05, "loss": 0.0217, "num_tokens": 20171036.0, "reward": 0.59375, "reward_std": 0.3061639964580536, "rewards/accuracy_reward/mean": 0.59375, "rewards/accuracy_reward/std": 0.49899089336395264, "sampling/importance_sampling_ratio/max": 1.8005858659744263, "sampling/importance_sampling_ratio/mean": 1.0003662109375, "sampling/importance_sampling_ratio/min": 0.6414482593536377, "sampling/sampling_logp_difference/max": 0.5881121158599854, "sampling/sampling_logp_difference/mean": 0.013058403506875038, "step": 1019 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 498.0, "completions/max_terminated_length": 498.0, "completions/mean_length": 350.5, "completions/mean_terminated_length": 350.5, "completions/min_length": 156.0, "completions/min_terminated_length": 156.0, "entropy": 0.31501710787415504, "epoch": 0.5460385438972163, "frac_reward_zero_std": 0.25, "grad_norm": 0.03650783374905586, "learning_rate": 1e-05, "loss": 0.0621, "num_tokens": 20185612.0, "reward": 0.78125, "reward_std": 0.3061639964580536, "rewards/accuracy_reward/mean": 0.78125, "rewards/accuracy_reward/std": 0.420013427734375, "sampling/importance_sampling_ratio/max": 1.5785146951675415, "sampling/importance_sampling_ratio/mean": 1.0001404285430908, "sampling/importance_sampling_ratio/min": 0.7512355446815491, "sampling/sampling_logp_difference/max": 0.456484317779541, "sampling/sampling_logp_difference/mean": 0.011222404427826405, "step": 1020 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09375, "completions/max_length": 768.0, "completions/max_terminated_length": 667.0, "completions/mean_length": 425.78125, "completions/mean_terminated_length": 390.3793029785156, "completions/min_length": 189.0, "completions/min_terminated_length": 189.0, "entropy": 0.44041524082422256, "epoch": 0.5465738758029979, "frac_reward_zero_std": 0.5, "grad_norm": 0.01822061836719513, "learning_rate": 1e-05, "loss": 0.0672, "num_tokens": 20203109.0, "reward": 0.5625, "reward_std": 0.249358132481575, "rewards/accuracy_reward/mean": 0.5625, "rewards/accuracy_reward/std": 0.504016101360321, "sampling/importance_sampling_ratio/max": 1.3275117874145508, "sampling/importance_sampling_ratio/mean": 0.9996468424797058, "sampling/importance_sampling_ratio/min": 0.7189075946807861, "sampling/sampling_logp_difference/max": 0.33002257347106934, "sampling/sampling_logp_difference/mean": 0.014490016736090183, "step": 1021 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 8.316699677379802e-05, "clip_ratio/low_min": 8.316699677379802e-05, "clip_ratio/region_mean": 8.316699677379802e-05, "completions/clipped_ratio": 0.03125, "completions/max_length": 768.0, "completions/max_terminated_length": 629.0, "completions/mean_length": 402.09375, "completions/mean_terminated_length": 390.2903137207031, "completions/min_length": 183.0, "completions/min_terminated_length": 183.0, "entropy": 0.37083595991134644, "epoch": 0.5471092077087795, "frac_reward_zero_std": 0.25, "grad_norm": 0.024714402854442596, "learning_rate": 1e-05, "loss": 0.0598, "num_tokens": 20219648.0, "reward": 0.6875, "reward_std": 0.3335031569004059, "rewards/accuracy_reward/mean": 0.6875, "rewards/accuracy_reward/std": 0.4709290862083435, "sampling/importance_sampling_ratio/max": 1.5523804426193237, "sampling/importance_sampling_ratio/mean": 0.9998931288719177, "sampling/importance_sampling_ratio/min": 0.6299923658370972, "sampling/sampling_logp_difference/max": 0.4620475769042969, "sampling/sampling_logp_difference/mean": 0.014000162482261658, "step": 1022 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 536.0, "completions/max_terminated_length": 536.0, "completions/mean_length": 378.03125, "completions/mean_terminated_length": 378.03125, "completions/min_length": 207.0, "completions/min_terminated_length": 207.0, "entropy": 0.2959277294576168, "epoch": 0.547644539614561, "frac_reward_zero_std": 0.75, "grad_norm": 0.0, "learning_rate": 1e-05, "loss": -0.0275, "num_tokens": 20236025.0, "reward": 0.96875, "reward_std": 0.0883883461356163, "rewards/accuracy_reward/mean": 0.96875, "rewards/accuracy_reward/std": 0.1767766922712326, "sampling/importance_sampling_ratio/max": 1.5157606601715088, "sampling/importance_sampling_ratio/mean": 0.9996615648269653, "sampling/importance_sampling_ratio/min": 0.6749536991119385, "sampling/sampling_logp_difference/max": 0.41591739654541016, "sampling/sampling_logp_difference/mean": 0.01128290593624115, "step": 1023 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 696.0, "completions/max_terminated_length": 696.0, "completions/mean_length": 427.8125, "completions/mean_terminated_length": 427.8125, "completions/min_length": 226.0, "completions/min_terminated_length": 226.0, "entropy": 0.31040345318615437, "epoch": 0.5481798715203426, "frac_reward_zero_std": 0.5, "grad_norm": 0.01015719585120678, "learning_rate": 1e-05, "loss": -0.0013, "num_tokens": 20253603.0, "reward": 0.46875, "reward_std": 0.24511480331420898, "rewards/accuracy_reward/mean": 0.46875, "rewards/accuracy_reward/std": 0.507007360458374, "sampling/importance_sampling_ratio/max": 1.4403468370437622, "sampling/importance_sampling_ratio/mean": 0.9997923970222473, "sampling/importance_sampling_ratio/min": 0.7165867686271667, "sampling/sampling_logp_difference/max": 0.3648838996887207, "sampling/sampling_logp_difference/mean": 0.011375730857253075, "step": 1024 } ], "logging_steps": 1, "max_steps": 1024, "num_input_tokens_seen": 20253603, "num_train_epochs": 1, "save_steps": 64, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": true, "should_training_stop": true }, "attributes": {} } }, "total_flos": 0.0, "train_batch_size": 4, "trial_name": null, "trial_params": null }