diff --git "a/trainer_state.json" "b/trainer_state.json" new file mode 100644--- /dev/null +++ "b/trainer_state.json" @@ -0,0 +1,8834 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.07108583614714768, + "eval_steps": 500, + "global_step": 400, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "completion_length": 427.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 460.0, + "completions/max_terminated_length": 460.0, + "completions/mean_length": 427.0, + "completions/mean_terminated_length": 427.0, + "completions/min_length": 386.0, + "completions/min_terminated_length": 386.0, + "epoch": 0.00017771459036786921, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.6094411611557007, + "kl": 1.0498238056898117, + "learning_rate": 0.0, + "loss": 0.001, + "num_tokens": 2836.0, + "reward": 0.6189357042312622, + "reward_std": 0.16637657582759857, + "rewards/check_answer/mean": 0.6189357042312622, + "rewards/check_answer/std": 0.16637657582759857, + "step": 1 + }, + { + "completion_length": 541.75, + "completions/clipped_ratio": 0.0, + "completions/max_length": 559.0, + "completions/max_terminated_length": 559.0, + "completions/mean_length": 541.75, + "completions/mean_terminated_length": 541.75, + "completions/min_length": 526.0, + "completions/min_terminated_length": 526.0, + "epoch": 0.00035542918073573843, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3810062408447266, + "kl": 1.1163947582244873, + "learning_rate": 7.142857142857144e-08, + "loss": 0.0011, + "num_tokens": 6171.0, + "reward": 0.6918560862541199, + "reward_std": 0.06921806931495667, + "rewards/check_answer/mean": 0.6918560862541199, + "rewards/check_answer/std": 0.06921806931495667, + "step": 2 + }, + { + "completion_length": 599.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 836.0, + "completions/max_terminated_length": 836.0, + "completions/mean_length": 599.0, + "completions/mean_terminated_length": 599.0, + "completions/min_length": 419.0, + "completions/min_terminated_length": 419.0, + "epoch": 0.0005331437711036076, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.256874442100525, + "kl": 0.801612138748169, + "learning_rate": 1.4285714285714287e-07, + "loss": 0.0008, + "num_tokens": 9807.0, + "reward": 0.6589475274085999, + "reward_std": 0.10396059602499008, + "rewards/check_answer/mean": 0.6589475274085999, + "rewards/check_answer/std": 0.10396059602499008, + "step": 3 + }, + { + "completion_length": 466.75, + "completions/clipped_ratio": 0.0, + "completions/max_length": 491.0, + "completions/max_terminated_length": 491.0, + "completions/mean_length": 466.75, + "completions/mean_terminated_length": 466.75, + "completions/min_length": 435.0, + "completions/min_terminated_length": 435.0, + "epoch": 0.0007108583614714769, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.8016468286514282, + "kl": 1.3025178015232086, + "learning_rate": 2.142857142857143e-07, + "loss": 0.0013, + "num_tokens": 12814.0, + "reward": 0.4898241460323334, + "reward_std": 0.0683608204126358, + "rewards/check_answer/mean": 0.4898241460323334, + "rewards/check_answer/std": 0.06836080551147461, + "step": 4 + }, + { + "completion_length": 596.25, + "completions/clipped_ratio": 0.0, + "completions/max_length": 665.0, + "completions/max_terminated_length": 665.0, + "completions/mean_length": 596.25, + "completions/mean_terminated_length": 596.25, + "completions/min_length": 504.0, + "completions/min_terminated_length": 504.0, + "epoch": 0.000888572951839346, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.4106318950653076, + "kl": 1.153193473815918, + "learning_rate": 2.8571428571428575e-07, + "loss": 0.0012, + "num_tokens": 16407.0, + "reward": 0.589160144329071, + "reward_std": 0.1366165727376938, + "rewards/check_answer/mean": 0.589160144329071, + "rewards/check_answer/std": 0.1366165727376938, + "step": 5 + }, + { + "completion_length": 521.75, + "completions/clipped_ratio": 0.0, + "completions/max_length": 566.0, + "completions/max_terminated_length": 566.0, + "completions/mean_length": 521.75, + "completions/mean_terminated_length": 521.75, + "completions/min_length": 491.0, + "completions/min_terminated_length": 491.0, + "epoch": 0.0010662875422072153, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.6702299118041992, + "kl": 1.1629396080970764, + "learning_rate": 3.5714285714285716e-07, + "loss": 0.0012, + "num_tokens": 19658.0, + "reward": 0.6594047546386719, + "reward_std": 0.03622956573963165, + "rewards/check_answer/mean": 0.6594047546386719, + "rewards/check_answer/std": 0.03622956946492195, + "step": 6 + }, + { + "completion_length": 557.75, + "completions/clipped_ratio": 0.0, + "completions/max_length": 574.0, + "completions/max_terminated_length": 574.0, + "completions/mean_length": 557.75, + "completions/mean_terminated_length": 557.75, + "completions/min_length": 537.0, + "completions/min_terminated_length": 537.0, + "epoch": 0.0012440021325750845, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.4264965057373047, + "kl": 1.3246853649616241, + "learning_rate": 4.285714285714286e-07, + "loss": 0.0013, + "num_tokens": 23093.0, + "reward": 0.5148710012435913, + "reward_std": 0.030068516731262207, + "rewards/check_answer/mean": 0.5148710012435913, + "rewards/check_answer/std": 0.030068522319197655, + "step": 7 + }, + { + "completion_length": 411.5, + "completions/clipped_ratio": 0.0, + "completions/max_length": 474.0, + "completions/max_terminated_length": 474.0, + "completions/mean_length": 411.5, + "completions/mean_terminated_length": 411.5, + "completions/min_length": 368.0, + "completions/min_terminated_length": 368.0, + "epoch": 0.0014217167229429537, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.6491725444793701, + "kl": 1.1441383361816406, + "learning_rate": 5.000000000000001e-07, + "loss": 0.0011, + "num_tokens": 25891.0, + "reward": 0.39803165197372437, + "reward_std": 0.1641732007265091, + "rewards/check_answer/mean": 0.39803165197372437, + "rewards/check_answer/std": 0.1641732156276703, + "step": 8 + }, + { + "completion_length": 432.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 499.0, + "completions/max_terminated_length": 499.0, + "completions/mean_length": 432.0, + "completions/mean_terminated_length": 432.0, + "completions/min_length": 385.0, + "completions/min_terminated_length": 385.0, + "epoch": 0.0015994313133108227, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.763498067855835, + "kl": 1.447030395269394, + "learning_rate": 5.714285714285715e-07, + "loss": 0.0014, + "num_tokens": 28743.0, + "reward": 0.6316161155700684, + "reward_std": 0.08715175837278366, + "rewards/check_answer/mean": 0.6316161155700684, + "rewards/check_answer/std": 0.08715175092220306, + "step": 9 + }, + { + "completion_length": 467.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 486.0, + "completions/max_terminated_length": 486.0, + "completions/mean_length": 467.0, + "completions/mean_terminated_length": 467.0, + "completions/min_length": 442.0, + "completions/min_terminated_length": 442.0, + "epoch": 0.001777145903678692, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.7400954961776733, + "kl": 2.312670022249222, + "learning_rate": 6.428571428571428e-07, + "loss": 0.0023, + "num_tokens": 31759.0, + "reward": 0.7970674633979797, + "reward_std": 0.18348881602287292, + "rewards/check_answer/mean": 0.7970674633979797, + "rewards/check_answer/std": 0.18348881602287292, + "step": 10 + }, + { + "completion_length": 405.75, + "completions/clipped_ratio": 0.0, + "completions/max_length": 495.0, + "completions/max_terminated_length": 495.0, + "completions/mean_length": 405.75, + "completions/mean_terminated_length": 405.75, + "completions/min_length": 339.0, + "completions/min_terminated_length": 339.0, + "epoch": 0.001954860494046561, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.9999518394470215, + "kl": 1.1593981087207794, + "learning_rate": 7.142857142857143e-07, + "loss": 0.0012, + "num_tokens": 34514.0, + "reward": 0.3714173436164856, + "reward_std": 0.06423555314540863, + "rewards/check_answer/mean": 0.3714173436164856, + "rewards/check_answer/std": 0.06423555314540863, + "step": 11 + }, + { + "completion_length": 434.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 493.0, + "completions/max_terminated_length": 493.0, + "completions/mean_length": 434.0, + "completions/mean_terminated_length": 434.0, + "completions/min_length": 354.0, + "completions/min_terminated_length": 354.0, + "epoch": 0.0021325750844144306, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.7830874919891357, + "kl": 2.1814075708389282, + "learning_rate": 7.857142857142857e-07, + "loss": 0.0022, + "num_tokens": 37398.0, + "reward": 0.6386286616325378, + "reward_std": 0.05019339174032211, + "rewards/check_answer/mean": 0.6386286616325378, + "rewards/check_answer/std": 0.05019339174032211, + "step": 12 + }, + { + "completion_length": 537.5, + "completions/clipped_ratio": 0.0, + "completions/max_length": 566.0, + "completions/max_terminated_length": 566.0, + "completions/mean_length": 537.5, + "completions/mean_terminated_length": 537.5, + "completions/min_length": 510.0, + "completions/min_terminated_length": 510.0, + "epoch": 0.0023102896747822996, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.486297369003296, + "kl": 1.0660419911146164, + "learning_rate": 8.571428571428572e-07, + "loss": 0.0011, + "num_tokens": 40724.0, + "reward": 0.35336199402809143, + "reward_std": 0.037462785840034485, + "rewards/check_answer/mean": 0.35336199402809143, + "rewards/check_answer/std": 0.03746278956532478, + "step": 13 + }, + { + "completion_length": 376.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 452.0, + "completions/max_terminated_length": 452.0, + "completions/mean_length": 376.0, + "completions/mean_terminated_length": 376.0, + "completions/min_length": 282.0, + "completions/min_terminated_length": 282.0, + "epoch": 0.002488004265150169, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.73249351978302, + "kl": 1.2243637442588806, + "learning_rate": 9.285714285714287e-07, + "loss": 0.0012, + "num_tokens": 43336.0, + "reward": 0.4773079752922058, + "reward_std": 0.18512390553951263, + "rewards/check_answer/mean": 0.4773079752922058, + "rewards/check_answer/std": 0.18512390553951263, + "step": 14 + }, + { + "completion_length": 410.25, + "completions/clipped_ratio": 0.0, + "completions/max_length": 440.0, + "completions/max_terminated_length": 440.0, + "completions/mean_length": 410.25, + "completions/mean_terminated_length": 410.25, + "completions/min_length": 348.0, + "completions/min_terminated_length": 348.0, + "epoch": 0.002665718855518038, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.818801999092102, + "kl": 1.239543467760086, + "learning_rate": 1.0000000000000002e-06, + "loss": 0.0012, + "num_tokens": 46113.0, + "reward": 0.8123418092727661, + "reward_std": 0.2969806492328644, + "rewards/check_answer/mean": 0.8123418092727661, + "rewards/check_answer/std": 0.2969806492328644, + "step": 15 + }, + { + "completion_length": 585.25, + "completions/clipped_ratio": 0.0, + "completions/max_length": 634.0, + "completions/max_terminated_length": 634.0, + "completions/mean_length": 585.25, + "completions/mean_terminated_length": 585.25, + "completions/min_length": 523.0, + "completions/min_terminated_length": 523.0, + "epoch": 0.0028434334458859074, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3664265871047974, + "kl": 1.164329156279564, + "learning_rate": 1.0714285714285714e-06, + "loss": 0.0012, + "num_tokens": 49654.0, + "reward": 0.7955901622772217, + "reward_std": 0.16037997603416443, + "rewards/check_answer/mean": 0.7955901622772217, + "rewards/check_answer/std": 0.16037999093532562, + "step": 16 + }, + { + "completion_length": 409.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 498.0, + "completions/max_terminated_length": 498.0, + "completions/mean_length": 409.0, + "completions/mean_terminated_length": 409.0, + "completions/min_length": 307.0, + "completions/min_terminated_length": 307.0, + "epoch": 0.0030211480362537764, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.7888686656951904, + "kl": 1.2454279661178589, + "learning_rate": 1.142857142857143e-06, + "loss": 0.0012, + "num_tokens": 52454.0, + "reward": 0.3565758168697357, + "reward_std": 0.19015806913375854, + "rewards/check_answer/mean": 0.3565758168697357, + "rewards/check_answer/std": 0.19015805423259735, + "step": 17 + }, + { + "completion_length": 492.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 615.0, + "completions/max_terminated_length": 615.0, + "completions/mean_length": 492.0, + "completions/mean_terminated_length": 492.0, + "completions/min_length": 426.0, + "completions/min_terminated_length": 426.0, + "epoch": 0.0031988626266216454, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.6943333148956299, + "kl": 1.0921899378299713, + "learning_rate": 1.2142857142857144e-06, + "loss": 0.0011, + "num_tokens": 55558.0, + "reward": 1.0070428848266602, + "reward_std": 0.08447255939245224, + "rewards/check_answer/mean": 1.0070428848266602, + "rewards/check_answer/std": 0.08447252959012985, + "step": 18 + }, + { + "completion_length": 574.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 628.0, + "completions/max_terminated_length": 628.0, + "completions/mean_length": 574.0, + "completions/mean_terminated_length": 574.0, + "completions/min_length": 524.0, + "completions/min_terminated_length": 524.0, + "epoch": 0.003376577216989515, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.5357227325439453, + "kl": 1.0007081925868988, + "learning_rate": 1.2857142857142856e-06, + "loss": 0.001, + "num_tokens": 59062.0, + "reward": 0.5813010931015015, + "reward_std": 0.17026397585868835, + "rewards/check_answer/mean": 0.5813010931015015, + "rewards/check_answer/std": 0.17026399075984955, + "step": 19 + }, + { + "completion_length": 422.75, + "completions/clipped_ratio": 0.0, + "completions/max_length": 441.0, + "completions/max_terminated_length": 441.0, + "completions/mean_length": 422.75, + "completions/mean_terminated_length": 422.75, + "completions/min_length": 408.0, + "completions/min_terminated_length": 408.0, + "epoch": 0.003554291807357384, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.447060227394104, + "kl": 1.0916816592216492, + "learning_rate": 1.3571428571428572e-06, + "loss": 0.0011, + "num_tokens": 61925.0, + "reward": 0.639136552810669, + "reward_std": 0.14528952538967133, + "rewards/check_answer/mean": 0.639136552810669, + "rewards/check_answer/std": 0.14528952538967133, + "step": 20 + }, + { + "completion_length": 491.5, + "completions/clipped_ratio": 0.0, + "completions/max_length": 507.0, + "completions/max_terminated_length": 507.0, + "completions/mean_length": 491.5, + "completions/mean_terminated_length": 491.5, + "completions/min_length": 463.0, + "completions/min_terminated_length": 463.0, + "epoch": 0.0037320063977252533, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.4165836572647095, + "kl": 0.9325059801340103, + "learning_rate": 1.4285714285714286e-06, + "loss": 0.0009, + "num_tokens": 65127.0, + "reward": 0.8968478441238403, + "reward_std": 0.07308956235647202, + "rewards/check_answer/mean": 0.8968478441238403, + "rewards/check_answer/std": 0.07308956235647202, + "step": 21 + }, + { + "completion_length": 308.75, + "completions/clipped_ratio": 0.0, + "completions/max_length": 337.0, + "completions/max_terminated_length": 337.0, + "completions/mean_length": 308.75, + "completions/mean_terminated_length": 308.75, + "completions/min_length": 251.0, + "completions/min_terminated_length": 251.0, + "epoch": 0.003909720988093122, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.006172445602715015, + "kl": 1.3253843784332275, + "learning_rate": 1.5e-06, + "loss": 0.0013, + "num_tokens": 67458.0, + "reward": 1.5, + "reward_std": 0.0, + "rewards/check_answer/mean": 1.5, + "rewards/check_answer/std": 0.0, + "step": 22 + }, + { + "completion_length": 457.75, + "completions/clipped_ratio": 0.0, + "completions/max_length": 521.0, + "completions/max_terminated_length": 521.0, + "completions/mean_length": 457.75, + "completions/mean_terminated_length": 457.75, + "completions/min_length": 405.0, + "completions/min_terminated_length": 405.0, + "epoch": 0.004087435578460991, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.757681965827942, + "kl": 1.0600132942199707, + "learning_rate": 1.5714285714285714e-06, + "loss": 0.0011, + "num_tokens": 70461.0, + "reward": 0.4985591769218445, + "reward_std": 0.18019117414951324, + "rewards/check_answer/mean": 0.4985591769218445, + "rewards/check_answer/std": 0.18019118905067444, + "step": 23 + }, + { + "completion_length": 409.25, + "completions/clipped_ratio": 0.0, + "completions/max_length": 423.0, + "completions/max_terminated_length": 423.0, + "completions/mean_length": 409.25, + "completions/mean_terminated_length": 409.25, + "completions/min_length": 383.0, + "completions/min_terminated_length": 383.0, + "epoch": 0.004265150168828861, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.7924888134002686, + "kl": 1.278127372264862, + "learning_rate": 1.642857142857143e-06, + "loss": 0.0013, + "num_tokens": 73210.0, + "reward": 0.7848971486091614, + "reward_std": 0.2511295974254608, + "rewards/check_answer/mean": 0.7848971486091614, + "rewards/check_answer/std": 0.2511295974254608, + "step": 24 + }, + { + "completion_length": 418.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 503.0, + "completions/max_terminated_length": 503.0, + "completions/mean_length": 418.0, + "completions/mean_terminated_length": 418.0, + "completions/min_length": 333.0, + "completions/min_terminated_length": 333.0, + "epoch": 0.00444286475919673, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.607702612876892, + "kl": 1.2083918154239655, + "learning_rate": 1.7142857142857145e-06, + "loss": 0.0012, + "num_tokens": 75998.0, + "reward": 0.9702531099319458, + "reward_std": 0.14307360351085663, + "rewards/check_answer/mean": 0.9702531099319458, + "rewards/check_answer/std": 0.14307361841201782, + "step": 25 + }, + { + "completion_length": 488.25, + "completions/clipped_ratio": 0.0, + "completions/max_length": 609.0, + "completions/max_terminated_length": 609.0, + "completions/mean_length": 488.25, + "completions/mean_terminated_length": 488.25, + "completions/min_length": 331.0, + "completions/min_terminated_length": 331.0, + "epoch": 0.004620579349564599, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.5748683214187622, + "kl": 1.0910266041755676, + "learning_rate": 1.7857142857142859e-06, + "loss": 0.0011, + "num_tokens": 79119.0, + "reward": 0.6022087335586548, + "reward_std": 0.10064248740673065, + "rewards/check_answer/mean": 0.6022087335586548, + "rewards/check_answer/std": 0.10064248740673065, + "step": 26 + }, + { + "completion_length": 632.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 674.0, + "completions/max_terminated_length": 674.0, + "completions/mean_length": 632.0, + "completions/mean_terminated_length": 632.0, + "completions/min_length": 600.0, + "completions/min_terminated_length": 600.0, + "epoch": 0.004798293939932468, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.4055097103118896, + "kl": 0.9649907499551773, + "learning_rate": 1.8571428571428573e-06, + "loss": 0.001, + "num_tokens": 82855.0, + "reward": 0.5038251876831055, + "reward_std": 0.07822742313146591, + "rewards/check_answer/mean": 0.5038251876831055, + "rewards/check_answer/std": 0.07822742313146591, + "step": 27 + }, + { + "completion_length": 555.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 578.0, + "completions/max_terminated_length": 578.0, + "completions/mean_length": 555.0, + "completions/mean_terminated_length": 555.0, + "completions/min_length": 519.0, + "completions/min_terminated_length": 519.0, + "epoch": 0.004976008530300338, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.5128551721572876, + "kl": 1.3766323626041412, + "learning_rate": 1.928571428571429e-06, + "loss": 0.0014, + "num_tokens": 86255.0, + "reward": 0.5897955894470215, + "reward_std": 0.14429643750190735, + "rewards/check_answer/mean": 0.5897955894470215, + "rewards/check_answer/std": 0.14429643750190735, + "step": 28 + }, + { + "completion_length": 351.5, + "completions/clipped_ratio": 0.0, + "completions/max_length": 398.0, + "completions/max_terminated_length": 398.0, + "completions/mean_length": 351.5, + "completions/mean_terminated_length": 351.5, + "completions/min_length": 269.0, + "completions/min_terminated_length": 269.0, + "epoch": 0.005153723120668207, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.9017515182495117, + "kl": 1.2128655910491943, + "learning_rate": 2.0000000000000003e-06, + "loss": 0.0012, + "num_tokens": 88761.0, + "reward": 0.24229314923286438, + "reward_std": 0.09404895454645157, + "rewards/check_answer/mean": 0.24229314923286438, + "rewards/check_answer/std": 0.09404894709587097, + "step": 29 + }, + { + "completion_length": 430.75, + "completions/clipped_ratio": 0.0, + "completions/max_length": 487.0, + "completions/max_terminated_length": 487.0, + "completions/mean_length": 430.75, + "completions/mean_terminated_length": 430.75, + "completions/min_length": 310.0, + "completions/min_terminated_length": 310.0, + "epoch": 0.005331437711036076, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.466556191444397, + "kl": 0.9134228527545929, + "learning_rate": 2.0714285714285717e-06, + "loss": 0.0009, + "num_tokens": 91664.0, + "reward": 0.7716047763824463, + "reward_std": 0.11445575952529907, + "rewards/check_answer/mean": 0.7716047763824463, + "rewards/check_answer/std": 0.11445576697587967, + "step": 30 + }, + { + "completion_length": 502.25, + "completions/clipped_ratio": 0.0, + "completions/max_length": 589.0, + "completions/max_terminated_length": 589.0, + "completions/mean_length": 502.25, + "completions/mean_terminated_length": 502.25, + "completions/min_length": 453.0, + "completions/min_terminated_length": 453.0, + "epoch": 0.005509152301403945, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.8003754615783691, + "kl": 1.0646594911813736, + "learning_rate": 2.1428571428571427e-06, + "loss": 0.0011, + "num_tokens": 94865.0, + "reward": 0.9116252064704895, + "reward_std": 0.06001972407102585, + "rewards/check_answer/mean": 0.9116252064704895, + "rewards/check_answer/std": 0.060019705444574356, + "step": 31 + }, + { + "completion_length": 454.25, + "completions/clipped_ratio": 0.0, + "completions/max_length": 549.0, + "completions/max_terminated_length": 549.0, + "completions/mean_length": 454.25, + "completions/mean_terminated_length": 454.25, + "completions/min_length": 416.0, + "completions/min_terminated_length": 416.0, + "epoch": 0.005686866891771815, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.75742506980896, + "kl": 1.6527438461780548, + "learning_rate": 2.2142857142857146e-06, + "loss": 0.0017, + "num_tokens": 97818.0, + "reward": 0.5714230537414551, + "reward_std": 0.1710004061460495, + "rewards/check_answer/mean": 0.5714230537414551, + "rewards/check_answer/std": 0.1710004061460495, + "step": 32 + }, + { + "completion_length": 282.75, + "completions/clipped_ratio": 0.0, + "completions/max_length": 313.0, + "completions/max_terminated_length": 313.0, + "completions/mean_length": 282.75, + "completions/mean_terminated_length": 282.75, + "completions/min_length": 255.0, + "completions/min_terminated_length": 255.0, + "epoch": 0.005864581482139684, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.9505561590194702, + "kl": 1.6063998937606812, + "learning_rate": 2.285714285714286e-06, + "loss": 0.0016, + "num_tokens": 100041.0, + "reward": 0.49875420331954956, + "reward_std": 0.2340739220380783, + "rewards/check_answer/mean": 0.49875420331954956, + "rewards/check_answer/std": 0.2340739369392395, + "step": 33 + }, + { + "completion_length": 376.5, + "completions/clipped_ratio": 0.0, + "completions/max_length": 416.0, + "completions/max_terminated_length": 416.0, + "completions/mean_length": 376.5, + "completions/mean_terminated_length": 376.5, + "completions/min_length": 345.0, + "completions/min_terminated_length": 345.0, + "epoch": 0.006042296072507553, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.888424277305603, + "kl": 1.435717135667801, + "learning_rate": 2.3571428571428574e-06, + "loss": 0.0014, + "num_tokens": 102667.0, + "reward": 0.9620379209518433, + "reward_std": 0.15259642899036407, + "rewards/check_answer/mean": 0.9620379209518433, + "rewards/check_answer/std": 0.15259642899036407, + "step": 34 + }, + { + "completion_length": 458.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 524.0, + "completions/max_terminated_length": 524.0, + "completions/mean_length": 458.0, + "completions/mean_terminated_length": 458.0, + "completions/min_length": 336.0, + "completions/min_terminated_length": 336.0, + "epoch": 0.006220010662875422, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.9206902980804443, + "kl": 2.231161266565323, + "learning_rate": 2.428571428571429e-06, + "loss": 0.0022, + "num_tokens": 105659.0, + "reward": 0.653532862663269, + "reward_std": 0.1333208829164505, + "rewards/check_answer/mean": 0.653532862663269, + "rewards/check_answer/std": 0.1333208829164505, + "step": 35 + }, + { + "completion_length": 402.75, + "completions/clipped_ratio": 0.0, + "completions/max_length": 440.0, + "completions/max_terminated_length": 440.0, + "completions/mean_length": 402.75, + "completions/mean_terminated_length": 402.75, + "completions/min_length": 333.0, + "completions/min_terminated_length": 333.0, + "epoch": 0.006397725253243291, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.7704061269760132, + "kl": 1.304360181093216, + "learning_rate": 2.5e-06, + "loss": 0.0013, + "num_tokens": 108382.0, + "reward": 0.6283889412879944, + "reward_std": 0.18885570764541626, + "rewards/check_answer/mean": 0.6283889412879944, + "rewards/check_answer/std": 0.18885570764541626, + "step": 36 + }, + { + "completion_length": 422.5, + "completions/clipped_ratio": 0.0, + "completions/max_length": 447.0, + "completions/max_terminated_length": 447.0, + "completions/mean_length": 422.5, + "completions/mean_terminated_length": 422.5, + "completions/min_length": 381.0, + "completions/min_terminated_length": 381.0, + "epoch": 0.006575439843611161, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.7185007333755493, + "kl": 1.1553759276866913, + "learning_rate": 2.571428571428571e-06, + "loss": 0.0012, + "num_tokens": 111188.0, + "reward": 0.7625912427902222, + "reward_std": 0.3398703336715698, + "rewards/check_answer/mean": 0.7625912427902222, + "rewards/check_answer/std": 0.3398703336715698, + "step": 37 + }, + { + "completion_length": 395.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 417.0, + "completions/max_terminated_length": 417.0, + "completions/mean_length": 395.0, + "completions/mean_terminated_length": 395.0, + "completions/min_length": 376.0, + "completions/min_terminated_length": 376.0, + "epoch": 0.00675315443397903, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.813489556312561, + "kl": 1.176555871963501, + "learning_rate": 2.642857142857143e-06, + "loss": 0.0012, + "num_tokens": 113896.0, + "reward": 0.4037095606327057, + "reward_std": 0.14651015400886536, + "rewards/check_answer/mean": 0.4037095606327057, + "rewards/check_answer/std": 0.14651015400886536, + "step": 38 + }, + { + "completion_length": 518.25, + "completions/clipped_ratio": 0.0, + "completions/max_length": 557.0, + "completions/max_terminated_length": 557.0, + "completions/mean_length": 518.25, + "completions/mean_terminated_length": 518.25, + "completions/min_length": 475.0, + "completions/min_terminated_length": 475.0, + "epoch": 0.006930869024346899, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.5206884145736694, + "kl": 1.066824734210968, + "learning_rate": 2.7142857142857144e-06, + "loss": 0.0011, + "num_tokens": 117121.0, + "reward": 0.6004906892776489, + "reward_std": 0.06248007342219353, + "rewards/check_answer/mean": 0.6004906892776489, + "rewards/check_answer/std": 0.06248008832335472, + "step": 39 + }, + { + "completion_length": 586.5, + "completions/clipped_ratio": 0.0, + "completions/max_length": 633.0, + "completions/max_terminated_length": 633.0, + "completions/mean_length": 586.5, + "completions/mean_terminated_length": 586.5, + "completions/min_length": 515.0, + "completions/min_terminated_length": 515.0, + "epoch": 0.007108583614714768, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.2672127485275269, + "kl": 1.0179399698972702, + "learning_rate": 2.785714285714286e-06, + "loss": 0.001, + "num_tokens": 120671.0, + "reward": 0.7895050048828125, + "reward_std": 0.058173682540655136, + "rewards/check_answer/mean": 0.7895050048828125, + "rewards/check_answer/std": 0.058173682540655136, + "step": 40 + }, + { + "completion_length": 474.5, + "completions/clipped_ratio": 0.0, + "completions/max_length": 567.0, + "completions/max_terminated_length": 567.0, + "completions/mean_length": 474.5, + "completions/mean_terminated_length": 474.5, + "completions/min_length": 424.0, + "completions/min_terminated_length": 424.0, + "epoch": 0.007286298205082638, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.59950852394104, + "kl": 1.0878016501665115, + "learning_rate": 2.8571428571428573e-06, + "loss": 0.0011, + "num_tokens": 123717.0, + "reward": 0.6761389374732971, + "reward_std": 0.04523153230547905, + "rewards/check_answer/mean": 0.6761389374732971, + "rewards/check_answer/std": 0.04523153603076935, + "step": 41 + }, + { + "completion_length": 589.25, + "completions/clipped_ratio": 0.0, + "completions/max_length": 659.0, + "completions/max_terminated_length": 659.0, + "completions/mean_length": 589.25, + "completions/mean_terminated_length": 589.25, + "completions/min_length": 549.0, + "completions/min_terminated_length": 549.0, + "epoch": 0.007464012795450507, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.543569564819336, + "kl": 1.1034990847110748, + "learning_rate": 2.928571428571429e-06, + "loss": 0.0011, + "num_tokens": 127262.0, + "reward": 0.5081104636192322, + "reward_std": 0.012905855663120747, + "rewards/check_answer/mean": 0.5081104636192322, + "rewards/check_answer/std": 0.012905837036669254, + "step": 42 + }, + { + "completion_length": 294.25, + "completions/clipped_ratio": 0.0, + "completions/max_length": 352.0, + "completions/max_terminated_length": 352.0, + "completions/mean_length": 294.25, + "completions/mean_terminated_length": 294.25, + "completions/min_length": 267.0, + "completions/min_terminated_length": 267.0, + "epoch": 0.007641727385818376, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.7369331121444702, + "kl": 1.1378049850463867, + "learning_rate": 3e-06, + "loss": 0.0011, + "num_tokens": 129555.0, + "reward": 1.392117977142334, + "reward_std": 0.21576404571533203, + "rewards/check_answer/mean": 1.392117977142334, + "rewards/check_answer/std": 0.21576404571533203, + "step": 43 + }, + { + "completion_length": 610.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 714.0, + "completions/max_terminated_length": 714.0, + "completions/mean_length": 610.0, + "completions/mean_terminated_length": 610.0, + "completions/min_length": 521.0, + "completions/min_terminated_length": 521.0, + "epoch": 0.007819441976186245, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3297312259674072, + "kl": 0.7992035746574402, + "learning_rate": 3.071428571428572e-06, + "loss": 0.0008, + "num_tokens": 133243.0, + "reward": 0.379222571849823, + "reward_std": 0.07391934096813202, + "rewards/check_answer/mean": 0.379222571849823, + "rewards/check_answer/std": 0.07391934841871262, + "step": 44 + }, + { + "completion_length": 526.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 602.0, + "completions/max_terminated_length": 602.0, + "completions/mean_length": 526.0, + "completions/mean_terminated_length": 526.0, + "completions/min_length": 445.0, + "completions/min_terminated_length": 445.0, + "epoch": 0.007997156566554114, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3624135255813599, + "kl": 1.055791214108467, + "learning_rate": 3.142857142857143e-06, + "loss": 0.0011, + "num_tokens": 136531.0, + "reward": 0.5060752630233765, + "reward_std": 0.07115290313959122, + "rewards/check_answer/mean": 0.5060752630233765, + "rewards/check_answer/std": 0.07115291059017181, + "step": 45 + }, + { + "completion_length": 347.75, + "completions/clipped_ratio": 0.0, + "completions/max_length": 435.0, + "completions/max_terminated_length": 435.0, + "completions/mean_length": 347.75, + "completions/mean_terminated_length": 347.75, + "completions/min_length": 205.0, + "completions/min_terminated_length": 205.0, + "epoch": 0.008174871156921983, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.5305379629135132, + "kl": 1.2677884995937347, + "learning_rate": 3.2142857142857147e-06, + "loss": 0.0013, + "num_tokens": 139030.0, + "reward": 0.5792946815490723, + "reward_std": 0.07226314395666122, + "rewards/check_answer/mean": 0.5792946815490723, + "rewards/check_answer/std": 0.07226314395666122, + "step": 46 + }, + { + "completion_length": 469.75, + "completions/clipped_ratio": 0.0, + "completions/max_length": 476.0, + "completions/max_terminated_length": 476.0, + "completions/mean_length": 469.75, + "completions/mean_terminated_length": 469.75, + "completions/min_length": 465.0, + "completions/min_terminated_length": 465.0, + "epoch": 0.008352585747289852, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.5030322074890137, + "kl": 1.171961322426796, + "learning_rate": 3.285714285714286e-06, + "loss": 0.0012, + "num_tokens": 142061.0, + "reward": 1.0066816806793213, + "reward_std": 0.16484001278877258, + "rewards/check_answer/mean": 1.0066816806793213, + "rewards/check_answer/std": 0.16484001278877258, + "step": 47 + }, + { + "completion_length": 340.5, + "completions/clipped_ratio": 0.0, + "completions/max_length": 382.0, + "completions/max_terminated_length": 382.0, + "completions/mean_length": 340.5, + "completions/mean_terminated_length": 340.5, + "completions/min_length": 312.0, + "completions/min_terminated_length": 312.0, + "epoch": 0.008530300337657722, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.6775590181350708, + "kl": 1.4609983712434769, + "learning_rate": 3.357142857142857e-06, + "loss": 0.0015, + "num_tokens": 144583.0, + "reward": 1.2281973361968994, + "reward_std": 0.18847818672657013, + "rewards/check_answer/mean": 1.2281973361968994, + "rewards/check_answer/std": 0.18847818672657013, + "step": 48 + }, + { + "completion_length": 298.75, + "completions/clipped_ratio": 0.0, + "completions/max_length": 315.0, + "completions/max_terminated_length": 315.0, + "completions/mean_length": 298.75, + "completions/mean_terminated_length": 298.75, + "completions/min_length": 290.0, + "completions/min_terminated_length": 290.0, + "epoch": 0.00870801492802559, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.6502373218536377, + "kl": 1.2542754113674164, + "learning_rate": 3.428571428571429e-06, + "loss": 0.0013, + "num_tokens": 146890.0, + "reward": 0.5080112814903259, + "reward_std": 0.03593429550528526, + "rewards/check_answer/mean": 0.5080112814903259, + "rewards/check_answer/std": 0.03593427687883377, + "step": 49 + }, + { + "completion_length": 541.25, + "completions/clipped_ratio": 0.0, + "completions/max_length": 612.0, + "completions/max_terminated_length": 612.0, + "completions/mean_length": 541.25, + "completions/mean_terminated_length": 541.25, + "completions/min_length": 496.0, + "completions/min_terminated_length": 496.0, + "epoch": 0.00888572951839346, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.5773475170135498, + "kl": 1.1764096915721893, + "learning_rate": 3.5e-06, + "loss": 0.0012, + "num_tokens": 150239.0, + "reward": 0.5298885107040405, + "reward_std": 0.062375664710998535, + "rewards/check_answer/mean": 0.5298885107040405, + "rewards/check_answer/std": 0.06237566098570824, + "step": 50 + }, + { + "completion_length": 473.25, + "completions/clipped_ratio": 0.0, + "completions/max_length": 491.0, + "completions/max_terminated_length": 491.0, + "completions/mean_length": 473.25, + "completions/mean_terminated_length": 473.25, + "completions/min_length": 445.0, + "completions/min_terminated_length": 445.0, + "epoch": 0.00906344410876133, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.5712735652923584, + "kl": 1.2196315228939056, + "learning_rate": 3.5714285714285718e-06, + "loss": 0.0012, + "num_tokens": 153272.0, + "reward": 0.9013558626174927, + "reward_std": 0.12637704610824585, + "rewards/check_answer/mean": 0.9013558626174927, + "rewards/check_answer/std": 0.12637704610824585, + "step": 51 + }, + { + "completion_length": 430.75, + "completions/clipped_ratio": 0.0, + "completions/max_length": 444.0, + "completions/max_terminated_length": 444.0, + "completions/mean_length": 430.75, + "completions/mean_terminated_length": 430.75, + "completions/min_length": 408.0, + "completions/min_terminated_length": 408.0, + "epoch": 0.009241158699129198, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.8517780303955078, + "kl": 1.4242485463619232, + "learning_rate": 3.642857142857143e-06, + "loss": 0.0014, + "num_tokens": 156111.0, + "reward": 0.48627614974975586, + "reward_std": 0.10064783692359924, + "rewards/check_answer/mean": 0.48627614974975586, + "rewards/check_answer/std": 0.10064783692359924, + "step": 52 + }, + { + "completion_length": 289.5, + "completions/clipped_ratio": 0.0, + "completions/max_length": 320.0, + "completions/max_terminated_length": 320.0, + "completions/mean_length": 289.5, + "completions/mean_terminated_length": 289.5, + "completions/min_length": 253.0, + "completions/min_terminated_length": 253.0, + "epoch": 0.009418873289497068, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.9996415376663208, + "kl": 1.5143893957138062, + "learning_rate": 3.7142857142857146e-06, + "loss": 0.0015, + "num_tokens": 158361.0, + "reward": 0.5390114188194275, + "reward_std": 0.6577240824699402, + "rewards/check_answer/mean": 0.5390114188194275, + "rewards/check_answer/std": 0.657724142074585, + "step": 53 + }, + { + "completion_length": 443.25, + "completions/clipped_ratio": 0.0, + "completions/max_length": 520.0, + "completions/max_terminated_length": 520.0, + "completions/mean_length": 443.25, + "completions/mean_terminated_length": 443.25, + "completions/min_length": 368.0, + "completions/min_terminated_length": 368.0, + "epoch": 0.009596587879864936, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3870877027511597, + "kl": 1.062680721282959, + "learning_rate": 3.785714285714286e-06, + "loss": 0.0011, + "num_tokens": 161294.0, + "reward": 0.5653637647628784, + "reward_std": 0.06738435477018356, + "rewards/check_answer/mean": 0.5653637647628784, + "rewards/check_answer/std": 0.06738435477018356, + "step": 54 + }, + { + "completion_length": 391.75, + "completions/clipped_ratio": 0.0, + "completions/max_length": 449.0, + "completions/max_terminated_length": 449.0, + "completions/mean_length": 391.75, + "completions/mean_terminated_length": 391.75, + "completions/min_length": 333.0, + "completions/min_terminated_length": 333.0, + "epoch": 0.009774302470232806, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.6596442461013794, + "kl": 0.9484704583883286, + "learning_rate": 3.857142857142858e-06, + "loss": 0.0009, + "num_tokens": 164025.0, + "reward": 0.7971336245536804, + "reward_std": 0.12026101350784302, + "rewards/check_answer/mean": 0.7971336245536804, + "rewards/check_answer/std": 0.12026101350784302, + "step": 55 + }, + { + "completion_length": 347.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 444.0, + "completions/max_terminated_length": 444.0, + "completions/mean_length": 347.0, + "completions/mean_terminated_length": 347.0, + "completions/min_length": 270.0, + "completions/min_terminated_length": 270.0, + "epoch": 0.009952017060600676, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.146622896194458, + "kl": 1.1937618106603622, + "learning_rate": 3.928571428571429e-06, + "loss": 0.0012, + "num_tokens": 166537.0, + "reward": 0.2618507444858551, + "reward_std": 0.054494522511959076, + "rewards/check_answer/mean": 0.2618507444858551, + "rewards/check_answer/std": 0.054494522511959076, + "step": 56 + }, + { + "completion_length": 520.5, + "completions/clipped_ratio": 0.0, + "completions/max_length": 584.0, + "completions/max_terminated_length": 584.0, + "completions/mean_length": 520.5, + "completions/mean_terminated_length": 520.5, + "completions/min_length": 436.0, + "completions/min_terminated_length": 436.0, + "epoch": 0.010129731650968544, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.5688644647598267, + "kl": 1.048645719885826, + "learning_rate": 4.000000000000001e-06, + "loss": 0.001, + "num_tokens": 169803.0, + "reward": 0.9876464605331421, + "reward_std": 0.25668615102767944, + "rewards/check_answer/mean": 0.9876464605331421, + "rewards/check_answer/std": 0.25668615102767944, + "step": 57 + }, + { + "completion_length": 541.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 610.0, + "completions/max_terminated_length": 610.0, + "completions/mean_length": 541.0, + "completions/mean_terminated_length": 541.0, + "completions/min_length": 483.0, + "completions/min_terminated_length": 483.0, + "epoch": 0.010307446241336414, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.2746442556381226, + "kl": 0.8855240046977997, + "learning_rate": 4.071428571428572e-06, + "loss": 0.0009, + "num_tokens": 173207.0, + "reward": 0.3755004405975342, + "reward_std": 0.049010019749403, + "rewards/check_answer/mean": 0.3755004405975342, + "rewards/check_answer/std": 0.04901003837585449, + "step": 58 + }, + { + "completion_length": 524.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 577.0, + "completions/max_terminated_length": 577.0, + "completions/mean_length": 524.0, + "completions/mean_terminated_length": 524.0, + "completions/min_length": 450.0, + "completions/min_terminated_length": 450.0, + "epoch": 0.010485160831704282, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.5979560613632202, + "kl": 1.034047693014145, + "learning_rate": 4.1428571428571435e-06, + "loss": 0.001, + "num_tokens": 176475.0, + "reward": 0.8849939703941345, + "reward_std": 0.11704953014850616, + "rewards/check_answer/mean": 0.8849939703941345, + "rewards/check_answer/std": 0.11704951524734497, + "step": 59 + }, + { + "completion_length": 471.25, + "completions/clipped_ratio": 0.0, + "completions/max_length": 558.0, + "completions/max_terminated_length": 558.0, + "completions/mean_length": 471.25, + "completions/mean_terminated_length": 471.25, + "completions/min_length": 372.0, + "completions/min_terminated_length": 372.0, + "epoch": 0.010662875422072152, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.5512830018997192, + "kl": 1.3353438973426819, + "learning_rate": 4.2142857142857145e-06, + "loss": 0.0013, + "num_tokens": 179544.0, + "reward": 0.4594007134437561, + "reward_std": 0.08352126181125641, + "rewards/check_answer/mean": 0.4594007134437561, + "rewards/check_answer/std": 0.08352126181125641, + "step": 60 + }, + { + "completion_length": 560.5, + "completions/clipped_ratio": 0.0, + "completions/max_length": 688.0, + "completions/max_terminated_length": 688.0, + "completions/mean_length": 560.5, + "completions/mean_terminated_length": 560.5, + "completions/min_length": 406.0, + "completions/min_terminated_length": 406.0, + "epoch": 0.010840590012440022, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.7048546075820923, + "kl": 0.9816142320632935, + "learning_rate": 4.2857142857142855e-06, + "loss": 0.001, + "num_tokens": 182994.0, + "reward": 0.38246601819992065, + "reward_std": 0.13571514189243317, + "rewards/check_answer/mean": 0.38246601819992065, + "rewards/check_answer/std": 0.13571515679359436, + "step": 61 + }, + { + "completion_length": 411.25, + "completions/clipped_ratio": 0.0, + "completions/max_length": 489.0, + "completions/max_terminated_length": 489.0, + "completions/mean_length": 411.25, + "completions/mean_terminated_length": 411.25, + "completions/min_length": 361.0, + "completions/min_terminated_length": 361.0, + "epoch": 0.01101830460280789, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.981577754020691, + "kl": 1.2702476382255554, + "learning_rate": 4.357142857142857e-06, + "loss": 0.0013, + "num_tokens": 185755.0, + "reward": 0.6234804391860962, + "reward_std": 0.2467672973871231, + "rewards/check_answer/mean": 0.6234804391860962, + "rewards/check_answer/std": 0.2467672973871231, + "step": 62 + }, + { + "completion_length": 511.5, + "completions/clipped_ratio": 0.0, + "completions/max_length": 589.0, + "completions/max_terminated_length": 589.0, + "completions/mean_length": 511.5, + "completions/mean_terminated_length": 511.5, + "completions/min_length": 388.0, + "completions/min_terminated_length": 388.0, + "epoch": 0.01119601919317576, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.365710735321045, + "kl": 0.9097413569688797, + "learning_rate": 4.428571428571429e-06, + "loss": 0.0009, + "num_tokens": 189005.0, + "reward": 0.4179466962814331, + "reward_std": 0.04861922189593315, + "rewards/check_answer/mean": 0.4179466962814331, + "rewards/check_answer/std": 0.04861922189593315, + "step": 63 + }, + { + "completion_length": 440.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 484.0, + "completions/max_terminated_length": 484.0, + "completions/mean_length": 440.0, + "completions/mean_terminated_length": 440.0, + "completions/min_length": 397.0, + "completions/min_terminated_length": 397.0, + "epoch": 0.01137373378354363, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.8046718835830688, + "kl": 1.3415338695049286, + "learning_rate": 4.5e-06, + "loss": 0.0013, + "num_tokens": 191909.0, + "reward": 0.4736880362033844, + "reward_std": 0.09703974425792694, + "rewards/check_answer/mean": 0.4736880362033844, + "rewards/check_answer/std": 0.09703974425792694, + "step": 64 + }, + { + "completion_length": 411.5, + "completions/clipped_ratio": 0.0, + "completions/max_length": 509.0, + "completions/max_terminated_length": 509.0, + "completions/mean_length": 411.5, + "completions/mean_terminated_length": 411.5, + "completions/min_length": 299.0, + "completions/min_terminated_length": 299.0, + "epoch": 0.011551448373911498, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.6717690229415894, + "kl": 1.1143593043088913, + "learning_rate": 4.571428571428572e-06, + "loss": 0.0011, + "num_tokens": 194675.0, + "reward": 0.7849711775779724, + "reward_std": 0.3310997784137726, + "rewards/check_answer/mean": 0.7849711775779724, + "rewards/check_answer/std": 0.3310997784137726, + "step": 65 + }, + { + "completion_length": 484.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 504.0, + "completions/max_terminated_length": 504.0, + "completions/mean_length": 484.0, + "completions/mean_terminated_length": 484.0, + "completions/min_length": 461.0, + "completions/min_terminated_length": 461.0, + "epoch": 0.011729162964279368, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3961551189422607, + "kl": 0.8092105090618134, + "learning_rate": 4.642857142857144e-06, + "loss": 0.0008, + "num_tokens": 197815.0, + "reward": 0.5569653511047363, + "reward_std": 0.11439482122659683, + "rewards/check_answer/mean": 0.5569653511047363, + "rewards/check_answer/std": 0.11439482867717743, + "step": 66 + }, + { + "completion_length": 580.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 598.0, + "completions/max_terminated_length": 598.0, + "completions/mean_length": 580.0, + "completions/mean_terminated_length": 580.0, + "completions/min_length": 553.0, + "completions/min_terminated_length": 553.0, + "epoch": 0.011906877554647236, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.20498788356781, + "kl": 0.9006475359201431, + "learning_rate": 4.714285714285715e-06, + "loss": 0.0009, + "num_tokens": 201331.0, + "reward": 0.5501281023025513, + "reward_std": 0.05327435955405235, + "rewards/check_answer/mean": 0.5501281023025513, + "rewards/check_answer/std": 0.053274352103471756, + "step": 67 + }, + { + "completion_length": 408.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 521.0, + "completions/max_terminated_length": 521.0, + "completions/mean_length": 408.0, + "completions/mean_terminated_length": 408.0, + "completions/min_length": 309.0, + "completions/min_terminated_length": 309.0, + "epoch": 0.012084592145015106, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.882188081741333, + "kl": 1.1671920120716095, + "learning_rate": 4.785714285714287e-06, + "loss": 0.0012, + "num_tokens": 204115.0, + "reward": 1.1435757875442505, + "reward_std": 0.22389693558216095, + "rewards/check_answer/mean": 1.1435757875442505, + "rewards/check_answer/std": 0.22389698028564453, + "step": 68 + }, + { + "completion_length": 481.25, + "completions/clipped_ratio": 0.0, + "completions/max_length": 590.0, + "completions/max_terminated_length": 590.0, + "completions/mean_length": 481.25, + "completions/mean_terminated_length": 481.25, + "completions/min_length": 363.0, + "completions/min_terminated_length": 363.0, + "epoch": 0.012262306735382976, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.533947229385376, + "kl": 1.1329405903816223, + "learning_rate": 4.857142857142858e-06, + "loss": 0.0011, + "num_tokens": 207200.0, + "reward": 0.44344037771224976, + "reward_std": 0.09549304842948914, + "rewards/check_answer/mean": 0.44344037771224976, + "rewards/check_answer/std": 0.09549304842948914, + "step": 69 + }, + { + "completion_length": 530.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 651.0, + "completions/max_terminated_length": 651.0, + "completions/mean_length": 530.0, + "completions/mean_terminated_length": 530.0, + "completions/min_length": 380.0, + "completions/min_terminated_length": 380.0, + "epoch": 0.012440021325750844, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.383154273033142, + "kl": 0.8603696227073669, + "learning_rate": 4.928571428571429e-06, + "loss": 0.0009, + "num_tokens": 210516.0, + "reward": 0.3628694415092468, + "reward_std": 0.04750685766339302, + "rewards/check_answer/mean": 0.3628694415092468, + "rewards/check_answer/std": 0.04750685766339302, + "step": 70 + }, + { + "completion_length": 467.25, + "completions/clipped_ratio": 0.0, + "completions/max_length": 501.0, + "completions/max_terminated_length": 501.0, + "completions/mean_length": 467.25, + "completions/mean_terminated_length": 467.25, + "completions/min_length": 428.0, + "completions/min_terminated_length": 428.0, + "epoch": 0.012617735916118714, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.7030493021011353, + "kl": 1.256488174200058, + "learning_rate": 5e-06, + "loss": 0.0013, + "num_tokens": 213513.0, + "reward": 0.9228349924087524, + "reward_std": 0.2585166394710541, + "rewards/check_answer/mean": 0.9228349924087524, + "rewards/check_answer/std": 0.2585166394710541, + "step": 71 + }, + { + "completion_length": 380.5, + "completions/clipped_ratio": 0.0, + "completions/max_length": 463.0, + "completions/max_terminated_length": 463.0, + "completions/mean_length": 380.5, + "completions/mean_terminated_length": 380.5, + "completions/min_length": 317.0, + "completions/min_terminated_length": 317.0, + "epoch": 0.012795450506486582, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.8185611963272095, + "kl": 1.088569387793541, + "learning_rate": 4.992063492063493e-06, + "loss": 0.0011, + "num_tokens": 216187.0, + "reward": 0.1358739137649536, + "reward_std": 0.03403108939528465, + "rewards/check_answer/mean": 0.1358739137649536, + "rewards/check_answer/std": 0.03403108939528465, + "step": 72 + }, + { + "completion_length": 391.5, + "completions/clipped_ratio": 0.0, + "completions/max_length": 472.0, + "completions/max_terminated_length": 472.0, + "completions/mean_length": 391.5, + "completions/mean_terminated_length": 391.5, + "completions/min_length": 352.0, + "completions/min_terminated_length": 352.0, + "epoch": 0.012973165096854452, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.7923240661621094, + "kl": 1.187307983636856, + "learning_rate": 4.9841269841269845e-06, + "loss": 0.0012, + "num_tokens": 218881.0, + "reward": 0.4668271541595459, + "reward_std": 0.28230786323547363, + "rewards/check_answer/mean": 0.4668271541595459, + "rewards/check_answer/std": 0.28230786323547363, + "step": 73 + }, + { + "completion_length": 459.5, + "completions/clipped_ratio": 0.0, + "completions/max_length": 560.0, + "completions/max_terminated_length": 560.0, + "completions/mean_length": 459.5, + "completions/mean_terminated_length": 459.5, + "completions/min_length": 314.0, + "completions/min_terminated_length": 314.0, + "epoch": 0.013150879687222321, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.5349751710891724, + "kl": 1.160512775182724, + "learning_rate": 4.976190476190477e-06, + "loss": 0.0012, + "num_tokens": 221875.0, + "reward": 0.5796365737915039, + "reward_std": 0.03070678748190403, + "rewards/check_answer/mean": 0.5796365737915039, + "rewards/check_answer/std": 0.03070676326751709, + "step": 74 + }, + { + "completion_length": 275.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 379.0, + "completions/max_terminated_length": 379.0, + "completions/mean_length": 275.0, + "completions/mean_terminated_length": 275.0, + "completions/min_length": 194.0, + "completions/min_terminated_length": 194.0, + "epoch": 0.01332859427759019, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.9937189817428589, + "kl": 1.2936433553695679, + "learning_rate": 4.968253968253969e-06, + "loss": 0.0013, + "num_tokens": 224087.0, + "reward": 0.618080735206604, + "reward_std": 0.028470057994127274, + "rewards/check_answer/mean": 0.618080735206604, + "rewards/check_answer/std": 0.028470059856772423, + "step": 75 + }, + { + "completion_length": 561.5, + "completions/clipped_ratio": 0.0, + "completions/max_length": 883.0, + "completions/max_terminated_length": 883.0, + "completions/mean_length": 561.5, + "completions/mean_terminated_length": 561.5, + "completions/min_length": 382.0, + "completions/min_terminated_length": 382.0, + "epoch": 0.01350630886795806, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.2865067720413208, + "kl": 1.0802096128463745, + "learning_rate": 4.960317460317461e-06, + "loss": 0.0011, + "num_tokens": 227501.0, + "reward": 0.45027637481689453, + "reward_std": 0.3139820694923401, + "rewards/check_answer/mean": 0.45027637481689453, + "rewards/check_answer/std": 0.3139820694923401, + "step": 76 + }, + { + "completion_length": 482.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 548.0, + "completions/max_terminated_length": 548.0, + "completions/mean_length": 482.0, + "completions/mean_terminated_length": 482.0, + "completions/min_length": 378.0, + "completions/min_terminated_length": 378.0, + "epoch": 0.01368402345832593, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.5967481136322021, + "kl": 2.116203159093857, + "learning_rate": 4.952380952380953e-06, + "loss": 0.0021, + "num_tokens": 230589.0, + "reward": 0.8557864427566528, + "reward_std": 0.07187496870756149, + "rewards/check_answer/mean": 0.8557864427566528, + "rewards/check_answer/std": 0.0718749538064003, + "step": 77 + }, + { + "completion_length": 435.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 450.0, + "completions/max_terminated_length": 450.0, + "completions/mean_length": 435.0, + "completions/mean_terminated_length": 435.0, + "completions/min_length": 410.0, + "completions/min_terminated_length": 410.0, + "epoch": 0.013861738048693797, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.7964273691177368, + "kl": 1.6678560972213745, + "learning_rate": 4.944444444444445e-06, + "loss": 0.0017, + "num_tokens": 233473.0, + "reward": 0.7173020839691162, + "reward_std": 0.1675439327955246, + "rewards/check_answer/mean": 0.7173020839691162, + "rewards/check_answer/std": 0.1675439327955246, + "step": 78 + }, + { + "completion_length": 329.5, + "completions/clipped_ratio": 0.0, + "completions/max_length": 370.0, + "completions/max_terminated_length": 370.0, + "completions/mean_length": 329.5, + "completions/mean_terminated_length": 329.5, + "completions/min_length": 297.0, + "completions/min_terminated_length": 297.0, + "epoch": 0.014039452639061667, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.785151481628418, + "kl": 1.2658391296863556, + "learning_rate": 4.936507936507937e-06, + "loss": 0.0013, + "num_tokens": 235919.0, + "reward": 0.3208128809928894, + "reward_std": 0.5496598482131958, + "rewards/check_answer/mean": 0.3208128809928894, + "rewards/check_answer/std": 0.5496599078178406, + "step": 79 + }, + { + "completion_length": 487.25, + "completions/clipped_ratio": 0.0, + "completions/max_length": 535.0, + "completions/max_terminated_length": 535.0, + "completions/mean_length": 487.25, + "completions/mean_terminated_length": 487.25, + "completions/min_length": 438.0, + "completions/min_terminated_length": 438.0, + "epoch": 0.014217167229429535, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.4646213054656982, + "kl": 1.082797721028328, + "learning_rate": 4.928571428571429e-06, + "loss": 0.0011, + "num_tokens": 239072.0, + "reward": 0.47449803352355957, + "reward_std": 0.05666935071349144, + "rewards/check_answer/mean": 0.47449803352355957, + "rewards/check_answer/std": 0.05666935071349144, + "step": 80 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 579.0, + "completions/max_terminated_length": 579.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 512.0, + "completions/min_length": 441.0, + "completions/min_terminated_length": 441.0, + "epoch": 0.014394881819797405, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.374875545501709, + "kl": 0.866473376750946, + "learning_rate": 4.920634920634921e-06, + "loss": 0.0009, + "num_tokens": 242296.0, + "reward": 0.9352073073387146, + "reward_std": 0.054083820432424545, + "rewards/check_answer/mean": 0.9352073073387146, + "rewards/check_answer/std": 0.054083820432424545, + "step": 81 + }, + { + "completion_length": 455.5, + "completions/clipped_ratio": 0.0, + "completions/max_length": 562.0, + "completions/max_terminated_length": 562.0, + "completions/mean_length": 455.5, + "completions/mean_terminated_length": 455.5, + "completions/min_length": 386.0, + "completions/min_terminated_length": 386.0, + "epoch": 0.014572596410165275, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3765476942062378, + "kl": 0.9197855442762375, + "learning_rate": 4.912698412698413e-06, + "loss": 0.0009, + "num_tokens": 245302.0, + "reward": 1.100656270980835, + "reward_std": 0.04409698769450188, + "rewards/check_answer/mean": 1.100656270980835, + "rewards/check_answer/std": 0.04409695789217949, + "step": 82 + }, + { + "completion_length": 396.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 412.0, + "completions/max_terminated_length": 412.0, + "completions/mean_length": 396.0, + "completions/mean_terminated_length": 396.0, + "completions/min_length": 377.0, + "completions/min_terminated_length": 377.0, + "epoch": 0.014750311000533143, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.643187165260315, + "kl": 1.173012375831604, + "learning_rate": 4.904761904761905e-06, + "loss": 0.0012, + "num_tokens": 248034.0, + "reward": 1.026364803314209, + "reward_std": 0.2264280915260315, + "rewards/check_answer/mean": 1.026364803314209, + "rewards/check_answer/std": 0.2264280915260315, + "step": 83 + }, + { + "completion_length": 508.25, + "completions/clipped_ratio": 0.0, + "completions/max_length": 557.0, + "completions/max_terminated_length": 557.0, + "completions/mean_length": 508.25, + "completions/mean_terminated_length": 508.25, + "completions/min_length": 484.0, + "completions/min_terminated_length": 484.0, + "epoch": 0.014928025590901013, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.608371376991272, + "kl": 1.112755537033081, + "learning_rate": 4.896825396825397e-06, + "loss": 0.0011, + "num_tokens": 251223.0, + "reward": 0.9159030914306641, + "reward_std": 0.09535390138626099, + "rewards/check_answer/mean": 0.9159030914306641, + "rewards/check_answer/std": 0.09535388648509979, + "step": 84 + }, + { + "completion_length": 377.25, + "completions/clipped_ratio": 0.0, + "completions/max_length": 478.0, + "completions/max_terminated_length": 478.0, + "completions/mean_length": 377.25, + "completions/mean_terminated_length": 377.25, + "completions/min_length": 260.0, + "completions/min_terminated_length": 260.0, + "epoch": 0.015105740181268883, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.8007338047027588, + "kl": 1.063372939825058, + "learning_rate": 4.888888888888889e-06, + "loss": 0.0011, + "num_tokens": 253880.0, + "reward": 0.5686249732971191, + "reward_std": 0.1699865609407425, + "rewards/check_answer/mean": 0.5686249732971191, + "rewards/check_answer/std": 0.1699865609407425, + "step": 85 + }, + { + "completion_length": 625.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 722.0, + "completions/max_terminated_length": 722.0, + "completions/mean_length": 625.0, + "completions/mean_terminated_length": 625.0, + "completions/min_length": 504.0, + "completions/min_terminated_length": 504.0, + "epoch": 0.015283454771636751, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.1691299676895142, + "kl": 0.7268833667039871, + "learning_rate": 4.880952380952381e-06, + "loss": 0.0007, + "num_tokens": 257636.0, + "reward": 0.6261911392211914, + "reward_std": 0.09556934982538223, + "rewards/check_answer/mean": 0.6261911392211914, + "rewards/check_answer/std": 0.09556934982538223, + "step": 86 + }, + { + "completion_length": 362.25, + "completions/clipped_ratio": 0.0, + "completions/max_length": 411.0, + "completions/max_terminated_length": 411.0, + "completions/mean_length": 362.25, + "completions/mean_terminated_length": 362.25, + "completions/min_length": 238.0, + "completions/min_terminated_length": 238.0, + "epoch": 0.015461169362004621, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.9808728694915771, + "kl": 1.2715768963098526, + "learning_rate": 4.8730158730158735e-06, + "loss": 0.0013, + "num_tokens": 260217.0, + "reward": 0.6537027359008789, + "reward_std": 0.23676364123821259, + "rewards/check_answer/mean": 0.6537027359008789, + "rewards/check_answer/std": 0.23676365613937378, + "step": 87 + }, + { + "completion_length": 342.5, + "completions/clipped_ratio": 0.0, + "completions/max_length": 396.0, + "completions/max_terminated_length": 396.0, + "completions/mean_length": 342.5, + "completions/mean_terminated_length": 342.5, + "completions/min_length": 252.0, + "completions/min_terminated_length": 252.0, + "epoch": 0.01563888395237249, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.8982717990875244, + "kl": 1.0811645984649658, + "learning_rate": 4.865079365079365e-06, + "loss": 0.0011, + "num_tokens": 262711.0, + "reward": 0.8981786966323853, + "reward_std": 0.07228042185306549, + "rewards/check_answer/mean": 0.8981786966323853, + "rewards/check_answer/std": 0.0722803920507431, + "step": 88 + }, + { + "completion_length": 317.25, + "completions/clipped_ratio": 0.0, + "completions/max_length": 469.0, + "completions/max_terminated_length": 469.0, + "completions/mean_length": 317.25, + "completions/mean_terminated_length": 317.25, + "completions/min_length": 227.0, + "completions/min_terminated_length": 227.0, + "epoch": 0.015816598542740357, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.1990370750427246, + "kl": 1.3140311539173126, + "learning_rate": 4.857142857142858e-06, + "loss": 0.0013, + "num_tokens": 265088.0, + "reward": 0.32814788818359375, + "reward_std": 0.029493404552340508, + "rewards/check_answer/mean": 0.32814788818359375, + "rewards/check_answer/std": 0.029493410140275955, + "step": 89 + }, + { + "completion_length": 323.5, + "completions/clipped_ratio": 0.0, + "completions/max_length": 406.0, + "completions/max_terminated_length": 406.0, + "completions/mean_length": 323.5, + "completions/mean_terminated_length": 323.5, + "completions/min_length": 250.0, + "completions/min_terminated_length": 250.0, + "epoch": 0.01599431313310823, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.9783636331558228, + "kl": 2.316634923219681, + "learning_rate": 4.849206349206349e-06, + "loss": 0.0023, + "num_tokens": 267474.0, + "reward": 1.1331236362457275, + "reward_std": 0.43865159153938293, + "rewards/check_answer/mean": 1.1331236362457275, + "rewards/check_answer/std": 0.4386516213417053, + "step": 90 + }, + { + "completion_length": 530.5, + "completions/clipped_ratio": 0.0, + "completions/max_length": 547.0, + "completions/max_terminated_length": 547.0, + "completions/mean_length": 530.5, + "completions/mean_terminated_length": 530.5, + "completions/min_length": 487.0, + "completions/min_terminated_length": 487.0, + "epoch": 0.016172027723476097, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.401001214981079, + "kl": 0.944896399974823, + "learning_rate": 4.841269841269842e-06, + "loss": 0.0009, + "num_tokens": 270800.0, + "reward": 0.6363397240638733, + "reward_std": 0.08994114398956299, + "rewards/check_answer/mean": 0.6363397240638733, + "rewards/check_answer/std": 0.08994113653898239, + "step": 91 + }, + { + "completion_length": 500.5, + "completions/clipped_ratio": 0.0, + "completions/max_length": 569.0, + "completions/max_terminated_length": 569.0, + "completions/mean_length": 500.5, + "completions/mean_terminated_length": 500.5, + "completions/min_length": 432.0, + "completions/min_terminated_length": 432.0, + "epoch": 0.016349742313843965, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.4314804077148438, + "kl": 0.9742838144302368, + "learning_rate": 4.833333333333333e-06, + "loss": 0.001, + "num_tokens": 273942.0, + "reward": 0.42001938819885254, + "reward_std": 0.056012097746133804, + "rewards/check_answer/mean": 0.42001938819885254, + "rewards/check_answer/std": 0.056012097746133804, + "step": 92 + }, + { + "completion_length": 535.5, + "completions/clipped_ratio": 0.0, + "completions/max_length": 596.0, + "completions/max_terminated_length": 596.0, + "completions/mean_length": 535.5, + "completions/mean_terminated_length": 535.5, + "completions/min_length": 408.0, + "completions/min_terminated_length": 408.0, + "epoch": 0.016527456904211837, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.5192902088165283, + "kl": 1.0595851242542267, + "learning_rate": 4.825396825396826e-06, + "loss": 0.0011, + "num_tokens": 277252.0, + "reward": 0.4132889211177826, + "reward_std": 0.030004048720002174, + "rewards/check_answer/mean": 0.4132889211177826, + "rewards/check_answer/std": 0.030004046857357025, + "step": 93 + }, + { + "completion_length": 585.25, + "completions/clipped_ratio": 0.0, + "completions/max_length": 723.0, + "completions/max_terminated_length": 723.0, + "completions/mean_length": 585.25, + "completions/mean_terminated_length": 585.25, + "completions/min_length": 510.0, + "completions/min_terminated_length": 510.0, + "epoch": 0.016705171494579705, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.1913037300109863, + "kl": 0.7135125249624252, + "learning_rate": 4.817460317460318e-06, + "loss": 0.0007, + "num_tokens": 280825.0, + "reward": 0.7782343626022339, + "reward_std": 0.09686165302991867, + "rewards/check_answer/mean": 0.7782343626022339, + "rewards/check_answer/std": 0.09686163812875748, + "step": 94 + }, + { + "completion_length": 390.75, + "completions/clipped_ratio": 0.0, + "completions/max_length": 437.0, + "completions/max_terminated_length": 437.0, + "completions/mean_length": 390.75, + "completions/mean_terminated_length": 390.75, + "completions/min_length": 344.0, + "completions/min_terminated_length": 344.0, + "epoch": 0.016882886084947573, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.8841480016708374, + "kl": 0.9209953248500824, + "learning_rate": 4.80952380952381e-06, + "loss": 0.0009, + "num_tokens": 283544.0, + "reward": 0.5185092687606812, + "reward_std": 0.04221953824162483, + "rewards/check_answer/mean": 0.5185092687606812, + "rewards/check_answer/std": 0.04221952706575394, + "step": 95 + }, + { + "completion_length": 441.75, + "completions/clipped_ratio": 0.0, + "completions/max_length": 476.0, + "completions/max_terminated_length": 476.0, + "completions/mean_length": 441.75, + "completions/mean_terminated_length": 441.75, + "completions/min_length": 415.0, + "completions/min_terminated_length": 415.0, + "epoch": 0.017060600675315445, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.6601427793502808, + "kl": 0.9485969841480255, + "learning_rate": 4.8015873015873025e-06, + "loss": 0.0009, + "num_tokens": 286475.0, + "reward": 0.5999540686607361, + "reward_std": 0.1883406788110733, + "rewards/check_answer/mean": 0.5999540686607361, + "rewards/check_answer/std": 0.1883406639099121, + "step": 96 + }, + { + "completion_length": 429.75, + "completions/clipped_ratio": 0.0, + "completions/max_length": 567.0, + "completions/max_terminated_length": 567.0, + "completions/mean_length": 429.75, + "completions/mean_terminated_length": 429.75, + "completions/min_length": 245.0, + "completions/min_terminated_length": 245.0, + "epoch": 0.017238315265683313, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.4426722526550293, + "kl": 1.0458677113056183, + "learning_rate": 4.793650793650794e-06, + "loss": 0.001, + "num_tokens": 289326.0, + "reward": 0.33017024397850037, + "reward_std": 0.11357025057077408, + "rewards/check_answer/mean": 0.33017024397850037, + "rewards/check_answer/std": 0.11357025802135468, + "step": 97 + }, + { + "completion_length": 481.75, + "completions/clipped_ratio": 0.0, + "completions/max_length": 531.0, + "completions/max_terminated_length": 531.0, + "completions/mean_length": 481.75, + "completions/mean_terminated_length": 481.75, + "completions/min_length": 434.0, + "completions/min_terminated_length": 434.0, + "epoch": 0.01741602985605118, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.5350953340530396, + "kl": 1.3473403006792068, + "learning_rate": 4.785714285714287e-06, + "loss": 0.0013, + "num_tokens": 292441.0, + "reward": 0.41716164350509644, + "reward_std": 0.06578143686056137, + "rewards/check_answer/mean": 0.41716164350509644, + "rewards/check_answer/std": 0.06578142195940018, + "step": 98 + }, + { + "completion_length": 493.25, + "completions/clipped_ratio": 0.0, + "completions/max_length": 555.0, + "completions/max_terminated_length": 555.0, + "completions/mean_length": 493.25, + "completions/mean_terminated_length": 493.25, + "completions/min_length": 416.0, + "completions/min_terminated_length": 416.0, + "epoch": 0.017593744446419052, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.5714409351348877, + "kl": 1.004945993423462, + "learning_rate": 4.777777777777778e-06, + "loss": 0.001, + "num_tokens": 295578.0, + "reward": 0.6212315559387207, + "reward_std": 0.050040777772665024, + "rewards/check_answer/mean": 0.6212315559387207, + "rewards/check_answer/std": 0.05004078894853592, + "step": 99 + }, + { + "completion_length": 310.75, + "completions/clipped_ratio": 0.0, + "completions/max_length": 426.0, + "completions/max_terminated_length": 426.0, + "completions/mean_length": 310.75, + "completions/mean_terminated_length": 310.75, + "completions/min_length": 232.0, + "completions/min_terminated_length": 232.0, + "epoch": 0.01777145903678692, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.5875662565231323, + "kl": 1.0317108184099197, + "learning_rate": 4.769841269841271e-06, + "loss": 0.001, + "num_tokens": 297969.0, + "reward": 0.5778508186340332, + "reward_std": 0.10223124176263809, + "rewards/check_answer/mean": 0.5778508186340332, + "rewards/check_answer/std": 0.10223124176263809, + "step": 100 + }, + { + "completion_length": 262.25, + "completions/clipped_ratio": 0.0, + "completions/max_length": 320.0, + "completions/max_terminated_length": 320.0, + "completions/mean_length": 262.25, + "completions/mean_terminated_length": 262.25, + "completions/min_length": 208.0, + "completions/min_terminated_length": 208.0, + "epoch": 0.01794917362715479, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.406073570251465, + "kl": 1.4292311072349548, + "learning_rate": 4.761904761904762e-06, + "loss": 0.0014, + "num_tokens": 300114.0, + "reward": 0.8254406452178955, + "reward_std": 0.555376410484314, + "rewards/check_answer/mean": 0.8254406452178955, + "rewards/check_answer/std": 0.555376410484314, + "step": 101 + }, + { + "completion_length": 281.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 407.0, + "completions/max_terminated_length": 407.0, + "completions/mean_length": 281.0, + "completions/mean_terminated_length": 281.0, + "completions/min_length": 218.0, + "completions/min_terminated_length": 218.0, + "epoch": 0.01812688821752266, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.7285429239273071, + "kl": 1.1764685809612274, + "learning_rate": 4.753968253968254e-06, + "loss": 0.0012, + "num_tokens": 302354.0, + "reward": 0.9168250560760498, + "reward_std": 0.23327459394931793, + "rewards/check_answer/mean": 0.9168250560760498, + "rewards/check_answer/std": 0.23327459394931793, + "step": 102 + }, + { + "completion_length": 454.25, + "completions/clipped_ratio": 0.0, + "completions/max_length": 471.0, + "completions/max_terminated_length": 471.0, + "completions/mean_length": 454.25, + "completions/mean_terminated_length": 454.25, + "completions/min_length": 434.0, + "completions/min_terminated_length": 434.0, + "epoch": 0.01830460280789053, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.7941714525222778, + "kl": 1.0800625532865524, + "learning_rate": 4.7460317460317465e-06, + "loss": 0.0011, + "num_tokens": 305319.0, + "reward": 0.644266664981842, + "reward_std": 0.14616478979587555, + "rewards/check_answer/mean": 0.644266664981842, + "rewards/check_answer/std": 0.14616480469703674, + "step": 103 + }, + { + "completion_length": 509.75, + "completions/clipped_ratio": 0.0, + "completions/max_length": 557.0, + "completions/max_terminated_length": 557.0, + "completions/mean_length": 509.75, + "completions/mean_terminated_length": 509.75, + "completions/min_length": 437.0, + "completions/min_terminated_length": 437.0, + "epoch": 0.018482317398258397, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3640716075897217, + "kl": 0.9189227521419525, + "learning_rate": 4.738095238095238e-06, + "loss": 0.0009, + "num_tokens": 308550.0, + "reward": 0.8741264939308167, + "reward_std": 0.15295472741127014, + "rewards/check_answer/mean": 0.8741264939308167, + "rewards/check_answer/std": 0.15295472741127014, + "step": 104 + }, + { + "completion_length": 343.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 467.0, + "completions/max_terminated_length": 467.0, + "completions/mean_length": 343.0, + "completions/mean_terminated_length": 343.0, + "completions/min_length": 265.0, + "completions/min_terminated_length": 265.0, + "epoch": 0.018660031988626265, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.171943187713623, + "kl": 1.0402073562145233, + "learning_rate": 4.730158730158731e-06, + "loss": 0.001, + "num_tokens": 311062.0, + "reward": 0.9081037044525146, + "reward_std": 0.18349121510982513, + "rewards/check_answer/mean": 0.9081037044525146, + "rewards/check_answer/std": 0.18349123001098633, + "step": 105 + }, + { + "completion_length": 411.25, + "completions/clipped_ratio": 0.0, + "completions/max_length": 511.0, + "completions/max_terminated_length": 511.0, + "completions/mean_length": 411.25, + "completions/mean_terminated_length": 411.25, + "completions/min_length": 228.0, + "completions/min_terminated_length": 228.0, + "epoch": 0.018837746578994136, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.5505229234695435, + "kl": 0.8422607332468033, + "learning_rate": 4.722222222222222e-06, + "loss": 0.0008, + "num_tokens": 313911.0, + "reward": 0.8211601972579956, + "reward_std": 0.10855837911367416, + "rewards/check_answer/mean": 0.8211601972579956, + "rewards/check_answer/std": 0.10855837911367416, + "step": 106 + }, + { + "completion_length": 345.75, + "completions/clipped_ratio": 0.0, + "completions/max_length": 484.0, + "completions/max_terminated_length": 484.0, + "completions/mean_length": 345.75, + "completions/mean_terminated_length": 345.75, + "completions/min_length": 278.0, + "completions/min_terminated_length": 278.0, + "epoch": 0.019015461169362004, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.7689751386642456, + "kl": 1.3255447894334793, + "learning_rate": 4.714285714285715e-06, + "loss": 0.0013, + "num_tokens": 316422.0, + "reward": 0.3924521505832672, + "reward_std": 0.11489391326904297, + "rewards/check_answer/mean": 0.3924521505832672, + "rewards/check_answer/std": 0.11489391326904297, + "step": 107 + }, + { + "completion_length": 577.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 633.0, + "completions/max_terminated_length": 633.0, + "completions/mean_length": 577.0, + "completions/mean_terminated_length": 577.0, + "completions/min_length": 520.0, + "completions/min_terminated_length": 520.0, + "epoch": 0.019193175759729873, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.4045451879501343, + "kl": 0.948733925819397, + "learning_rate": 4.706349206349206e-06, + "loss": 0.0009, + "num_tokens": 319942.0, + "reward": 0.7583677768707275, + "reward_std": 0.09759147465229034, + "rewards/check_answer/mean": 0.7583677768707275, + "rewards/check_answer/std": 0.09759148210287094, + "step": 108 + }, + { + "completion_length": 488.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 522.0, + "completions/max_terminated_length": 522.0, + "completions/mean_length": 488.0, + "completions/mean_terminated_length": 488.0, + "completions/min_length": 408.0, + "completions/min_terminated_length": 408.0, + "epoch": 0.019370890350097744, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.5084129571914673, + "kl": 0.9847242683172226, + "learning_rate": 4.698412698412699e-06, + "loss": 0.001, + "num_tokens": 323078.0, + "reward": 0.5234397649765015, + "reward_std": 0.09028327465057373, + "rewards/check_answer/mean": 0.5234397649765015, + "rewards/check_answer/std": 0.09028327465057373, + "step": 109 + }, + { + "completion_length": 471.75, + "completions/clipped_ratio": 0.0, + "completions/max_length": 501.0, + "completions/max_terminated_length": 501.0, + "completions/mean_length": 471.75, + "completions/mean_terminated_length": 471.75, + "completions/min_length": 422.0, + "completions/min_terminated_length": 422.0, + "epoch": 0.019548604940465612, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.8055229187011719, + "kl": 1.0391763895750046, + "learning_rate": 4.6904761904761905e-06, + "loss": 0.001, + "num_tokens": 326121.0, + "reward": 0.2687339186668396, + "reward_std": 0.06397005170583725, + "rewards/check_answer/mean": 0.2687339186668396, + "rewards/check_answer/std": 0.06397005915641785, + "step": 110 + }, + { + "completion_length": 353.5, + "completions/clipped_ratio": 0.0, + "completions/max_length": 481.0, + "completions/max_terminated_length": 481.0, + "completions/mean_length": 353.5, + "completions/mean_terminated_length": 353.5, + "completions/min_length": 287.0, + "completions/min_terminated_length": 287.0, + "epoch": 0.01972631953083348, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.7900550365447998, + "kl": 1.1994563341140747, + "learning_rate": 4.682539682539683e-06, + "loss": 0.0012, + "num_tokens": 328703.0, + "reward": 0.7745100259780884, + "reward_std": 0.07118824124336243, + "rewards/check_answer/mean": 0.7745100259780884, + "rewards/check_answer/std": 0.07118824124336243, + "step": 111 + }, + { + "completion_length": 314.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 395.0, + "completions/max_terminated_length": 395.0, + "completions/mean_length": 314.0, + "completions/mean_terminated_length": 314.0, + "completions/min_length": 221.0, + "completions/min_terminated_length": 221.0, + "epoch": 0.019904034121201352, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.7322038412094116, + "kl": 1.1014239341020584, + "learning_rate": 4.674603174603175e-06, + "loss": 0.0011, + "num_tokens": 331067.0, + "reward": 0.712478518486023, + "reward_std": 0.2585718333721161, + "rewards/check_answer/mean": 0.712478518486023, + "rewards/check_answer/std": 0.2585718333721161, + "step": 112 + }, + { + "completion_length": 440.75, + "completions/clipped_ratio": 0.0, + "completions/max_length": 488.0, + "completions/max_terminated_length": 488.0, + "completions/mean_length": 440.75, + "completions/mean_terminated_length": 440.75, + "completions/min_length": 353.0, + "completions/min_terminated_length": 353.0, + "epoch": 0.02008174871156922, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.374124526977539, + "kl": 1.009152129292488, + "learning_rate": 4.666666666666667e-06, + "loss": 0.001, + "num_tokens": 334014.0, + "reward": 0.6918644905090332, + "reward_std": 0.28659263253211975, + "rewards/check_answer/mean": 0.6918644905090332, + "rewards/check_answer/std": 0.28659263253211975, + "step": 113 + }, + { + "completion_length": 325.25, + "completions/clipped_ratio": 0.0, + "completions/max_length": 361.0, + "completions/max_terminated_length": 361.0, + "completions/mean_length": 325.25, + "completions/mean_terminated_length": 325.25, + "completions/min_length": 298.0, + "completions/min_terminated_length": 298.0, + "epoch": 0.02025946330193709, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.688226580619812, + "kl": 0.8765446245670319, + "learning_rate": 4.658730158730159e-06, + "loss": 0.0009, + "num_tokens": 336467.0, + "reward": 0.7640917301177979, + "reward_std": 0.07848616689443588, + "rewards/check_answer/mean": 0.7640917301177979, + "rewards/check_answer/std": 0.07848618924617767, + "step": 114 + }, + { + "completion_length": 225.75, + "completions/clipped_ratio": 0.0, + "completions/max_length": 252.0, + "completions/max_terminated_length": 252.0, + "completions/mean_length": 225.75, + "completions/mean_terminated_length": 225.75, + "completions/min_length": 211.0, + "completions/min_terminated_length": 211.0, + "epoch": 0.02043717789230496, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.8820006847381592, + "kl": 1.1795692443847656, + "learning_rate": 4.650793650793651e-06, + "loss": 0.0012, + "num_tokens": 338490.0, + "reward": 1.0758620500564575, + "reward_std": 0.014311790466308594, + "rewards/check_answer/mean": 1.0758620500564575, + "rewards/check_answer/std": 0.014311791397631168, + "step": 115 + }, + { + "completion_length": 326.5, + "completions/clipped_ratio": 0.0, + "completions/max_length": 360.0, + "completions/max_terminated_length": 360.0, + "completions/mean_length": 326.5, + "completions/mean_terminated_length": 326.5, + "completions/min_length": 247.0, + "completions/min_terminated_length": 247.0, + "epoch": 0.020614892482672828, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.018541097640991, + "kl": 1.2944844365119934, + "learning_rate": 4.642857142857144e-06, + "loss": 0.0013, + "num_tokens": 340904.0, + "reward": 0.8421782851219177, + "reward_std": 0.5801041722297668, + "rewards/check_answer/mean": 0.8421782851219177, + "rewards/check_answer/std": 0.5801041722297668, + "step": 116 + }, + { + "completion_length": 850.75, + "completions/clipped_ratio": 0.25, + "completions/max_length": 1733.0, + "completions/max_terminated_length": 584.0, + "completions/mean_length": 850.75, + "completions/mean_terminated_length": 556.6666870117188, + "completions/min_length": 539.0, + "completions/min_terminated_length": 539.0, + "epoch": 0.020792607073040696, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0379717350006104, + "kl": 1.2277846336364746, + "learning_rate": 4.634920634920635e-06, + "loss": 0.0012, + "num_tokens": 345531.0, + "reward": 0.8824595808982849, + "reward_std": 0.3745229244232178, + "rewards/check_answer/mean": 0.8824595808982849, + "rewards/check_answer/std": 0.3745229244232178, + "step": 117 + }, + { + "completion_length": 429.75, + "completions/clipped_ratio": 0.0, + "completions/max_length": 585.0, + "completions/max_terminated_length": 585.0, + "completions/mean_length": 429.75, + "completions/mean_terminated_length": 429.75, + "completions/min_length": 282.0, + "completions/min_terminated_length": 282.0, + "epoch": 0.020970321663408564, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.5808711051940918, + "kl": 1.2089886367321014, + "learning_rate": 4.626984126984128e-06, + "loss": 0.0012, + "num_tokens": 348426.0, + "reward": 0.49039697647094727, + "reward_std": 0.13966494798660278, + "rewards/check_answer/mean": 0.49039697647094727, + "rewards/check_answer/std": 0.1396649330854416, + "step": 118 + }, + { + "completion_length": 332.5, + "completions/clipped_ratio": 0.0, + "completions/max_length": 408.0, + "completions/max_terminated_length": 408.0, + "completions/mean_length": 332.5, + "completions/mean_terminated_length": 332.5, + "completions/min_length": 267.0, + "completions/min_terminated_length": 267.0, + "epoch": 0.021148036253776436, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.896164894104004, + "kl": 1.2581031918525696, + "learning_rate": 4.6190476190476196e-06, + "loss": 0.0013, + "num_tokens": 350868.0, + "reward": 0.5684536695480347, + "reward_std": 0.17921964824199677, + "rewards/check_answer/mean": 0.5684536695480347, + "rewards/check_answer/std": 0.17921964824199677, + "step": 119 + }, + { + "completion_length": 419.75, + "completions/clipped_ratio": 0.0, + "completions/max_length": 510.0, + "completions/max_terminated_length": 510.0, + "completions/mean_length": 419.75, + "completions/mean_terminated_length": 419.75, + "completions/min_length": 318.0, + "completions/min_terminated_length": 318.0, + "epoch": 0.021325750844144304, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3449409008026123, + "kl": 0.9334018975496292, + "learning_rate": 4.611111111111112e-06, + "loss": 0.0009, + "num_tokens": 353731.0, + "reward": 0.9145520329475403, + "reward_std": 0.16757537424564362, + "rewards/check_answer/mean": 0.9145520329475403, + "rewards/check_answer/std": 0.16757537424564362, + "step": 120 + }, + { + "completion_length": 644.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 843.0, + "completions/max_terminated_length": 843.0, + "completions/mean_length": 644.0, + "completions/mean_terminated_length": 644.0, + "completions/min_length": 508.0, + "completions/min_terminated_length": 508.0, + "epoch": 0.021503465434512172, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.2333385944366455, + "kl": 1.0253196954727173, + "learning_rate": 4.603174603174604e-06, + "loss": 0.001, + "num_tokens": 357515.0, + "reward": 0.5978646874427795, + "reward_std": 0.23829804360866547, + "rewards/check_answer/mean": 0.5978646874427795, + "rewards/check_answer/std": 0.23829805850982666, + "step": 121 + }, + { + "completion_length": 589.25, + "completions/clipped_ratio": 0.0, + "completions/max_length": 744.0, + "completions/max_terminated_length": 744.0, + "completions/mean_length": 589.25, + "completions/mean_terminated_length": 589.25, + "completions/min_length": 368.0, + "completions/min_terminated_length": 368.0, + "epoch": 0.021681180024880044, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.2892918586730957, + "kl": 0.852889820933342, + "learning_rate": 4.595238095238095e-06, + "loss": 0.0009, + "num_tokens": 361076.0, + "reward": 0.32557737827301025, + "reward_std": 0.07417508214712143, + "rewards/check_answer/mean": 0.32557737827301025, + "rewards/check_answer/std": 0.07417508214712143, + "step": 122 + }, + { + "completion_length": 238.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 379.0, + "completions/max_terminated_length": 379.0, + "completions/mean_length": 238.0, + "completions/mean_terminated_length": 238.0, + "completions/min_length": 166.0, + "completions/min_terminated_length": 166.0, + "epoch": 0.021858894615247912, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.9720977544784546, + "kl": 1.2939183413982391, + "learning_rate": 4.587301587301588e-06, + "loss": 0.0013, + "num_tokens": 363136.0, + "reward": 1.2107616662979126, + "reward_std": 0.3746259808540344, + "rewards/check_answer/mean": 1.2107616662979126, + "rewards/check_answer/std": 0.3746259808540344, + "step": 123 + }, + { + "completion_length": 520.5, + "completions/clipped_ratio": 0.0, + "completions/max_length": 610.0, + "completions/max_terminated_length": 610.0, + "completions/mean_length": 520.5, + "completions/mean_terminated_length": 520.5, + "completions/min_length": 442.0, + "completions/min_terminated_length": 442.0, + "epoch": 0.02203660920561578, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.2540651559829712, + "kl": 0.7220829427242279, + "learning_rate": 4.5793650793650795e-06, + "loss": 0.0007, + "num_tokens": 366438.0, + "reward": 0.4332646131515503, + "reward_std": 0.06124412640929222, + "rewards/check_answer/mean": 0.4332646131515503, + "rewards/check_answer/std": 0.06124412640929222, + "step": 124 + }, + { + "completion_length": 424.25, + "completions/clipped_ratio": 0.0, + "completions/max_length": 611.0, + "completions/max_terminated_length": 611.0, + "completions/mean_length": 424.25, + "completions/mean_terminated_length": 424.25, + "completions/min_length": 299.0, + "completions/min_terminated_length": 299.0, + "epoch": 0.02221432379598365, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.7561167478561401, + "kl": 0.9426107108592987, + "learning_rate": 4.571428571428572e-06, + "loss": 0.0009, + "num_tokens": 369263.0, + "reward": 0.47949886322021484, + "reward_std": 0.014681631699204445, + "rewards/check_answer/mean": 0.47949886322021484, + "rewards/check_answer/std": 0.014681617729365826, + "step": 125 + }, + { + "completion_length": 447.25, + "completions/clipped_ratio": 0.0, + "completions/max_length": 513.0, + "completions/max_terminated_length": 513.0, + "completions/mean_length": 447.25, + "completions/mean_terminated_length": 447.25, + "completions/min_length": 354.0, + "completions/min_terminated_length": 354.0, + "epoch": 0.02239203838635152, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.5910497903823853, + "kl": 0.9867564290761948, + "learning_rate": 4.563492063492064e-06, + "loss": 0.001, + "num_tokens": 372228.0, + "reward": 0.41330093145370483, + "reward_std": 0.061453063040971756, + "rewards/check_answer/mean": 0.41330093145370483, + "rewards/check_answer/std": 0.061453066766262054, + "step": 126 + }, + { + "completion_length": 385.5, + "completions/clipped_ratio": 0.0, + "completions/max_length": 450.0, + "completions/max_terminated_length": 450.0, + "completions/mean_length": 385.5, + "completions/mean_terminated_length": 385.5, + "completions/min_length": 241.0, + "completions/min_terminated_length": 241.0, + "epoch": 0.022569752976719388, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.7781590223312378, + "kl": 1.2027154564857483, + "learning_rate": 4.555555555555556e-06, + "loss": 0.0012, + "num_tokens": 374906.0, + "reward": 1.0481634140014648, + "reward_std": 0.38712385296821594, + "rewards/check_answer/mean": 1.0481634140014648, + "rewards/check_answer/std": 0.38712385296821594, + "step": 127 + }, + { + "completion_length": 479.75, + "completions/clipped_ratio": 0.0, + "completions/max_length": 570.0, + "completions/max_terminated_length": 570.0, + "completions/mean_length": 479.75, + "completions/mean_terminated_length": 479.75, + "completions/min_length": 417.0, + "completions/min_terminated_length": 417.0, + "epoch": 0.02274746756708726, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.4399559497833252, + "kl": 1.1581873446702957, + "learning_rate": 4.547619047619048e-06, + "loss": 0.0012, + "num_tokens": 378005.0, + "reward": 0.5218023061752319, + "reward_std": 0.10244939476251602, + "rewards/check_answer/mean": 0.5218023061752319, + "rewards/check_answer/std": 0.10244939476251602, + "step": 128 + }, + { + "completion_length": 314.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 376.0, + "completions/max_terminated_length": 376.0, + "completions/mean_length": 314.0, + "completions/mean_terminated_length": 314.0, + "completions/min_length": 214.0, + "completions/min_terminated_length": 214.0, + "epoch": 0.022925182157455128, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.065781831741333, + "kl": 1.0792067795991898, + "learning_rate": 4.53968253968254e-06, + "loss": 0.0011, + "num_tokens": 380365.0, + "reward": 0.14184531569480896, + "reward_std": 0.11691629886627197, + "rewards/check_answer/mean": 0.14184531569480896, + "rewards/check_answer/std": 0.11691630631685257, + "step": 129 + }, + { + "completion_length": 394.5, + "completions/clipped_ratio": 0.0, + "completions/max_length": 459.0, + "completions/max_terminated_length": 459.0, + "completions/mean_length": 394.5, + "completions/mean_terminated_length": 394.5, + "completions/min_length": 305.0, + "completions/min_terminated_length": 305.0, + "epoch": 0.023102896747822996, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.7899744510650635, + "kl": 1.202794149518013, + "learning_rate": 4.531746031746032e-06, + "loss": 0.0012, + "num_tokens": 383103.0, + "reward": 0.9240409731864929, + "reward_std": 0.19024822115898132, + "rewards/check_answer/mean": 0.9240409731864929, + "rewards/check_answer/std": 0.19024820625782013, + "step": 130 + }, + { + "completion_length": 525.75, + "completions/clipped_ratio": 0.0, + "completions/max_length": 636.0, + "completions/max_terminated_length": 636.0, + "completions/mean_length": 525.75, + "completions/mean_terminated_length": 525.75, + "completions/min_length": 425.0, + "completions/min_terminated_length": 425.0, + "epoch": 0.023280611338190864, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.5444499254226685, + "kl": 1.0851371437311172, + "learning_rate": 4.523809523809524e-06, + "loss": 0.0011, + "num_tokens": 386370.0, + "reward": 0.5510776042938232, + "reward_std": 0.14302389323711395, + "rewards/check_answer/mean": 0.5510776042938232, + "rewards/check_answer/std": 0.14302392303943634, + "step": 131 + }, + { + "completion_length": 391.5, + "completions/clipped_ratio": 0.0, + "completions/max_length": 467.0, + "completions/max_terminated_length": 467.0, + "completions/mean_length": 391.5, + "completions/mean_terminated_length": 391.5, + "completions/min_length": 324.0, + "completions/min_terminated_length": 324.0, + "epoch": 0.023458325928558735, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.977134346961975, + "kl": 1.2055960595607758, + "learning_rate": 4.515873015873016e-06, + "loss": 0.0012, + "num_tokens": 389068.0, + "reward": 0.630324125289917, + "reward_std": 0.14643849432468414, + "rewards/check_answer/mean": 0.630324125289917, + "rewards/check_answer/std": 0.14643850922584534, + "step": 132 + }, + { + "completion_length": 365.75, + "completions/clipped_ratio": 0.0, + "completions/max_length": 426.0, + "completions/max_terminated_length": 426.0, + "completions/mean_length": 365.75, + "completions/mean_terminated_length": 365.75, + "completions/min_length": 282.0, + "completions/min_terminated_length": 282.0, + "epoch": 0.023636040518926604, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.7458113431930542, + "kl": 1.1333343386650085, + "learning_rate": 4.5079365079365085e-06, + "loss": 0.0011, + "num_tokens": 391643.0, + "reward": 0.3864520192146301, + "reward_std": 0.10917352885007858, + "rewards/check_answer/mean": 0.3864520192146301, + "rewards/check_answer/std": 0.10917352885007858, + "step": 133 + }, + { + "completion_length": 313.75, + "completions/clipped_ratio": 0.0, + "completions/max_length": 367.0, + "completions/max_terminated_length": 367.0, + "completions/mean_length": 313.75, + "completions/mean_terminated_length": 313.75, + "completions/min_length": 215.0, + "completions/min_terminated_length": 215.0, + "epoch": 0.02381375510929447, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.8480489253997803, + "kl": 1.1135710626840591, + "learning_rate": 4.5e-06, + "loss": 0.0011, + "num_tokens": 394014.0, + "reward": 0.4650374948978424, + "reward_std": 0.16275101900100708, + "rewards/check_answer/mean": 0.4650374948978424, + "rewards/check_answer/std": 0.16275101900100708, + "step": 134 + }, + { + "completion_length": 202.5, + "completions/clipped_ratio": 0.0, + "completions/max_length": 253.0, + "completions/max_terminated_length": 253.0, + "completions/mean_length": 202.5, + "completions/mean_terminated_length": 202.5, + "completions/min_length": 172.0, + "completions/min_terminated_length": 172.0, + "epoch": 0.023991469699662343, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.3134193420410156, + "kl": 1.4941602051258087, + "learning_rate": 4.492063492063493e-06, + "loss": 0.0015, + "num_tokens": 395924.0, + "reward": 0.2674972414970398, + "reward_std": 0.05775442719459534, + "rewards/check_answer/mean": 0.2674972414970398, + "rewards/check_answer/std": 0.05775442719459534, + "step": 135 + }, + { + "completion_length": 546.5, + "completions/clipped_ratio": 0.0, + "completions/max_length": 603.0, + "completions/max_terminated_length": 603.0, + "completions/mean_length": 546.5, + "completions/mean_terminated_length": 546.5, + "completions/min_length": 477.0, + "completions/min_terminated_length": 477.0, + "epoch": 0.02416918429003021, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.2359838485717773, + "kl": 0.8067422658205032, + "learning_rate": 4.484126984126984e-06, + "loss": 0.0008, + "num_tokens": 399342.0, + "reward": 0.5291463136672974, + "reward_std": 0.05505036190152168, + "rewards/check_answer/mean": 0.5291463136672974, + "rewards/check_answer/std": 0.05505036562681198, + "step": 136 + }, + { + "completion_length": 515.5, + "completions/clipped_ratio": 0.0, + "completions/max_length": 612.0, + "completions/max_terminated_length": 612.0, + "completions/mean_length": 515.5, + "completions/mean_terminated_length": 515.5, + "completions/min_length": 431.0, + "completions/min_terminated_length": 431.0, + "epoch": 0.02434689888039808, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.452023983001709, + "kl": 1.1316813081502914, + "learning_rate": 4.476190476190477e-06, + "loss": 0.0011, + "num_tokens": 402580.0, + "reward": 0.7063450813293457, + "reward_std": 0.05188734456896782, + "rewards/check_answer/mean": 0.7063450813293457, + "rewards/check_answer/std": 0.05188734456896782, + "step": 137 + }, + { + "completion_length": 164.25, + "completions/clipped_ratio": 0.0, + "completions/max_length": 180.0, + "completions/max_terminated_length": 180.0, + "completions/mean_length": 164.25, + "completions/mean_terminated_length": 164.25, + "completions/min_length": 148.0, + "completions/min_terminated_length": 148.0, + "epoch": 0.02452461347076595, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.583338975906372, + "kl": 1.4289655685424805, + "learning_rate": 4.468253968253969e-06, + "loss": 0.0014, + "num_tokens": 404341.0, + "reward": 0.7204106450080872, + "reward_std": 0.08577322959899902, + "rewards/check_answer/mean": 0.7204106450080872, + "rewards/check_answer/std": 0.08577322959899902, + "step": 138 + }, + { + "completion_length": 324.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 381.0, + "completions/max_terminated_length": 381.0, + "completions/mean_length": 324.0, + "completions/mean_terminated_length": 324.0, + "completions/min_length": 295.0, + "completions/min_terminated_length": 295.0, + "epoch": 0.02470232806113382, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.7330161333084106, + "kl": 0.9842696934938431, + "learning_rate": 4.460317460317461e-06, + "loss": 0.001, + "num_tokens": 406773.0, + "reward": 0.48003125190734863, + "reward_std": 0.07409851998090744, + "rewards/check_answer/mean": 0.48003125190734863, + "rewards/check_answer/std": 0.07409852743148804, + "step": 139 + }, + { + "completion_length": 252.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 284.0, + "completions/max_terminated_length": 284.0, + "completions/mean_length": 252.0, + "completions/mean_terminated_length": 252.0, + "completions/min_length": 207.0, + "completions/min_terminated_length": 207.0, + "epoch": 0.024880042651501687, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.1670877933502197, + "kl": 1.2738738059997559, + "learning_rate": 4.4523809523809525e-06, + "loss": 0.0013, + "num_tokens": 408881.0, + "reward": 0.5589589476585388, + "reward_std": 0.3076104521751404, + "rewards/check_answer/mean": 0.5589589476585388, + "rewards/check_answer/std": 0.30761048197746277, + "step": 140 + }, + { + "completion_length": 485.75, + "completions/clipped_ratio": 0.0, + "completions/max_length": 536.0, + "completions/max_terminated_length": 536.0, + "completions/mean_length": 485.75, + "completions/mean_terminated_length": 485.75, + "completions/min_length": 424.0, + "completions/min_terminated_length": 424.0, + "epoch": 0.02505775724186956, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.4860774278640747, + "kl": 1.0862714499235153, + "learning_rate": 4.444444444444444e-06, + "loss": 0.0011, + "num_tokens": 412028.0, + "reward": 1.1310088634490967, + "reward_std": 0.0607450008392334, + "rewards/check_answer/mean": 1.1310088634490967, + "rewards/check_answer/std": 0.0607449971139431, + "step": 141 + }, + { + "completion_length": 357.5, + "completions/clipped_ratio": 0.0, + "completions/max_length": 442.0, + "completions/max_terminated_length": 442.0, + "completions/mean_length": 357.5, + "completions/mean_terminated_length": 357.5, + "completions/min_length": 279.0, + "completions/min_terminated_length": 279.0, + "epoch": 0.025235471832237427, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.857495665550232, + "kl": 0.9955440014600754, + "learning_rate": 4.436507936507937e-06, + "loss": 0.001, + "num_tokens": 414582.0, + "reward": 0.371147096157074, + "reward_std": 0.03809575363993645, + "rewards/check_answer/mean": 0.371147096157074, + "rewards/check_answer/std": 0.03809575363993645, + "step": 142 + }, + { + "completion_length": 512.5, + "completions/clipped_ratio": 0.0, + "completions/max_length": 556.0, + "completions/max_terminated_length": 556.0, + "completions/mean_length": 512.5, + "completions/mean_terminated_length": 512.5, + "completions/min_length": 478.0, + "completions/min_terminated_length": 478.0, + "epoch": 0.025413186422605295, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3737552165985107, + "kl": 0.8718635737895966, + "learning_rate": 4.428571428571429e-06, + "loss": 0.0009, + "num_tokens": 417828.0, + "reward": 0.7945088148117065, + "reward_std": 0.10661814361810684, + "rewards/check_answer/mean": 0.7945088148117065, + "rewards/check_answer/std": 0.10661814361810684, + "step": 143 + }, + { + "completion_length": 400.5, + "completions/clipped_ratio": 0.0, + "completions/max_length": 514.0, + "completions/max_terminated_length": 514.0, + "completions/mean_length": 400.5, + "completions/mean_terminated_length": 400.5, + "completions/min_length": 279.0, + "completions/min_terminated_length": 279.0, + "epoch": 0.025590901012973163, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.7630726099014282, + "kl": 1.0710228085517883, + "learning_rate": 4.420634920634921e-06, + "loss": 0.0011, + "num_tokens": 420578.0, + "reward": 0.47816359996795654, + "reward_std": 0.0742780864238739, + "rewards/check_answer/mean": 0.47816359996795654, + "rewards/check_answer/std": 0.0742780864238739, + "step": 144 + }, + { + "completion_length": 344.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 401.0, + "completions/max_terminated_length": 401.0, + "completions/mean_length": 344.0, + "completions/mean_terminated_length": 344.0, + "completions/min_length": 289.0, + "completions/min_terminated_length": 289.0, + "epoch": 0.025768615603341035, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.808454155921936, + "kl": 1.1346302181482315, + "learning_rate": 4.412698412698413e-06, + "loss": 0.0011, + "num_tokens": 423110.0, + "reward": 1.2456703186035156, + "reward_std": 0.18238836526870728, + "rewards/check_answer/mean": 1.2456703186035156, + "rewards/check_answer/std": 0.18238838016986847, + "step": 145 + }, + { + "completion_length": 563.25, + "completions/clipped_ratio": 0.0, + "completions/max_length": 614.0, + "completions/max_terminated_length": 614.0, + "completions/mean_length": 563.25, + "completions/mean_terminated_length": 563.25, + "completions/min_length": 447.0, + "completions/min_terminated_length": 447.0, + "epoch": 0.025946330193708903, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.4395103454589844, + "kl": 0.9940735101699829, + "learning_rate": 4.404761904761905e-06, + "loss": 0.001, + "num_tokens": 426535.0, + "reward": 0.4605726897716522, + "reward_std": 0.018929272890090942, + "rewards/check_answer/mean": 0.4605726897716522, + "rewards/check_answer/std": 0.018929272890090942, + "step": 146 + }, + { + "completion_length": 404.75, + "completions/clipped_ratio": 0.0, + "completions/max_length": 463.0, + "completions/max_terminated_length": 463.0, + "completions/mean_length": 404.75, + "completions/mean_terminated_length": 404.75, + "completions/min_length": 320.0, + "completions/min_terminated_length": 320.0, + "epoch": 0.02612404478407677, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.565238118171692, + "kl": 0.8450873494148254, + "learning_rate": 4.396825396825397e-06, + "loss": 0.0008, + "num_tokens": 429318.0, + "reward": 0.655875027179718, + "reward_std": 0.012029829435050488, + "rewards/check_answer/mean": 0.655875027179718, + "rewards/check_answer/std": 0.012029829435050488, + "step": 147 + }, + { + "completion_length": 401.25, + "completions/clipped_ratio": 0.0, + "completions/max_length": 442.0, + "completions/max_terminated_length": 442.0, + "completions/mean_length": 401.25, + "completions/mean_terminated_length": 401.25, + "completions/min_length": 358.0, + "completions/min_terminated_length": 358.0, + "epoch": 0.026301759374444643, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.7787821292877197, + "kl": 1.0799409300088882, + "learning_rate": 4.388888888888889e-06, + "loss": 0.0011, + "num_tokens": 432043.0, + "reward": 0.3119197487831116, + "reward_std": 0.10698849707841873, + "rewards/check_answer/mean": 0.3119197487831116, + "rewards/check_answer/std": 0.10698850452899933, + "step": 148 + }, + { + "completion_length": 631.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 713.0, + "completions/max_terminated_length": 713.0, + "completions/mean_length": 631.0, + "completions/mean_terminated_length": 631.0, + "completions/min_length": 548.0, + "completions/min_terminated_length": 548.0, + "epoch": 0.02647947396481251, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.177681565284729, + "kl": 1.2260465174913406, + "learning_rate": 4.3809523809523815e-06, + "loss": 0.0012, + "num_tokens": 435819.0, + "reward": 0.6100305318832397, + "reward_std": 0.04862495884299278, + "rewards/check_answer/mean": 0.6100305318832397, + "rewards/check_answer/std": 0.048624955117702484, + "step": 149 + }, + { + "completion_length": 546.5, + "completions/clipped_ratio": 0.0, + "completions/max_length": 620.0, + "completions/max_terminated_length": 620.0, + "completions/mean_length": 546.5, + "completions/mean_terminated_length": 546.5, + "completions/min_length": 404.0, + "completions/min_terminated_length": 404.0, + "epoch": 0.02665718855518038, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.6101106405258179, + "kl": 0.8679392486810684, + "learning_rate": 4.373015873015873e-06, + "loss": 0.0009, + "num_tokens": 439197.0, + "reward": 0.461988627910614, + "reward_std": 0.08495597541332245, + "rewards/check_answer/mean": 0.461988627910614, + "rewards/check_answer/std": 0.08495597541332245, + "step": 150 + }, + { + "completion_length": 471.5, + "completions/clipped_ratio": 0.0, + "completions/max_length": 567.0, + "completions/max_terminated_length": 567.0, + "completions/mean_length": 471.5, + "completions/mean_terminated_length": 471.5, + "completions/min_length": 432.0, + "completions/min_terminated_length": 432.0, + "epoch": 0.02683490314554825, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.6391887664794922, + "kl": 0.7990295141935349, + "learning_rate": 4.365079365079366e-06, + "loss": 0.0008, + "num_tokens": 442279.0, + "reward": 0.4876329302787781, + "reward_std": 0.05170879140496254, + "rewards/check_answer/mean": 0.4876329302787781, + "rewards/check_answer/std": 0.05170879140496254, + "step": 151 + }, + { + "completion_length": 340.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 390.0, + "completions/max_terminated_length": 390.0, + "completions/mean_length": 340.0, + "completions/mean_terminated_length": 340.0, + "completions/min_length": 277.0, + "completions/min_terminated_length": 277.0, + "epoch": 0.02701261773591612, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.9967378377914429, + "kl": 1.0871484875679016, + "learning_rate": 4.357142857142857e-06, + "loss": 0.0011, + "num_tokens": 444783.0, + "reward": 0.7888412475585938, + "reward_std": 0.09264621883630753, + "rewards/check_answer/mean": 0.7888412475585938, + "rewards/check_answer/std": 0.09264620393514633, + "step": 152 + }, + { + "completion_length": 443.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 535.0, + "completions/max_terminated_length": 535.0, + "completions/mean_length": 443.0, + "completions/mean_terminated_length": 443.0, + "completions/min_length": 377.0, + "completions/min_terminated_length": 377.0, + "epoch": 0.027190332326283987, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.5320669412612915, + "kl": 1.216535747051239, + "learning_rate": 4.34920634920635e-06, + "loss": 0.0012, + "num_tokens": 447747.0, + "reward": 0.4390728175640106, + "reward_std": 0.09861871600151062, + "rewards/check_answer/mean": 0.4390728175640106, + "rewards/check_answer/std": 0.09861870110034943, + "step": 153 + }, + { + "completion_length": 496.5, + "completions/clipped_ratio": 0.0, + "completions/max_length": 552.0, + "completions/max_terminated_length": 552.0, + "completions/mean_length": 496.5, + "completions/mean_terminated_length": 496.5, + "completions/min_length": 459.0, + "completions/min_terminated_length": 459.0, + "epoch": 0.02736804691665186, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.5672584772109985, + "kl": 0.9466253966093063, + "learning_rate": 4.3412698412698414e-06, + "loss": 0.0009, + "num_tokens": 450913.0, + "reward": 0.4815061390399933, + "reward_std": 0.04056670889258385, + "rewards/check_answer/mean": 0.4815061390399933, + "rewards/check_answer/std": 0.04056670516729355, + "step": 154 + }, + { + "completion_length": 432.25, + "completions/clipped_ratio": 0.0, + "completions/max_length": 448.0, + "completions/max_terminated_length": 448.0, + "completions/mean_length": 432.25, + "completions/mean_terminated_length": 432.25, + "completions/min_length": 400.0, + "completions/min_terminated_length": 400.0, + "epoch": 0.027545761507019727, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.9660674333572388, + "kl": 0.9921428710222244, + "learning_rate": 4.333333333333334e-06, + "loss": 0.001, + "num_tokens": 453814.0, + "reward": 0.32661956548690796, + "reward_std": 0.08909963071346283, + "rewards/check_answer/mean": 0.32661956548690796, + "rewards/check_answer/std": 0.08909963071346283, + "step": 155 + }, + { + "completion_length": 394.25, + "completions/clipped_ratio": 0.0, + "completions/max_length": 474.0, + "completions/max_terminated_length": 474.0, + "completions/mean_length": 394.25, + "completions/mean_terminated_length": 394.25, + "completions/min_length": 334.0, + "completions/min_terminated_length": 334.0, + "epoch": 0.027723476097387595, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.7710676193237305, + "kl": 1.1431185454130173, + "learning_rate": 4.3253968253968256e-06, + "loss": 0.0011, + "num_tokens": 456531.0, + "reward": 0.5441449880599976, + "reward_std": 0.27911967039108276, + "rewards/check_answer/mean": 0.5441449880599976, + "rewards/check_answer/std": 0.27911967039108276, + "step": 156 + }, + { + "completion_length": 351.75, + "completions/clipped_ratio": 0.0, + "completions/max_length": 457.0, + "completions/max_terminated_length": 457.0, + "completions/mean_length": 351.75, + "completions/mean_terminated_length": 351.75, + "completions/min_length": 224.0, + "completions/min_terminated_length": 224.0, + "epoch": 0.027901190687755463, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.7665750980377197, + "kl": 0.970624715089798, + "learning_rate": 4.317460317460318e-06, + "loss": 0.001, + "num_tokens": 459094.0, + "reward": 0.7966454029083252, + "reward_std": 0.2309737652540207, + "rewards/check_answer/mean": 0.7966454029083252, + "rewards/check_answer/std": 0.2309737652540207, + "step": 157 + }, + { + "completion_length": 237.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 340.0, + "completions/max_terminated_length": 340.0, + "completions/mean_length": 237.0, + "completions/mean_terminated_length": 237.0, + "completions/min_length": 183.0, + "completions/min_terminated_length": 183.0, + "epoch": 0.028078905278123335, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.2502260208129883, + "kl": 1.3823933005332947, + "learning_rate": 4.30952380952381e-06, + "loss": 0.0014, + "num_tokens": 461150.0, + "reward": 1.2577112913131714, + "reward_std": 0.3310515582561493, + "rewards/check_answer/mean": 1.2577112913131714, + "rewards/check_answer/std": 0.3310515582561493, + "step": 158 + }, + { + "completion_length": 435.25, + "completions/clipped_ratio": 0.0, + "completions/max_length": 502.0, + "completions/max_terminated_length": 502.0, + "completions/mean_length": 435.25, + "completions/mean_terminated_length": 435.25, + "completions/min_length": 334.0, + "completions/min_terminated_length": 334.0, + "epoch": 0.028256619868491203, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.5836832523345947, + "kl": 1.281766653060913, + "learning_rate": 4.301587301587302e-06, + "loss": 0.0013, + "num_tokens": 464043.0, + "reward": 0.7175251841545105, + "reward_std": 0.11627113074064255, + "rewards/check_answer/mean": 0.7175251841545105, + "rewards/check_answer/std": 0.11627113074064255, + "step": 159 + }, + { + "completion_length": 478.25, + "completions/clipped_ratio": 0.0, + "completions/max_length": 559.0, + "completions/max_terminated_length": 559.0, + "completions/mean_length": 478.25, + "completions/mean_terminated_length": 478.25, + "completions/min_length": 378.0, + "completions/min_terminated_length": 378.0, + "epoch": 0.02843433445885907, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.5781432390213013, + "kl": 2.0126918852329254, + "learning_rate": 4.293650793650794e-06, + "loss": 0.002, + "num_tokens": 467112.0, + "reward": 1.0377893447875977, + "reward_std": 0.17348872125148773, + "rewards/check_answer/mean": 1.0377893447875977, + "rewards/check_answer/std": 0.17348873615264893, + "step": 160 + }, + { + "completion_length": 353.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 534.0, + "completions/max_terminated_length": 534.0, + "completions/mean_length": 353.0, + "completions/mean_terminated_length": 353.0, + "completions/min_length": 245.0, + "completions/min_terminated_length": 245.0, + "epoch": 0.028612049049226942, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.5405149459838867, + "kl": 1.1099863350391388, + "learning_rate": 4.2857142857142855e-06, + "loss": 0.0011, + "num_tokens": 469672.0, + "reward": 0.7634687423706055, + "reward_std": 0.16830436885356903, + "rewards/check_answer/mean": 0.7634687423706055, + "rewards/check_answer/std": 0.16830436885356903, + "step": 161 + }, + { + "completion_length": 463.25, + "completions/clipped_ratio": 0.0, + "completions/max_length": 582.0, + "completions/max_terminated_length": 582.0, + "completions/mean_length": 463.25, + "completions/mean_terminated_length": 463.25, + "completions/min_length": 376.0, + "completions/min_terminated_length": 376.0, + "epoch": 0.02878976363959481, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3183034658432007, + "kl": 1.1439854502677917, + "learning_rate": 4.277777777777778e-06, + "loss": 0.0011, + "num_tokens": 472685.0, + "reward": 0.5173400640487671, + "reward_std": 0.17256943881511688, + "rewards/check_answer/mean": 0.5173400640487671, + "rewards/check_answer/std": 0.1725694090127945, + "step": 162 + }, + { + "completion_length": 531.5, + "completions/clipped_ratio": 0.0, + "completions/max_length": 546.0, + "completions/max_terminated_length": 546.0, + "completions/mean_length": 531.5, + "completions/mean_terminated_length": 531.5, + "completions/min_length": 511.0, + "completions/min_terminated_length": 511.0, + "epoch": 0.02896747822996268, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.347712755203247, + "kl": 0.7350674420595169, + "learning_rate": 4.26984126984127e-06, + "loss": 0.0007, + "num_tokens": 476047.0, + "reward": 0.6685494184494019, + "reward_std": 0.031588803976774216, + "rewards/check_answer/mean": 0.6685494184494019, + "rewards/check_answer/std": 0.031588807702064514, + "step": 163 + }, + { + "completion_length": 556.75, + "completions/clipped_ratio": 0.0, + "completions/max_length": 631.0, + "completions/max_terminated_length": 631.0, + "completions/mean_length": 556.75, + "completions/mean_terminated_length": 556.75, + "completions/min_length": 470.0, + "completions/min_terminated_length": 470.0, + "epoch": 0.02914519282033055, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.4339076280593872, + "kl": 0.8411607146263123, + "learning_rate": 4.261904761904762e-06, + "loss": 0.0008, + "num_tokens": 479498.0, + "reward": 0.6374480724334717, + "reward_std": 0.13143180310726166, + "rewards/check_answer/mean": 0.6374480724334717, + "rewards/check_answer/std": 0.13143180310726166, + "step": 164 + }, + { + "completion_length": 331.75, + "completions/clipped_ratio": 0.0, + "completions/max_length": 445.0, + "completions/max_terminated_length": 445.0, + "completions/mean_length": 331.75, + "completions/mean_terminated_length": 331.75, + "completions/min_length": 241.0, + "completions/min_terminated_length": 241.0, + "epoch": 0.02932290741069842, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.8252785205841064, + "kl": 1.183883011341095, + "learning_rate": 4.2539682539682546e-06, + "loss": 0.0012, + "num_tokens": 481929.0, + "reward": 0.7619638442993164, + "reward_std": 0.16213203966617584, + "rewards/check_answer/mean": 0.7619638442993164, + "rewards/check_answer/std": 0.16213203966617584, + "step": 165 + }, + { + "completion_length": 503.5, + "completions/clipped_ratio": 0.0, + "completions/max_length": 649.0, + "completions/max_terminated_length": 649.0, + "completions/mean_length": 503.5, + "completions/mean_terminated_length": 503.5, + "completions/min_length": 391.0, + "completions/min_terminated_length": 391.0, + "epoch": 0.029500622001066287, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3157219886779785, + "kl": 0.7668842077255249, + "learning_rate": 4.246031746031746e-06, + "loss": 0.0008, + "num_tokens": 485155.0, + "reward": 0.7842249870300293, + "reward_std": 0.1028033196926117, + "rewards/check_answer/mean": 0.7842249870300293, + "rewards/check_answer/std": 0.1028033196926117, + "step": 166 + }, + { + "completion_length": 480.5, + "completions/clipped_ratio": 0.0, + "completions/max_length": 685.0, + "completions/max_terminated_length": 685.0, + "completions/mean_length": 480.5, + "completions/mean_terminated_length": 480.5, + "completions/min_length": 305.0, + "completions/min_terminated_length": 305.0, + "epoch": 0.029678336591434158, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.7447102069854736, + "kl": 1.1376843452453613, + "learning_rate": 4.238095238095239e-06, + "loss": 0.0011, + "num_tokens": 488257.0, + "reward": 0.468927800655365, + "reward_std": 0.09710308164358139, + "rewards/check_answer/mean": 0.468927800655365, + "rewards/check_answer/std": 0.09710308164358139, + "step": 167 + }, + { + "completion_length": 690.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 811.0, + "completions/max_terminated_length": 811.0, + "completions/mean_length": 690.0, + "completions/mean_terminated_length": 690.0, + "completions/min_length": 599.0, + "completions/min_terminated_length": 599.0, + "epoch": 0.029856051181802026, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0533298254013062, + "kl": 0.6091443598270416, + "learning_rate": 4.23015873015873e-06, + "loss": 0.0006, + "num_tokens": 492269.0, + "reward": 0.565061628818512, + "reward_std": 0.0684511587023735, + "rewards/check_answer/mean": 0.565061628818512, + "rewards/check_answer/std": 0.0684511587023735, + "step": 168 + }, + { + "completion_length": 280.25, + "completions/clipped_ratio": 0.0, + "completions/max_length": 387.0, + "completions/max_terminated_length": 387.0, + "completions/mean_length": 280.25, + "completions/mean_terminated_length": 280.25, + "completions/min_length": 167.0, + "completions/min_terminated_length": 167.0, + "epoch": 0.030033765772169894, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.046386480331421, + "kl": 1.14199760556221, + "learning_rate": 4.222222222222223e-06, + "loss": 0.0011, + "num_tokens": 494498.0, + "reward": 0.30173179507255554, + "reward_std": 0.22188961505889893, + "rewards/check_answer/mean": 0.30173179507255554, + "rewards/check_answer/std": 0.22188962996006012, + "step": 169 + }, + { + "completion_length": 378.75, + "completions/clipped_ratio": 0.0, + "completions/max_length": 448.0, + "completions/max_terminated_length": 448.0, + "completions/mean_length": 378.75, + "completions/mean_terminated_length": 378.75, + "completions/min_length": 234.0, + "completions/min_terminated_length": 234.0, + "epoch": 0.030211480362537766, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.9076892137527466, + "kl": 1.1338582932949066, + "learning_rate": 4.2142857142857145e-06, + "loss": 0.0011, + "num_tokens": 497121.0, + "reward": 0.22424927353858948, + "reward_std": 0.06156895309686661, + "rewards/check_answer/mean": 0.22424927353858948, + "rewards/check_answer/std": 0.06156895309686661, + "step": 170 + }, + { + "completion_length": 424.75, + "completions/clipped_ratio": 0.0, + "completions/max_length": 444.0, + "completions/max_terminated_length": 444.0, + "completions/mean_length": 424.75, + "completions/mean_terminated_length": 424.75, + "completions/min_length": 386.0, + "completions/min_terminated_length": 386.0, + "epoch": 0.030389194952905634, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.9031684398651123, + "kl": 1.1977380812168121, + "learning_rate": 4.206349206349207e-06, + "loss": 0.0012, + "num_tokens": 499936.0, + "reward": 0.8130784034729004, + "reward_std": 0.12363550066947937, + "rewards/check_answer/mean": 0.8130784034729004, + "rewards/check_answer/std": 0.12363548576831818, + "step": 171 + }, + { + "completion_length": 510.75, + "completions/clipped_ratio": 0.0, + "completions/max_length": 552.0, + "completions/max_terminated_length": 552.0, + "completions/mean_length": 510.75, + "completions/mean_terminated_length": 510.75, + "completions/min_length": 470.0, + "completions/min_terminated_length": 470.0, + "epoch": 0.030566909543273502, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3281104564666748, + "kl": 1.2302511632442474, + "learning_rate": 4.198412698412699e-06, + "loss": 0.0012, + "num_tokens": 503151.0, + "reward": 0.8297035694122314, + "reward_std": 0.059518009424209595, + "rewards/check_answer/mean": 0.8297035694122314, + "rewards/check_answer/std": 0.059518001973629, + "step": 172 + }, + { + "completion_length": 467.25, + "completions/clipped_ratio": 0.0, + "completions/max_length": 603.0, + "completions/max_terminated_length": 603.0, + "completions/mean_length": 467.25, + "completions/mean_terminated_length": 467.25, + "completions/min_length": 321.0, + "completions/min_terminated_length": 321.0, + "epoch": 0.03074462413364137, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.6283422708511353, + "kl": 1.2639419585466385, + "learning_rate": 4.190476190476191e-06, + "loss": 0.0013, + "num_tokens": 506156.0, + "reward": 0.8152176737785339, + "reward_std": 0.14423207938671112, + "rewards/check_answer/mean": 0.8152176737785339, + "rewards/check_answer/std": 0.14423207938671112, + "step": 173 + }, + { + "completion_length": 297.75, + "completions/clipped_ratio": 0.0, + "completions/max_length": 405.0, + "completions/max_terminated_length": 405.0, + "completions/mean_length": 297.75, + "completions/mean_terminated_length": 297.75, + "completions/min_length": 168.0, + "completions/min_terminated_length": 168.0, + "epoch": 0.030922338724009242, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.8512938022613525, + "kl": 1.276896983385086, + "learning_rate": 4.182539682539683e-06, + "loss": 0.0013, + "num_tokens": 508459.0, + "reward": 0.8983228206634521, + "reward_std": 0.23461507260799408, + "rewards/check_answer/mean": 0.8983228206634521, + "rewards/check_answer/std": 0.23461505770683289, + "step": 174 + }, + { + "completion_length": 326.25, + "completions/clipped_ratio": 0.0, + "completions/max_length": 411.0, + "completions/max_terminated_length": 411.0, + "completions/mean_length": 326.25, + "completions/mean_terminated_length": 326.25, + "completions/min_length": 282.0, + "completions/min_terminated_length": 282.0, + "epoch": 0.03110005331437711, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.8056871891021729, + "kl": 1.1819824874401093, + "learning_rate": 4.174603174603175e-06, + "loss": 0.0012, + "num_tokens": 510888.0, + "reward": 0.6271681785583496, + "reward_std": 0.08832673728466034, + "rewards/check_answer/mean": 0.6271681785583496, + "rewards/check_answer/std": 0.08832673728466034, + "step": 175 + }, + { + "completion_length": 531.5, + "completions/clipped_ratio": 0.0, + "completions/max_length": 577.0, + "completions/max_terminated_length": 577.0, + "completions/mean_length": 531.5, + "completions/mean_terminated_length": 531.5, + "completions/min_length": 408.0, + "completions/min_terminated_length": 408.0, + "epoch": 0.03127776790474498, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.4564107656478882, + "kl": 1.3183227330446243, + "learning_rate": 4.166666666666667e-06, + "loss": 0.0013, + "num_tokens": 514166.0, + "reward": 0.6827693581581116, + "reward_std": 0.08819714188575745, + "rewards/check_answer/mean": 0.6827693581581116, + "rewards/check_answer/std": 0.08819711953401566, + "step": 176 + }, + { + "completion_length": 314.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 395.0, + "completions/max_terminated_length": 395.0, + "completions/mean_length": 314.0, + "completions/mean_terminated_length": 314.0, + "completions/min_length": 233.0, + "completions/min_terminated_length": 233.0, + "epoch": 0.031455482495112846, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.986071228981018, + "kl": 1.2774607837200165, + "learning_rate": 4.158730158730159e-06, + "loss": 0.0013, + "num_tokens": 516530.0, + "reward": 0.16138049960136414, + "reward_std": 0.04136866331100464, + "rewards/check_answer/mean": 0.16138049960136414, + "rewards/check_answer/std": 0.04136866703629494, + "step": 177 + }, + { + "completion_length": 636.25, + "completions/clipped_ratio": 0.0, + "completions/max_length": 869.0, + "completions/max_terminated_length": 869.0, + "completions/mean_length": 636.25, + "completions/mean_terminated_length": 636.25, + "completions/min_length": 491.0, + "completions/min_terminated_length": 491.0, + "epoch": 0.031633197085480715, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3712565898895264, + "kl": 0.7792616635560989, + "learning_rate": 4.150793650793651e-06, + "loss": 0.0008, + "num_tokens": 520303.0, + "reward": 0.5512293577194214, + "reward_std": 0.06619084626436234, + "rewards/check_answer/mean": 0.5512293577194214, + "rewards/check_answer/std": 0.06619084626436234, + "step": 178 + }, + { + "completion_length": 293.5, + "completions/clipped_ratio": 0.0, + "completions/max_length": 492.0, + "completions/max_terminated_length": 492.0, + "completions/mean_length": 293.5, + "completions/mean_terminated_length": 293.5, + "completions/min_length": 222.0, + "completions/min_terminated_length": 222.0, + "epoch": 0.03181091167584859, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.8168277740478516, + "kl": 1.2853178530931473, + "learning_rate": 4.1428571428571435e-06, + "loss": 0.0013, + "num_tokens": 522605.0, + "reward": 0.7641668319702148, + "reward_std": 0.2912050485610962, + "rewards/check_answer/mean": 0.7641668319702148, + "rewards/check_answer/std": 0.2912050187587738, + "step": 179 + }, + { + "completion_length": 455.25, + "completions/clipped_ratio": 0.0, + "completions/max_length": 530.0, + "completions/max_terminated_length": 530.0, + "completions/mean_length": 455.25, + "completions/mean_terminated_length": 455.25, + "completions/min_length": 354.0, + "completions/min_terminated_length": 354.0, + "epoch": 0.03198862626621646, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.6546411514282227, + "kl": 0.9170921891927719, + "learning_rate": 4.134920634920635e-06, + "loss": 0.0009, + "num_tokens": 525582.0, + "reward": 0.5996488928794861, + "reward_std": 0.16385574638843536, + "rewards/check_answer/mean": 0.5996488928794861, + "rewards/check_answer/std": 0.16385574638843536, + "step": 180 + }, + { + "completion_length": 413.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 532.0, + "completions/max_terminated_length": 532.0, + "completions/mean_length": 413.0, + "completions/mean_terminated_length": 413.0, + "completions/min_length": 357.0, + "completions/min_terminated_length": 357.0, + "epoch": 0.032166340856584326, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.8733916282653809, + "kl": 1.0652090162038803, + "learning_rate": 4.126984126984127e-06, + "loss": 0.0011, + "num_tokens": 528350.0, + "reward": 0.8629907369613647, + "reward_std": 0.19258101284503937, + "rewards/check_answer/mean": 0.8629907369613647, + "rewards/check_answer/std": 0.19258098304271698, + "step": 181 + }, + { + "completion_length": 315.75, + "completions/clipped_ratio": 0.0, + "completions/max_length": 341.0, + "completions/max_terminated_length": 341.0, + "completions/mean_length": 315.75, + "completions/mean_terminated_length": 315.75, + "completions/min_length": 253.0, + "completions/min_terminated_length": 253.0, + "epoch": 0.032344055446952194, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.031298875808716, + "kl": 1.131786733865738, + "learning_rate": 4.119047619047619e-06, + "loss": 0.0011, + "num_tokens": 530741.0, + "reward": 0.45975661277770996, + "reward_std": 0.13406918942928314, + "rewards/check_answer/mean": 0.45975661277770996, + "rewards/check_answer/std": 0.13406920433044434, + "step": 182 + }, + { + "completion_length": 321.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 331.0, + "completions/max_terminated_length": 331.0, + "completions/mean_length": 321.0, + "completions/mean_terminated_length": 321.0, + "completions/min_length": 305.0, + "completions/min_terminated_length": 305.0, + "epoch": 0.03252177003732006, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.075031042098999, + "kl": 1.3818604350090027, + "learning_rate": 4.111111111111111e-06, + "loss": 0.0014, + "num_tokens": 533125.0, + "reward": 0.633269190788269, + "reward_std": 0.10433237999677658, + "rewards/check_answer/mean": 0.633269190788269, + "rewards/check_answer/std": 0.10433237999677658, + "step": 183 + }, + { + "completion_length": 331.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 389.0, + "completions/max_terminated_length": 389.0, + "completions/mean_length": 331.0, + "completions/mean_terminated_length": 331.0, + "completions/min_length": 212.0, + "completions/min_terminated_length": 212.0, + "epoch": 0.03269948462768793, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.916518211364746, + "kl": 1.230439841747284, + "learning_rate": 4.103174603174603e-06, + "loss": 0.0012, + "num_tokens": 535565.0, + "reward": 0.5834702253341675, + "reward_std": 0.1800461858510971, + "rewards/check_answer/mean": 0.5834702253341675, + "rewards/check_answer/std": 0.1800461858510971, + "step": 184 + }, + { + "completion_length": 484.75, + "completions/clipped_ratio": 0.0, + "completions/max_length": 616.0, + "completions/max_terminated_length": 616.0, + "completions/mean_length": 484.75, + "completions/mean_terminated_length": 484.75, + "completions/min_length": 284.0, + "completions/min_terminated_length": 284.0, + "epoch": 0.032877199218055805, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.5022894144058228, + "kl": 0.8969959318637848, + "learning_rate": 4.095238095238096e-06, + "loss": 0.0009, + "num_tokens": 538676.0, + "reward": 0.4728669822216034, + "reward_std": 0.3648776710033417, + "rewards/check_answer/mean": 0.4728669822216034, + "rewards/check_answer/std": 0.3648776710033417, + "step": 185 + }, + { + "completion_length": 520.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 580.0, + "completions/max_terminated_length": 580.0, + "completions/mean_length": 520.0, + "completions/mean_terminated_length": 520.0, + "completions/min_length": 443.0, + "completions/min_terminated_length": 443.0, + "epoch": 0.033054913808423673, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.616834044456482, + "kl": 0.8219860643148422, + "learning_rate": 4.0873015873015875e-06, + "loss": 0.0008, + "num_tokens": 541912.0, + "reward": 0.5659242868423462, + "reward_std": 0.0763397291302681, + "rewards/check_answer/mean": 0.5659242868423462, + "rewards/check_answer/std": 0.0763397291302681, + "step": 186 + }, + { + "completion_length": 630.25, + "completions/clipped_ratio": 0.0, + "completions/max_length": 716.0, + "completions/max_terminated_length": 716.0, + "completions/mean_length": 630.25, + "completions/mean_terminated_length": 630.25, + "completions/min_length": 514.0, + "completions/min_terminated_length": 514.0, + "epoch": 0.03323262839879154, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3261500597000122, + "kl": 0.6475310772657394, + "learning_rate": 4.07936507936508e-06, + "loss": 0.0006, + "num_tokens": 545637.0, + "reward": 0.5103482604026794, + "reward_std": 0.04495608061552048, + "rewards/check_answer/mean": 0.5103482604026794, + "rewards/check_answer/std": 0.04495606571435928, + "step": 187 + }, + { + "completion_length": 656.75, + "completions/clipped_ratio": 0.0, + "completions/max_length": 983.0, + "completions/max_terminated_length": 983.0, + "completions/mean_length": 656.75, + "completions/mean_terminated_length": 656.75, + "completions/min_length": 469.0, + "completions/min_terminated_length": 469.0, + "epoch": 0.03341034298915941, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.2920455932617188, + "kl": 0.8283561021089554, + "learning_rate": 4.071428571428572e-06, + "loss": 0.0008, + "num_tokens": 549436.0, + "reward": 0.5403710603713989, + "reward_std": 0.24990840256214142, + "rewards/check_answer/mean": 0.5403710603713989, + "rewards/check_answer/std": 0.2499084323644638, + "step": 188 + }, + { + "completion_length": 410.5, + "completions/clipped_ratio": 0.0, + "completions/max_length": 520.0, + "completions/max_terminated_length": 520.0, + "completions/mean_length": 410.5, + "completions/mean_terminated_length": 410.5, + "completions/min_length": 275.0, + "completions/min_terminated_length": 275.0, + "epoch": 0.03358805757952728, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.776705026626587, + "kl": 0.9838277697563171, + "learning_rate": 4.063492063492064e-06, + "loss": 0.001, + "num_tokens": 552222.0, + "reward": 0.6380422115325928, + "reward_std": 0.1814417541027069, + "rewards/check_answer/mean": 0.6380422115325928, + "rewards/check_answer/std": 0.1814417541027069, + "step": 189 + }, + { + "completion_length": 551.5, + "completions/clipped_ratio": 0.0, + "completions/max_length": 637.0, + "completions/max_terminated_length": 637.0, + "completions/mean_length": 551.5, + "completions/mean_terminated_length": 551.5, + "completions/min_length": 469.0, + "completions/min_terminated_length": 469.0, + "epoch": 0.033765772169895146, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.4350589513778687, + "kl": 0.8946151733398438, + "learning_rate": 4.055555555555556e-06, + "loss": 0.0009, + "num_tokens": 555628.0, + "reward": 0.6631824374198914, + "reward_std": 0.0526728481054306, + "rewards/check_answer/mean": 0.6631824374198914, + "rewards/check_answer/std": 0.05267287418246269, + "step": 190 + }, + { + "completion_length": 335.25, + "completions/clipped_ratio": 0.0, + "completions/max_length": 372.0, + "completions/max_terminated_length": 372.0, + "completions/mean_length": 335.25, + "completions/mean_terminated_length": 335.25, + "completions/min_length": 290.0, + "completions/min_terminated_length": 290.0, + "epoch": 0.033943486760263014, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.6080844402313232, + "kl": 1.0406917929649353, + "learning_rate": 4.047619047619048e-06, + "loss": 0.001, + "num_tokens": 558101.0, + "reward": 0.8645050525665283, + "reward_std": 0.16412971913814545, + "rewards/check_answer/mean": 0.8645050525665283, + "rewards/check_answer/std": 0.16412973403930664, + "step": 191 + }, + { + "completion_length": 295.5, + "completions/clipped_ratio": 0.0, + "completions/max_length": 321.0, + "completions/max_terminated_length": 321.0, + "completions/mean_length": 295.5, + "completions/mean_terminated_length": 295.5, + "completions/min_length": 279.0, + "completions/min_terminated_length": 279.0, + "epoch": 0.03412120135063089, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.0053203105926514, + "kl": 1.3715750873088837, + "learning_rate": 4.03968253968254e-06, + "loss": 0.0014, + "num_tokens": 560415.0, + "reward": 0.37511610984802246, + "reward_std": 0.12241348624229431, + "rewards/check_answer/mean": 0.37511610984802246, + "rewards/check_answer/std": 0.12241347879171371, + "step": 192 + }, + { + "completion_length": 351.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 426.0, + "completions/max_terminated_length": 426.0, + "completions/mean_length": 351.0, + "completions/mean_terminated_length": 351.0, + "completions/min_length": 254.0, + "completions/min_terminated_length": 254.0, + "epoch": 0.03429891594099876, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.6852611303329468, + "kl": 0.9062083661556244, + "learning_rate": 4.031746031746032e-06, + "loss": 0.0009, + "num_tokens": 562943.0, + "reward": 0.7322611808776855, + "reward_std": 0.10273779183626175, + "rewards/check_answer/mean": 0.7322611808776855, + "rewards/check_answer/std": 0.10273779183626175, + "step": 193 + }, + { + "completion_length": 387.5, + "completions/clipped_ratio": 0.0, + "completions/max_length": 441.0, + "completions/max_terminated_length": 441.0, + "completions/mean_length": 387.5, + "completions/mean_terminated_length": 387.5, + "completions/min_length": 258.0, + "completions/min_terminated_length": 258.0, + "epoch": 0.034476630531366625, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.495898962020874, + "kl": 0.9667275995016098, + "learning_rate": 4.023809523809524e-06, + "loss": 0.001, + "num_tokens": 565661.0, + "reward": 1.0379855632781982, + "reward_std": 0.013653420843183994, + "rewards/check_answer/mean": 1.0379855632781982, + "rewards/check_answer/std": 0.013653420843183994, + "step": 194 + }, + { + "completion_length": 516.25, + "completions/clipped_ratio": 0.0, + "completions/max_length": 655.0, + "completions/max_terminated_length": 655.0, + "completions/mean_length": 516.25, + "completions/mean_terminated_length": 516.25, + "completions/min_length": 363.0, + "completions/min_terminated_length": 363.0, + "epoch": 0.034654345121734494, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.8335789442062378, + "kl": 2.6260423213243484, + "learning_rate": 4.0158730158730165e-06, + "loss": 0.0026, + "num_tokens": 568954.0, + "reward": 0.4649602472782135, + "reward_std": 0.05998750403523445, + "rewards/check_answer/mean": 0.4649602472782135, + "rewards/check_answer/std": 0.05998750776052475, + "step": 195 + }, + { + "completion_length": 486.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 562.0, + "completions/max_terminated_length": 562.0, + "completions/mean_length": 486.0, + "completions/mean_terminated_length": 486.0, + "completions/min_length": 445.0, + "completions/min_terminated_length": 445.0, + "epoch": 0.03483205971210236, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.7766040563583374, + "kl": 1.0821466594934464, + "learning_rate": 4.007936507936508e-06, + "loss": 0.0011, + "num_tokens": 572042.0, + "reward": 0.5732154846191406, + "reward_std": 0.08260545134544373, + "rewards/check_answer/mean": 0.5732154846191406, + "rewards/check_answer/std": 0.08260544389486313, + "step": 196 + }, + { + "completion_length": 681.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 823.0, + "completions/max_terminated_length": 823.0, + "completions/mean_length": 681.0, + "completions/mean_terminated_length": 681.0, + "completions/min_length": 593.0, + "completions/min_terminated_length": 593.0, + "epoch": 0.03500977430247023, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.1446372270584106, + "kl": 0.685902014374733, + "learning_rate": 4.000000000000001e-06, + "loss": 0.0007, + "num_tokens": 576022.0, + "reward": 0.714977502822876, + "reward_std": 0.0700133666396141, + "rewards/check_answer/mean": 0.714977502822876, + "rewards/check_answer/std": 0.0700133740901947, + "step": 197 + }, + { + "completion_length": 267.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 280.0, + "completions/max_terminated_length": 280.0, + "completions/mean_length": 267.0, + "completions/mean_terminated_length": 267.0, + "completions/min_length": 238.0, + "completions/min_terminated_length": 238.0, + "epoch": 0.035187488892838105, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.380545139312744, + "kl": 1.7545035779476166, + "learning_rate": 3.992063492063492e-06, + "loss": 0.0018, + "num_tokens": 578194.0, + "reward": 0.5900921821594238, + "reward_std": 0.35790684819221497, + "rewards/check_answer/mean": 0.5900921821594238, + "rewards/check_answer/std": 0.3579068183898926, + "step": 198 + }, + { + "completion_length": 466.5, + "completions/clipped_ratio": 0.0, + "completions/max_length": 545.0, + "completions/max_terminated_length": 545.0, + "completions/mean_length": 466.5, + "completions/mean_terminated_length": 466.5, + "completions/min_length": 384.0, + "completions/min_terminated_length": 384.0, + "epoch": 0.03536520348320597, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.8136112689971924, + "kl": 0.944290354847908, + "learning_rate": 3.984126984126984e-06, + "loss": 0.0009, + "num_tokens": 581228.0, + "reward": 0.5919873714447021, + "reward_std": 0.09145790338516235, + "rewards/check_answer/mean": 0.5919873714447021, + "rewards/check_answer/std": 0.09145790338516235, + "step": 199 + }, + { + "completion_length": 232.75, + "completions/clipped_ratio": 0.0, + "completions/max_length": 285.0, + "completions/max_terminated_length": 285.0, + "completions/mean_length": 232.75, + "completions/mean_terminated_length": 232.75, + "completions/min_length": 179.0, + "completions/min_terminated_length": 179.0, + "epoch": 0.03554291807357384, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.2700116634368896, + "kl": 1.3578459918498993, + "learning_rate": 3.9761904761904764e-06, + "loss": 0.0014, + "num_tokens": 583263.0, + "reward": 0.6666097640991211, + "reward_std": 0.313691109418869, + "rewards/check_answer/mean": 0.6666097640991211, + "rewards/check_answer/std": 0.3136911392211914, + "step": 200 + }, + { + "completion_length": 492.75, + "completions/clipped_ratio": 0.0, + "completions/max_length": 568.0, + "completions/max_terminated_length": 568.0, + "completions/mean_length": 492.75, + "completions/mean_terminated_length": 492.75, + "completions/min_length": 426.0, + "completions/min_terminated_length": 426.0, + "epoch": 0.03572063266394171, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.5221089124679565, + "kl": 0.8189870119094849, + "learning_rate": 3.968253968253968e-06, + "loss": 0.0008, + "num_tokens": 586426.0, + "reward": 0.5162832736968994, + "reward_std": 0.054980941116809845, + "rewards/check_answer/mean": 0.5162832736968994, + "rewards/check_answer/std": 0.05498094484210014, + "step": 201 + }, + { + "completion_length": 405.5, + "completions/clipped_ratio": 0.0, + "completions/max_length": 463.0, + "completions/max_terminated_length": 463.0, + "completions/mean_length": 405.5, + "completions/mean_terminated_length": 405.5, + "completions/min_length": 347.0, + "completions/min_terminated_length": 347.0, + "epoch": 0.03589834725430958, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.6586710214614868, + "kl": 1.192984789609909, + "learning_rate": 3.9603174603174606e-06, + "loss": 0.0012, + "num_tokens": 589196.0, + "reward": 0.8408722281455994, + "reward_std": 0.12209592759609222, + "rewards/check_answer/mean": 0.8408722281455994, + "rewards/check_answer/std": 0.12209593504667282, + "step": 202 + }, + { + "completion_length": 551.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 675.0, + "completions/max_terminated_length": 675.0, + "completions/mean_length": 551.0, + "completions/mean_terminated_length": 551.0, + "completions/min_length": 466.0, + "completions/min_terminated_length": 466.0, + "epoch": 0.036076061844677446, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3964484930038452, + "kl": 0.7525434195995331, + "learning_rate": 3.952380952380952e-06, + "loss": 0.0008, + "num_tokens": 592620.0, + "reward": 0.43263351917266846, + "reward_std": 0.058862458914518356, + "rewards/check_answer/mean": 0.43263351917266846, + "rewards/check_answer/std": 0.05886244773864746, + "step": 203 + }, + { + "completion_length": 329.75, + "completions/clipped_ratio": 0.0, + "completions/max_length": 385.0, + "completions/max_terminated_length": 385.0, + "completions/mean_length": 329.75, + "completions/mean_terminated_length": 329.75, + "completions/min_length": 294.0, + "completions/min_terminated_length": 294.0, + "epoch": 0.03625377643504532, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.8947840929031372, + "kl": 1.2776507586240768, + "learning_rate": 3.944444444444445e-06, + "loss": 0.0013, + "num_tokens": 595067.0, + "reward": 0.8257237076759338, + "reward_std": 0.09456206113100052, + "rewards/check_answer/mean": 0.8257237076759338, + "rewards/check_answer/std": 0.09456204622983932, + "step": 204 + }, + { + "completion_length": 331.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 424.0, + "completions/max_terminated_length": 424.0, + "completions/mean_length": 331.0, + "completions/mean_terminated_length": 331.0, + "completions/min_length": 284.0, + "completions/min_terminated_length": 284.0, + "epoch": 0.03643149102541319, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.9775497913360596, + "kl": 1.0914415419101715, + "learning_rate": 3.936507936507936e-06, + "loss": 0.0011, + "num_tokens": 597503.0, + "reward": 0.4486619532108307, + "reward_std": 0.20394665002822876, + "rewards/check_answer/mean": 0.4486619532108307, + "rewards/check_answer/std": 0.20394665002822876, + "step": 205 + }, + { + "completion_length": 525.5, + "completions/clipped_ratio": 0.0, + "completions/max_length": 612.0, + "completions/max_terminated_length": 612.0, + "completions/mean_length": 525.5, + "completions/mean_terminated_length": 525.5, + "completions/min_length": 480.0, + "completions/min_terminated_length": 480.0, + "epoch": 0.03660920561578106, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.4579870700836182, + "kl": 0.8831206411123276, + "learning_rate": 3.928571428571429e-06, + "loss": 0.0009, + "num_tokens": 600773.0, + "reward": 0.5694155693054199, + "reward_std": 0.0706477239727974, + "rewards/check_answer/mean": 0.5694155693054199, + "rewards/check_answer/std": 0.0706477090716362, + "step": 206 + }, + { + "completion_length": 397.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 495.0, + "completions/max_terminated_length": 495.0, + "completions/mean_length": 397.0, + "completions/mean_terminated_length": 397.0, + "completions/min_length": 309.0, + "completions/min_terminated_length": 309.0, + "epoch": 0.036786920206148925, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.9087179899215698, + "kl": 1.082411751151085, + "learning_rate": 3.920634920634921e-06, + "loss": 0.0011, + "num_tokens": 603501.0, + "reward": 0.6026151180267334, + "reward_std": 0.09127797186374664, + "rewards/check_answer/mean": 0.6026151180267334, + "rewards/check_answer/std": 0.09127800166606903, + "step": 207 + }, + { + "completion_length": 541.5, + "completions/clipped_ratio": 0.0, + "completions/max_length": 618.0, + "completions/max_terminated_length": 618.0, + "completions/mean_length": 541.5, + "completions/mean_terminated_length": 541.5, + "completions/min_length": 487.0, + "completions/min_terminated_length": 487.0, + "epoch": 0.03696463479651679, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.5631201267242432, + "kl": 0.7632221281528473, + "learning_rate": 3.912698412698413e-06, + "loss": 0.0008, + "num_tokens": 606823.0, + "reward": 0.46171265840530396, + "reward_std": 0.0858674943447113, + "rewards/check_answer/mean": 0.46171265840530396, + "rewards/check_answer/std": 0.0858675092458725, + "step": 208 + }, + { + "completion_length": 299.25, + "completions/clipped_ratio": 0.0, + "completions/max_length": 358.0, + "completions/max_terminated_length": 358.0, + "completions/mean_length": 299.25, + "completions/mean_terminated_length": 299.25, + "completions/min_length": 217.0, + "completions/min_terminated_length": 217.0, + "epoch": 0.03714234938688466, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.7001370191574097, + "kl": 1.1468262076377869, + "learning_rate": 3.9047619047619055e-06, + "loss": 0.0011, + "num_tokens": 609140.0, + "reward": 1.1788017749786377, + "reward_std": 0.23657526075839996, + "rewards/check_answer/mean": 1.1788017749786377, + "rewards/check_answer/std": 0.23657529056072235, + "step": 209 + }, + { + "completion_length": 321.25, + "completions/clipped_ratio": 0.0, + "completions/max_length": 377.0, + "completions/max_terminated_length": 377.0, + "completions/mean_length": 321.25, + "completions/mean_terminated_length": 321.25, + "completions/min_length": 252.0, + "completions/min_terminated_length": 252.0, + "epoch": 0.03732006397725253, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.0018045902252197, + "kl": 1.2047775089740753, + "learning_rate": 3.896825396825397e-06, + "loss": 0.0012, + "num_tokens": 611549.0, + "reward": 0.4826478362083435, + "reward_std": 0.2732682228088379, + "rewards/check_answer/mean": 0.4826478362083435, + "rewards/check_answer/std": 0.2732682228088379, + "step": 210 + }, + { + "completion_length": 553.5, + "completions/clipped_ratio": 0.0, + "completions/max_length": 622.0, + "completions/max_terminated_length": 622.0, + "completions/mean_length": 553.5, + "completions/mean_terminated_length": 553.5, + "completions/min_length": 421.0, + "completions/min_terminated_length": 421.0, + "epoch": 0.037497778567620405, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.4428868293762207, + "kl": 1.2566351145505905, + "learning_rate": 3.88888888888889e-06, + "loss": 0.0013, + "num_tokens": 614959.0, + "reward": 0.562415361404419, + "reward_std": 0.03217893838882446, + "rewards/check_answer/mean": 0.562415361404419, + "rewards/check_answer/std": 0.03217893838882446, + "step": 211 + }, + { + "completion_length": 533.75, + "completions/clipped_ratio": 0.0, + "completions/max_length": 616.0, + "completions/max_terminated_length": 616.0, + "completions/mean_length": 533.75, + "completions/mean_terminated_length": 533.75, + "completions/min_length": 441.0, + "completions/min_terminated_length": 441.0, + "epoch": 0.03767549315798827, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.474406361579895, + "kl": 0.8861031532287598, + "learning_rate": 3.880952380952381e-06, + "loss": 0.0009, + "num_tokens": 618286.0, + "reward": 0.716295599937439, + "reward_std": 0.08145526796579361, + "rewards/check_answer/mean": 0.716295599937439, + "rewards/check_answer/std": 0.08145526051521301, + "step": 212 + }, + { + "completion_length": 603.75, + "completions/clipped_ratio": 0.0, + "completions/max_length": 669.0, + "completions/max_terminated_length": 669.0, + "completions/mean_length": 603.75, + "completions/mean_terminated_length": 603.75, + "completions/min_length": 454.0, + "completions/min_terminated_length": 454.0, + "epoch": 0.03785320774835614, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3624635934829712, + "kl": 0.817508801817894, + "learning_rate": 3.873015873015874e-06, + "loss": 0.0008, + "num_tokens": 621881.0, + "reward": 0.3234153985977173, + "reward_std": 0.07267056405544281, + "rewards/check_answer/mean": 0.3234153985977173, + "rewards/check_answer/std": 0.07267057150602341, + "step": 213 + }, + { + "completion_length": 549.5, + "completions/clipped_ratio": 0.0, + "completions/max_length": 658.0, + "completions/max_terminated_length": 658.0, + "completions/mean_length": 549.5, + "completions/mean_terminated_length": 549.5, + "completions/min_length": 462.0, + "completions/min_terminated_length": 462.0, + "epoch": 0.03803092233872401, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.5851331949234009, + "kl": 0.9887559115886688, + "learning_rate": 3.865079365079365e-06, + "loss": 0.001, + "num_tokens": 625263.0, + "reward": 0.7112226486206055, + "reward_std": 0.02053823135793209, + "rewards/check_answer/mean": 0.7112226486206055, + "rewards/check_answer/std": 0.02053823322057724, + "step": 214 + }, + { + "completion_length": 545.25, + "completions/clipped_ratio": 0.0, + "completions/max_length": 621.0, + "completions/max_terminated_length": 621.0, + "completions/mean_length": 545.25, + "completions/mean_terminated_length": 545.25, + "completions/min_length": 425.0, + "completions/min_terminated_length": 425.0, + "epoch": 0.03820863692909188, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.423057198524475, + "kl": 1.0859228074550629, + "learning_rate": 3.857142857142858e-06, + "loss": 0.0011, + "num_tokens": 628648.0, + "reward": 0.5066788196563721, + "reward_std": 0.026953205466270447, + "rewards/check_answer/mean": 0.5066788196563721, + "rewards/check_answer/std": 0.02695320174098015, + "step": 215 + }, + { + "completion_length": 483.75, + "completions/clipped_ratio": 0.0, + "completions/max_length": 613.0, + "completions/max_terminated_length": 613.0, + "completions/mean_length": 483.75, + "completions/mean_terminated_length": 483.75, + "completions/min_length": 411.0, + "completions/min_terminated_length": 411.0, + "epoch": 0.038386351519459745, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.6435989141464233, + "kl": 0.8813229352235794, + "learning_rate": 3.8492063492063495e-06, + "loss": 0.0009, + "num_tokens": 631751.0, + "reward": 0.7219762802124023, + "reward_std": 0.17963886260986328, + "rewards/check_answer/mean": 0.7219762802124023, + "rewards/check_answer/std": 0.17963886260986328, + "step": 216 + }, + { + "completion_length": 537.5, + "completions/clipped_ratio": 0.0, + "completions/max_length": 697.0, + "completions/max_terminated_length": 697.0, + "completions/mean_length": 537.5, + "completions/mean_terminated_length": 537.5, + "completions/min_length": 435.0, + "completions/min_terminated_length": 435.0, + "epoch": 0.03856406610982762, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.4791259765625, + "kl": 0.9588049501180649, + "learning_rate": 3.841269841269842e-06, + "loss": 0.001, + "num_tokens": 635053.0, + "reward": 0.5066652894020081, + "reward_std": 0.0735304206609726, + "rewards/check_answer/mean": 0.5066652894020081, + "rewards/check_answer/std": 0.07353042811155319, + "step": 217 + }, + { + "completion_length": 392.75, + "completions/clipped_ratio": 0.0, + "completions/max_length": 492.0, + "completions/max_terminated_length": 492.0, + "completions/mean_length": 392.75, + "completions/mean_terminated_length": 392.75, + "completions/min_length": 245.0, + "completions/min_terminated_length": 245.0, + "epoch": 0.03874178070019549, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.9984493255615234, + "kl": 1.0702104568481445, + "learning_rate": 3.833333333333334e-06, + "loss": 0.0011, + "num_tokens": 637744.0, + "reward": 0.7809625267982483, + "reward_std": 0.5125808119773865, + "rewards/check_answer/mean": 0.7809625267982483, + "rewards/check_answer/std": 0.5125808119773865, + "step": 218 + }, + { + "completion_length": 447.75, + "completions/clipped_ratio": 0.0, + "completions/max_length": 499.0, + "completions/max_terminated_length": 499.0, + "completions/mean_length": 447.75, + "completions/mean_terminated_length": 447.75, + "completions/min_length": 360.0, + "completions/min_terminated_length": 360.0, + "epoch": 0.038919495290563356, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.494664192199707, + "kl": 0.837522029876709, + "learning_rate": 3.825396825396825e-06, + "loss": 0.0008, + "num_tokens": 640751.0, + "reward": 1.227266550064087, + "reward_std": 0.04486949369311333, + "rewards/check_answer/mean": 1.227266550064087, + "rewards/check_answer/std": 0.04486951231956482, + "step": 219 + }, + { + "completion_length": 407.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 558.0, + "completions/max_terminated_length": 558.0, + "completions/mean_length": 407.0, + "completions/mean_terminated_length": 407.0, + "completions/min_length": 277.0, + "completions/min_terminated_length": 277.0, + "epoch": 0.039097209880931225, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.7991682291030884, + "kl": 1.061606004834175, + "learning_rate": 3.817460317460318e-06, + "loss": 0.0011, + "num_tokens": 643523.0, + "reward": 0.7400416135787964, + "reward_std": 0.11785461008548737, + "rewards/check_answer/mean": 0.7400416135787964, + "rewards/check_answer/std": 0.11785461753606796, + "step": 220 + }, + { + "completion_length": 523.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 602.0, + "completions/max_terminated_length": 602.0, + "completions/mean_length": 523.0, + "completions/mean_terminated_length": 523.0, + "completions/min_length": 450.0, + "completions/min_terminated_length": 450.0, + "epoch": 0.03927492447129909, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.4204972982406616, + "kl": 1.000058576464653, + "learning_rate": 3.80952380952381e-06, + "loss": 0.001, + "num_tokens": 646795.0, + "reward": 0.7960029244422913, + "reward_std": 0.11042596399784088, + "rewards/check_answer/mean": 0.7960029244422913, + "rewards/check_answer/std": 0.11042595654726028, + "step": 221 + }, + { + "completion_length": 509.75, + "completions/clipped_ratio": 0.0, + "completions/max_length": 586.0, + "completions/max_terminated_length": 586.0, + "completions/mean_length": 509.75, + "completions/mean_terminated_length": 509.75, + "completions/min_length": 406.0, + "completions/min_terminated_length": 406.0, + "epoch": 0.03945263906166696, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.4458086490631104, + "kl": 1.0492773056030273, + "learning_rate": 3.801587301587302e-06, + "loss": 0.001, + "num_tokens": 650042.0, + "reward": 0.8759781718254089, + "reward_std": 0.07902158796787262, + "rewards/check_answer/mean": 0.8759781718254089, + "rewards/check_answer/std": 0.07902161031961441, + "step": 222 + }, + { + "completion_length": 422.25, + "completions/clipped_ratio": 0.0, + "completions/max_length": 494.0, + "completions/max_terminated_length": 494.0, + "completions/mean_length": 422.25, + "completions/mean_terminated_length": 422.25, + "completions/min_length": 330.0, + "completions/min_terminated_length": 330.0, + "epoch": 0.03963035365203483, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.7075461149215698, + "kl": 1.1013980060815811, + "learning_rate": 3.793650793650794e-06, + "loss": 0.0011, + "num_tokens": 652883.0, + "reward": 0.6839185953140259, + "reward_std": 0.16940036416053772, + "rewards/check_answer/mean": 0.6839185953140259, + "rewards/check_answer/std": 0.1694003790616989, + "step": 223 + }, + { + "completion_length": 410.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 459.0, + "completions/max_terminated_length": 459.0, + "completions/mean_length": 410.0, + "completions/mean_terminated_length": 410.0, + "completions/min_length": 368.0, + "completions/min_terminated_length": 368.0, + "epoch": 0.039808068242402704, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.579961895942688, + "kl": 1.1135330200195312, + "learning_rate": 3.785714285714286e-06, + "loss": 0.0011, + "num_tokens": 655659.0, + "reward": 0.7339705228805542, + "reward_std": 0.09850845485925674, + "rewards/check_answer/mean": 0.7339705228805542, + "rewards/check_answer/std": 0.09850846230983734, + "step": 224 + }, + { + "completion_length": 405.75, + "completions/clipped_ratio": 0.0, + "completions/max_length": 492.0, + "completions/max_terminated_length": 492.0, + "completions/mean_length": 405.75, + "completions/mean_terminated_length": 405.75, + "completions/min_length": 327.0, + "completions/min_terminated_length": 327.0, + "epoch": 0.03998578283277057, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.5088939666748047, + "kl": 1.0870481431484222, + "learning_rate": 3.777777777777778e-06, + "loss": 0.0011, + "num_tokens": 658394.0, + "reward": 0.6195275783538818, + "reward_std": 0.32846853137016296, + "rewards/check_answer/mean": 0.6195275783538818, + "rewards/check_answer/std": 0.32846853137016296, + "step": 225 + }, + { + "completion_length": 425.75, + "completions/clipped_ratio": 0.0, + "completions/max_length": 504.0, + "completions/max_terminated_length": 504.0, + "completions/mean_length": 425.75, + "completions/mean_terminated_length": 425.75, + "completions/min_length": 352.0, + "completions/min_terminated_length": 352.0, + "epoch": 0.04016349742313844, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.6888916492462158, + "kl": 1.0393297523260117, + "learning_rate": 3.76984126984127e-06, + "loss": 0.001, + "num_tokens": 661237.0, + "reward": 0.5425740480422974, + "reward_std": 0.14747239649295807, + "rewards/check_answer/mean": 0.5425740480422974, + "rewards/check_answer/std": 0.14747241139411926, + "step": 226 + }, + { + "completion_length": 430.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 551.0, + "completions/max_terminated_length": 551.0, + "completions/mean_length": 430.0, + "completions/mean_terminated_length": 430.0, + "completions/min_length": 325.0, + "completions/min_terminated_length": 325.0, + "epoch": 0.04034121201350631, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.497071385383606, + "kl": 0.8925988674163818, + "learning_rate": 3.761904761904762e-06, + "loss": 0.0009, + "num_tokens": 664129.0, + "reward": 0.5838330984115601, + "reward_std": 0.14572261273860931, + "rewards/check_answer/mean": 0.5838330984115601, + "rewards/check_answer/std": 0.14572261273860931, + "step": 227 + }, + { + "completion_length": 387.75, + "completions/clipped_ratio": 0.0, + "completions/max_length": 466.0, + "completions/max_terminated_length": 466.0, + "completions/mean_length": 387.75, + "completions/mean_terminated_length": 387.75, + "completions/min_length": 284.0, + "completions/min_terminated_length": 284.0, + "epoch": 0.04051892660387418, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.95894193649292, + "kl": 1.2442685663700104, + "learning_rate": 3.7539682539682543e-06, + "loss": 0.0012, + "num_tokens": 666784.0, + "reward": 0.3317446708679199, + "reward_std": 0.12045136839151382, + "rewards/check_answer/mean": 0.3317446708679199, + "rewards/check_answer/std": 0.12045136094093323, + "step": 228 + }, + { + "completion_length": 627.25, + "completions/clipped_ratio": 0.0, + "completions/max_length": 788.0, + "completions/max_terminated_length": 788.0, + "completions/mean_length": 627.25, + "completions/mean_terminated_length": 627.25, + "completions/min_length": 499.0, + "completions/min_terminated_length": 499.0, + "epoch": 0.040696641194242045, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.2475444078445435, + "kl": 0.8898588418960571, + "learning_rate": 3.7460317460317463e-06, + "loss": 0.0009, + "num_tokens": 670541.0, + "reward": 0.5734164714813232, + "reward_std": 0.06777976453304291, + "rewards/check_answer/mean": 0.5734164714813232, + "rewards/check_answer/std": 0.0677797719836235, + "step": 229 + }, + { + "completion_length": 528.25, + "completions/clipped_ratio": 0.0, + "completions/max_length": 586.0, + "completions/max_terminated_length": 586.0, + "completions/mean_length": 528.25, + "completions/mean_terminated_length": 528.25, + "completions/min_length": 443.0, + "completions/min_terminated_length": 443.0, + "epoch": 0.04087435578460992, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.5907459259033203, + "kl": 0.9156730771064758, + "learning_rate": 3.7380952380952384e-06, + "loss": 0.0009, + "num_tokens": 673814.0, + "reward": 0.6122113466262817, + "reward_std": 0.06687498092651367, + "rewards/check_answer/mean": 0.6122113466262817, + "rewards/check_answer/std": 0.06687495857477188, + "step": 230 + }, + { + "completion_length": 572.25, + "completions/clipped_ratio": 0.0, + "completions/max_length": 606.0, + "completions/max_terminated_length": 606.0, + "completions/mean_length": 572.25, + "completions/mean_terminated_length": 572.25, + "completions/min_length": 514.0, + "completions/min_terminated_length": 514.0, + "epoch": 0.04105207037497779, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.419886589050293, + "kl": 0.9749362170696259, + "learning_rate": 3.7301587301587305e-06, + "loss": 0.001, + "num_tokens": 677267.0, + "reward": 0.8499937653541565, + "reward_std": 0.16837330162525177, + "rewards/check_answer/mean": 0.8499937653541565, + "rewards/check_answer/std": 0.16837328672409058, + "step": 231 + }, + { + "completion_length": 461.75, + "completions/clipped_ratio": 0.0, + "completions/max_length": 542.0, + "completions/max_terminated_length": 542.0, + "completions/mean_length": 461.75, + "completions/mean_terminated_length": 461.75, + "completions/min_length": 377.0, + "completions/min_terminated_length": 377.0, + "epoch": 0.041229784965345656, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.645576000213623, + "kl": 0.988830104470253, + "learning_rate": 3.7222222222222225e-06, + "loss": 0.001, + "num_tokens": 680234.0, + "reward": 0.9341785311698914, + "reward_std": 0.23470927774906158, + "rewards/check_answer/mean": 0.9341785311698914, + "rewards/check_answer/std": 0.23470927774906158, + "step": 232 + }, + { + "completion_length": 610.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 754.0, + "completions/max_terminated_length": 754.0, + "completions/mean_length": 610.0, + "completions/mean_terminated_length": 610.0, + "completions/min_length": 448.0, + "completions/min_terminated_length": 448.0, + "epoch": 0.041407499555713524, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.2507656812667847, + "kl": 0.7385053485631943, + "learning_rate": 3.7142857142857146e-06, + "loss": 0.0007, + "num_tokens": 683910.0, + "reward": 0.5352281332015991, + "reward_std": 0.04549167677760124, + "rewards/check_answer/mean": 0.5352281332015991, + "rewards/check_answer/std": 0.04549166187644005, + "step": 233 + }, + { + "completion_length": 609.75, + "completions/clipped_ratio": 0.0, + "completions/max_length": 648.0, + "completions/max_terminated_length": 648.0, + "completions/mean_length": 609.75, + "completions/mean_terminated_length": 609.75, + "completions/min_length": 558.0, + "completions/min_terminated_length": 558.0, + "epoch": 0.04158521414608139, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3870525360107422, + "kl": 0.7475785911083221, + "learning_rate": 3.7063492063492067e-06, + "loss": 0.0007, + "num_tokens": 687569.0, + "reward": 0.4529799520969391, + "reward_std": 0.045896902680397034, + "rewards/check_answer/mean": 0.4529799520969391, + "rewards/check_answer/std": 0.04589690640568733, + "step": 234 + }, + { + "completion_length": 511.75, + "completions/clipped_ratio": 0.0, + "completions/max_length": 583.0, + "completions/max_terminated_length": 583.0, + "completions/mean_length": 511.75, + "completions/mean_terminated_length": 511.75, + "completions/min_length": 420.0, + "completions/min_terminated_length": 420.0, + "epoch": 0.04176292873644926, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.6288220882415771, + "kl": 0.9887201339006424, + "learning_rate": 3.6984126984126987e-06, + "loss": 0.001, + "num_tokens": 690784.0, + "reward": 0.705742359161377, + "reward_std": 0.1110423356294632, + "rewards/check_answer/mean": 0.705742359161377, + "rewards/check_answer/std": 0.11104235798120499, + "step": 235 + }, + { + "completion_length": 531.5, + "completions/clipped_ratio": 0.0, + "completions/max_length": 603.0, + "completions/max_terminated_length": 603.0, + "completions/mean_length": 531.5, + "completions/mean_terminated_length": 531.5, + "completions/min_length": 350.0, + "completions/min_terminated_length": 350.0, + "epoch": 0.04194064332681713, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.2495639324188232, + "kl": 0.7813977897167206, + "learning_rate": 3.690476190476191e-06, + "loss": 0.0008, + "num_tokens": 694166.0, + "reward": 1.2415096759796143, + "reward_std": 0.19065117835998535, + "rewards/check_answer/mean": 1.2415096759796143, + "rewards/check_answer/std": 0.19065116345882416, + "step": 236 + }, + { + "completion_length": 403.25, + "completions/clipped_ratio": 0.0, + "completions/max_length": 450.0, + "completions/max_terminated_length": 450.0, + "completions/mean_length": 403.25, + "completions/mean_terminated_length": 403.25, + "completions/min_length": 345.0, + "completions/min_terminated_length": 345.0, + "epoch": 0.042118357917185004, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.8500796556472778, + "kl": 1.0187488496303558, + "learning_rate": 3.6825396825396833e-06, + "loss": 0.001, + "num_tokens": 696903.0, + "reward": 0.6371432542800903, + "reward_std": 0.07777383923530579, + "rewards/check_answer/mean": 0.6371432542800903, + "rewards/check_answer/std": 0.07777383923530579, + "step": 237 + }, + { + "completion_length": 454.5, + "completions/clipped_ratio": 0.0, + "completions/max_length": 533.0, + "completions/max_terminated_length": 533.0, + "completions/mean_length": 454.5, + "completions/mean_terminated_length": 454.5, + "completions/min_length": 386.0, + "completions/min_terminated_length": 386.0, + "epoch": 0.04229607250755287, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.4619567394256592, + "kl": 0.7681007236242294, + "learning_rate": 3.6746031746031754e-06, + "loss": 0.0008, + "num_tokens": 699897.0, + "reward": 0.6196759939193726, + "reward_std": 0.051524724811315536, + "rewards/check_answer/mean": 0.6196759939193726, + "rewards/check_answer/std": 0.051524724811315536, + "step": 238 + }, + { + "completion_length": 376.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 462.0, + "completions/max_terminated_length": 462.0, + "completions/mean_length": 376.0, + "completions/mean_terminated_length": 376.0, + "completions/min_length": 311.0, + "completions/min_terminated_length": 311.0, + "epoch": 0.04247378709792074, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.5873010158538818, + "kl": 0.9846309572458267, + "learning_rate": 3.6666666666666666e-06, + "loss": 0.001, + "num_tokens": 702549.0, + "reward": 0.5979254245758057, + "reward_std": 0.08328143507242203, + "rewards/check_answer/mean": 0.5979254245758057, + "rewards/check_answer/std": 0.08328142762184143, + "step": 239 + }, + { + "completion_length": 417.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 492.0, + "completions/max_terminated_length": 492.0, + "completions/mean_length": 417.0, + "completions/mean_terminated_length": 417.0, + "completions/min_length": 343.0, + "completions/min_terminated_length": 343.0, + "epoch": 0.04265150168828861, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.7484502792358398, + "kl": 1.022307112812996, + "learning_rate": 3.6587301587301586e-06, + "loss": 0.001, + "num_tokens": 705357.0, + "reward": 0.35202890634536743, + "reward_std": 0.06691336631774902, + "rewards/check_answer/mean": 0.35202890634536743, + "rewards/check_answer/std": 0.06691337376832962, + "step": 240 + }, + { + "completion_length": 423.5, + "completions/clipped_ratio": 0.0, + "completions/max_length": 551.0, + "completions/max_terminated_length": 551.0, + "completions/mean_length": 423.5, + "completions/mean_terminated_length": 423.5, + "completions/min_length": 363.0, + "completions/min_terminated_length": 363.0, + "epoch": 0.042829216278656476, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.7925474643707275, + "kl": 1.7321273684501648, + "learning_rate": 3.6507936507936507e-06, + "loss": 0.0017, + "num_tokens": 708235.0, + "reward": 0.6433290243148804, + "reward_std": 0.17733269929885864, + "rewards/check_answer/mean": 0.6433290243148804, + "rewards/check_answer/std": 0.17733269929885864, + "step": 241 + }, + { + "completion_length": 344.75, + "completions/clipped_ratio": 0.0, + "completions/max_length": 444.0, + "completions/max_terminated_length": 444.0, + "completions/mean_length": 344.75, + "completions/mean_terminated_length": 344.75, + "completions/min_length": 244.0, + "completions/min_terminated_length": 244.0, + "epoch": 0.043006930869024344, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.7820135354995728, + "kl": 1.646221399307251, + "learning_rate": 3.642857142857143e-06, + "loss": 0.0016, + "num_tokens": 710738.0, + "reward": 0.4637607932090759, + "reward_std": 0.2009269893169403, + "rewards/check_answer/mean": 0.4637607932090759, + "rewards/check_answer/std": 0.2009270042181015, + "step": 242 + }, + { + "completion_length": 279.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 336.0, + "completions/max_terminated_length": 336.0, + "completions/mean_length": 279.0, + "completions/mean_terminated_length": 279.0, + "completions/min_length": 233.0, + "completions/min_terminated_length": 233.0, + "epoch": 0.04318464545939222, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.884946346282959, + "kl": 1.4627968966960907, + "learning_rate": 3.6349206349206353e-06, + "loss": 0.0015, + "num_tokens": 712954.0, + "reward": 0.08365356922149658, + "reward_std": 0.048354603350162506, + "rewards/check_answer/mean": 0.08365356922149658, + "rewards/check_answer/std": 0.048354603350162506, + "step": 243 + }, + { + "completion_length": 521.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 646.0, + "completions/max_terminated_length": 646.0, + "completions/mean_length": 521.0, + "completions/mean_terminated_length": 521.0, + "completions/min_length": 433.0, + "completions/min_terminated_length": 433.0, + "epoch": 0.04336236004976009, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3385543823242188, + "kl": 0.8651070147752762, + "learning_rate": 3.6269841269841273e-06, + "loss": 0.0009, + "num_tokens": 716238.0, + "reward": 0.8709990382194519, + "reward_std": 0.025177743285894394, + "rewards/check_answer/mean": 0.8709990382194519, + "rewards/check_answer/std": 0.025177743285894394, + "step": 244 + }, + { + "completion_length": 534.75, + "completions/clipped_ratio": 0.0, + "completions/max_length": 588.0, + "completions/max_terminated_length": 588.0, + "completions/mean_length": 534.75, + "completions/mean_terminated_length": 534.75, + "completions/min_length": 391.0, + "completions/min_terminated_length": 391.0, + "epoch": 0.043540074640127956, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.4057061672210693, + "kl": 1.1244578957557678, + "learning_rate": 3.6190476190476194e-06, + "loss": 0.0011, + "num_tokens": 719533.0, + "reward": 0.9694785475730896, + "reward_std": 0.1145370677113533, + "rewards/check_answer/mean": 0.9694785475730896, + "rewards/check_answer/std": 0.1145370677113533, + "step": 245 + }, + { + "completion_length": 387.25, + "completions/clipped_ratio": 0.0, + "completions/max_length": 416.0, + "completions/max_terminated_length": 416.0, + "completions/mean_length": 387.25, + "completions/mean_terminated_length": 387.25, + "completions/min_length": 325.0, + "completions/min_terminated_length": 325.0, + "epoch": 0.043717789230495824, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.820640206336975, + "kl": 1.5689309239387512, + "learning_rate": 3.6111111111111115e-06, + "loss": 0.0016, + "num_tokens": 722202.0, + "reward": 0.4090389609336853, + "reward_std": 0.09788350760936737, + "rewards/check_answer/mean": 0.4090389609336853, + "rewards/check_answer/std": 0.09788351505994797, + "step": 246 + }, + { + "completion_length": 524.75, + "completions/clipped_ratio": 0.0, + "completions/max_length": 615.0, + "completions/max_terminated_length": 615.0, + "completions/mean_length": 524.75, + "completions/mean_terminated_length": 524.75, + "completions/min_length": 416.0, + "completions/min_terminated_length": 416.0, + "epoch": 0.04389550382086369, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3757284879684448, + "kl": 0.7826510369777679, + "learning_rate": 3.6031746031746035e-06, + "loss": 0.0008, + "num_tokens": 725489.0, + "reward": 0.9417409896850586, + "reward_std": 0.39931824803352356, + "rewards/check_answer/mean": 0.9417409896850586, + "rewards/check_answer/std": 0.39931821823120117, + "step": 247 + }, + { + "completion_length": 692.75, + "completions/clipped_ratio": 0.0, + "completions/max_length": 981.0, + "completions/max_terminated_length": 981.0, + "completions/mean_length": 692.75, + "completions/mean_terminated_length": 692.75, + "completions/min_length": 486.0, + "completions/min_terminated_length": 486.0, + "epoch": 0.04407321841123156, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.1299306154251099, + "kl": 0.7229564934968948, + "learning_rate": 3.5952380952380956e-06, + "loss": 0.0007, + "num_tokens": 729500.0, + "reward": 0.6503614783287048, + "reward_std": 0.045035649091005325, + "rewards/check_answer/mean": 0.6503614783287048, + "rewards/check_answer/std": 0.04503565654158592, + "step": 248 + }, + { + "completion_length": 480.5, + "completions/clipped_ratio": 0.0, + "completions/max_length": 591.0, + "completions/max_terminated_length": 591.0, + "completions/mean_length": 480.5, + "completions/mean_terminated_length": 480.5, + "completions/min_length": 351.0, + "completions/min_terminated_length": 351.0, + "epoch": 0.04425093300159943, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3678374290466309, + "kl": 0.8728310763835907, + "learning_rate": 3.5873015873015877e-06, + "loss": 0.0009, + "num_tokens": 732590.0, + "reward": 0.8919079303741455, + "reward_std": 0.10072972625494003, + "rewards/check_answer/mean": 0.8919079303741455, + "rewards/check_answer/std": 0.10072971135377884, + "step": 249 + }, + { + "completion_length": 466.75, + "completions/clipped_ratio": 0.0, + "completions/max_length": 505.0, + "completions/max_terminated_length": 505.0, + "completions/mean_length": 466.75, + "completions/mean_terminated_length": 466.75, + "completions/min_length": 437.0, + "completions/min_terminated_length": 437.0, + "epoch": 0.0444286475919673, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.570044994354248, + "kl": 0.9534753710031509, + "learning_rate": 3.5793650793650797e-06, + "loss": 0.001, + "num_tokens": 735681.0, + "reward": 0.6919865012168884, + "reward_std": 0.014157108031213284, + "rewards/check_answer/mean": 0.6919865012168884, + "rewards/check_answer/std": 0.014157108031213284, + "step": 250 + }, + { + "completion_length": 376.5, + "completions/clipped_ratio": 0.0, + "completions/max_length": 505.0, + "completions/max_terminated_length": 505.0, + "completions/mean_length": 376.5, + "completions/mean_terminated_length": 376.5, + "completions/min_length": 261.0, + "completions/min_terminated_length": 261.0, + "epoch": 0.04460636218233517, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.9166706800460815, + "kl": 0.9647689759731293, + "learning_rate": 3.5714285714285718e-06, + "loss": 0.001, + "num_tokens": 738319.0, + "reward": 0.44414740800857544, + "reward_std": 0.13158369064331055, + "rewards/check_answer/mean": 0.44414740800857544, + "rewards/check_answer/std": 0.13158370554447174, + "step": 251 + }, + { + "completion_length": 319.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 406.0, + "completions/max_terminated_length": 406.0, + "completions/mean_length": 319.0, + "completions/mean_terminated_length": 319.0, + "completions/min_length": 231.0, + "completions/min_terminated_length": 231.0, + "epoch": 0.04478407677270304, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.1092257499694824, + "kl": 1.1556675136089325, + "learning_rate": 3.563492063492064e-06, + "loss": 0.0012, + "num_tokens": 740707.0, + "reward": 0.5653150081634521, + "reward_std": 0.1314965784549713, + "rewards/check_answer/mean": 0.5653150081634521, + "rewards/check_answer/std": 0.1314965784549713, + "step": 252 + }, + { + "completion_length": 321.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 339.0, + "completions/max_terminated_length": 339.0, + "completions/mean_length": 321.0, + "completions/mean_terminated_length": 321.0, + "completions/min_length": 305.0, + "completions/min_terminated_length": 305.0, + "epoch": 0.04496179136307091, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.977848768234253, + "kl": 1.1421026587486267, + "learning_rate": 3.555555555555556e-06, + "loss": 0.0011, + "num_tokens": 743111.0, + "reward": 1.2765793800354004, + "reward_std": 0.37045302987098694, + "rewards/check_answer/mean": 1.2765793800354004, + "rewards/check_answer/std": 0.3704530596733093, + "step": 253 + }, + { + "completion_length": 491.5, + "completions/clipped_ratio": 0.0, + "completions/max_length": 574.0, + "completions/max_terminated_length": 574.0, + "completions/mean_length": 491.5, + "completions/mean_terminated_length": 491.5, + "completions/min_length": 415.0, + "completions/min_terminated_length": 415.0, + "epoch": 0.045139505953438776, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.4590531587600708, + "kl": 1.0585539489984512, + "learning_rate": 3.547619047619048e-06, + "loss": 0.0011, + "num_tokens": 746269.0, + "reward": 0.9823049306869507, + "reward_std": 0.07934407889842987, + "rewards/check_answer/mean": 0.9823049306869507, + "rewards/check_answer/std": 0.07934407889842987, + "step": 254 + }, + { + "completion_length": 429.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 472.0, + "completions/max_terminated_length": 472.0, + "completions/mean_length": 429.0, + "completions/mean_terminated_length": 429.0, + "completions/min_length": 353.0, + "completions/min_terminated_length": 353.0, + "epoch": 0.045317220543806644, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.9310379028320312, + "kl": 1.1345547437667847, + "learning_rate": 3.53968253968254e-06, + "loss": 0.0011, + "num_tokens": 749121.0, + "reward": 0.45538854598999023, + "reward_std": 0.10226347297430038, + "rewards/check_answer/mean": 0.45538854598999023, + "rewards/check_answer/std": 0.10226348042488098, + "step": 255 + }, + { + "completion_length": 404.5, + "completions/clipped_ratio": 0.0, + "completions/max_length": 415.0, + "completions/max_terminated_length": 415.0, + "completions/mean_length": 404.5, + "completions/mean_terminated_length": 404.5, + "completions/min_length": 387.0, + "completions/min_terminated_length": 387.0, + "epoch": 0.04549493513417452, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.4988094568252563, + "kl": 1.2277261018753052, + "learning_rate": 3.531746031746032e-06, + "loss": 0.0012, + "num_tokens": 751847.0, + "reward": 0.504880428314209, + "reward_std": 0.26580870151519775, + "rewards/check_answer/mean": 0.504880428314209, + "rewards/check_answer/std": 0.26580870151519775, + "step": 256 + }, + { + "completion_length": 606.75, + "completions/clipped_ratio": 0.0, + "completions/max_length": 704.0, + "completions/max_terminated_length": 704.0, + "completions/mean_length": 606.75, + "completions/mean_terminated_length": 606.75, + "completions/min_length": 545.0, + "completions/min_terminated_length": 545.0, + "epoch": 0.04567264972454239, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3845511674880981, + "kl": 0.7865335196256638, + "learning_rate": 3.523809523809524e-06, + "loss": 0.0008, + "num_tokens": 755498.0, + "reward": 0.4449111521244049, + "reward_std": 0.04709044471383095, + "rewards/check_answer/mean": 0.4449111521244049, + "rewards/check_answer/std": 0.04709044098854065, + "step": 257 + }, + { + "completion_length": 568.25, + "completions/clipped_ratio": 0.0, + "completions/max_length": 588.0, + "completions/max_terminated_length": 588.0, + "completions/mean_length": 568.25, + "completions/mean_terminated_length": 568.25, + "completions/min_length": 536.0, + "completions/min_terminated_length": 536.0, + "epoch": 0.045850364314910255, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.5005916357040405, + "kl": 0.961561843752861, + "learning_rate": 3.5158730158730162e-06, + "loss": 0.001, + "num_tokens": 758955.0, + "reward": 0.9184130430221558, + "reward_std": 0.08506201952695847, + "rewards/check_answer/mean": 0.9184130430221558, + "rewards/check_answer/std": 0.08506201952695847, + "step": 258 + }, + { + "completion_length": 488.25, + "completions/clipped_ratio": 0.0, + "completions/max_length": 599.0, + "completions/max_terminated_length": 599.0, + "completions/mean_length": 488.25, + "completions/mean_terminated_length": 488.25, + "completions/min_length": 409.0, + "completions/min_terminated_length": 409.0, + "epoch": 0.04602807890527812, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.433339238166809, + "kl": 1.8387483060359955, + "learning_rate": 3.507936507936508e-06, + "loss": 0.0018, + "num_tokens": 762080.0, + "reward": 0.5739443302154541, + "reward_std": 0.04967578500509262, + "rewards/check_answer/mean": 0.5739443302154541, + "rewards/check_answer/std": 0.04967578500509262, + "step": 259 + }, + { + "completion_length": 665.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 779.0, + "completions/max_terminated_length": 779.0, + "completions/mean_length": 665.0, + "completions/mean_terminated_length": 665.0, + "completions/min_length": 594.0, + "completions/min_terminated_length": 594.0, + "epoch": 0.04620579349564599, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3149513006210327, + "kl": 0.5900047421455383, + "learning_rate": 3.5e-06, + "loss": 0.0006, + "num_tokens": 765968.0, + "reward": 0.5332039594650269, + "reward_std": 0.03528818115592003, + "rewards/check_answer/mean": 0.5332039594650269, + "rewards/check_answer/std": 0.03528818115592003, + "step": 260 + }, + { + "completion_length": 470.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 551.0, + "completions/max_terminated_length": 551.0, + "completions/mean_length": 470.0, + "completions/mean_terminated_length": 470.0, + "completions/min_length": 378.0, + "completions/min_terminated_length": 378.0, + "epoch": 0.04638350808601386, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.5507720708847046, + "kl": 0.8859493583440781, + "learning_rate": 3.492063492063492e-06, + "loss": 0.0009, + "num_tokens": 769020.0, + "reward": 1.028480052947998, + "reward_std": 0.039744578301906586, + "rewards/check_answer/mean": 1.028480052947998, + "rewards/check_answer/std": 0.03974457457661629, + "step": 261 + }, + { + "completion_length": 562.5, + "completions/clipped_ratio": 0.0, + "completions/max_length": 594.0, + "completions/max_terminated_length": 594.0, + "completions/mean_length": 562.5, + "completions/mean_terminated_length": 562.5, + "completions/min_length": 518.0, + "completions/min_terminated_length": 518.0, + "epoch": 0.04656122267638173, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.4958258867263794, + "kl": 0.9075725972652435, + "learning_rate": 3.484126984126984e-06, + "loss": 0.0009, + "num_tokens": 772478.0, + "reward": 0.5773640871047974, + "reward_std": 0.07390649616718292, + "rewards/check_answer/mean": 0.5773640871047974, + "rewards/check_answer/std": 0.07390650361776352, + "step": 262 + }, + { + "completion_length": 534.5, + "completions/clipped_ratio": 0.0, + "completions/max_length": 592.0, + "completions/max_terminated_length": 592.0, + "completions/mean_length": 534.5, + "completions/mean_terminated_length": 534.5, + "completions/min_length": 404.0, + "completions/min_terminated_length": 404.0, + "epoch": 0.0467389372667496, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.40358567237854, + "kl": 0.9262001812458038, + "learning_rate": 3.476190476190476e-06, + "loss": 0.0009, + "num_tokens": 775804.0, + "reward": 0.7191150784492493, + "reward_std": 0.18629220128059387, + "rewards/check_answer/mean": 0.7191150784492493, + "rewards/check_answer/std": 0.18629220128059387, + "step": 263 + }, + { + "completion_length": 729.5, + "completions/clipped_ratio": 0.0, + "completions/max_length": 931.0, + "completions/max_terminated_length": 931.0, + "completions/mean_length": 729.5, + "completions/mean_terminated_length": 729.5, + "completions/min_length": 573.0, + "completions/min_terminated_length": 573.0, + "epoch": 0.04691665185711747, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.1973987817764282, + "kl": 0.7220532447099686, + "learning_rate": 3.4682539682539686e-06, + "loss": 0.0007, + "num_tokens": 779970.0, + "reward": 0.8442820310592651, + "reward_std": 0.10566776990890503, + "rewards/check_answer/mean": 0.8442820310592651, + "rewards/check_answer/std": 0.10566778481006622, + "step": 264 + }, + { + "completion_length": 417.5, + "completions/clipped_ratio": 0.0, + "completions/max_length": 596.0, + "completions/max_terminated_length": 596.0, + "completions/mean_length": 417.5, + "completions/mean_terminated_length": 417.5, + "completions/min_length": 335.0, + "completions/min_terminated_length": 335.0, + "epoch": 0.04709436644748534, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.7977854013442993, + "kl": 1.008522853255272, + "learning_rate": 3.4603174603174607e-06, + "loss": 0.001, + "num_tokens": 782792.0, + "reward": 0.4289383888244629, + "reward_std": 0.09566648304462433, + "rewards/check_answer/mean": 0.4289383888244629, + "rewards/check_answer/std": 0.09566648304462433, + "step": 265 + }, + { + "completion_length": 324.5, + "completions/clipped_ratio": 0.0, + "completions/max_length": 376.0, + "completions/max_terminated_length": 376.0, + "completions/mean_length": 324.5, + "completions/mean_terminated_length": 324.5, + "completions/min_length": 201.0, + "completions/min_terminated_length": 201.0, + "epoch": 0.04727208103785321, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.0535054206848145, + "kl": 1.2962539196014404, + "learning_rate": 3.4523809523809528e-06, + "loss": 0.0013, + "num_tokens": 785202.0, + "reward": 0.7310268878936768, + "reward_std": 0.12941241264343262, + "rewards/check_answer/mean": 0.7310268878936768, + "rewards/check_answer/std": 0.12941241264343262, + "step": 266 + }, + { + "completion_length": 397.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 506.0, + "completions/max_terminated_length": 506.0, + "completions/mean_length": 397.0, + "completions/mean_terminated_length": 397.0, + "completions/min_length": 305.0, + "completions/min_terminated_length": 305.0, + "epoch": 0.047449795628221075, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.6865134239196777, + "kl": 1.0350788980722427, + "learning_rate": 3.444444444444445e-06, + "loss": 0.001, + "num_tokens": 787926.0, + "reward": 0.6125245094299316, + "reward_std": 0.042323432862758636, + "rewards/check_answer/mean": 0.6125245094299316, + "rewards/check_answer/std": 0.04232344776391983, + "step": 267 + }, + { + "completion_length": 452.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 496.0, + "completions/max_terminated_length": 496.0, + "completions/mean_length": 452.0, + "completions/mean_terminated_length": 452.0, + "completions/min_length": 372.0, + "completions/min_terminated_length": 372.0, + "epoch": 0.04762751021858894, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.8140619993209839, + "kl": 1.1835395395755768, + "learning_rate": 3.436507936507937e-06, + "loss": 0.0012, + "num_tokens": 790898.0, + "reward": 0.7142941951751709, + "reward_std": 0.04723852872848511, + "rewards/check_answer/mean": 0.7142941951751709, + "rewards/check_answer/std": 0.04723851755261421, + "step": 268 + }, + { + "completion_length": 423.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 485.0, + "completions/max_terminated_length": 485.0, + "completions/mean_length": 423.0, + "completions/mean_terminated_length": 423.0, + "completions/min_length": 310.0, + "completions/min_terminated_length": 310.0, + "epoch": 0.04780522480895682, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.790672779083252, + "kl": 0.9827256351709366, + "learning_rate": 3.428571428571429e-06, + "loss": 0.001, + "num_tokens": 793762.0, + "reward": 0.8307985067367554, + "reward_std": 0.3221786618232727, + "rewards/check_answer/mean": 0.8307985067367554, + "rewards/check_answer/std": 0.3221786618232727, + "step": 269 + }, + { + "completion_length": 358.25, + "completions/clipped_ratio": 0.0, + "completions/max_length": 426.0, + "completions/max_terminated_length": 426.0, + "completions/mean_length": 358.25, + "completions/mean_terminated_length": 358.25, + "completions/min_length": 307.0, + "completions/min_terminated_length": 307.0, + "epoch": 0.04798293939932469, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.941380500793457, + "kl": 1.0881590843200684, + "learning_rate": 3.420634920634921e-06, + "loss": 0.0011, + "num_tokens": 796307.0, + "reward": 0.4849153161048889, + "reward_std": 0.29613256454467773, + "rewards/check_answer/mean": 0.4849153161048889, + "rewards/check_answer/std": 0.2961325943470001, + "step": 270 + }, + { + "completion_length": 326.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 385.0, + "completions/max_terminated_length": 385.0, + "completions/mean_length": 326.0, + "completions/mean_terminated_length": 326.0, + "completions/min_length": 215.0, + "completions/min_terminated_length": 215.0, + "epoch": 0.048160653989692555, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.9679803848266602, + "kl": 1.3071230351924896, + "learning_rate": 3.412698412698413e-06, + "loss": 0.0013, + "num_tokens": 798719.0, + "reward": 0.22918394207954407, + "reward_std": 0.062074821442365646, + "rewards/check_answer/mean": 0.22918394207954407, + "rewards/check_answer/std": 0.062074825167655945, + "step": 271 + }, + { + "completion_length": 398.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 448.0, + "completions/max_terminated_length": 448.0, + "completions/mean_length": 398.0, + "completions/mean_terminated_length": 398.0, + "completions/min_length": 349.0, + "completions/min_terminated_length": 349.0, + "epoch": 0.04833836858006042, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.018653392791748, + "kl": 1.0570762902498245, + "learning_rate": 3.404761904761905e-06, + "loss": 0.0011, + "num_tokens": 801451.0, + "reward": 0.6168738603591919, + "reward_std": 0.2640104293823242, + "rewards/check_answer/mean": 0.6168738603591919, + "rewards/check_answer/std": 0.26401039958000183, + "step": 272 + }, + { + "completion_length": 683.75, + "completions/clipped_ratio": 0.0, + "completions/max_length": 757.0, + "completions/max_terminated_length": 757.0, + "completions/mean_length": 683.75, + "completions/mean_terminated_length": 683.75, + "completions/min_length": 593.0, + "completions/min_terminated_length": 593.0, + "epoch": 0.04851608317042829, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.255545973777771, + "kl": 0.656520888209343, + "learning_rate": 3.3968253968253972e-06, + "loss": 0.0007, + "num_tokens": 805418.0, + "reward": 0.600953221321106, + "reward_std": 0.053118567913770676, + "rewards/check_answer/mean": 0.600953221321106, + "rewards/check_answer/std": 0.053118567913770676, + "step": 273 + }, + { + "completion_length": 410.25, + "completions/clipped_ratio": 0.0, + "completions/max_length": 464.0, + "completions/max_terminated_length": 464.0, + "completions/mean_length": 410.25, + "completions/mean_terminated_length": 410.25, + "completions/min_length": 371.0, + "completions/min_terminated_length": 371.0, + "epoch": 0.04869379776079616, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.6982554197311401, + "kl": 1.035290315747261, + "learning_rate": 3.3888888888888893e-06, + "loss": 0.001, + "num_tokens": 808167.0, + "reward": 0.4320813715457916, + "reward_std": 0.07345610857009888, + "rewards/check_answer/mean": 0.4320813715457916, + "rewards/check_answer/std": 0.07345611602067947, + "step": 274 + }, + { + "completion_length": 362.25, + "completions/clipped_ratio": 0.0, + "completions/max_length": 449.0, + "completions/max_terminated_length": 449.0, + "completions/mean_length": 362.25, + "completions/mean_terminated_length": 362.25, + "completions/min_length": 277.0, + "completions/min_terminated_length": 277.0, + "epoch": 0.04887151235116403, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.048283100128174, + "kl": 1.2046183943748474, + "learning_rate": 3.3809523809523814e-06, + "loss": 0.0012, + "num_tokens": 810744.0, + "reward": 0.3853052854537964, + "reward_std": 0.17090635001659393, + "rewards/check_answer/mean": 0.3853052854537964, + "rewards/check_answer/std": 0.17090635001659393, + "step": 275 + }, + { + "completion_length": 305.25, + "completions/clipped_ratio": 0.0, + "completions/max_length": 355.0, + "completions/max_terminated_length": 355.0, + "completions/mean_length": 305.25, + "completions/mean_terminated_length": 305.25, + "completions/min_length": 209.0, + "completions/min_terminated_length": 209.0, + "epoch": 0.0490492269415319, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.6985806226730347, + "kl": 1.415896326303482, + "learning_rate": 3.3730158730158734e-06, + "loss": 0.0014, + "num_tokens": 813081.0, + "reward": 0.8497557640075684, + "reward_std": 0.18619711697101593, + "rewards/check_answer/mean": 0.8497557640075684, + "rewards/check_answer/std": 0.18619710206985474, + "step": 276 + }, + { + "completion_length": 553.25, + "completions/clipped_ratio": 0.0, + "completions/max_length": 627.0, + "completions/max_terminated_length": 627.0, + "completions/mean_length": 553.25, + "completions/mean_terminated_length": 553.25, + "completions/min_length": 489.0, + "completions/min_terminated_length": 489.0, + "epoch": 0.04922694153189977, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.2501318454742432, + "kl": 0.8245724886655807, + "learning_rate": 3.3650793650793655e-06, + "loss": 0.0008, + "num_tokens": 816462.0, + "reward": 0.7029004096984863, + "reward_std": 0.06054462492465973, + "rewards/check_answer/mean": 0.7029004096984863, + "rewards/check_answer/std": 0.060544636100530624, + "step": 277 + }, + { + "completion_length": 511.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 577.0, + "completions/max_terminated_length": 577.0, + "completions/mean_length": 511.0, + "completions/mean_terminated_length": 511.0, + "completions/min_length": 341.0, + "completions/min_terminated_length": 341.0, + "epoch": 0.04940465612226764, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.5694273710250854, + "kl": 0.9416662156581879, + "learning_rate": 3.357142857142857e-06, + "loss": 0.0009, + "num_tokens": 819678.0, + "reward": 0.6121678948402405, + "reward_std": 0.11947009712457657, + "rewards/check_answer/mean": 0.6121678948402405, + "rewards/check_answer/std": 0.11947011202573776, + "step": 278 + }, + { + "completion_length": 588.25, + "completions/clipped_ratio": 0.0, + "completions/max_length": 637.0, + "completions/max_terminated_length": 637.0, + "completions/mean_length": 588.25, + "completions/mean_terminated_length": 588.25, + "completions/min_length": 566.0, + "completions/min_terminated_length": 566.0, + "epoch": 0.04958237071263551, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.327501893043518, + "kl": 0.7060510069131851, + "learning_rate": 3.349206349206349e-06, + "loss": 0.0007, + "num_tokens": 823227.0, + "reward": 0.8100916147232056, + "reward_std": 0.054197296500205994, + "rewards/check_answer/mean": 0.8100916147232056, + "rewards/check_answer/std": 0.0541972815990448, + "step": 279 + }, + { + "completion_length": 620.25, + "completions/clipped_ratio": 0.0, + "completions/max_length": 661.0, + "completions/max_terminated_length": 661.0, + "completions/mean_length": 620.25, + "completions/mean_terminated_length": 620.25, + "completions/min_length": 580.0, + "completions/min_terminated_length": 580.0, + "epoch": 0.049760085303003375, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3387950658798218, + "kl": 0.7631412148475647, + "learning_rate": 3.3412698412698413e-06, + "loss": 0.0008, + "num_tokens": 826916.0, + "reward": 0.5462270975112915, + "reward_std": 0.060185160487890244, + "rewards/check_answer/mean": 0.5462270975112915, + "rewards/check_answer/std": 0.06018515303730965, + "step": 280 + }, + { + "completion_length": 428.75, + "completions/clipped_ratio": 0.0, + "completions/max_length": 487.0, + "completions/max_terminated_length": 487.0, + "completions/mean_length": 428.75, + "completions/mean_terminated_length": 428.75, + "completions/min_length": 378.0, + "completions/min_terminated_length": 378.0, + "epoch": 0.04993779989337124, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.5694767236709595, + "kl": 1.0430829524993896, + "learning_rate": 3.3333333333333333e-06, + "loss": 0.001, + "num_tokens": 829767.0, + "reward": 1.1746463775634766, + "reward_std": 0.15259021520614624, + "rewards/check_answer/mean": 1.1746463775634766, + "rewards/check_answer/std": 0.15259020030498505, + "step": 281 + }, + { + "completion_length": 363.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 405.0, + "completions/max_terminated_length": 405.0, + "completions/mean_length": 363.0, + "completions/mean_terminated_length": 363.0, + "completions/min_length": 301.0, + "completions/min_terminated_length": 301.0, + "epoch": 0.05011551448373912, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.8097895383834839, + "kl": 1.2064782232046127, + "learning_rate": 3.3253968253968254e-06, + "loss": 0.0012, + "num_tokens": 832327.0, + "reward": 0.6995459794998169, + "reward_std": 0.19229966402053833, + "rewards/check_answer/mean": 0.6995459794998169, + "rewards/check_answer/std": 0.19229967892169952, + "step": 282 + }, + { + "completion_length": 564.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 601.0, + "completions/max_terminated_length": 601.0, + "completions/mean_length": 564.0, + "completions/mean_terminated_length": 564.0, + "completions/min_length": 499.0, + "completions/min_terminated_length": 499.0, + "epoch": 0.050293229074106986, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3872801065444946, + "kl": 0.9052465409040451, + "learning_rate": 3.3174603174603175e-06, + "loss": 0.0009, + "num_tokens": 835767.0, + "reward": 0.8402754068374634, + "reward_std": 0.1533222496509552, + "rewards/check_answer/mean": 0.8402754068374634, + "rewards/check_answer/std": 0.1533222496509552, + "step": 283 + }, + { + "completion_length": 567.25, + "completions/clipped_ratio": 0.0, + "completions/max_length": 637.0, + "completions/max_terminated_length": 637.0, + "completions/mean_length": 567.25, + "completions/mean_terminated_length": 567.25, + "completions/min_length": 483.0, + "completions/min_terminated_length": 483.0, + "epoch": 0.050470943664474854, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.4436535835266113, + "kl": 0.6898186206817627, + "learning_rate": 3.3095238095238095e-06, + "loss": 0.0007, + "num_tokens": 839224.0, + "reward": 0.372622549533844, + "reward_std": 0.02591540291905403, + "rewards/check_answer/mean": 0.372622549533844, + "rewards/check_answer/std": 0.02591540291905403, + "step": 284 + }, + { + "completion_length": 371.75, + "completions/clipped_ratio": 0.0, + "completions/max_length": 427.0, + "completions/max_terminated_length": 427.0, + "completions/mean_length": 371.75, + "completions/mean_terminated_length": 371.75, + "completions/min_length": 307.0, + "completions/min_terminated_length": 307.0, + "epoch": 0.05064865825484272, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.879496693611145, + "kl": 1.3733893036842346, + "learning_rate": 3.3015873015873016e-06, + "loss": 0.0014, + "num_tokens": 841827.0, + "reward": 0.5645827651023865, + "reward_std": 0.12172205001115799, + "rewards/check_answer/mean": 0.5645827651023865, + "rewards/check_answer/std": 0.1217220351099968, + "step": 285 + }, + { + "completion_length": 399.5, + "completions/clipped_ratio": 0.0, + "completions/max_length": 494.0, + "completions/max_terminated_length": 494.0, + "completions/mean_length": 399.5, + "completions/mean_terminated_length": 399.5, + "completions/min_length": 210.0, + "completions/min_terminated_length": 210.0, + "epoch": 0.05082637284521059, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.6859583854675293, + "kl": 1.535703033208847, + "learning_rate": 3.293650793650794e-06, + "loss": 0.0015, + "num_tokens": 844553.0, + "reward": 1.0446977615356445, + "reward_std": 0.1805015206336975, + "rewards/check_answer/mean": 1.0446977615356445, + "rewards/check_answer/std": 0.1805015206336975, + "step": 286 + }, + { + "completion_length": 450.5, + "completions/clipped_ratio": 0.0, + "completions/max_length": 504.0, + "completions/max_terminated_length": 504.0, + "completions/mean_length": 450.5, + "completions/mean_terminated_length": 450.5, + "completions/min_length": 394.0, + "completions/min_terminated_length": 394.0, + "epoch": 0.05100408743557846, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.5410058498382568, + "kl": 0.8857669532299042, + "learning_rate": 3.285714285714286e-06, + "loss": 0.0009, + "num_tokens": 847511.0, + "reward": 0.3833571672439575, + "reward_std": 0.0725344717502594, + "rewards/check_answer/mean": 0.3833571672439575, + "rewards/check_answer/std": 0.0725344717502594, + "step": 287 + }, + { + "completion_length": 525.5, + "completions/clipped_ratio": 0.0, + "completions/max_length": 545.0, + "completions/max_terminated_length": 545.0, + "completions/mean_length": 525.5, + "completions/mean_terminated_length": 525.5, + "completions/min_length": 507.0, + "completions/min_terminated_length": 507.0, + "epoch": 0.05118180202594633, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.534903645515442, + "kl": 0.9601296484470367, + "learning_rate": 3.277777777777778e-06, + "loss": 0.001, + "num_tokens": 850789.0, + "reward": 0.9623134732246399, + "reward_std": 0.11412428319454193, + "rewards/check_answer/mean": 0.9623134732246399, + "rewards/check_answer/std": 0.11412427574396133, + "step": 288 + }, + { + "completion_length": 501.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 532.0, + "completions/max_terminated_length": 532.0, + "completions/mean_length": 501.0, + "completions/mean_terminated_length": 501.0, + "completions/min_length": 464.0, + "completions/min_terminated_length": 464.0, + "epoch": 0.0513595166163142, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.619199275970459, + "kl": 1.0911777317523956, + "learning_rate": 3.2698412698412703e-06, + "loss": 0.0011, + "num_tokens": 853957.0, + "reward": 0.3744562864303589, + "reward_std": 0.029305126518011093, + "rewards/check_answer/mean": 0.3744562864303589, + "rewards/check_answer/std": 0.029305126518011093, + "step": 289 + }, + { + "completion_length": 615.5, + "completions/clipped_ratio": 0.0, + "completions/max_length": 705.0, + "completions/max_terminated_length": 705.0, + "completions/mean_length": 615.5, + "completions/mean_terminated_length": 615.5, + "completions/min_length": 537.0, + "completions/min_terminated_length": 537.0, + "epoch": 0.05153723120668207, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3343628644943237, + "kl": 0.6739598959684372, + "learning_rate": 3.2619047619047623e-06, + "loss": 0.0007, + "num_tokens": 857651.0, + "reward": 1.0102436542510986, + "reward_std": 0.10770290344953537, + "rewards/check_answer/mean": 1.0102436542510986, + "rewards/check_answer/std": 0.10770291090011597, + "step": 290 + }, + { + "completion_length": 428.75, + "completions/clipped_ratio": 0.0, + "completions/max_length": 513.0, + "completions/max_terminated_length": 513.0, + "completions/mean_length": 428.75, + "completions/mean_terminated_length": 428.75, + "completions/min_length": 345.0, + "completions/min_terminated_length": 345.0, + "epoch": 0.05171494579704994, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.6610342264175415, + "kl": 1.076801985502243, + "learning_rate": 3.2539682539682544e-06, + "loss": 0.0011, + "num_tokens": 860514.0, + "reward": 0.9736640453338623, + "reward_std": 0.10310596972703934, + "rewards/check_answer/mean": 0.9736640453338623, + "rewards/check_answer/std": 0.10310596972703934, + "step": 291 + }, + { + "completion_length": 526.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 622.0, + "completions/max_terminated_length": 622.0, + "completions/mean_length": 526.0, + "completions/mean_terminated_length": 526.0, + "completions/min_length": 438.0, + "completions/min_terminated_length": 438.0, + "epoch": 0.051892660387417806, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3932909965515137, + "kl": 0.8913145214319229, + "learning_rate": 3.2460317460317465e-06, + "loss": 0.0009, + "num_tokens": 863774.0, + "reward": 0.6207559704780579, + "reward_std": 0.0433323010802269, + "rewards/check_answer/mean": 0.6207559704780579, + "rewards/check_answer/std": 0.04333232715725899, + "step": 292 + }, + { + "completion_length": 550.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 607.0, + "completions/max_terminated_length": 607.0, + "completions/mean_length": 550.0, + "completions/mean_terminated_length": 550.0, + "completions/min_length": 488.0, + "completions/min_terminated_length": 488.0, + "epoch": 0.052070374977785674, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.4882313013076782, + "kl": 0.8690508455038071, + "learning_rate": 3.2380952380952385e-06, + "loss": 0.0009, + "num_tokens": 867162.0, + "reward": 0.3711192011833191, + "reward_std": 0.09456292539834976, + "rewards/check_answer/mean": 0.3711192011833191, + "rewards/check_answer/std": 0.09456291794776917, + "step": 293 + }, + { + "completion_length": 310.5, + "completions/clipped_ratio": 0.0, + "completions/max_length": 377.0, + "completions/max_terminated_length": 377.0, + "completions/mean_length": 310.5, + "completions/mean_terminated_length": 310.5, + "completions/min_length": 241.0, + "completions/min_terminated_length": 241.0, + "epoch": 0.05224808956815354, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.9652611017227173, + "kl": 2.249081164598465, + "learning_rate": 3.2301587301587306e-06, + "loss": 0.0022, + "num_tokens": 869524.0, + "reward": 0.4758301377296448, + "reward_std": 0.16247986257076263, + "rewards/check_answer/mean": 0.4758301377296448, + "rewards/check_answer/std": 0.16247986257076263, + "step": 294 + }, + { + "completion_length": 476.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 581.0, + "completions/max_terminated_length": 581.0, + "completions/mean_length": 476.0, + "completions/mean_terminated_length": 476.0, + "completions/min_length": 418.0, + "completions/min_terminated_length": 418.0, + "epoch": 0.05242580415852142, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3406497240066528, + "kl": 0.8857537657022476, + "learning_rate": 3.2222222222222227e-06, + "loss": 0.0009, + "num_tokens": 872568.0, + "reward": 0.6054399013519287, + "reward_std": 0.11192188411951065, + "rewards/check_answer/mean": 0.6054399013519287, + "rewards/check_answer/std": 0.11192188411951065, + "step": 295 + }, + { + "completion_length": 510.75, + "completions/clipped_ratio": 0.0, + "completions/max_length": 547.0, + "completions/max_terminated_length": 547.0, + "completions/mean_length": 510.75, + "completions/mean_terminated_length": 510.75, + "completions/min_length": 476.0, + "completions/min_terminated_length": 476.0, + "epoch": 0.052603518748889286, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.6864010095596313, + "kl": 1.1410781145095825, + "learning_rate": 3.2142857142857147e-06, + "loss": 0.0011, + "num_tokens": 875763.0, + "reward": 0.5221328735351562, + "reward_std": 0.12563703954219818, + "rewards/check_answer/mean": 0.5221328735351562, + "rewards/check_answer/std": 0.12563705444335938, + "step": 296 + }, + { + "completion_length": 376.25, + "completions/clipped_ratio": 0.0, + "completions/max_length": 542.0, + "completions/max_terminated_length": 542.0, + "completions/mean_length": 376.25, + "completions/mean_terminated_length": 376.25, + "completions/min_length": 249.0, + "completions/min_terminated_length": 249.0, + "epoch": 0.052781233339257154, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.528670310974121, + "kl": 1.0501147210597992, + "learning_rate": 3.206349206349207e-06, + "loss": 0.0011, + "num_tokens": 878412.0, + "reward": 0.9965839385986328, + "reward_std": 0.08146949857473373, + "rewards/check_answer/mean": 0.9965839385986328, + "rewards/check_answer/std": 0.08146951347589493, + "step": 297 + }, + { + "completion_length": 502.75, + "completions/clipped_ratio": 0.0, + "completions/max_length": 584.0, + "completions/max_terminated_length": 584.0, + "completions/mean_length": 502.75, + "completions/mean_terminated_length": 502.75, + "completions/min_length": 399.0, + "completions/min_terminated_length": 399.0, + "epoch": 0.05295894792962502, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.4674580097198486, + "kl": 0.8372730612754822, + "learning_rate": 3.1984126984126984e-06, + "loss": 0.0008, + "num_tokens": 881591.0, + "reward": 0.4154725670814514, + "reward_std": 0.09623111039400101, + "rewards/check_answer/mean": 0.4154725670814514, + "rewards/check_answer/std": 0.0962311252951622, + "step": 298 + }, + { + "completion_length": 787.75, + "completions/clipped_ratio": 0.0, + "completions/max_length": 877.0, + "completions/max_terminated_length": 877.0, + "completions/mean_length": 787.75, + "completions/mean_terminated_length": 787.75, + "completions/min_length": 755.0, + "completions/min_terminated_length": 755.0, + "epoch": 0.05313666251999289, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0730735063552856, + "kl": 0.6790334284305573, + "learning_rate": 3.1904761904761905e-06, + "loss": 0.0007, + "num_tokens": 885982.0, + "reward": 0.7593076229095459, + "reward_std": 0.01759275607764721, + "rewards/check_answer/mean": 0.7593076229095459, + "rewards/check_answer/std": 0.017592772841453552, + "step": 299 + }, + { + "completion_length": 235.5, + "completions/clipped_ratio": 0.0, + "completions/max_length": 280.0, + "completions/max_terminated_length": 280.0, + "completions/mean_length": 235.5, + "completions/mean_terminated_length": 235.5, + "completions/min_length": 212.0, + "completions/min_terminated_length": 212.0, + "epoch": 0.05331437711036076, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.377175807952881, + "kl": 1.5857573747634888, + "learning_rate": 3.1825396825396826e-06, + "loss": 0.0016, + "num_tokens": 888016.0, + "reward": 0.1881876289844513, + "reward_std": 0.01520258467644453, + "rewards/check_answer/mean": 0.1881876289844513, + "rewards/check_answer/std": 0.015202581882476807, + "step": 300 + }, + { + "completion_length": 577.5, + "completions/clipped_ratio": 0.0, + "completions/max_length": 659.0, + "completions/max_terminated_length": 659.0, + "completions/mean_length": 577.5, + "completions/mean_terminated_length": 577.5, + "completions/min_length": 440.0, + "completions/min_terminated_length": 440.0, + "epoch": 0.053492091700728626, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3808391094207764, + "kl": 0.893027126789093, + "learning_rate": 3.1746031746031746e-06, + "loss": 0.0009, + "num_tokens": 891490.0, + "reward": 0.6694183945655823, + "reward_std": 0.14587241411209106, + "rewards/check_answer/mean": 0.6694183945655823, + "rewards/check_answer/std": 0.14587242901325226, + "step": 301 + }, + { + "completion_length": 638.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 711.0, + "completions/max_terminated_length": 711.0, + "completions/mean_length": 638.0, + "completions/mean_terminated_length": 638.0, + "completions/min_length": 533.0, + "completions/min_terminated_length": 533.0, + "epoch": 0.0536698062910965, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.2870622873306274, + "kl": 0.876729428768158, + "learning_rate": 3.1666666666666667e-06, + "loss": 0.0009, + "num_tokens": 895242.0, + "reward": 0.8270393013954163, + "reward_std": 0.07924500852823257, + "rewards/check_answer/mean": 0.8270393013954163, + "rewards/check_answer/std": 0.07924499362707138, + "step": 302 + }, + { + "completion_length": 597.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 624.0, + "completions/max_terminated_length": 624.0, + "completions/mean_length": 597.0, + "completions/mean_terminated_length": 597.0, + "completions/min_length": 555.0, + "completions/min_terminated_length": 555.0, + "epoch": 0.05384752088146437, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.332850694656372, + "kl": 0.7819689065217972, + "learning_rate": 3.1587301587301588e-06, + "loss": 0.0008, + "num_tokens": 898822.0, + "reward": 0.515304684638977, + "reward_std": 0.06797716021537781, + "rewards/check_answer/mean": 0.515304684638977, + "rewards/check_answer/std": 0.06797716021537781, + "step": 303 + }, + { + "completion_length": 539.25, + "completions/clipped_ratio": 0.0, + "completions/max_length": 551.0, + "completions/max_terminated_length": 551.0, + "completions/mean_length": 539.25, + "completions/mean_terminated_length": 539.25, + "completions/min_length": 517.0, + "completions/min_terminated_length": 517.0, + "epoch": 0.05402523547183224, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3149205446243286, + "kl": 0.7560958564281464, + "learning_rate": 3.150793650793651e-06, + "loss": 0.0008, + "num_tokens": 902183.0, + "reward": 0.7622121572494507, + "reward_std": 0.14100457727909088, + "rewards/check_answer/mean": 0.7622121572494507, + "rewards/check_answer/std": 0.14100460708141327, + "step": 304 + }, + { + "completion_length": 611.5, + "completions/clipped_ratio": 0.0, + "completions/max_length": 677.0, + "completions/max_terminated_length": 677.0, + "completions/mean_length": 611.5, + "completions/mean_terminated_length": 611.5, + "completions/min_length": 548.0, + "completions/min_terminated_length": 548.0, + "epoch": 0.054202950062200106, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.2738730907440186, + "kl": 0.7394523322582245, + "learning_rate": 3.142857142857143e-06, + "loss": 0.0007, + "num_tokens": 905865.0, + "reward": 0.9216010570526123, + "reward_std": 0.15905454754829407, + "rewards/check_answer/mean": 0.9216010570526123, + "rewards/check_answer/std": 0.15905457735061646, + "step": 305 + }, + { + "completion_length": 576.25, + "completions/clipped_ratio": 0.0, + "completions/max_length": 693.0, + "completions/max_terminated_length": 693.0, + "completions/mean_length": 576.25, + "completions/mean_terminated_length": 576.25, + "completions/min_length": 375.0, + "completions/min_terminated_length": 375.0, + "epoch": 0.054380664652567974, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.4615154266357422, + "kl": 0.8034448176622391, + "learning_rate": 3.134920634920635e-06, + "loss": 0.0008, + "num_tokens": 909398.0, + "reward": 0.6731904149055481, + "reward_std": 0.10627910494804382, + "rewards/check_answer/mean": 0.6731904149055481, + "rewards/check_answer/std": 0.10627911239862442, + "step": 306 + }, + { + "completion_length": 289.25, + "completions/clipped_ratio": 0.0, + "completions/max_length": 318.0, + "completions/max_terminated_length": 318.0, + "completions/mean_length": 289.25, + "completions/mean_terminated_length": 289.25, + "completions/min_length": 273.0, + "completions/min_terminated_length": 273.0, + "epoch": 0.05455837924293584, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.1155736446380615, + "kl": 1.6435502469539642, + "learning_rate": 3.126984126984127e-06, + "loss": 0.0016, + "num_tokens": 911655.0, + "reward": 1.1246044635772705, + "reward_std": 0.39093536138534546, + "rewards/check_answer/mean": 1.1246044635772705, + "rewards/check_answer/std": 0.39093536138534546, + "step": 307 + }, + { + "completion_length": 434.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 524.0, + "completions/max_terminated_length": 524.0, + "completions/mean_length": 434.0, + "completions/mean_terminated_length": 434.0, + "completions/min_length": 315.0, + "completions/min_terminated_length": 315.0, + "epoch": 0.05473609383330372, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.6189072132110596, + "kl": 1.117174044251442, + "learning_rate": 3.1190476190476195e-06, + "loss": 0.0011, + "num_tokens": 914515.0, + "reward": 0.6435281038284302, + "reward_std": 0.17675022780895233, + "rewards/check_answer/mean": 0.6435281038284302, + "rewards/check_answer/std": 0.17675024271011353, + "step": 308 + }, + { + "completion_length": 328.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 430.0, + "completions/max_terminated_length": 430.0, + "completions/mean_length": 328.0, + "completions/mean_terminated_length": 328.0, + "completions/min_length": 281.0, + "completions/min_terminated_length": 281.0, + "epoch": 0.054913808423671585, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.084550380706787, + "kl": 1.2165972590446472, + "learning_rate": 3.1111111111111116e-06, + "loss": 0.0012, + "num_tokens": 916951.0, + "reward": 0.33441269397735596, + "reward_std": 0.0777365118265152, + "rewards/check_answer/mean": 0.33441269397735596, + "rewards/check_answer/std": 0.0777365043759346, + "step": 309 + }, + { + "completion_length": 450.75, + "completions/clipped_ratio": 0.0, + "completions/max_length": 568.0, + "completions/max_terminated_length": 568.0, + "completions/mean_length": 450.75, + "completions/mean_terminated_length": 450.75, + "completions/min_length": 283.0, + "completions/min_terminated_length": 283.0, + "epoch": 0.055091523014039454, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.5784587860107422, + "kl": 1.0402155667543411, + "learning_rate": 3.1031746031746036e-06, + "loss": 0.001, + "num_tokens": 919910.0, + "reward": 0.29483291506767273, + "reward_std": 0.32212281227111816, + "rewards/check_answer/mean": 0.29483291506767273, + "rewards/check_answer/std": 0.32212281227111816, + "step": 310 + }, + { + "completion_length": 428.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 452.0, + "completions/max_terminated_length": 452.0, + "completions/mean_length": 428.0, + "completions/mean_terminated_length": 428.0, + "completions/min_length": 392.0, + "completions/min_terminated_length": 392.0, + "epoch": 0.05526923760440732, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.693036675453186, + "kl": 0.9303410053253174, + "learning_rate": 3.0952380952380957e-06, + "loss": 0.0009, + "num_tokens": 922766.0, + "reward": 0.5544993877410889, + "reward_std": 0.06014230474829674, + "rewards/check_answer/mean": 0.5544993877410889, + "rewards/check_answer/std": 0.060142308473587036, + "step": 311 + }, + { + "completion_length": 584.25, + "completions/clipped_ratio": 0.0, + "completions/max_length": 624.0, + "completions/max_terminated_length": 624.0, + "completions/mean_length": 584.25, + "completions/mean_terminated_length": 584.25, + "completions/min_length": 511.0, + "completions/min_terminated_length": 511.0, + "epoch": 0.05544695219477519, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.1301647424697876, + "kl": 0.8142163455486298, + "learning_rate": 3.0873015873015878e-06, + "loss": 0.0008, + "num_tokens": 926339.0, + "reward": 0.5951245427131653, + "reward_std": 0.08697230368852615, + "rewards/check_answer/mean": 0.5951245427131653, + "rewards/check_answer/std": 0.08697231858968735, + "step": 312 + }, + { + "completion_length": 525.75, + "completions/clipped_ratio": 0.0, + "completions/max_length": 562.0, + "completions/max_terminated_length": 562.0, + "completions/mean_length": 525.75, + "completions/mean_terminated_length": 525.75, + "completions/min_length": 430.0, + "completions/min_terminated_length": 430.0, + "epoch": 0.05562466678514306, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.6353572607040405, + "kl": 0.8869746625423431, + "learning_rate": 3.07936507936508e-06, + "loss": 0.0009, + "num_tokens": 929606.0, + "reward": 0.534943163394928, + "reward_std": 0.09478342533111572, + "rewards/check_answer/mean": 0.534943163394928, + "rewards/check_answer/std": 0.09478344023227692, + "step": 313 + }, + { + "completion_length": 526.5, + "completions/clipped_ratio": 0.0, + "completions/max_length": 632.0, + "completions/max_terminated_length": 632.0, + "completions/mean_length": 526.5, + "completions/mean_terminated_length": 526.5, + "completions/min_length": 387.0, + "completions/min_terminated_length": 387.0, + "epoch": 0.055802381375510926, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.2205736637115479, + "kl": 0.8127477318048477, + "learning_rate": 3.071428571428572e-06, + "loss": 0.0008, + "num_tokens": 932924.0, + "reward": 0.8197857141494751, + "reward_std": 0.06929592788219452, + "rewards/check_answer/mean": 0.8197857141494751, + "rewards/check_answer/std": 0.06929592788219452, + "step": 314 + }, + { + "completion_length": 300.25, + "completions/clipped_ratio": 0.0, + "completions/max_length": 366.0, + "completions/max_terminated_length": 366.0, + "completions/mean_length": 300.25, + "completions/mean_terminated_length": 300.25, + "completions/min_length": 244.0, + "completions/min_terminated_length": 244.0, + "epoch": 0.0559800959658788, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.8808090686798096, + "kl": 1.1378408521413803, + "learning_rate": 3.063492063492064e-06, + "loss": 0.0011, + "num_tokens": 935261.0, + "reward": 0.6404327154159546, + "reward_std": 0.09276391565799713, + "rewards/check_answer/mean": 0.6404327154159546, + "rewards/check_answer/std": 0.09276392310857773, + "step": 315 + }, + { + "completion_length": 489.25, + "completions/clipped_ratio": 0.0, + "completions/max_length": 611.0, + "completions/max_terminated_length": 611.0, + "completions/mean_length": 489.25, + "completions/mean_terminated_length": 489.25, + "completions/min_length": 427.0, + "completions/min_terminated_length": 427.0, + "epoch": 0.05615781055624667, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.6700539588928223, + "kl": 1.1479647308588028, + "learning_rate": 3.055555555555556e-06, + "loss": 0.0011, + "num_tokens": 938350.0, + "reward": 0.3595621585845947, + "reward_std": 0.045500971376895905, + "rewards/check_answer/mean": 0.3595621585845947, + "rewards/check_answer/std": 0.045500971376895905, + "step": 316 + }, + { + "completion_length": 603.5, + "completions/clipped_ratio": 0.0, + "completions/max_length": 698.0, + "completions/max_terminated_length": 698.0, + "completions/mean_length": 603.5, + "completions/mean_terminated_length": 603.5, + "completions/min_length": 467.0, + "completions/min_terminated_length": 467.0, + "epoch": 0.05633552514661454, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.1136270761489868, + "kl": 0.8240077495574951, + "learning_rate": 3.047619047619048e-06, + "loss": 0.0008, + "num_tokens": 941984.0, + "reward": 0.6625612378120422, + "reward_std": 0.088188037276268, + "rewards/check_answer/mean": 0.6625612378120422, + "rewards/check_answer/std": 0.08818802982568741, + "step": 317 + }, + { + "completion_length": 625.25, + "completions/clipped_ratio": 0.0, + "completions/max_length": 688.0, + "completions/max_terminated_length": 688.0, + "completions/mean_length": 625.25, + "completions/mean_terminated_length": 625.25, + "completions/min_length": 553.0, + "completions/min_terminated_length": 553.0, + "epoch": 0.056513239736982405, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.1915112733840942, + "kl": 0.6459822058677673, + "learning_rate": 3.0396825396825397e-06, + "loss": 0.0006, + "num_tokens": 945721.0, + "reward": 0.6700223684310913, + "reward_std": 0.042193882167339325, + "rewards/check_answer/mean": 0.6700223684310913, + "rewards/check_answer/std": 0.04219387471675873, + "step": 318 + }, + { + "completion_length": 387.75, + "completions/clipped_ratio": 0.0, + "completions/max_length": 414.0, + "completions/max_terminated_length": 414.0, + "completions/mean_length": 387.75, + "completions/mean_terminated_length": 387.75, + "completions/min_length": 332.0, + "completions/min_terminated_length": 332.0, + "epoch": 0.056690954327350274, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.0014514923095703, + "kl": 1.993764191865921, + "learning_rate": 3.031746031746032e-06, + "loss": 0.002, + "num_tokens": 948400.0, + "reward": 0.6778929233551025, + "reward_std": 0.08552777767181396, + "rewards/check_answer/mean": 0.6778929233551025, + "rewards/check_answer/std": 0.08552776277065277, + "step": 319 + }, + { + "completion_length": 314.25, + "completions/clipped_ratio": 0.0, + "completions/max_length": 351.0, + "completions/max_terminated_length": 351.0, + "completions/mean_length": 314.25, + "completions/mean_terminated_length": 314.25, + "completions/min_length": 301.0, + "completions/min_terminated_length": 301.0, + "epoch": 0.05686866891771814, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.8478176593780518, + "kl": 1.2670190036296844, + "learning_rate": 3.023809523809524e-06, + "loss": 0.0013, + "num_tokens": 950769.0, + "reward": 0.9119844436645508, + "reward_std": 0.3170468211174011, + "rewards/check_answer/mean": 0.9119844436645508, + "rewards/check_answer/std": 0.3170468807220459, + "step": 320 + }, + { + "completion_length": 522.25, + "completions/clipped_ratio": 0.0, + "completions/max_length": 629.0, + "completions/max_terminated_length": 629.0, + "completions/mean_length": 522.25, + "completions/mean_terminated_length": 522.25, + "completions/min_length": 403.0, + "completions/min_terminated_length": 403.0, + "epoch": 0.05704638350808602, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.6379070281982422, + "kl": 0.9933887124061584, + "learning_rate": 3.015873015873016e-06, + "loss": 0.001, + "num_tokens": 954010.0, + "reward": 0.41864892840385437, + "reward_std": 0.18404975533485413, + "rewards/check_answer/mean": 0.41864892840385437, + "rewards/check_answer/std": 0.18404975533485413, + "step": 321 + }, + { + "completion_length": 539.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 624.0, + "completions/max_terminated_length": 624.0, + "completions/mean_length": 539.0, + "completions/mean_terminated_length": 539.0, + "completions/min_length": 466.0, + "completions/min_terminated_length": 466.0, + "epoch": 0.057224098098453885, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3833321332931519, + "kl": 0.9609391242265701, + "learning_rate": 3.007936507936508e-06, + "loss": 0.001, + "num_tokens": 957386.0, + "reward": 0.9422582387924194, + "reward_std": 0.21418391168117523, + "rewards/check_answer/mean": 0.9422582387924194, + "rewards/check_answer/std": 0.21418389678001404, + "step": 322 + }, + { + "completion_length": 376.75, + "completions/clipped_ratio": 0.0, + "completions/max_length": 447.0, + "completions/max_terminated_length": 447.0, + "completions/mean_length": 376.75, + "completions/mean_terminated_length": 376.75, + "completions/min_length": 261.0, + "completions/min_terminated_length": 261.0, + "epoch": 0.05740181268882175, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.7104628086090088, + "kl": 1.065696358680725, + "learning_rate": 3e-06, + "loss": 0.0011, + "num_tokens": 960033.0, + "reward": 0.8748215436935425, + "reward_std": 0.06767245382070541, + "rewards/check_answer/mean": 0.8748215436935425, + "rewards/check_answer/std": 0.06767246872186661, + "step": 323 + }, + { + "completion_length": 251.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 319.0, + "completions/max_terminated_length": 319.0, + "completions/mean_length": 251.0, + "completions/mean_terminated_length": 251.0, + "completions/min_length": 177.0, + "completions/min_terminated_length": 177.0, + "epoch": 0.05757952727918962, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.016751289367676, + "kl": 1.2200357019901276, + "learning_rate": 2.992063492063492e-06, + "loss": 0.0012, + "num_tokens": 962141.0, + "reward": 0.9851218461990356, + "reward_std": 0.5946353673934937, + "rewards/check_answer/mean": 0.9851218461990356, + "rewards/check_answer/std": 0.5946353673934937, + "step": 324 + }, + { + "completion_length": 499.25, + "completions/clipped_ratio": 0.0, + "completions/max_length": 596.0, + "completions/max_terminated_length": 596.0, + "completions/mean_length": 499.25, + "completions/mean_terminated_length": 499.25, + "completions/min_length": 433.0, + "completions/min_terminated_length": 433.0, + "epoch": 0.05775724186955749, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.48799467086792, + "kl": 0.8197063505649567, + "learning_rate": 2.984126984126984e-06, + "loss": 0.0008, + "num_tokens": 965306.0, + "reward": 0.5132365226745605, + "reward_std": 0.08968237042427063, + "rewards/check_answer/mean": 0.5132365226745605, + "rewards/check_answer/std": 0.08968238532543182, + "step": 325 + }, + { + "completion_length": 347.25, + "completions/clipped_ratio": 0.0, + "completions/max_length": 420.0, + "completions/max_terminated_length": 420.0, + "completions/mean_length": 347.25, + "completions/mean_terminated_length": 347.25, + "completions/min_length": 250.0, + "completions/min_terminated_length": 250.0, + "epoch": 0.05793495645992536, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.0635764598846436, + "kl": 1.091228261590004, + "learning_rate": 2.9761904761904763e-06, + "loss": 0.0011, + "num_tokens": 967815.0, + "reward": 0.2954791486263275, + "reward_std": 0.09739544987678528, + "rewards/check_answer/mean": 0.2954791486263275, + "rewards/check_answer/std": 0.09739544987678528, + "step": 326 + }, + { + "completion_length": 586.75, + "completions/clipped_ratio": 0.0, + "completions/max_length": 683.0, + "completions/max_terminated_length": 683.0, + "completions/mean_length": 586.75, + "completions/mean_terminated_length": 586.75, + "completions/min_length": 445.0, + "completions/min_terminated_length": 445.0, + "epoch": 0.058112671050293226, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3021204471588135, + "kl": 0.7141899466514587, + "learning_rate": 2.9682539682539683e-06, + "loss": 0.0007, + "num_tokens": 971330.0, + "reward": 0.5484046936035156, + "reward_std": 0.08494994789361954, + "rewards/check_answer/mean": 0.5484046936035156, + "rewards/check_answer/std": 0.08494995534420013, + "step": 327 + }, + { + "completion_length": 618.5, + "completions/clipped_ratio": 0.0, + "completions/max_length": 637.0, + "completions/max_terminated_length": 637.0, + "completions/mean_length": 618.5, + "completions/mean_terminated_length": 618.5, + "completions/min_length": 591.0, + "completions/min_terminated_length": 591.0, + "epoch": 0.0582903856406611, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.261189341545105, + "kl": 0.7775250673294067, + "learning_rate": 2.9603174603174604e-06, + "loss": 0.0008, + "num_tokens": 975020.0, + "reward": 0.6435350775718689, + "reward_std": 0.07737845182418823, + "rewards/check_answer/mean": 0.6435350775718689, + "rewards/check_answer/std": 0.07737845182418823, + "step": 328 + }, + { + "completion_length": 318.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 399.0, + "completions/max_terminated_length": 399.0, + "completions/mean_length": 318.0, + "completions/mean_terminated_length": 318.0, + "completions/min_length": 216.0, + "completions/min_terminated_length": 216.0, + "epoch": 0.05846810023102897, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.718347191810608, + "kl": 0.993873119354248, + "learning_rate": 2.9523809523809525e-06, + "loss": 0.001, + "num_tokens": 977440.0, + "reward": 1.4608731269836426, + "reward_std": 0.07825374603271484, + "rewards/check_answer/mean": 1.4608731269836426, + "rewards/check_answer/std": 0.07825374603271484, + "step": 329 + }, + { + "completion_length": 290.75, + "completions/clipped_ratio": 0.0, + "completions/max_length": 319.0, + "completions/max_terminated_length": 319.0, + "completions/mean_length": 290.75, + "completions/mean_terminated_length": 290.75, + "completions/min_length": 240.0, + "completions/min_terminated_length": 240.0, + "epoch": 0.05864581482139684, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.06520308554172516, + "kl": 1.8041629493236542, + "learning_rate": 2.944444444444445e-06, + "loss": 0.0018, + "num_tokens": 979699.0, + "reward": 0.5776078701019287, + "reward_std": 0.0, + "rewards/check_answer/mean": 0.5776078701019287, + "rewards/check_answer/std": 0.0, + "step": 330 + }, + { + "completion_length": 491.5, + "completions/clipped_ratio": 0.0, + "completions/max_length": 573.0, + "completions/max_terminated_length": 573.0, + "completions/mean_length": 491.5, + "completions/mean_terminated_length": 491.5, + "completions/min_length": 366.0, + "completions/min_terminated_length": 366.0, + "epoch": 0.058823529411764705, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.6860709190368652, + "kl": 3.7682925313711166, + "learning_rate": 2.936507936507937e-06, + "loss": 0.0038, + "num_tokens": 982845.0, + "reward": 0.6522830724716187, + "reward_std": 0.05248499661684036, + "rewards/check_answer/mean": 0.6522830724716187, + "rewards/check_answer/std": 0.052484992891550064, + "step": 331 + }, + { + "completion_length": 365.75, + "completions/clipped_ratio": 0.0, + "completions/max_length": 423.0, + "completions/max_terminated_length": 423.0, + "completions/mean_length": 365.75, + "completions/mean_terminated_length": 365.75, + "completions/min_length": 298.0, + "completions/min_terminated_length": 298.0, + "epoch": 0.05900124400213257, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.921233892440796, + "kl": 1.0943335890769958, + "learning_rate": 2.928571428571429e-06, + "loss": 0.0011, + "num_tokens": 985436.0, + "reward": 0.9379060864448547, + "reward_std": 0.23881644010543823, + "rewards/check_answer/mean": 0.9379060864448547, + "rewards/check_answer/std": 0.23881642520427704, + "step": 332 + }, + { + "completion_length": 380.5, + "completions/clipped_ratio": 0.0, + "completions/max_length": 465.0, + "completions/max_terminated_length": 465.0, + "completions/mean_length": 380.5, + "completions/mean_terminated_length": 380.5, + "completions/min_length": 285.0, + "completions/min_terminated_length": 285.0, + "epoch": 0.05917895859250044, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3847863674163818, + "kl": 0.9714604318141937, + "learning_rate": 2.920634920634921e-06, + "loss": 0.001, + "num_tokens": 988110.0, + "reward": 0.865478515625, + "reward_std": 0.14856794476509094, + "rewards/check_answer/mean": 0.865478515625, + "rewards/check_answer/std": 0.14856792986392975, + "step": 333 + }, + { + "completion_length": 450.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 527.0, + "completions/max_terminated_length": 527.0, + "completions/mean_length": 450.0, + "completions/mean_terminated_length": 450.0, + "completions/min_length": 325.0, + "completions/min_terminated_length": 325.0, + "epoch": 0.059356673182868316, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.4313362836837769, + "kl": 1.003603994846344, + "learning_rate": 2.9126984126984132e-06, + "loss": 0.001, + "num_tokens": 991110.0, + "reward": 1.0104808807373047, + "reward_std": 0.12709029018878937, + "rewards/check_answer/mean": 1.0104808807373047, + "rewards/check_answer/std": 0.12709030508995056, + "step": 334 + }, + { + "completion_length": 479.25, + "completions/clipped_ratio": 0.0, + "completions/max_length": 657.0, + "completions/max_terminated_length": 657.0, + "completions/mean_length": 479.25, + "completions/mean_terminated_length": 479.25, + "completions/min_length": 387.0, + "completions/min_terminated_length": 387.0, + "epoch": 0.059534387773236185, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.5928345918655396, + "kl": 0.8456375598907471, + "learning_rate": 2.9047619047619053e-06, + "loss": 0.0008, + "num_tokens": 994223.0, + "reward": 0.6031970977783203, + "reward_std": 0.11164192855358124, + "rewards/check_answer/mean": 0.6031970977783203, + "rewards/check_answer/std": 0.11164193600416183, + "step": 335 + }, + { + "completion_length": 638.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 685.0, + "completions/max_terminated_length": 685.0, + "completions/mean_length": 638.0, + "completions/mean_terminated_length": 638.0, + "completions/min_length": 569.0, + "completions/min_terminated_length": 569.0, + "epoch": 0.05971210236360405, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3519030809402466, + "kl": 0.8861069083213806, + "learning_rate": 2.8968253968253974e-06, + "loss": 0.0009, + "num_tokens": 997995.0, + "reward": 0.587228000164032, + "reward_std": 0.04346587508916855, + "rewards/check_answer/mean": 0.587228000164032, + "rewards/check_answer/std": 0.043465860188007355, + "step": 336 + }, + { + "completion_length": 543.75, + "completions/clipped_ratio": 0.0, + "completions/max_length": 603.0, + "completions/max_terminated_length": 603.0, + "completions/mean_length": 543.75, + "completions/mean_terminated_length": 543.75, + "completions/min_length": 416.0, + "completions/min_terminated_length": 416.0, + "epoch": 0.05988981695397192, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.4519314765930176, + "kl": 0.9474461674690247, + "learning_rate": 2.888888888888889e-06, + "loss": 0.0009, + "num_tokens": 1001334.0, + "reward": 0.4379974603652954, + "reward_std": 0.038793571293354034, + "rewards/check_answer/mean": 0.4379974603652954, + "rewards/check_answer/std": 0.038793571293354034, + "step": 337 + }, + { + "completion_length": 462.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 510.0, + "completions/max_terminated_length": 510.0, + "completions/mean_length": 462.0, + "completions/mean_terminated_length": 462.0, + "completions/min_length": 430.0, + "completions/min_terminated_length": 430.0, + "epoch": 0.06006753154433979, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.6436101198196411, + "kl": 0.9353700429201126, + "learning_rate": 2.880952380952381e-06, + "loss": 0.0009, + "num_tokens": 1004330.0, + "reward": 0.7715719938278198, + "reward_std": 0.09180789440870285, + "rewards/check_answer/mean": 0.7715719938278198, + "rewards/check_answer/std": 0.09180789440870285, + "step": 338 + }, + { + "completion_length": 566.25, + "completions/clipped_ratio": 0.0, + "completions/max_length": 584.0, + "completions/max_terminated_length": 584.0, + "completions/mean_length": 566.25, + "completions/mean_terminated_length": 566.25, + "completions/min_length": 525.0, + "completions/min_terminated_length": 525.0, + "epoch": 0.06024524613470766, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.5423588752746582, + "kl": 0.9218340069055557, + "learning_rate": 2.873015873015873e-06, + "loss": 0.0009, + "num_tokens": 1007771.0, + "reward": 0.6818769574165344, + "reward_std": 0.07576140016317368, + "rewards/check_answer/mean": 0.6818769574165344, + "rewards/check_answer/std": 0.07576142251491547, + "step": 339 + }, + { + "completion_length": 282.25, + "completions/clipped_ratio": 0.0, + "completions/max_length": 313.0, + "completions/max_terminated_length": 313.0, + "completions/mean_length": 282.25, + "completions/mean_terminated_length": 282.25, + "completions/min_length": 256.0, + "completions/min_terminated_length": 256.0, + "epoch": 0.06042296072507553, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.2459816932678223, + "kl": 1.3345037996768951, + "learning_rate": 2.865079365079365e-06, + "loss": 0.0013, + "num_tokens": 1009996.0, + "reward": 0.8702346086502075, + "reward_std": 0.32417604327201843, + "rewards/check_answer/mean": 0.8702346086502075, + "rewards/check_answer/std": 0.32417601346969604, + "step": 340 + }, + { + "completion_length": 390.5, + "completions/clipped_ratio": 0.0, + "completions/max_length": 503.0, + "completions/max_terminated_length": 503.0, + "completions/mean_length": 390.5, + "completions/mean_terminated_length": 390.5, + "completions/min_length": 276.0, + "completions/min_terminated_length": 276.0, + "epoch": 0.0606006753154434, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.4166250228881836, + "kl": 1.2890968769788742, + "learning_rate": 2.8571428571428573e-06, + "loss": 0.0013, + "num_tokens": 1012746.0, + "reward": 0.7851279973983765, + "reward_std": 0.10244892537593842, + "rewards/check_answer/mean": 0.7851279973983765, + "rewards/check_answer/std": 0.10244891792535782, + "step": 341 + }, + { + "completion_length": 459.75, + "completions/clipped_ratio": 0.0, + "completions/max_length": 510.0, + "completions/max_terminated_length": 510.0, + "completions/mean_length": 459.75, + "completions/mean_terminated_length": 459.75, + "completions/min_length": 402.0, + "completions/min_terminated_length": 402.0, + "epoch": 0.06077838990581127, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3638399839401245, + "kl": 1.0932938754558563, + "learning_rate": 2.8492063492063493e-06, + "loss": 0.0011, + "num_tokens": 1015749.0, + "reward": 0.7626172304153442, + "reward_std": 0.11766716837882996, + "rewards/check_answer/mean": 0.7626172304153442, + "rewards/check_answer/std": 0.11766716837882996, + "step": 342 + }, + { + "completion_length": 524.5, + "completions/clipped_ratio": 0.0, + "completions/max_length": 598.0, + "completions/max_terminated_length": 598.0, + "completions/mean_length": 524.5, + "completions/mean_terminated_length": 524.5, + "completions/min_length": 341.0, + "completions/min_terminated_length": 341.0, + "epoch": 0.060956104496179137, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.5445250272750854, + "kl": 0.9230055660009384, + "learning_rate": 2.8412698412698414e-06, + "loss": 0.0009, + "num_tokens": 1019039.0, + "reward": 0.6102451086044312, + "reward_std": 0.03914730250835419, + "rewards/check_answer/mean": 0.6102451086044312, + "rewards/check_answer/std": 0.03914730250835419, + "step": 343 + }, + { + "completion_length": 360.5, + "completions/clipped_ratio": 0.0, + "completions/max_length": 459.0, + "completions/max_terminated_length": 459.0, + "completions/mean_length": 360.5, + "completions/mean_terminated_length": 360.5, + "completions/min_length": 273.0, + "completions/min_terminated_length": 273.0, + "epoch": 0.061133819086547005, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.7623164653778076, + "kl": 2.2060094326734543, + "learning_rate": 2.8333333333333335e-06, + "loss": 0.0022, + "num_tokens": 1021613.0, + "reward": 0.5367588996887207, + "reward_std": 0.09137478470802307, + "rewards/check_answer/mean": 0.5367588996887207, + "rewards/check_answer/std": 0.09137478470802307, + "step": 344 + }, + { + "completion_length": 386.75, + "completions/clipped_ratio": 0.0, + "completions/max_length": 481.0, + "completions/max_terminated_length": 481.0, + "completions/mean_length": 386.75, + "completions/mean_terminated_length": 386.75, + "completions/min_length": 254.0, + "completions/min_terminated_length": 254.0, + "epoch": 0.06131153367691487, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.6934525966644287, + "kl": 0.9983743131160736, + "learning_rate": 2.8253968253968255e-06, + "loss": 0.001, + "num_tokens": 1024292.0, + "reward": 0.8309897184371948, + "reward_std": 0.049490053206682205, + "rewards/check_answer/mean": 0.8309897184371948, + "rewards/check_answer/std": 0.049490053206682205, + "step": 345 + }, + { + "completion_length": 679.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 1733.0, + "completions/max_terminated_length": 495.0, + "completions/mean_length": 679.0, + "completions/mean_terminated_length": 327.66668701171875, + "completions/min_length": 240.0, + "completions/min_terminated_length": 240.0, + "epoch": 0.06148924826728274, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.2086621522903442, + "kl": 0.9801141023635864, + "learning_rate": 2.8174603174603176e-06, + "loss": 0.001, + "num_tokens": 1028160.0, + "reward": 0.40882742404937744, + "reward_std": 0.5656044483184814, + "rewards/check_answer/mean": 0.40882742404937744, + "rewards/check_answer/std": 0.5656045079231262, + "step": 346 + }, + { + "completion_length": 499.5, + "completions/clipped_ratio": 0.0, + "completions/max_length": 670.0, + "completions/max_terminated_length": 670.0, + "completions/mean_length": 499.5, + "completions/mean_terminated_length": 499.5, + "completions/min_length": 398.0, + "completions/min_terminated_length": 398.0, + "epoch": 0.061666962857650616, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.5193108320236206, + "kl": 0.8783832341432571, + "learning_rate": 2.8095238095238096e-06, + "loss": 0.0009, + "num_tokens": 1031374.0, + "reward": 0.5393548011779785, + "reward_std": 0.18399730324745178, + "rewards/check_answer/mean": 0.5393548011779785, + "rewards/check_answer/std": 0.18399730324745178, + "step": 347 + }, + { + "completion_length": 426.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 537.0, + "completions/max_terminated_length": 537.0, + "completions/mean_length": 426.0, + "completions/mean_terminated_length": 426.0, + "completions/min_length": 353.0, + "completions/min_terminated_length": 353.0, + "epoch": 0.061844677448018484, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.5595829486846924, + "kl": 2.0859084129333496, + "learning_rate": 2.8015873015873017e-06, + "loss": 0.0021, + "num_tokens": 1034230.0, + "reward": 0.35466447472572327, + "reward_std": 0.007600174751132727, + "rewards/check_answer/mean": 0.35466447472572327, + "rewards/check_answer/std": 0.007600174751132727, + "step": 348 + }, + { + "completion_length": 242.25, + "completions/clipped_ratio": 0.0, + "completions/max_length": 296.0, + "completions/max_terminated_length": 296.0, + "completions/mean_length": 242.25, + "completions/mean_terminated_length": 242.25, + "completions/min_length": 168.0, + "completions/min_terminated_length": 168.0, + "epoch": 0.06202239203838635, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.0255789756774902, + "kl": 1.20065575838089, + "learning_rate": 2.7936507936507938e-06, + "loss": 0.0012, + "num_tokens": 1036303.0, + "reward": 1.0878612995147705, + "reward_std": 0.5185619592666626, + "rewards/check_answer/mean": 1.0878612995147705, + "rewards/check_answer/std": 0.5185619592666626, + "step": 349 + }, + { + "completion_length": 493.25, + "completions/clipped_ratio": 0.0, + "completions/max_length": 607.0, + "completions/max_terminated_length": 607.0, + "completions/mean_length": 493.25, + "completions/mean_terminated_length": 493.25, + "completions/min_length": 254.0, + "completions/min_terminated_length": 254.0, + "epoch": 0.06220010662875422, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.5794947147369385, + "kl": 0.7937883138656616, + "learning_rate": 2.785714285714286e-06, + "loss": 0.0008, + "num_tokens": 1039448.0, + "reward": 0.552399218082428, + "reward_std": 0.027781648561358452, + "rewards/check_answer/mean": 0.552399218082428, + "rewards/check_answer/std": 0.027781641110777855, + "step": 350 + }, + { + "completion_length": 412.5, + "completions/clipped_ratio": 0.0, + "completions/max_length": 506.0, + "completions/max_terminated_length": 506.0, + "completions/mean_length": 412.5, + "completions/mean_terminated_length": 412.5, + "completions/min_length": 317.0, + "completions/min_terminated_length": 317.0, + "epoch": 0.06237782121912209, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.685268759727478, + "kl": 1.08314049243927, + "learning_rate": 2.7777777777777783e-06, + "loss": 0.0011, + "num_tokens": 1042238.0, + "reward": 0.6266283392906189, + "reward_std": 0.12399213761091232, + "rewards/check_answer/mean": 0.6266283392906189, + "rewards/check_answer/std": 0.12399215251207352, + "step": 351 + }, + { + "completion_length": 261.75, + "completions/clipped_ratio": 0.0, + "completions/max_length": 311.0, + "completions/max_terminated_length": 311.0, + "completions/mean_length": 261.75, + "completions/mean_terminated_length": 261.75, + "completions/min_length": 182.0, + "completions/min_terminated_length": 182.0, + "epoch": 0.06255553580948996, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.057827949523926, + "kl": 1.1714966595172882, + "learning_rate": 2.7698412698412704e-06, + "loss": 0.0012, + "num_tokens": 1044401.0, + "reward": 1.1655079126358032, + "reward_std": 0.1512351781129837, + "rewards/check_answer/mean": 1.1655079126358032, + "rewards/check_answer/std": 0.1512351632118225, + "step": 352 + }, + { + "completion_length": 454.75, + "completions/clipped_ratio": 0.0, + "completions/max_length": 590.0, + "completions/max_terminated_length": 590.0, + "completions/mean_length": 454.75, + "completions/mean_terminated_length": 454.75, + "completions/min_length": 349.0, + "completions/min_terminated_length": 349.0, + "epoch": 0.06273325039985783, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.6537492275238037, + "kl": 0.836554616689682, + "learning_rate": 2.7619047619047625e-06, + "loss": 0.0008, + "num_tokens": 1047384.0, + "reward": 0.5927922129631042, + "reward_std": 0.03192928433418274, + "rewards/check_answer/mean": 0.5927922129631042, + "rewards/check_answer/std": 0.031929273158311844, + "step": 353 + }, + { + "completion_length": 449.5, + "completions/clipped_ratio": 0.0, + "completions/max_length": 516.0, + "completions/max_terminated_length": 516.0, + "completions/mean_length": 449.5, + "completions/mean_terminated_length": 449.5, + "completions/min_length": 358.0, + "completions/min_terminated_length": 358.0, + "epoch": 0.06291096499022569, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.5732802152633667, + "kl": 0.9828883409500122, + "learning_rate": 2.7539682539682545e-06, + "loss": 0.001, + "num_tokens": 1050310.0, + "reward": 0.7062097787857056, + "reward_std": 0.07088971138000488, + "rewards/check_answer/mean": 0.7062097787857056, + "rewards/check_answer/std": 0.07088971138000488, + "step": 354 + }, + { + "completion_length": 416.5, + "completions/clipped_ratio": 0.0, + "completions/max_length": 466.0, + "completions/max_terminated_length": 466.0, + "completions/mean_length": 416.5, + "completions/mean_terminated_length": 416.5, + "completions/min_length": 377.0, + "completions/min_terminated_length": 377.0, + "epoch": 0.06308867958059357, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.5273946523666382, + "kl": 0.7866203188896179, + "learning_rate": 2.7460317460317466e-06, + "loss": 0.0008, + "num_tokens": 1053128.0, + "reward": 0.5653597116470337, + "reward_std": 0.10099492967128754, + "rewards/check_answer/mean": 0.5653597116470337, + "rewards/check_answer/std": 0.10099492967128754, + "step": 355 + }, + { + "completion_length": 303.75, + "completions/clipped_ratio": 0.0, + "completions/max_length": 431.0, + "completions/max_terminated_length": 431.0, + "completions/mean_length": 303.75, + "completions/mean_terminated_length": 303.75, + "completions/min_length": 238.0, + "completions/min_terminated_length": 238.0, + "epoch": 0.06326639417096143, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.6983771324157715, + "kl": 1.0617644041776657, + "learning_rate": 2.7380952380952387e-06, + "loss": 0.0011, + "num_tokens": 1055463.0, + "reward": 0.7611073851585388, + "reward_std": 0.21248966455459595, + "rewards/check_answer/mean": 0.7611073851585388, + "rewards/check_answer/std": 0.21248964965343475, + "step": 356 + }, + { + "completion_length": 429.75, + "completions/clipped_ratio": 0.0, + "completions/max_length": 554.0, + "completions/max_terminated_length": 554.0, + "completions/mean_length": 429.75, + "completions/mean_terminated_length": 429.75, + "completions/min_length": 357.0, + "completions/min_terminated_length": 357.0, + "epoch": 0.0634441087613293, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.7513980865478516, + "kl": 0.9004791676998138, + "learning_rate": 2.7301587301587303e-06, + "loss": 0.0009, + "num_tokens": 1058338.0, + "reward": 0.7837266325950623, + "reward_std": 0.41704121232032776, + "rewards/check_answer/mean": 0.7837266325950623, + "rewards/check_answer/std": 0.41704121232032776, + "step": 357 + }, + { + "completion_length": 329.75, + "completions/clipped_ratio": 0.0, + "completions/max_length": 368.0, + "completions/max_terminated_length": 368.0, + "completions/mean_length": 329.75, + "completions/mean_terminated_length": 329.75, + "completions/min_length": 278.0, + "completions/min_terminated_length": 278.0, + "epoch": 0.06362182335169718, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.628482699394226, + "kl": 1.0090267211198807, + "learning_rate": 2.7222222222222224e-06, + "loss": 0.001, + "num_tokens": 1060765.0, + "reward": 0.7090367078781128, + "reward_std": 0.26245641708374023, + "rewards/check_answer/mean": 0.7090367078781128, + "rewards/check_answer/std": 0.26245638728141785, + "step": 358 + }, + { + "completion_length": 407.5, + "completions/clipped_ratio": 0.0, + "completions/max_length": 488.0, + "completions/max_terminated_length": 488.0, + "completions/mean_length": 407.5, + "completions/mean_terminated_length": 407.5, + "completions/min_length": 284.0, + "completions/min_terminated_length": 284.0, + "epoch": 0.06379953794206504, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.618558406829834, + "kl": 2.1619486063718796, + "learning_rate": 2.7142857142857144e-06, + "loss": 0.0022, + "num_tokens": 1063575.0, + "reward": 0.6832454800605774, + "reward_std": 0.13225074112415314, + "rewards/check_answer/mean": 0.6832454800605774, + "rewards/check_answer/std": 0.13225072622299194, + "step": 359 + }, + { + "completion_length": 356.5, + "completions/clipped_ratio": 0.0, + "completions/max_length": 409.0, + "completions/max_terminated_length": 409.0, + "completions/mean_length": 356.5, + "completions/mean_terminated_length": 356.5, + "completions/min_length": 289.0, + "completions/min_terminated_length": 289.0, + "epoch": 0.06397725253243292, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.7556419372558594, + "kl": 0.9982209950685501, + "learning_rate": 2.7063492063492065e-06, + "loss": 0.001, + "num_tokens": 1066133.0, + "reward": 0.9417442083358765, + "reward_std": 0.0924437865614891, + "rewards/check_answer/mean": 0.9417442083358765, + "rewards/check_answer/std": 0.0924437940120697, + "step": 360 + }, + { + "completion_length": 438.75, + "completions/clipped_ratio": 0.0, + "completions/max_length": 506.0, + "completions/max_terminated_length": 506.0, + "completions/mean_length": 438.75, + "completions/mean_terminated_length": 438.75, + "completions/min_length": 383.0, + "completions/min_terminated_length": 383.0, + "epoch": 0.06415496712280078, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.6464420557022095, + "kl": 1.1058107018470764, + "learning_rate": 2.6984126984126986e-06, + "loss": 0.0011, + "num_tokens": 1069028.0, + "reward": 0.5221630930900574, + "reward_std": 0.08845002204179764, + "rewards/check_answer/mean": 0.5221630930900574, + "rewards/check_answer/std": 0.08845002204179764, + "step": 361 + }, + { + "completion_length": 395.25, + "completions/clipped_ratio": 0.0, + "completions/max_length": 545.0, + "completions/max_terminated_length": 545.0, + "completions/mean_length": 395.25, + "completions/mean_terminated_length": 395.25, + "completions/min_length": 289.0, + "completions/min_terminated_length": 289.0, + "epoch": 0.06433268171316865, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.5931658744812012, + "kl": 3.2120852321386337, + "learning_rate": 2.6904761904761906e-06, + "loss": 0.0032, + "num_tokens": 1071749.0, + "reward": 1.010395884513855, + "reward_std": 0.39355483651161194, + "rewards/check_answer/mean": 1.010395884513855, + "rewards/check_answer/std": 0.3935548663139343, + "step": 362 + }, + { + "completion_length": 366.75, + "completions/clipped_ratio": 0.0, + "completions/max_length": 516.0, + "completions/max_terminated_length": 516.0, + "completions/mean_length": 366.75, + "completions/mean_terminated_length": 366.75, + "completions/min_length": 257.0, + "completions/min_terminated_length": 257.0, + "epoch": 0.06451039630353653, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.682044506072998, + "kl": 1.1539453864097595, + "learning_rate": 2.6825396825396827e-06, + "loss": 0.0012, + "num_tokens": 1074356.0, + "reward": 0.6987335681915283, + "reward_std": 0.0868241935968399, + "rewards/check_answer/mean": 0.6987335681915283, + "rewards/check_answer/std": 0.0868242084980011, + "step": 363 + }, + { + "completion_length": 361.75, + "completions/clipped_ratio": 0.0, + "completions/max_length": 570.0, + "completions/max_terminated_length": 570.0, + "completions/mean_length": 361.75, + "completions/mean_terminated_length": 361.75, + "completions/min_length": 237.0, + "completions/min_terminated_length": 237.0, + "epoch": 0.06468811089390439, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.6390739679336548, + "kl": 1.1847519278526306, + "learning_rate": 2.6746031746031748e-06, + "loss": 0.0012, + "num_tokens": 1076923.0, + "reward": 0.8292936682701111, + "reward_std": 0.20514844357967377, + "rewards/check_answer/mean": 0.8292936682701111, + "rewards/check_answer/std": 0.20514844357967377, + "step": 364 + }, + { + "completion_length": 292.5, + "completions/clipped_ratio": 0.0, + "completions/max_length": 390.0, + "completions/max_terminated_length": 390.0, + "completions/mean_length": 292.5, + "completions/mean_terminated_length": 292.5, + "completions/min_length": 226.0, + "completions/min_terminated_length": 226.0, + "epoch": 0.06486582548427226, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.981711745262146, + "kl": 1.2689296901226044, + "learning_rate": 2.666666666666667e-06, + "loss": 0.0013, + "num_tokens": 1079209.0, + "reward": 0.9906752705574036, + "reward_std": 0.468296080827713, + "rewards/check_answer/mean": 0.9906752705574036, + "rewards/check_answer/std": 0.468296080827713, + "step": 365 + }, + { + "completion_length": 304.25, + "completions/clipped_ratio": 0.0, + "completions/max_length": 353.0, + "completions/max_terminated_length": 353.0, + "completions/mean_length": 304.25, + "completions/mean_terminated_length": 304.25, + "completions/min_length": 236.0, + "completions/min_terminated_length": 236.0, + "epoch": 0.06504354007464012, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.9962329864501953, + "kl": 1.0153531730175018, + "learning_rate": 2.658730158730159e-06, + "loss": 0.001, + "num_tokens": 1081590.0, + "reward": 0.4718705713748932, + "reward_std": 0.1988232433795929, + "rewards/check_answer/mean": 0.4718705713748932, + "rewards/check_answer/std": 0.1988232433795929, + "step": 366 + }, + { + "completion_length": 586.5, + "completions/clipped_ratio": 0.0, + "completions/max_length": 696.0, + "completions/max_terminated_length": 696.0, + "completions/mean_length": 586.5, + "completions/mean_terminated_length": 586.5, + "completions/min_length": 508.0, + "completions/min_terminated_length": 508.0, + "epoch": 0.065221254665008, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.2512035369873047, + "kl": 1.0086099058389664, + "learning_rate": 2.650793650793651e-06, + "loss": 0.001, + "num_tokens": 1085176.0, + "reward": 0.49589335918426514, + "reward_std": 0.02800540067255497, + "rewards/check_answer/mean": 0.49589335918426514, + "rewards/check_answer/std": 0.02800540253520012, + "step": 367 + }, + { + "completion_length": 653.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 811.0, + "completions/max_terminated_length": 811.0, + "completions/mean_length": 653.0, + "completions/mean_terminated_length": 653.0, + "completions/min_length": 457.0, + "completions/min_terminated_length": 457.0, + "epoch": 0.06539896925537586, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.2003167867660522, + "kl": 0.7881456017494202, + "learning_rate": 2.642857142857143e-06, + "loss": 0.0008, + "num_tokens": 1089024.0, + "reward": 0.7394192218780518, + "reward_std": 0.09663909673690796, + "rewards/check_answer/mean": 0.7394192218780518, + "rewards/check_answer/std": 0.09663910418748856, + "step": 368 + }, + { + "completion_length": 456.25, + "completions/clipped_ratio": 0.0, + "completions/max_length": 541.0, + "completions/max_terminated_length": 541.0, + "completions/mean_length": 456.25, + "completions/mean_terminated_length": 456.25, + "completions/min_length": 340.0, + "completions/min_terminated_length": 340.0, + "epoch": 0.06557668384574374, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.5003619194030762, + "kl": 0.8792458027601242, + "learning_rate": 2.634920634920635e-06, + "loss": 0.0009, + "num_tokens": 1092085.0, + "reward": 0.271815687417984, + "reward_std": 0.048961468040943146, + "rewards/check_answer/mean": 0.271815687417984, + "rewards/check_answer/std": 0.04896145686507225, + "step": 369 + }, + { + "completion_length": 241.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 281.0, + "completions/max_terminated_length": 281.0, + "completions/mean_length": 241.0, + "completions/mean_terminated_length": 241.0, + "completions/min_length": 185.0, + "completions/min_terminated_length": 185.0, + "epoch": 0.06575439843611161, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.5538017749786377, + "kl": 1.2450654804706573, + "learning_rate": 2.626984126984127e-06, + "loss": 0.0012, + "num_tokens": 1094145.0, + "reward": 1.040844202041626, + "reward_std": 0.6358775496482849, + "rewards/check_answer/mean": 1.040844202041626, + "rewards/check_answer/std": 0.6358775496482849, + "step": 370 + }, + { + "completion_length": 413.75, + "completions/clipped_ratio": 0.0, + "completions/max_length": 517.0, + "completions/max_terminated_length": 517.0, + "completions/mean_length": 413.75, + "completions/mean_terminated_length": 413.75, + "completions/min_length": 310.0, + "completions/min_terminated_length": 310.0, + "epoch": 0.06593211302647947, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.8476204872131348, + "kl": 1.6918334662914276, + "learning_rate": 2.6190476190476192e-06, + "loss": 0.0017, + "num_tokens": 1096956.0, + "reward": 1.0639305114746094, + "reward_std": 0.0493079274892807, + "rewards/check_answer/mean": 1.0639305114746094, + "rewards/check_answer/std": 0.049307931214571, + "step": 371 + }, + { + "completion_length": 253.25, + "completions/clipped_ratio": 0.0, + "completions/max_length": 295.0, + "completions/max_terminated_length": 295.0, + "completions/mean_length": 253.25, + "completions/mean_terminated_length": 253.25, + "completions/min_length": 205.0, + "completions/min_terminated_length": 205.0, + "epoch": 0.06610982761684735, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.1900248527526855, + "kl": 1.0722286999225616, + "learning_rate": 2.6111111111111113e-06, + "loss": 0.0011, + "num_tokens": 1099077.0, + "reward": 0.3582559823989868, + "reward_std": 0.037198010832071304, + "rewards/check_answer/mean": 0.3582559823989868, + "rewards/check_answer/std": 0.037198010832071304, + "step": 372 + }, + { + "completion_length": 339.5, + "completions/clipped_ratio": 0.0, + "completions/max_length": 406.0, + "completions/max_terminated_length": 406.0, + "completions/mean_length": 339.5, + "completions/mean_terminated_length": 339.5, + "completions/min_length": 286.0, + "completions/min_terminated_length": 286.0, + "epoch": 0.06628754220721521, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.416177749633789, + "kl": 0.759627029299736, + "learning_rate": 2.6031746031746038e-06, + "loss": 0.0008, + "num_tokens": 1101567.0, + "reward": 0.5204699039459229, + "reward_std": 0.20935100317001343, + "rewards/check_answer/mean": 0.5204699039459229, + "rewards/check_answer/std": 0.20935101807117462, + "step": 373 + }, + { + "completion_length": 581.75, + "completions/clipped_ratio": 0.0, + "completions/max_length": 717.0, + "completions/max_terminated_length": 717.0, + "completions/mean_length": 581.75, + "completions/mean_terminated_length": 581.75, + "completions/min_length": 361.0, + "completions/min_terminated_length": 361.0, + "epoch": 0.06646525679758308, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3829495906829834, + "kl": 0.7596141695976257, + "learning_rate": 2.595238095238096e-06, + "loss": 0.0008, + "num_tokens": 1105114.0, + "reward": 0.6244233846664429, + "reward_std": 0.1169346272945404, + "rewards/check_answer/mean": 0.6244233846664429, + "rewards/check_answer/std": 0.1169346421957016, + "step": 374 + }, + { + "completion_length": 553.25, + "completions/clipped_ratio": 0.0, + "completions/max_length": 615.0, + "completions/max_terminated_length": 615.0, + "completions/mean_length": 553.25, + "completions/mean_terminated_length": 553.25, + "completions/min_length": 532.0, + "completions/min_terminated_length": 532.0, + "epoch": 0.06664297138795094, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.049517273902893, + "kl": 0.731134757399559, + "learning_rate": 2.587301587301588e-06, + "loss": 0.0007, + "num_tokens": 1108555.0, + "reward": 0.9090848565101624, + "reward_std": 0.053088050335645676, + "rewards/check_answer/mean": 0.9090848565101624, + "rewards/check_answer/std": 0.05308803915977478, + "step": 375 + }, + { + "completion_length": 581.5, + "completions/clipped_ratio": 0.0, + "completions/max_length": 793.0, + "completions/max_terminated_length": 793.0, + "completions/mean_length": 581.5, + "completions/mean_terminated_length": 581.5, + "completions/min_length": 442.0, + "completions/min_terminated_length": 442.0, + "epoch": 0.06682068597831882, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.1811550855636597, + "kl": 0.7464872002601624, + "learning_rate": 2.57936507936508e-06, + "loss": 0.0007, + "num_tokens": 1112113.0, + "reward": 0.5068844556808472, + "reward_std": 0.08424738049507141, + "rewards/check_answer/mean": 0.5068844556808472, + "rewards/check_answer/std": 0.08424738049507141, + "step": 376 + }, + { + "completion_length": 456.25, + "completions/clipped_ratio": 0.0, + "completions/max_length": 531.0, + "completions/max_terminated_length": 531.0, + "completions/mean_length": 456.25, + "completions/mean_terminated_length": 456.25, + "completions/min_length": 331.0, + "completions/min_terminated_length": 331.0, + "epoch": 0.0669984005686867, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.6065852642059326, + "kl": 1.0195069313049316, + "learning_rate": 2.571428571428571e-06, + "loss": 0.001, + "num_tokens": 1115090.0, + "reward": 0.9967861771583557, + "reward_std": 0.1993342638015747, + "rewards/check_answer/mean": 0.9967861771583557, + "rewards/check_answer/std": 0.1993342638015747, + "step": 377 + }, + { + "completion_length": 570.25, + "completions/clipped_ratio": 0.0, + "completions/max_length": 772.0, + "completions/max_terminated_length": 772.0, + "completions/mean_length": 570.25, + "completions/mean_terminated_length": 570.25, + "completions/min_length": 417.0, + "completions/min_terminated_length": 417.0, + "epoch": 0.06717611515905456, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.288891315460205, + "kl": 0.9126172959804535, + "learning_rate": 2.5634920634920637e-06, + "loss": 0.0009, + "num_tokens": 1118543.0, + "reward": 0.9744613170623779, + "reward_std": 0.10629359632730484, + "rewards/check_answer/mean": 0.9744613170623779, + "rewards/check_answer/std": 0.10629359632730484, + "step": 378 + }, + { + "completion_length": 584.25, + "completions/clipped_ratio": 0.0, + "completions/max_length": 758.0, + "completions/max_terminated_length": 758.0, + "completions/mean_length": 584.25, + "completions/mean_terminated_length": 584.25, + "completions/min_length": 336.0, + "completions/min_terminated_length": 336.0, + "epoch": 0.06735382974942243, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.518787145614624, + "kl": 1.7292539477348328, + "learning_rate": 2.5555555555555557e-06, + "loss": 0.0017, + "num_tokens": 1122052.0, + "reward": 0.9780303835868835, + "reward_std": 0.11002466827630997, + "rewards/check_answer/mean": 0.9780303835868835, + "rewards/check_answer/std": 0.11002466827630997, + "step": 379 + }, + { + "completion_length": 464.75, + "completions/clipped_ratio": 0.0, + "completions/max_length": 674.0, + "completions/max_terminated_length": 674.0, + "completions/mean_length": 464.75, + "completions/mean_terminated_length": 464.75, + "completions/min_length": 313.0, + "completions/min_terminated_length": 313.0, + "epoch": 0.06753154433979029, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.394750952720642, + "kl": 0.9494964927434921, + "learning_rate": 2.547619047619048e-06, + "loss": 0.0009, + "num_tokens": 1125083.0, + "reward": 0.49630308151245117, + "reward_std": 0.14733171463012695, + "rewards/check_answer/mean": 0.49630308151245117, + "rewards/check_answer/std": 0.14733172953128815, + "step": 380 + }, + { + "completion_length": 366.75, + "completions/clipped_ratio": 0.0, + "completions/max_length": 422.0, + "completions/max_terminated_length": 422.0, + "completions/mean_length": 366.75, + "completions/mean_terminated_length": 366.75, + "completions/min_length": 228.0, + "completions/min_terminated_length": 228.0, + "epoch": 0.06770925893015817, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.5692752599716187, + "kl": 1.070998653769493, + "learning_rate": 2.53968253968254e-06, + "loss": 0.0011, + "num_tokens": 1127670.0, + "reward": 0.8239761590957642, + "reward_std": 0.22099705040454865, + "rewards/check_answer/mean": 0.8239761590957642, + "rewards/check_answer/std": 0.22099705040454865, + "step": 381 + }, + { + "completion_length": 559.75, + "completions/clipped_ratio": 0.0, + "completions/max_length": 689.0, + "completions/max_terminated_length": 689.0, + "completions/mean_length": 559.75, + "completions/mean_terminated_length": 559.75, + "completions/min_length": 372.0, + "completions/min_terminated_length": 372.0, + "epoch": 0.06788697352052603, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.4717862606048584, + "kl": 0.7960640043020248, + "learning_rate": 2.531746031746032e-06, + "loss": 0.0008, + "num_tokens": 1131065.0, + "reward": 0.7636763453483582, + "reward_std": 0.03722745552659035, + "rewards/check_answer/mean": 0.7636763453483582, + "rewards/check_answer/std": 0.03722746670246124, + "step": 382 + }, + { + "completion_length": 446.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 593.0, + "completions/max_terminated_length": 593.0, + "completions/mean_length": 446.0, + "completions/mean_terminated_length": 446.0, + "completions/min_length": 374.0, + "completions/min_terminated_length": 374.0, + "epoch": 0.0680646881108939, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.5540060997009277, + "kl": 0.8313327878713608, + "learning_rate": 2.523809523809524e-06, + "loss": 0.0008, + "num_tokens": 1134045.0, + "reward": 0.6946367621421814, + "reward_std": 0.10725139826536179, + "rewards/check_answer/mean": 0.6946367621421814, + "rewards/check_answer/std": 0.10725140571594238, + "step": 383 + }, + { + "completion_length": 614.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 701.0, + "completions/max_terminated_length": 701.0, + "completions/mean_length": 614.0, + "completions/mean_terminated_length": 614.0, + "completions/min_length": 552.0, + "completions/min_terminated_length": 552.0, + "epoch": 0.06824240270126178, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.311665415763855, + "kl": 0.8095138221979141, + "learning_rate": 2.515873015873016e-06, + "loss": 0.0008, + "num_tokens": 1137665.0, + "reward": 0.4693688750267029, + "reward_std": 0.06689858436584473, + "rewards/check_answer/mean": 0.4693688750267029, + "rewards/check_answer/std": 0.06689856946468353, + "step": 384 + }, + { + "completion_length": 582.25, + "completions/clipped_ratio": 0.0, + "completions/max_length": 657.0, + "completions/max_terminated_length": 657.0, + "completions/mean_length": 582.25, + "completions/mean_terminated_length": 582.25, + "completions/min_length": 468.0, + "completions/min_terminated_length": 468.0, + "epoch": 0.06842011729162964, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0779439210891724, + "kl": 0.6766816079616547, + "learning_rate": 2.507936507936508e-06, + "loss": 0.0007, + "num_tokens": 1141214.0, + "reward": 0.6747899055480957, + "reward_std": 0.07833781838417053, + "rewards/check_answer/mean": 0.6747899055480957, + "rewards/check_answer/std": 0.07833781838417053, + "step": 385 + }, + { + "completion_length": 403.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 469.0, + "completions/max_terminated_length": 469.0, + "completions/mean_length": 403.0, + "completions/mean_terminated_length": 403.0, + "completions/min_length": 289.0, + "completions/min_terminated_length": 289.0, + "epoch": 0.06859783188199751, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.6302034854888916, + "kl": 0.8939348012208939, + "learning_rate": 2.5e-06, + "loss": 0.0009, + "num_tokens": 1143954.0, + "reward": 0.3099154233932495, + "reward_std": 0.15409527719020844, + "rewards/check_answer/mean": 0.3099154233932495, + "rewards/check_answer/std": 0.15409527719020844, + "step": 386 + }, + { + "completion_length": 601.75, + "completions/clipped_ratio": 0.0, + "completions/max_length": 738.0, + "completions/max_terminated_length": 738.0, + "completions/mean_length": 601.75, + "completions/mean_terminated_length": 601.75, + "completions/min_length": 368.0, + "completions/min_terminated_length": 368.0, + "epoch": 0.06877554647236538, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.424817442893982, + "kl": 0.8362443447113037, + "learning_rate": 2.4920634920634923e-06, + "loss": 0.0008, + "num_tokens": 1147545.0, + "reward": 0.7676776647567749, + "reward_std": 0.20846693217754364, + "rewards/check_answer/mean": 0.7676776647567749, + "rewards/check_answer/std": 0.20846691727638245, + "step": 387 + }, + { + "completion_length": 360.5, + "completions/clipped_ratio": 0.0, + "completions/max_length": 424.0, + "completions/max_terminated_length": 424.0, + "completions/mean_length": 360.5, + "completions/mean_terminated_length": 360.5, + "completions/min_length": 276.0, + "completions/min_terminated_length": 276.0, + "epoch": 0.06895326106273325, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.7432420253753662, + "kl": 1.0870371013879776, + "learning_rate": 2.4841269841269843e-06, + "loss": 0.0011, + "num_tokens": 1150095.0, + "reward": 0.5128135085105896, + "reward_std": 0.12844218313694, + "rewards/check_answer/mean": 0.5128135085105896, + "rewards/check_answer/std": 0.12844218313694, + "step": 388 + }, + { + "completion_length": 562.25, + "completions/clipped_ratio": 0.0, + "completions/max_length": 687.0, + "completions/max_terminated_length": 687.0, + "completions/mean_length": 562.25, + "completions/mean_terminated_length": 562.25, + "completions/min_length": 448.0, + "completions/min_terminated_length": 448.0, + "epoch": 0.06913097565310113, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3039882183074951, + "kl": 0.8053465038537979, + "learning_rate": 2.4761904761904764e-06, + "loss": 0.0008, + "num_tokens": 1153600.0, + "reward": 0.6226294040679932, + "reward_std": 0.10893689841032028, + "rewards/check_answer/mean": 0.6226294040679932, + "rewards/check_answer/std": 0.10893691331148148, + "step": 389 + }, + { + "completion_length": 392.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 545.0, + "completions/max_terminated_length": 545.0, + "completions/mean_length": 392.0, + "completions/mean_terminated_length": 392.0, + "completions/min_length": 334.0, + "completions/min_terminated_length": 334.0, + "epoch": 0.06930869024346899, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.7045742273330688, + "kl": 0.9221175163984299, + "learning_rate": 2.4682539682539685e-06, + "loss": 0.0009, + "num_tokens": 1156316.0, + "reward": 0.5222982168197632, + "reward_std": 0.054217997938394547, + "rewards/check_answer/mean": 0.5222982168197632, + "rewards/check_answer/std": 0.054218001663684845, + "step": 390 + }, + { + "completion_length": 370.75, + "completions/clipped_ratio": 0.0, + "completions/max_length": 535.0, + "completions/max_terminated_length": 535.0, + "completions/mean_length": 370.75, + "completions/mean_terminated_length": 370.75, + "completions/min_length": 246.0, + "completions/min_terminated_length": 246.0, + "epoch": 0.06948640483383686, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.7914966344833374, + "kl": 1.043500304222107, + "learning_rate": 2.4603174603174605e-06, + "loss": 0.001, + "num_tokens": 1158939.0, + "reward": 0.39007383584976196, + "reward_std": 0.06512384861707687, + "rewards/check_answer/mean": 0.39007383584976196, + "rewards/check_answer/std": 0.06512384861707687, + "step": 391 + }, + { + "completion_length": 403.25, + "completions/clipped_ratio": 0.0, + "completions/max_length": 500.0, + "completions/max_terminated_length": 500.0, + "completions/mean_length": 403.25, + "completions/mean_terminated_length": 403.25, + "completions/min_length": 302.0, + "completions/min_terminated_length": 302.0, + "epoch": 0.06966411942420472, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.3422685861587524, + "kl": 1.0109106451272964, + "learning_rate": 2.4523809523809526e-06, + "loss": 0.001, + "num_tokens": 1161676.0, + "reward": 0.7319890856742859, + "reward_std": 0.07778804004192352, + "rewards/check_answer/mean": 0.7319890856742859, + "rewards/check_answer/std": 0.07778804004192352, + "step": 392 + }, + { + "completion_length": 450.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 605.0, + "completions/max_terminated_length": 605.0, + "completions/mean_length": 450.0, + "completions/mean_terminated_length": 450.0, + "completions/min_length": 350.0, + "completions/min_terminated_length": 350.0, + "epoch": 0.0698418340145726, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.531283974647522, + "kl": 0.8451306223869324, + "learning_rate": 2.4444444444444447e-06, + "loss": 0.0008, + "num_tokens": 1164652.0, + "reward": 0.8435445427894592, + "reward_std": 0.14115361869335175, + "rewards/check_answer/mean": 0.8435445427894592, + "rewards/check_answer/std": 0.14115363359451294, + "step": 393 + }, + { + "completion_length": 396.5, + "completions/clipped_ratio": 0.0, + "completions/max_length": 510.0, + "completions/max_terminated_length": 510.0, + "completions/mean_length": 396.5, + "completions/mean_terminated_length": 396.5, + "completions/min_length": 213.0, + "completions/min_terminated_length": 213.0, + "epoch": 0.07001954860494046, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.4916499853134155, + "kl": 0.9246963560581207, + "learning_rate": 2.4365079365079367e-06, + "loss": 0.0009, + "num_tokens": 1167410.0, + "reward": 0.6280086636543274, + "reward_std": 0.1793169528245926, + "rewards/check_answer/mean": 0.6280086636543274, + "rewards/check_answer/std": 0.1793169379234314, + "step": 394 + }, + { + "completion_length": 379.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 521.0, + "completions/max_terminated_length": 521.0, + "completions/mean_length": 379.0, + "completions/mean_terminated_length": 379.0, + "completions/min_length": 266.0, + "completions/min_terminated_length": 266.0, + "epoch": 0.07019726319530833, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.7005797624588013, + "kl": 1.0196750611066818, + "learning_rate": 2.428571428571429e-06, + "loss": 0.001, + "num_tokens": 1170094.0, + "reward": 0.8843287229537964, + "reward_std": 0.1384207308292389, + "rewards/check_answer/mean": 0.8843287229537964, + "rewards/check_answer/std": 0.13842076063156128, + "step": 395 + }, + { + "completion_length": 586.75, + "completions/clipped_ratio": 0.0, + "completions/max_length": 690.0, + "completions/max_terminated_length": 690.0, + "completions/mean_length": 586.75, + "completions/mean_terminated_length": 586.75, + "completions/min_length": 441.0, + "completions/min_terminated_length": 441.0, + "epoch": 0.07037497778567621, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.447705626487732, + "kl": 0.9538527727127075, + "learning_rate": 2.420634920634921e-06, + "loss": 0.001, + "num_tokens": 1173633.0, + "reward": 0.8358705043792725, + "reward_std": 0.10273723304271698, + "rewards/check_answer/mean": 0.8358705043792725, + "rewards/check_answer/std": 0.10273724794387817, + "step": 396 + }, + { + "completion_length": 242.75, + "completions/clipped_ratio": 0.0, + "completions/max_length": 313.0, + "completions/max_terminated_length": 313.0, + "completions/mean_length": 242.75, + "completions/mean_terminated_length": 242.75, + "completions/min_length": 185.0, + "completions/min_terminated_length": 185.0, + "epoch": 0.07055269237604407, + "frac_reward_zero_std": 0.0, + "grad_norm": 2.0574350357055664, + "kl": 1.2170641124248505, + "learning_rate": 2.412698412698413e-06, + "loss": 0.0012, + "num_tokens": 1175724.0, + "reward": 0.4298151135444641, + "reward_std": 0.20456048846244812, + "rewards/check_answer/mean": 0.4298151135444641, + "rewards/check_answer/std": 0.20456050336360931, + "step": 397 + }, + { + "completion_length": 594.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 826.0, + "completions/max_terminated_length": 826.0, + "completions/mean_length": 594.0, + "completions/mean_terminated_length": 594.0, + "completions/min_length": 389.0, + "completions/min_terminated_length": 389.0, + "epoch": 0.07073040696641195, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.2806570529937744, + "kl": 0.8345987200737, + "learning_rate": 2.404761904761905e-06, + "loss": 0.0008, + "num_tokens": 1179312.0, + "reward": 0.6557444930076599, + "reward_std": 0.09894954413175583, + "rewards/check_answer/mean": 0.6557444930076599, + "rewards/check_answer/std": 0.09894953668117523, + "step": 398 + }, + { + "completion_length": 528.5, + "completions/clipped_ratio": 0.0, + "completions/max_length": 689.0, + "completions/max_terminated_length": 689.0, + "completions/mean_length": 528.5, + "completions/mean_terminated_length": 528.5, + "completions/min_length": 418.0, + "completions/min_terminated_length": 418.0, + "epoch": 0.07090812155677981, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.4865775108337402, + "kl": 0.9594889581203461, + "learning_rate": 2.396825396825397e-06, + "loss": 0.001, + "num_tokens": 1182602.0, + "reward": 0.5999321937561035, + "reward_std": 0.1533043533563614, + "rewards/check_answer/mean": 0.5999321937561035, + "rewards/check_answer/std": 0.1533043384552002, + "step": 399 + }, + { + "completion_length": 696.75, + "completions/clipped_ratio": 0.0, + "completions/max_length": 733.0, + "completions/max_terminated_length": 733.0, + "completions/mean_length": 696.75, + "completions/mean_terminated_length": 696.75, + "completions/min_length": 639.0, + "completions/min_terminated_length": 639.0, + "epoch": 0.07108583614714768, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.2060387134552002, + "kl": 0.9064538776874542, + "learning_rate": 2.388888888888889e-06, + "loss": 0.0009, + "num_tokens": 1186613.0, + "reward": 0.5622410774230957, + "reward_std": 0.04867928475141525, + "rewards/check_answer/mean": 0.5622410774230957, + "rewards/check_answer/std": 0.04867929220199585, + "step": 400 + } + ], + "logging_steps": 1, + "max_steps": 700, + "num_input_tokens_seen": 1186613, + "num_train_epochs": 1, + "save_steps": 100, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 0.0, + "train_batch_size": 1, + "trial_name": null, + "trial_params": null +}