{ "best_global_step": null, "best_metric": null, "best_model_checkpoint": null, "epoch": 1.0, "eval_steps": 500, "global_step": 252, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2012.0, "completions/max_terminated_length": 2012.0, "completions/mean_length": 931.857421875, "completions/mean_terminated_length": 931.857421875, "completions/min_length": 388.0, "completions/min_terminated_length": 388.0, "entropy": 0.4899652898311615, "epoch": 0.003968253968253968, "frac_reward_zero_std": 0.40625, "grad_norm": 0.019391125068068504, "learning_rate": 1e-05, "loss": -0.0091, "num_tokens": 833719.0, "reward": 0.783251941204071, "reward_std": 0.15391692519187927, "rewards/ngram_repetition2/mean": 0.0, "rewards/ngram_repetition2/std": 0.0, "rewards/ngram_repetition3/mean": 0.0, "rewards/ngram_repetition3/std": 0.0, "rewards/sentence_repetition/mean": 0.0, "rewards/sentence_repetition/std": 0.0, "rewards/symbolic_reward_accuracy/mean": 0.732421875, "rewards/symbolic_reward_accuracy/std": 0.4431293308734894, "rewards/symbolic_reward_partial_score/mean": 0.90185546875, "rewards/symbolic_reward_partial_score/std": 0.19786402583122253, "rewards/tag_count_reward/mean": 0.0, "rewards/tag_count_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.1348549127578735, "sampling/importance_sampling_ratio/min": 2.5190163796651177e-05, "sampling/sampling_logp_difference/max": 10.589056968688965, "sampling/sampling_logp_difference/mean": 0.2086421549320221, "step": 1 }, { "clip_ratio/high_max": 0.25, "clip_ratio/high_mean": 0.0390625, "clip_ratio/low_mean": 0.0234375, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0625, "entropy": 0.4904091954231262, "epoch": 0.007936507936507936, "grad_norm": 0.02842220850288868, "learning_rate": 1e-05, "loss": -0.0081, "step": 2 }, { "clip_ratio/high_max": 0.25, "clip_ratio/high_mean": 0.03125, "clip_ratio/low_mean": 0.078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.109375, "entropy": 0.493672251701355, "epoch": 0.011904761904761904, "grad_norm": 0.017648357897996902, "learning_rate": 1e-05, "loss": -0.0048, "step": 3 }, { "clip_ratio/high_max": 0.25, "clip_ratio/high_mean": 0.03125, "clip_ratio/low_mean": 0.1875, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.21875, "entropy": 0.4953870475292206, "epoch": 0.015873015873015872, "grad_norm": 0.015915699303150177, "learning_rate": 1e-05, "loss": 0.0265, "step": 4 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2061.0, "completions/max_terminated_length": 2061.0, "completions/mean_length": 1004.126953125, "completions/mean_terminated_length": 1004.126953125, "completions/min_length": 367.0, "completions/min_terminated_length": 367.0, "entropy": 0.5080731809139252, "epoch": 0.01984126984126984, "frac_reward_zero_std": 0.25, "grad_norm": 0.028012007474899292, "learning_rate": 1e-05, "loss": 0.0088, "num_tokens": 1718456.0, "reward": 0.7163079380989075, "reward_std": 0.20483723282814026, "rewards/ngram_repetition2/mean": 0.0, "rewards/ngram_repetition2/std": 0.0, "rewards/ngram_repetition3/mean": -6.187678809510544e-05, "rewards/ngram_repetition3/std": 0.0010427762754261494, "rewards/sentence_repetition/mean": 0.0, "rewards/sentence_repetition/std": 0.0, "rewards/symbolic_reward_accuracy/mean": 0.65625, "rewards/symbolic_reward_accuracy/std": 0.4754233956336975, "rewards/symbolic_reward_partial_score/mean": 0.8564453125, "rewards/symbolic_reward_partial_score/std": 0.25247785449028015, "rewards/tag_count_reward/mean": 0.0, "rewards/tag_count_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.1413912773132324, "sampling/importance_sampling_ratio/min": 0.00015259887732099742, "sampling/sampling_logp_difference/max": 8.787697792053223, "sampling/sampling_logp_difference/mean": 0.21632857620716095, "step": 5 }, { "clip_ratio/high_max": 0.125, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0859375, "entropy": 0.5091428160667419, "epoch": 0.023809523809523808, "grad_norm": 0.027424518018960953, "learning_rate": 1e-05, "loss": -0.0081, "step": 6 }, { "clip_ratio/high_max": 0.375, "clip_ratio/high_mean": 0.046875, "clip_ratio/low_mean": 0.09375, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.140625, "entropy": 0.5064189434051514, "epoch": 0.027777777777777776, "grad_norm": 0.03622233122587204, "learning_rate": 1e-05, "loss": -0.0081, "step": 7 }, { "clip_ratio/high_max": 0.25, "clip_ratio/high_mean": 0.015625, "clip_ratio/low_mean": 0.1953125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.2109375, "entropy": 0.5119710564613342, "epoch": 0.031746031746031744, "grad_norm": 0.03297780826687813, "learning_rate": 1e-05, "loss": 0.0062, "step": 8 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.001953125, "completions/max_length": 8192.0, "completions/max_terminated_length": 2167.0, "completions/mean_length": 1066.048828125, "completions/mean_terminated_length": 1052.103759765625, "completions/min_length": 409.0, "completions/min_terminated_length": 409.0, "entropy": 0.5385481119155884, "epoch": 0.03571428571428571, "frac_reward_zero_std": 0.375, "grad_norm": 0.036817729473114014, "learning_rate": 1e-05, "loss": 0.0082, "num_tokens": 2589137.0, "reward": 0.8728970289230347, "reward_std": 0.16464269161224365, "rewards/ngram_repetition2/mean": -8.738729957258329e-05, "rewards/ngram_repetition2/std": 0.0017841163789853454, "rewards/ngram_repetition3/mean": -0.0002455631329212338, "rewards/ngram_repetition3/std": 0.0039971922524273396, "rewards/sentence_repetition/mean": 0.0, "rewards/sentence_repetition/std": 0.0, "rewards/symbolic_reward_accuracy/mean": 0.8515625, "rewards/symbolic_reward_accuracy/std": 0.35588082671165466, "rewards/symbolic_reward_partial_score/mean": 0.92333984375, "rewards/symbolic_reward_partial_score/std": 0.19933150708675385, "rewards/tag_count_reward/mean": -0.001953125, "rewards/tag_count_reward/std": 0.04419417306780815, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.1490129232406616, "sampling/importance_sampling_ratio/min": 0.0015110719250515103, "sampling/sampling_logp_difference/max": 6.494935989379883, "sampling/sampling_logp_difference/mean": 0.22550542652606964, "step": 9 }, { "clip_ratio/high_max": 0.125, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.046875, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0546875, "entropy": 0.5337882339954376, "epoch": 0.03968253968253968, "grad_norm": 0.02156970649957657, "learning_rate": 1e-05, "loss": 0.0113, "step": 10 }, { "clip_ratio/high_max": 0.125, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.1015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.109375, "entropy": 0.5329746603965759, "epoch": 0.04365079365079365, "grad_norm": 0.022666223347187042, "learning_rate": 1e-05, "loss": 0.0226, "step": 11 }, { "clip_ratio/high_max": 0.125, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.09375, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.1015625, "entropy": 0.5298546552658081, "epoch": 0.047619047619047616, "grad_norm": 0.021358540281653404, "learning_rate": 1e-05, "loss": -0.0066, "step": 12 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2184.0, "completions/max_terminated_length": 2184.0, "completions/mean_length": 1149.2109375, "completions/mean_terminated_length": 1149.2109375, "completions/min_length": 327.0, "completions/min_terminated_length": 327.0, "entropy": 0.5508022904396057, "epoch": 0.051587301587301584, "frac_reward_zero_std": 0.25, "grad_norm": 0.03623722121119499, "learning_rate": 1e-05, "loss": 0.0104, "num_tokens": 3519101.0, "reward": 0.8040039539337158, "reward_std": 0.23532113432884216, "rewards/ngram_repetition2/mean": 0.0, "rewards/ngram_repetition2/std": 0.0, "rewards/ngram_repetition3/mean": 0.0, "rewards/ngram_repetition3/std": 0.0, "rewards/sentence_repetition/mean": 0.0, "rewards/sentence_repetition/std": 0.0, "rewards/symbolic_reward_accuracy/mean": 0.765625, "rewards/symbolic_reward_accuracy/std": 0.42402184009552, "rewards/symbolic_reward_partial_score/mean": 0.8935546875, "rewards/symbolic_reward_partial_score/std": 0.22642776370048523, "rewards/tag_count_reward/mean": 0.0, "rewards/tag_count_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.1566712856292725, "sampling/importance_sampling_ratio/min": 4.5738033804809675e-05, "sampling/sampling_logp_difference/max": 9.99258041381836, "sampling/sampling_logp_difference/mean": 0.23426732420921326, "step": 13 }, { "clip_ratio/high_max": 0.25, "clip_ratio/high_mean": 0.0390625, "clip_ratio/low_mean": 0.03125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0703125, "entropy": 0.5531112849712372, "epoch": 0.05555555555555555, "grad_norm": 0.030078258365392685, "learning_rate": 1e-05, "loss": 0.0027, "step": 14 }, { "clip_ratio/high_max": 0.25, "clip_ratio/high_mean": 0.0703125, "clip_ratio/low_mean": 0.03125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.1015625, "entropy": 0.5538643896579742, "epoch": 0.05952380952380952, "grad_norm": 0.036736875772476196, "learning_rate": 1e-05, "loss": -0.0078, "step": 15 }, { "clip_ratio/high_max": 0.25, "clip_ratio/high_mean": 0.015625, "clip_ratio/low_mean": 0.1328125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.1484375, "entropy": 0.5511457026004791, "epoch": 0.06349206349206349, "grad_norm": 0.025234857574105263, "learning_rate": 1e-05, "loss": 0.002, "step": 16 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.005859375, "completions/max_length": 8192.0, "completions/max_terminated_length": 2413.0, "completions/mean_length": 1241.208984375, "completions/mean_terminated_length": 1200.24169921875, "completions/min_length": 443.0, "completions/min_terminated_length": 443.0, "entropy": 0.5505756735801697, "epoch": 0.06746031746031746, "frac_reward_zero_std": 0.21875, "grad_norm": 0.048497121781110764, "learning_rate": 1e-05, "loss": 0.015, "num_tokens": 4505512.0, "reward": 0.7780177593231201, "reward_std": 0.2124088853597641, "rewards/ngram_repetition2/mean": -0.00207839859649539, "rewards/ngram_repetition2/std": 0.0309219378978014, "rewards/ngram_repetition3/mean": -0.0022493049036711454, "rewards/ngram_repetition3/std": 0.03252926468849182, "rewards/sentence_repetition/mean": -0.001518026809208095, "rewards/sentence_repetition/std": 0.025519903749227524, "rewards/symbolic_reward_accuracy/mean": 0.736328125, "rewards/symbolic_reward_accuracy/std": 0.4410543739795685, "rewards/symbolic_reward_partial_score/mean": 0.87744140625, "rewards/symbolic_reward_partial_score/std": 0.23091304302215576, "rewards/tag_count_reward/mean": -0.005859375, "rewards/tag_count_reward/std": 0.07639661431312561, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.1530300378799438, "sampling/importance_sampling_ratio/min": 1.4715351426275447e-05, "sampling/sampling_logp_difference/max": 11.126619338989258, "sampling/sampling_logp_difference/mean": 0.22909365594387054, "step": 17 }, { "clip_ratio/high_max": 0.125, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0859375, "entropy": 0.551996260881424, "epoch": 0.07142857142857142, "grad_norm": 0.02824103645980358, "learning_rate": 1e-05, "loss": -0.0008, "step": 18 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.109375, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.109375, "entropy": 0.5422624349594116, "epoch": 0.07539682539682539, "grad_norm": 0.03899236023426056, "learning_rate": 1e-05, "loss": 0.0115, "step": 19 }, { "clip_ratio/high_max": 0.125, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.1875, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.1953125, "entropy": 0.5322679281234741, "epoch": 0.07936507936507936, "grad_norm": 0.03367049992084503, "learning_rate": 1e-05, "loss": 0.0086, "step": 20 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0078125, "completions/max_length": 8192.0, "completions/max_terminated_length": 2317.0, "completions/mean_length": 1262.94921875, "completions/mean_terminated_length": 1208.3897705078125, "completions/min_length": 282.0, "completions/min_terminated_length": 282.0, "entropy": 0.5361554622650146, "epoch": 0.08333333333333333, "frac_reward_zero_std": 0.21875, "grad_norm": 0.024270186200737953, "learning_rate": 1e-05, "loss": 0.0029, "num_tokens": 5511758.0, "reward": 0.6900801658630371, "reward_std": 0.2088010311126709, "rewards/ngram_repetition2/mean": -0.004531805403530598, "rewards/ngram_repetition2/std": 0.05988180264830589, "rewards/ngram_repetition3/mean": -0.004582167137414217, "rewards/ngram_repetition3/std": 0.06044253706932068, "rewards/sentence_repetition/mean": -0.0014235757989808917, "rewards/sentence_repetition/std": 0.03221184387803078, "rewards/symbolic_reward_accuracy/mean": 0.62890625, "rewards/symbolic_reward_accuracy/std": 0.4835699498653412, "rewards/symbolic_reward_partial_score/mean": 0.83642578125, "rewards/symbolic_reward_partial_score/std": 0.24724769592285156, "rewards/tag_count_reward/mean": -0.009765625, "rewards/tag_count_reward/std": 0.09843364357948303, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.1515878438949585, "sampling/importance_sampling_ratio/min": 3.4871933166868985e-05, "sampling/sampling_logp_difference/max": 10.26382827758789, "sampling/sampling_logp_difference/mean": 0.22689393162727356, "step": 21 }, { "clip_ratio/high_max": 0.125, "clip_ratio/high_mean": 0.015625, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.03125, "entropy": 0.5295783281326294, "epoch": 0.0873015873015873, "grad_norm": 0.0383714959025383, "learning_rate": 1e-05, "loss": 0.0261, "step": 22 }, { "clip_ratio/high_max": 0.375, "clip_ratio/high_mean": 0.0234375, "clip_ratio/low_mean": 0.125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.1484375, "entropy": 0.5451477766036987, "epoch": 0.09126984126984126, "grad_norm": 0.045755717903375626, "learning_rate": 1e-05, "loss": 0.0227, "step": 23 }, { "clip_ratio/high_max": 0.125, "clip_ratio/high_mean": 0.015625, "clip_ratio/low_mean": 0.21875, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.234375, "entropy": 0.5578975677490234, "epoch": 0.09523809523809523, "grad_norm": 0.03128146380186081, "learning_rate": 1e-05, "loss": 0.0064, "step": 24 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0078125, "completions/max_length": 8192.0, "completions/max_terminated_length": 2148.0, "completions/mean_length": 1290.79296875, "completions/mean_terminated_length": 1236.4527587890625, "completions/min_length": 345.0, "completions/min_terminated_length": 345.0, "entropy": 0.5296036005020142, "epoch": 0.0992063492063492, "frac_reward_zero_std": 0.21875, "grad_norm": 0.0418429858982563, "learning_rate": 1e-05, "loss": 0.0174, "num_tokens": 6531236.0, "reward": 0.8133755922317505, "reward_std": 0.19003061950206757, "rewards/ngram_repetition2/mean": -0.004084443673491478, "rewards/ngram_repetition2/std": 0.05170508474111557, "rewards/ngram_repetition3/mean": -0.004495682194828987, "rewards/ngram_repetition3/std": 0.0532810278236866, "rewards/sentence_repetition/mean": -0.0015145307406783104, "rewards/sentence_repetition/std": 0.026236727833747864, "rewards/symbolic_reward_accuracy/mean": 0.77734375, "rewards/symbolic_reward_accuracy/std": 0.41643625497817993, "rewards/symbolic_reward_partial_score/mean": 0.900390625, "rewards/symbolic_reward_partial_score/std": 0.21748392283916473, "rewards/tag_count_reward/mean": -0.0078125, "rewards/tag_count_reward/std": 0.08812850713729858, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.1531308889389038, "sampling/importance_sampling_ratio/min": 6.273036888160277e-06, "sampling/sampling_logp_difference/max": 11.979249954223633, "sampling/sampling_logp_difference/mean": 0.2282177358865738, "step": 25 }, { "clip_ratio/high_max": 0.25, "clip_ratio/high_mean": 0.015625, "clip_ratio/low_mean": 0.0234375, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0390625, "entropy": 0.5391701757907867, "epoch": 0.10317460317460317, "grad_norm": 0.024703076109290123, "learning_rate": 1e-05, "loss": 0.001, "step": 26 }, { "clip_ratio/high_max": 0.25, "clip_ratio/high_mean": 0.015625, "clip_ratio/low_mean": 0.0703125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0859375, "entropy": 0.5502187609672546, "epoch": 0.10714285714285714, "grad_norm": 0.042478591203689575, "learning_rate": 1e-05, "loss": -0.0058, "step": 27 }, { "clip_ratio/high_max": 0.125, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.140625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.1484375, "entropy": 0.5443172454833984, "epoch": 0.1111111111111111, "grad_norm": 0.027527781203389168, "learning_rate": 1e-05, "loss": 0.0107, "step": 28 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.00390625, "completions/max_length": 8192.0, "completions/max_terminated_length": 2558.0, "completions/mean_length": 1305.12109375, "completions/mean_terminated_length": 1278.11376953125, "completions/min_length": 434.0, "completions/min_terminated_length": 434.0, "entropy": 0.5285843014717102, "epoch": 0.11507936507936507, "frac_reward_zero_std": 0.1875, "grad_norm": 0.0555814690887928, "learning_rate": 1e-05, "loss": 0.0447, "num_tokens": 7544034.0, "reward": 0.8156405091285706, "reward_std": 0.23784983158111572, "rewards/ngram_repetition2/mean": -0.0016481123166158795, "rewards/ngram_repetition2/std": 0.03445202484726906, "rewards/ngram_repetition3/mean": -0.0016825615894049406, "rewards/ngram_repetition3/std": 0.03473168984055519, "rewards/sentence_repetition/mean": 0.0, "rewards/sentence_repetition/std": 0.0, "rewards/symbolic_reward_accuracy/mean": 0.78515625, "rewards/symbolic_reward_accuracy/std": 0.4111155867576599, "rewards/symbolic_reward_partial_score/mean": 0.88818359375, "rewards/symbolic_reward_partial_score/std": 0.26044148206710815, "rewards/tag_count_reward/mean": -0.00390625, "rewards/tag_count_reward/std": 0.06243881583213806, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.1548700332641602, "sampling/importance_sampling_ratio/min": 0.0003772444906644523, "sampling/sampling_logp_difference/max": 7.882616996765137, "sampling/sampling_logp_difference/mean": 0.23007941246032715, "step": 29 }, { "clip_ratio/high_max": 0.125, "clip_ratio/high_mean": 0.015625, "clip_ratio/low_mean": 0.046875, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0625, "entropy": 0.5250012874603271, "epoch": 0.11904761904761904, "grad_norm": 0.032588765025138855, "learning_rate": 1e-05, "loss": -0.0088, "step": 30 }, { "clip_ratio/high_max": 0.125, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.1484375, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.15625, "entropy": 0.5342312455177307, "epoch": 0.12301587301587301, "grad_norm": 0.027414429932832718, "learning_rate": 1e-05, "loss": 0.0002, "step": 31 }, { "clip_ratio/high_max": 0.125, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.1328125, "entropy": 0.5392702221870422, "epoch": 0.12698412698412698, "grad_norm": 0.03042636252939701, "learning_rate": 1e-05, "loss": -0.0075, "step": 32 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.017578125, "completions/max_length": 8192.0, "completions/max_terminated_length": 2801.0, "completions/mean_length": 1494.390625, "completions/mean_terminated_length": 1374.5526123046875, "completions/min_length": 433.0, "completions/min_terminated_length": 433.0, "entropy": 0.532558023929596, "epoch": 0.13095238095238096, "frac_reward_zero_std": 0.125, "grad_norm": 0.049848731607198715, "learning_rate": 1e-05, "loss": 0.0197, "num_tokens": 8662762.0, "reward": 0.7535891532897949, "reward_std": 0.20455370843410492, "rewards/ngram_repetition2/mean": -0.01040959358215332, "rewards/ngram_repetition2/std": 0.08168824017047882, "rewards/ngram_repetition3/mean": -0.01096004992723465, "rewards/ngram_repetition3/std": 0.08340611308813095, "rewards/sentence_repetition/mean": -0.005458846688270569, "rewards/sentence_repetition/std": 0.05533413216471672, "rewards/symbolic_reward_accuracy/mean": 0.69921875, "rewards/symbolic_reward_accuracy/std": 0.45904624462127686, "rewards/symbolic_reward_partial_score/mean": 0.88525390625, "rewards/symbolic_reward_partial_score/std": 0.2078271061182022, "rewards/tag_count_reward/mean": -0.01171875, "rewards/tag_count_reward/std": 0.10772226005792618, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.1470518112182617, "sampling/importance_sampling_ratio/min": 0.0012160565238445997, "sampling/sampling_logp_difference/max": 6.712141990661621, "sampling/sampling_logp_difference/mean": 0.21713975071907043, "step": 33 }, { "clip_ratio/high_max": 0.25, "clip_ratio/high_mean": 0.046875, "clip_ratio/low_mean": 0.0234375, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0703125, "entropy": 0.5083911120891571, "epoch": 0.1349206349206349, "grad_norm": 0.03026898391544819, "learning_rate": 1e-05, "loss": 0.0223, "step": 34 }, { "clip_ratio/high_max": 0.25, "clip_ratio/high_mean": 0.03125, "clip_ratio/low_mean": 0.0625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.09375, "entropy": 0.5368916988372803, "epoch": 0.1388888888888889, "grad_norm": 0.0356556698679924, "learning_rate": 1e-05, "loss": -0.0176, "step": 35 }, { "clip_ratio/high_max": 0.25, "clip_ratio/high_mean": 0.015625, "clip_ratio/low_mean": 0.171875, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.1875, "entropy": 0.49885277450084686, "epoch": 0.14285714285714285, "grad_norm": 0.03617777302861214, "learning_rate": 1e-05, "loss": 0.0415, "step": 36 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.009765625, "completions/max_length": 8192.0, "completions/max_terminated_length": 3265.0, "completions/mean_length": 1645.283203125, "completions/mean_terminated_length": 1580.719970703125, "completions/min_length": 392.0, "completions/min_terminated_length": 392.0, "entropy": 0.526048332452774, "epoch": 0.14682539682539683, "frac_reward_zero_std": 0.125, "grad_norm": 0.04523659124970436, "learning_rate": 1e-05, "loss": 0.0238, "num_tokens": 9865083.0, "reward": 0.7424488067626953, "reward_std": 0.215779647231102, "rewards/ngram_repetition2/mean": -0.004797599744051695, "rewards/ngram_repetition2/std": 0.04601871222257614, "rewards/ngram_repetition3/mean": -0.005221162457019091, "rewards/ngram_repetition3/std": 0.0470595583319664, "rewards/sentence_repetition/mean": -0.0029129728209227324, "rewards/sentence_repetition/std": 0.03621043637394905, "rewards/symbolic_reward_accuracy/mean": 0.689453125, "rewards/symbolic_reward_accuracy/std": 0.46317005157470703, "rewards/symbolic_reward_partial_score/mean": 0.869140625, "rewards/symbolic_reward_partial_score/std": 0.23032106459140778, "rewards/tag_count_reward/mean": -0.0078125, "rewards/tag_count_reward/std": 0.08812850713729858, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.1516501903533936, "sampling/importance_sampling_ratio/min": 0.0013894386356696486, "sampling/sampling_logp_difference/max": 6.578855514526367, "sampling/sampling_logp_difference/mean": 0.22245685756206512, "step": 37 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0859375, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0859375, "entropy": 0.5196961164474487, "epoch": 0.15079365079365079, "grad_norm": 0.04294053092598915, "learning_rate": 1e-05, "loss": -0.0014, "step": 38 }, { "clip_ratio/high_max": 0.125, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.2890625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.296875, "entropy": 0.5212567150592804, "epoch": 0.15476190476190477, "grad_norm": 0.029605941846966743, "learning_rate": 1e-05, "loss": 0.0182, "step": 39 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.296875, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.296875, "entropy": 0.5307926535606384, "epoch": 0.15873015873015872, "grad_norm": 0.04217367246747017, "learning_rate": 1e-05, "loss": 0.0009, "step": 40 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02734375, "completions/max_length": 8192.0, "completions/max_terminated_length": 3303.0, "completions/mean_length": 1848.79296875, "completions/mean_terminated_length": 1670.4698486328125, "completions/min_length": 474.0, "completions/min_terminated_length": 474.0, "entropy": 0.5088708996772766, "epoch": 0.1626984126984127, "frac_reward_zero_std": 0.03125, "grad_norm": 0.04686456546187401, "learning_rate": 1e-05, "loss": 0.018, "num_tokens": 11173265.0, "reward": 0.7183312773704529, "reward_std": 0.21814408898353577, "rewards/ngram_repetition2/mean": -0.018662545830011368, "rewards/ngram_repetition2/std": 0.10112260282039642, "rewards/ngram_repetition3/mean": -0.01979563757777214, "rewards/ngram_repetition3/std": 0.10316861420869827, "rewards/sentence_repetition/mean": -0.013179361820220947, "rewards/sentence_repetition/std": 0.08368713408708572, "rewards/symbolic_reward_accuracy/mean": 0.666015625, "rewards/symbolic_reward_accuracy/std": 0.47209542989730835, "rewards/symbolic_reward_partial_score/mean": 0.8505859375, "rewards/symbolic_reward_partial_score/std": 0.2455865740776062, "rewards/tag_count_reward/mean": -0.025390625, "rewards/tag_count_reward/std": 0.15746226906776428, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.142880916595459, "sampling/importance_sampling_ratio/min": 0.0013725899625569582, "sampling/sampling_logp_difference/max": 6.591055870056152, "sampling/sampling_logp_difference/mean": 0.2087078094482422, "step": 41 }, { "clip_ratio/high_max": 0.25, "clip_ratio/high_mean": 0.0234375, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0234375, "entropy": 0.4749659299850464, "epoch": 0.16666666666666666, "grad_norm": 0.03617702051997185, "learning_rate": 1e-05, "loss": 0.0713, "step": 42 }, { "clip_ratio/high_max": 0.375, "clip_ratio/high_mean": 0.0390625, "clip_ratio/low_mean": 0.1328125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.171875, "entropy": 0.5151703655719757, "epoch": 0.17063492063492064, "grad_norm": 0.03681587800383568, "learning_rate": 1e-05, "loss": 0.0141, "step": 43 }, { "clip_ratio/high_max": 0.125, "clip_ratio/high_mean": 0.0234375, "clip_ratio/low_mean": 0.1875, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.2109375, "entropy": 0.5055911093950272, "epoch": 0.1746031746031746, "grad_norm": 0.04378470033407211, "learning_rate": 1e-05, "loss": 0.0434, "step": 44 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.095703125, "completions/max_length": 8192.0, "completions/max_terminated_length": 6065.0, "completions/mean_length": 2366.3671875, "completions/mean_terminated_length": 1749.83154296875, "completions/min_length": 543.0, "completions/min_terminated_length": 543.0, "entropy": 0.45105186104774475, "epoch": 0.17857142857142858, "frac_reward_zero_std": 0.0, "grad_norm": 0.04162848740816116, "learning_rate": 1e-05, "loss": 0.0521, "num_tokens": 12760141.0, "reward": 0.7157611846923828, "reward_std": 0.20244178175926208, "rewards/ngram_repetition2/mean": -0.0591588020324707, "rewards/ngram_repetition2/std": 0.1800626665353775, "rewards/ngram_repetition3/mean": -0.062470559030771255, "rewards/ngram_repetition3/std": 0.18351468443870544, "rewards/sentence_repetition/mean": -0.02100549452006817, "rewards/sentence_repetition/std": 0.0797034353017807, "rewards/symbolic_reward_accuracy/mean": 0.65625, "rewards/symbolic_reward_accuracy/std": 0.4754233956336975, "rewards/symbolic_reward_partial_score/mean": 0.87109375, "rewards/symbolic_reward_partial_score/std": 0.22553345561027527, "rewards/tag_count_reward/mean": -0.03515625, "rewards/tag_count_reward/std": 0.1843547374010086, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.1190338134765625, "sampling/importance_sampling_ratio/min": 2.9442677259794436e-05, "sampling/sampling_logp_difference/max": 10.433065414428711, "sampling/sampling_logp_difference/mean": 0.17251503467559814, "step": 45 }, { "clip_ratio/high_max": 0.25, "clip_ratio/high_mean": 0.0625, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0703125, "entropy": 0.45093561708927155, "epoch": 0.18253968253968253, "grad_norm": 0.0957750529050827, "learning_rate": 1e-05, "loss": 0.0587, "step": 46 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.1328125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.1328125, "entropy": 0.46573543548583984, "epoch": 0.1865079365079365, "grad_norm": 0.04806223884224892, "learning_rate": 1e-05, "loss": -0.0216, "step": 47 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.2578125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.2578125, "entropy": 0.4004691243171692, "epoch": 0.19047619047619047, "grad_norm": 0.0406065471470356, "learning_rate": 1e-05, "loss": 0.2019, "step": 48 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.087890625, "completions/max_length": 8192.0, "completions/max_terminated_length": 5248.0, "completions/mean_length": 2243.021484375, "completions/mean_terminated_length": 1669.7794189453125, "completions/min_length": 584.0, "completions/min_terminated_length": 584.0, "entropy": 0.41885489225387573, "epoch": 0.19444444444444445, "frac_reward_zero_std": 0.0, "grad_norm": 0.03534400463104248, "learning_rate": 1e-05, "loss": 0.1239, "num_tokens": 14236120.0, "reward": 0.8248867988586426, "reward_std": 0.18499471247196198, "rewards/ngram_repetition2/mean": -0.06148881837725639, "rewards/ngram_repetition2/std": 0.1797143816947937, "rewards/ngram_repetition3/mean": -0.06444543600082397, "rewards/ngram_repetition3/std": 0.18241557478904724, "rewards/sentence_repetition/mean": -0.02210882306098938, "rewards/sentence_repetition/std": 0.09284410625696182, "rewards/symbolic_reward_accuracy/mean": 0.8046875, "rewards/symbolic_reward_accuracy/std": 0.3968288004398346, "rewards/symbolic_reward_partial_score/mean": 0.892578125, "rewards/symbolic_reward_partial_score/std": 0.24765750765800476, "rewards/tag_count_reward/mean": -0.046875, "rewards/tag_count_reward/std": 0.21157780289649963, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.1206936836242676, "sampling/importance_sampling_ratio/min": 3.9870826640253654e-07, "sampling/sampling_logp_difference/max": 14.73503589630127, "sampling/sampling_logp_difference/mean": 0.1745004951953888, "step": 49 }, { "clip_ratio/high_max": 0.25, "clip_ratio/high_mean": 0.0390625, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0390625, "entropy": 0.4192962348461151, "epoch": 0.1984126984126984, "grad_norm": 0.04903830960392952, "learning_rate": 1e-05, "loss": 0.1424, "step": 50 }, { "clip_ratio/high_max": 0.25, "clip_ratio/high_mean": 0.0546875, "clip_ratio/low_mean": 0.0546875, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.109375, "entropy": 0.4624395966529846, "epoch": 0.20238095238095238, "grad_norm": 0.049799274653196335, "learning_rate": 1e-05, "loss": 0.0653, "step": 51 }, { "clip_ratio/high_max": 0.125, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.1015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.109375, "entropy": 0.47439947724342346, "epoch": 0.20634920634920634, "grad_norm": 0.04301155358552933, "learning_rate": 1e-05, "loss": 0.0432, "step": 52 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.287109375, "completions/max_length": 8192.0, "completions/max_terminated_length": 4432.0, "completions/mean_length": 3765.40625, "completions/mean_terminated_length": 1982.64111328125, "completions/min_length": 458.0, "completions/min_terminated_length": 458.0, "entropy": 0.29511846601963043, "epoch": 0.21031746031746032, "frac_reward_zero_std": 0.0, "grad_norm": 0.060881707817316055, "learning_rate": 1e-05, "loss": 0.1699, "num_tokens": 16522600.0, "reward": 0.7429922223091125, "reward_std": 0.2635670304298401, "rewards/ngram_repetition2/mean": -0.17410045862197876, "rewards/ngram_repetition2/std": 0.25672969222068787, "rewards/ngram_repetition3/mean": -0.17981290817260742, "rewards/ngram_repetition3/std": 0.26004987955093384, "rewards/sentence_repetition/mean": -0.10760784149169922, "rewards/sentence_repetition/std": 0.1995861679315567, "rewards/symbolic_reward_accuracy/mean": 0.732421875, "rewards/symbolic_reward_accuracy/std": 0.4431293308734894, "rewards/symbolic_reward_partial_score/mean": 0.85205078125, "rewards/symbolic_reward_partial_score/std": 0.28318965435028076, "rewards/tag_count_reward/mean": -0.20703125, "rewards/tag_count_reward/std": 0.40557438135147095, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0907182693481445, "sampling/importance_sampling_ratio/min": 1.0083810956379446e-12, "sampling/sampling_logp_difference/max": 27.6226749420166, "sampling/sampling_logp_difference/mean": 0.13107267022132874, "step": 53 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.1015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.1015625, "entropy": 0.35645565390586853, "epoch": 0.21428571428571427, "grad_norm": 0.055166009813547134, "learning_rate": 1e-05, "loss": 0.0949, "step": 54 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.2421875, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.2421875, "entropy": 0.35095688700675964, "epoch": 0.21825396825396826, "grad_norm": 0.08587633818387985, "learning_rate": 1e-05, "loss": 0.1527, "step": 55 }, { "clip_ratio/high_max": 0.25, "clip_ratio/high_mean": 0.015625, "clip_ratio/low_mean": 0.1328125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.1484375, "entropy": 0.33085887134075165, "epoch": 0.2222222222222222, "grad_norm": 0.04242081195116043, "learning_rate": 1e-05, "loss": 0.0807, "step": 56 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.37109375, "completions/max_length": 8192.0, "completions/max_terminated_length": 5908.0, "completions/mean_length": 4147.17578125, "completions/mean_terminated_length": 1760.478271484375, "completions/min_length": 543.0, "completions/min_terminated_length": 543.0, "entropy": 0.24479665607213974, "epoch": 0.2261904761904762, "frac_reward_zero_std": 0.0, "grad_norm": 0.04463326185941696, "learning_rate": 1e-05, "loss": 0.0396, "num_tokens": 18987522.0, "reward": 0.648320198059082, "reward_std": 0.2978951930999756, "rewards/ngram_repetition2/mean": -0.23350077867507935, "rewards/ngram_repetition2/std": 0.27949315309524536, "rewards/ngram_repetition3/mean": -0.24011221528053284, "rewards/ngram_repetition3/std": 0.28325703740119934, "rewards/sentence_repetition/mean": -0.13870351016521454, "rewards/sentence_repetition/std": 0.2181253284215927, "rewards/symbolic_reward_accuracy/mean": 0.6328125, "rewards/symbolic_reward_accuracy/std": 0.48250964283943176, "rewards/symbolic_reward_partial_score/mean": 0.78564453125, "rewards/symbolic_reward_partial_score/std": 0.3289281129837036, "rewards/tag_count_reward/mean": -0.2421875, "rewards/tag_count_reward/std": 0.42882615327835083, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.076377034187317, "sampling/importance_sampling_ratio/min": 0.0011872592149302363, "sampling/sampling_logp_difference/max": 6.73610782623291, "sampling/sampling_logp_difference/mean": 0.1094931811094284, "step": 57 }, { "clip_ratio/high_max": 0.375, "clip_ratio/high_mean": 0.109375, "clip_ratio/low_mean": 0.0234375, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.1328125, "entropy": 0.31162580847740173, "epoch": 0.23015873015873015, "grad_norm": 0.061540767550468445, "learning_rate": 1e-05, "loss": 0.1548, "step": 58 }, { "clip_ratio/high_max": 0.125, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.140625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.1484375, "entropy": 0.24130023270845413, "epoch": 0.23412698412698413, "grad_norm": 0.04283663630485535, "learning_rate": 1e-05, "loss": 0.0592, "step": 59 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.296875, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.296875, "entropy": 0.29536642134189606, "epoch": 0.23809523809523808, "grad_norm": 0.10053296387195587, "learning_rate": 1e-05, "loss": 0.12, "step": 60 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.537109375, "completions/max_length": 8192.0, "completions/max_terminated_length": 6267.0, "completions/mean_length": 5152.953125, "completions/mean_terminated_length": 1626.6328125, "completions/min_length": 365.0, "completions/min_terminated_length": 365.0, "entropy": 0.21371906995773315, "epoch": 0.24206349206349206, "frac_reward_zero_std": 0.0, "grad_norm": 0.05521790683269501, "learning_rate": 1e-05, "loss": 0.1195, "num_tokens": 21976746.0, "reward": 0.6653094291687012, "reward_std": 0.35927891731262207, "rewards/ngram_repetition2/mean": -0.3568421006202698, "rewards/ngram_repetition2/std": 0.3149164021015167, "rewards/ngram_repetition3/mean": -0.3595767617225647, "rewards/ngram_repetition3/std": 0.32074519991874695, "rewards/sentence_repetition/mean": -0.22334471344947815, "rewards/sentence_repetition/std": 0.25914594531059265, "rewards/symbolic_reward_accuracy/mean": 0.66015625, "rewards/symbolic_reward_accuracy/std": 0.4741191864013672, "rewards/symbolic_reward_partial_score/mean": 0.8037109375, "rewards/symbolic_reward_partial_score/std": 0.32702162861824036, "rewards/tag_count_reward/mean": -0.28515625, "rewards/tag_count_reward/std": 0.45193037390708923, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0554835796356201, "sampling/importance_sampling_ratio/min": 1.5679698606163583e-07, "sampling/sampling_logp_difference/max": 15.668313980102539, "sampling/sampling_logp_difference/mean": 0.08012787997722626, "step": 61 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.1953125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.1953125, "entropy": 0.21423833817243576, "epoch": 0.24603174603174602, "grad_norm": 0.32476258277893066, "learning_rate": 1e-05, "loss": 0.1036, "step": 62 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.265625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.265625, "entropy": 0.19419603049755096, "epoch": 0.25, "grad_norm": 0.11604978889226913, "learning_rate": 1e-05, "loss": -0.0477, "step": 63 }, { "clip_ratio/high_max": 0.125, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.15625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.1640625, "entropy": 0.1795002520084381, "epoch": 0.25396825396825395, "grad_norm": 0.05532268434762955, "learning_rate": 1e-05, "loss": 0.0911, "step": 64 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.642578125, "completions/max_length": 8192.0, "completions/max_terminated_length": 4607.0, "completions/mean_length": 5810.994140625, "completions/mean_terminated_length": 1530.387939453125, "completions/min_length": 701.0, "completions/min_terminated_length": 701.0, "entropy": 0.1565011814236641, "epoch": 0.25793650793650796, "frac_reward_zero_std": 0.0, "grad_norm": 0.04580986872315407, "learning_rate": 1e-05, "loss": 0.1438, "num_tokens": 25304551.0, "reward": 0.584717869758606, "reward_std": 0.36316779255867004, "rewards/ngram_repetition2/mean": -0.432647705078125, "rewards/ngram_repetition2/std": 0.3042435944080353, "rewards/ngram_repetition3/mean": -0.4355693757534027, "rewards/ngram_repetition3/std": 0.31115105748176575, "rewards/sentence_repetition/mean": -0.279133141040802, "rewards/sentence_repetition/std": 0.26221224665641785, "rewards/symbolic_reward_accuracy/mean": 0.599609375, "rewards/symbolic_reward_accuracy/std": 0.4904567301273346, "rewards/symbolic_reward_partial_score/mean": 0.7314453125, "rewards/symbolic_reward_partial_score/std": 0.3726162910461426, "rewards/tag_count_reward/mean": -0.4296875, "rewards/tag_count_reward/std": 0.4955156147480011, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0471299886703491, "sampling/importance_sampling_ratio/min": 0.0008837437490001321, "sampling/sampling_logp_difference/max": 7.031343460083008, "sampling/sampling_logp_difference/mean": 0.066426582634449, "step": 65 }, { "clip_ratio/high_max": 0.25, "clip_ratio/high_mean": 0.015625, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.03125, "entropy": 0.16354916989803314, "epoch": 0.2619047619047619, "grad_norm": 0.055534061044454575, "learning_rate": 1e-05, "loss": 0.0153, "step": 66 }, { "clip_ratio/high_max": 0.25, "clip_ratio/high_mean": 0.015625, "clip_ratio/low_mean": 0.0703125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0859375, "entropy": 0.1632969081401825, "epoch": 0.26587301587301587, "grad_norm": 0.0415620431303978, "learning_rate": 1e-05, "loss": 0.0778, "step": 67 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.1640625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.1640625, "entropy": 0.15156294405460358, "epoch": 0.2698412698412698, "grad_norm": 0.06093261390924454, "learning_rate": 1e-05, "loss": 0.0967, "step": 68 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.505859375, "completions/max_length": 8192.0, "completions/max_terminated_length": 6508.0, "completions/mean_length": 4834.21875, "completions/mean_terminated_length": 1396.806396484375, "completions/min_length": 567.0, "completions/min_terminated_length": 567.0, "entropy": 0.18089503049850464, "epoch": 0.27380952380952384, "frac_reward_zero_std": 0.0, "grad_norm": 0.0946880504488945, "learning_rate": 1e-05, "loss": -0.0206, "num_tokens": 28131927.0, "reward": 0.5640771389007568, "reward_std": 0.39757049083709717, "rewards/ngram_repetition2/mean": -0.36212357878685, "rewards/ngram_repetition2/std": 0.32067063450813293, "rewards/ngram_repetition3/mean": -0.361666202545166, "rewards/ngram_repetition3/std": 0.32757768034935, "rewards/sentence_repetition/mean": -0.2073608636856079, "rewards/sentence_repetition/std": 0.26262417435646057, "rewards/symbolic_reward_accuracy/mean": 0.560546875, "rewards/symbolic_reward_accuracy/std": 0.49680593609809875, "rewards/symbolic_reward_partial_score/mean": 0.70751953125, "rewards/symbolic_reward_partial_score/std": 0.38821572065353394, "rewards/tag_count_reward/mean": -0.3125, "rewards/tag_count_reward/std": 0.4639657139778137, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0472054481506348, "sampling/importance_sampling_ratio/min": 3.90792820326169e-06, "sampling/sampling_logp_difference/max": 12.452503204345703, "sampling/sampling_logp_difference/mean": 0.06639708578586578, "step": 69 }, { "clip_ratio/high_max": 0.125, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.046875, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0546875, "entropy": 0.1630459502339363, "epoch": 0.2777777777777778, "grad_norm": 0.05245819315314293, "learning_rate": 1e-05, "loss": -0.031, "step": 70 }, { "clip_ratio/high_max": 0.25, "clip_ratio/high_mean": 0.015625, "clip_ratio/low_mean": 0.078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.09375, "entropy": 0.15807346999645233, "epoch": 0.28174603174603174, "grad_norm": 0.04118438810110092, "learning_rate": 1e-05, "loss": 0.1522, "step": 71 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.1953125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.1953125, "entropy": 0.18679703772068024, "epoch": 0.2857142857142857, "grad_norm": 0.0393151119351387, "learning_rate": 1e-05, "loss": 0.1071, "step": 72 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.408203125, "completions/max_length": 8192.0, "completions/max_terminated_length": 7169.0, "completions/mean_length": 4186.994140625, "completions/mean_terminated_length": 1424.46533203125, "completions/min_length": 359.0, "completions/min_terminated_length": 359.0, "entropy": 0.2277718335390091, "epoch": 0.2896825396825397, "frac_reward_zero_std": 0.0, "grad_norm": 0.051285628229379654, "learning_rate": 1e-05, "loss": -0.008, "num_tokens": 30621908.0, "reward": 0.7025977373123169, "reward_std": 0.2931853234767914, "rewards/ngram_repetition2/mean": -0.301375150680542, "rewards/ngram_repetition2/std": 0.32986778020858765, "rewards/ngram_repetition3/mean": -0.3006427586078644, "rewards/ngram_repetition3/std": 0.3338293135166168, "rewards/sentence_repetition/mean": -0.10988512635231018, "rewards/sentence_repetition/std": 0.21824848651885986, "rewards/symbolic_reward_accuracy/mean": 0.71484375, "rewards/symbolic_reward_accuracy/std": 0.45193037390708923, "rewards/symbolic_reward_partial_score/mean": 0.82470703125, "rewards/symbolic_reward_partial_score/std": 0.31770938634872437, "rewards/tag_count_reward/mean": -0.380859375, "rewards/tag_count_reward/std": 0.48607301712036133, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0573573112487793, "sampling/importance_sampling_ratio/min": 1.8034556487078746e-11, "sampling/sampling_logp_difference/max": 24.738731384277344, "sampling/sampling_logp_difference/mean": 0.07999817281961441, "step": 73 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0625, "entropy": 0.21516019105911255, "epoch": 0.29365079365079366, "grad_norm": 0.0444764606654644, "learning_rate": 1e-05, "loss": 0.0934, "step": 74 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.09375, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.09375, "entropy": 0.18979737162590027, "epoch": 0.2976190476190476, "grad_norm": 0.05204957723617554, "learning_rate": 1e-05, "loss": -0.011, "step": 75 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.1328125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.1328125, "entropy": 0.19768279790878296, "epoch": 0.30158730158730157, "grad_norm": 0.04237138852477074, "learning_rate": 1e-05, "loss": 0.0128, "step": 76 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.349609375, "completions/max_length": 8192.0, "completions/max_terminated_length": 5723.0, "completions/mean_length": 3712.126953125, "completions/mean_terminated_length": 1304.027099609375, "completions/min_length": 448.0, "completions/min_terminated_length": 448.0, "entropy": 0.19913502782583237, "epoch": 0.3055555555555556, "frac_reward_zero_std": 0.0, "grad_norm": 0.2402896136045456, "learning_rate": 1e-05, "loss": 0.1194, "num_tokens": 32862421.0, "reward": 0.682103157043457, "reward_std": 0.25951170921325684, "rewards/ngram_repetition2/mean": -0.28036749362945557, "rewards/ngram_repetition2/std": 0.34457746148109436, "rewards/ngram_repetition3/mean": -0.2796359658241272, "rewards/ngram_repetition3/std": 0.34700778126716614, "rewards/sentence_repetition/mean": -0.09198664873838425, "rewards/sentence_repetition/std": 0.20152416825294495, "rewards/symbolic_reward_accuracy/mean": 0.658203125, "rewards/symbolic_reward_accuracy/std": 0.4747757613658905, "rewards/symbolic_reward_partial_score/mean": 0.81298828125, "rewards/symbolic_reward_partial_score/std": 0.30135416984558105, "rewards/tag_count_reward/mean": -0.16015625, "rewards/tag_count_reward/std": 0.3671095669269562, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.065352201461792, "sampling/importance_sampling_ratio/min": 1.866979071341035e-10, "sampling/sampling_logp_difference/max": 22.40152931213379, "sampling/sampling_logp_difference/mean": 0.0924573689699173, "step": 77 }, { "clip_ratio/high_max": 0.375, "clip_ratio/high_mean": 0.03125, "clip_ratio/low_mean": 0.03125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0625, "entropy": 0.2694672644138336, "epoch": 0.30952380952380953, "grad_norm": 0.30339595675468445, "learning_rate": 1e-05, "loss": -0.0008, "step": 78 }, { "clip_ratio/high_max": 0.375, "clip_ratio/high_mean": 0.046875, "clip_ratio/low_mean": 0.078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.125, "entropy": 0.2688329368829727, "epoch": 0.3134920634920635, "grad_norm": 0.042341988533735275, "learning_rate": 1e-05, "loss": 0.0767, "step": 79 }, { "clip_ratio/high_max": 0.125, "clip_ratio/high_mean": 0.015625, "clip_ratio/low_mean": 0.1484375, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.1640625, "entropy": 0.26124274730682373, "epoch": 0.31746031746031744, "grad_norm": 0.040945865213871, "learning_rate": 1e-05, "loss": 0.0259, "step": 80 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.4609375, "completions/max_length": 8192.0, "completions/max_terminated_length": 8089.0, "completions/mean_length": 4418.654296875, "completions/mean_terminated_length": 1192.1702880859375, "completions/min_length": 409.0, "completions/min_terminated_length": 409.0, "entropy": 0.24447178095579147, "epoch": 0.32142857142857145, "frac_reward_zero_std": 0.0, "grad_norm": 0.3140353262424469, "learning_rate": 1e-05, "loss": -0.1155, "num_tokens": 35466340.0, "reward": 0.7915980219841003, "reward_std": 0.29427048563957214, "rewards/ngram_repetition2/mean": -0.3955739736557007, "rewards/ngram_repetition2/std": 0.4065479636192322, "rewards/ngram_repetition3/mean": -0.39375513792037964, "rewards/ngram_repetition3/std": 0.4088459610939026, "rewards/sentence_repetition/mean": -0.06551767885684967, "rewards/sentence_repetition/std": 0.1914827823638916, "rewards/symbolic_reward_accuracy/mean": 0.78125, "rewards/symbolic_reward_accuracy/std": 0.41380295157432556, "rewards/symbolic_reward_partial_score/mean": 0.88330078125, "rewards/symbolic_reward_partial_score/std": 0.2501065135002136, "rewards/tag_count_reward/mean": -0.1171875, "rewards/tag_count_reward/std": 0.32195815443992615, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0622519254684448, "sampling/importance_sampling_ratio/min": 9.177392353010294e-12, "sampling/sampling_logp_difference/max": 25.414278030395508, "sampling/sampling_logp_difference/mean": 0.08255760371685028, "step": 81 }, { "clip_ratio/high_max": 0.125, "clip_ratio/high_mean": 0.0234375, "clip_ratio/low_mean": 0.0234375, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.046875, "entropy": 0.19228526204824448, "epoch": 0.3253968253968254, "grad_norm": 0.14637868106365204, "learning_rate": 1e-05, "loss": 0.1577, "step": 82 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.1015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.1015625, "entropy": 0.20319241285324097, "epoch": 0.32936507936507936, "grad_norm": 0.4751107394695282, "learning_rate": 1e-05, "loss": 0.0756, "step": 83 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.1015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.1015625, "entropy": 0.18567422777414322, "epoch": 0.3333333333333333, "grad_norm": 0.12354305386543274, "learning_rate": 1e-05, "loss": -0.0674, "step": 84 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.2890625, "completions/max_length": 8192.0, "completions/max_terminated_length": 8067.0, "completions/mean_length": 3320.478515625, "completions/mean_terminated_length": 1339.75, "completions/min_length": 431.0, "completions/min_terminated_length": 431.0, "entropy": 0.2971004843711853, "epoch": 0.3373015873015873, "frac_reward_zero_std": 0.0, "grad_norm": 0.04326535761356354, "learning_rate": 1e-05, "loss": 0.1504, "num_tokens": 37517337.0, "reward": 0.7201089859008789, "reward_std": 0.28874659538269043, "rewards/ngram_repetition2/mean": -0.25622648000717163, "rewards/ngram_repetition2/std": 0.3613777458667755, "rewards/ngram_repetition3/mean": -0.25260815024375916, "rewards/ngram_repetition3/std": 0.3636447489261627, "rewards/sentence_repetition/mean": -0.06815455108880997, "rewards/sentence_repetition/std": 0.19773656129837036, "rewards/symbolic_reward_accuracy/mean": 0.689453125, "rewards/symbolic_reward_accuracy/std": 0.46317005157470703, "rewards/symbolic_reward_partial_score/mean": 0.8369140625, "rewards/symbolic_reward_partial_score/std": 0.2806803584098816, "rewards/tag_count_reward/mean": -0.078125, "rewards/tag_count_reward/std": 0.26863065361976624, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.0809917449951172, "sampling/importance_sampling_ratio/min": 3.4648691793308373e-13, "sampling/sampling_logp_difference/max": 28.69093132019043, "sampling/sampling_logp_difference/mean": 0.10925842821598053, "step": 85 }, { "clip_ratio/high_max": 0.125, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.03125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0390625, "entropy": 0.3483195900917053, "epoch": 0.3412698412698413, "grad_norm": 0.03285548835992813, "learning_rate": 1e-05, "loss": 0.0623, "step": 86 }, { "clip_ratio/high_max": 0.375, "clip_ratio/high_mean": 0.046875, "clip_ratio/low_mean": 0.0625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.109375, "entropy": 0.31874437630176544, "epoch": 0.34523809523809523, "grad_norm": 0.03926711529493332, "learning_rate": 1e-05, "loss": 0.0114, "step": 87 }, { "clip_ratio/high_max": 0.25, "clip_ratio/high_mean": 0.0546875, "clip_ratio/low_mean": 0.1484375, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.203125, "entropy": 0.2724745571613312, "epoch": 0.3492063492063492, "grad_norm": 0.03760918602347374, "learning_rate": 1e-05, "loss": 0.14, "step": 88 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.083984375, "completions/max_length": 8192.0, "completions/max_terminated_length": 4730.0, "completions/mean_length": 1832.2734375, "completions/mean_terminated_length": 1249.185546875, "completions/min_length": 378.0, "completions/min_terminated_length": 378.0, "entropy": 0.4735376834869385, "epoch": 0.3531746031746032, "frac_reward_zero_std": 0.0, "grad_norm": 0.059152498841285706, "learning_rate": 1e-05, "loss": 0.0559, "num_tokens": 38812389.0, "reward": 0.7257625460624695, "reward_std": 0.25111252069473267, "rewards/ngram_repetition2/mean": -0.09366865456104279, "rewards/ngram_repetition2/std": 0.22025002539157867, "rewards/ngram_repetition3/mean": -0.08607886731624603, "rewards/ngram_repetition3/std": 0.22113096714019775, "rewards/sentence_repetition/mean": -0.017435554414987564, "rewards/sentence_repetition/std": 0.0973522961139679, "rewards/symbolic_reward_accuracy/mean": 0.689453125, "rewards/symbolic_reward_accuracy/std": 0.46317005157470703, "rewards/symbolic_reward_partial_score/mean": 0.8359375, "rewards/symbolic_reward_partial_score/std": 0.2861573100090027, "rewards/tag_count_reward/mean": -0.056640625, "rewards/tag_count_reward/std": 0.23138070106506348, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.1368573904037476, "sampling/importance_sampling_ratio/min": 2.736367132882833e-09, "sampling/sampling_logp_difference/max": 19.71663475036621, "sampling/sampling_logp_difference/mean": 0.18504942953586578, "step": 89 }, { "clip_ratio/high_max": 0.125, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.0234375, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.03125, "entropy": 0.5180395841598511, "epoch": 0.35714285714285715, "grad_norm": 0.056337956339120865, "learning_rate": 1e-05, "loss": -0.0054, "step": 90 }, { "clip_ratio/high_max": 0.25, "clip_ratio/high_mean": 0.046875, "clip_ratio/low_mean": 0.046875, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.09375, "entropy": 0.4704781621694565, "epoch": 0.3611111111111111, "grad_norm": 0.058693937957286835, "learning_rate": 1e-05, "loss": 0.0624, "step": 91 }, { "clip_ratio/high_max": 0.125, "clip_ratio/high_mean": 0.0234375, "clip_ratio/low_mean": 0.1015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.125, "entropy": 0.5089136958122253, "epoch": 0.36507936507936506, "grad_norm": 0.03134845569729805, "learning_rate": 1e-05, "loss": 0.041, "step": 92 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.044921875, "completions/max_length": 8192.0, "completions/max_terminated_length": 5155.0, "completions/mean_length": 1543.939453125, "completions/mean_terminated_length": 1231.24951171875, "completions/min_length": 446.0, "completions/min_terminated_length": 446.0, "entropy": 0.5548083782196045, "epoch": 0.36904761904761907, "frac_reward_zero_std": 0.0, "grad_norm": 0.049277447164058685, "learning_rate": 1e-05, "loss": 0.0202, "num_tokens": 39971846.0, "reward": 0.6843430995941162, "reward_std": 0.26696306467056274, "rewards/ngram_repetition2/mean": -0.07940564304590225, "rewards/ngram_repetition2/std": 0.17461377382278442, "rewards/ngram_repetition3/mean": -0.0676058903336525, "rewards/ngram_repetition3/std": 0.17480789124965668, "rewards/sentence_repetition/mean": -0.007548762485384941, "rewards/sentence_repetition/std": 0.051889773458242416, "rewards/symbolic_reward_accuracy/mean": 0.625, "rewards/symbolic_reward_accuracy/std": 0.4845963716506958, "rewards/symbolic_reward_partial_score/mean": 0.83642578125, "rewards/symbolic_reward_partial_score/std": 0.25837501883506775, "rewards/tag_count_reward/mean": -0.025390625, "rewards/tag_count_reward/std": 0.15746226906776428, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.1613116264343262, "sampling/importance_sampling_ratio/min": 0.0015013572992756963, "sampling/sampling_logp_difference/max": 6.501385688781738, "sampling/sampling_logp_difference/mean": 0.21706466376781464, "step": 93 }, { "clip_ratio/high_max": 0.375, "clip_ratio/high_mean": 0.0234375, "clip_ratio/low_mean": 0.03125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0546875, "entropy": 0.5276340395212173, "epoch": 0.373015873015873, "grad_norm": 0.04870341718196869, "learning_rate": 1e-05, "loss": 0.0208, "step": 94 }, { "clip_ratio/high_max": 0.125, "clip_ratio/high_mean": 0.03125, "clip_ratio/low_mean": 0.125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.15625, "entropy": 0.5200925171375275, "epoch": 0.376984126984127, "grad_norm": 0.05106513574719429, "learning_rate": 1e-05, "loss": 0.0423, "step": 95 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.21875, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.21875, "entropy": 0.5299715399742126, "epoch": 0.38095238095238093, "grad_norm": 0.036806683987379074, "learning_rate": 1e-05, "loss": -0.0269, "step": 96 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0390625, "completions/max_length": 8192.0, "completions/max_terminated_length": 5702.0, "completions/mean_length": 1364.59375, "completions/mean_terminated_length": 1087.056884765625, "completions/min_length": 368.0, "completions/min_terminated_length": 368.0, "entropy": 0.5400635600090027, "epoch": 0.38492063492063494, "frac_reward_zero_std": 0.0, "grad_norm": 0.05168670415878296, "learning_rate": 1e-05, "loss": 0.0347, "num_tokens": 41033782.0, "reward": 0.7434127926826477, "reward_std": 0.21698029339313507, "rewards/ngram_repetition2/mean": -0.0588124617934227, "rewards/ngram_repetition2/std": 0.16793230175971985, "rewards/ngram_repetition3/mean": -0.05034123361110687, "rewards/ngram_repetition3/std": 0.16768810153007507, "rewards/sentence_repetition/mean": -0.0026930849999189377, "rewards/sentence_repetition/std": 0.03430738300085068, "rewards/symbolic_reward_accuracy/mean": 0.69921875, "rewards/symbolic_reward_accuracy/std": 0.45904624462127686, "rewards/symbolic_reward_partial_score/mean": 0.861328125, "rewards/symbolic_reward_partial_score/std": 0.24888910353183746, "rewards/tag_count_reward/mean": -0.033203125, "rewards/tag_count_reward/std": 0.17934183776378632, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.1639056205749512, "sampling/importance_sampling_ratio/min": 0.000760988739784807, "sampling/sampling_logp_difference/max": 7.180891990661621, "sampling/sampling_logp_difference/mean": 0.22075198590755463, "step": 97 }, { "clip_ratio/high_max": 0.5, "clip_ratio/high_mean": 0.1015625, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.1171875, "entropy": 0.5744107961654663, "epoch": 0.3888888888888889, "grad_norm": 0.04336725175380707, "learning_rate": 1e-05, "loss": 0.0736, "step": 98 }, { "clip_ratio/high_max": 0.25, "clip_ratio/high_mean": 0.0390625, "clip_ratio/low_mean": 0.140625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.1796875, "entropy": 0.5691567063331604, "epoch": 0.39285714285714285, "grad_norm": 0.039867136627435684, "learning_rate": 1e-05, "loss": 0.0537, "step": 99 }, { "clip_ratio/high_max": 0.125, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.171875, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.1796875, "entropy": 0.5634361505508423, "epoch": 0.3968253968253968, "grad_norm": 0.04052422568202019, "learning_rate": 1e-05, "loss": 0.021, "step": 100 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.015625, "completions/max_length": 8192.0, "completions/max_terminated_length": 4878.0, "completions/mean_length": 1174.89453125, "completions/mean_terminated_length": 1063.511962890625, "completions/min_length": 147.0, "completions/min_terminated_length": 147.0, "entropy": 0.5878070592880249, "epoch": 0.4007936507936508, "frac_reward_zero_std": 0.0, "grad_norm": 0.04968047887086868, "learning_rate": 1e-05, "loss": 0.0287, "num_tokens": 41978560.0, "reward": 0.7900087833404541, "reward_std": 0.19058659672737122, "rewards/ngram_repetition2/mean": -0.035881444811820984, "rewards/ngram_repetition2/std": 0.11486562341451645, "rewards/ngram_repetition3/mean": -0.029830805957317352, "rewards/ngram_repetition3/std": 0.11308922618627548, "rewards/sentence_repetition/mean": -0.0007943843957036734, "rewards/sentence_repetition/std": 0.006330643780529499, "rewards/symbolic_reward_accuracy/mean": 0.75, "rewards/symbolic_reward_accuracy/std": 0.43343618512153625, "rewards/symbolic_reward_partial_score/mean": 0.89013671875, "rewards/symbolic_reward_partial_score/std": 0.22066742181777954, "rewards/tag_count_reward/mean": -0.013671875, "rewards/tag_count_reward/std": 0.1162383034825325, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.186119556427002, "sampling/importance_sampling_ratio/min": 0.0010022756177932024, "sampling/sampling_logp_difference/max": 6.905482292175293, "sampling/sampling_logp_difference/mean": 0.24954494833946228, "step": 101 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.140625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.140625, "entropy": 0.5914620459079742, "epoch": 0.40476190476190477, "grad_norm": 0.0457477830350399, "learning_rate": 1e-05, "loss": 0.0321, "step": 102 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.2421875, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.2421875, "entropy": 0.6051645874977112, "epoch": 0.4087301587301587, "grad_norm": 0.03900279104709625, "learning_rate": 1e-05, "loss": 0.0139, "step": 103 }, { "clip_ratio/high_max": 0.125, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.2109375, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.21875, "entropy": 0.6001529097557068, "epoch": 0.4126984126984127, "grad_norm": 0.04870934411883354, "learning_rate": 1e-05, "loss": -0.0142, "step": 104 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.013671875, "completions/max_length": 8192.0, "completions/max_terminated_length": 4081.0, "completions/mean_length": 1070.7265625, "completions/mean_terminated_length": 972.015869140625, "completions/min_length": 282.0, "completions/min_terminated_length": 282.0, "entropy": 0.6071796119213104, "epoch": 0.4166666666666667, "frac_reward_zero_std": 0.0, "grad_norm": 0.0434139147400856, "learning_rate": 1e-05, "loss": 0.0415, "num_tokens": 42860660.0, "reward": 0.7837015390396118, "reward_std": 0.17486105859279633, "rewards/ngram_repetition2/mean": -0.02740011364221573, "rewards/ngram_repetition2/std": 0.1060570552945137, "rewards/ngram_repetition3/mean": -0.024946145713329315, "rewards/ngram_repetition3/std": 0.10461968928575516, "rewards/sentence_repetition/mean": -0.00035471213050186634, "rewards/sentence_repetition/std": 0.004889630246907473, "rewards/symbolic_reward_accuracy/mean": 0.73828125, "rewards/symbolic_reward_accuracy/std": 0.44000017642974854, "rewards/symbolic_reward_partial_score/mean": 0.89599609375, "rewards/symbolic_reward_partial_score/std": 0.20819459855556488, "rewards/tag_count_reward/mean": -0.013671875, "rewards/tag_count_reward/std": 0.1162383034825325, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.189232587814331, "sampling/importance_sampling_ratio/min": 0.0019420855678617954, "sampling/sampling_logp_difference/max": 6.243992805480957, "sampling/sampling_logp_difference/mean": 0.25198525190353394, "step": 105 }, { "clip_ratio/high_max": 0.375, "clip_ratio/high_mean": 0.109375, "clip_ratio/low_mean": 0.0234375, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.1328125, "entropy": 0.6282003223896027, "epoch": 0.42063492063492064, "grad_norm": 0.04214237257838249, "learning_rate": 1e-05, "loss": 0.0287, "step": 106 }, { "clip_ratio/high_max": 0.25, "clip_ratio/high_mean": 0.078125, "clip_ratio/low_mean": 0.1640625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.2421875, "entropy": 0.5875833630561829, "epoch": 0.4246031746031746, "grad_norm": 0.0386333242058754, "learning_rate": 1e-05, "loss": 0.0299, "step": 107 }, { "clip_ratio/high_max": 0.125, "clip_ratio/high_mean": 0.015625, "clip_ratio/low_mean": 0.21875, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.234375, "entropy": 0.6417720317840576, "epoch": 0.42857142857142855, "grad_norm": 0.059936199337244034, "learning_rate": 1e-05, "loss": -0.0119, "step": 108 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.017578125, "completions/max_length": 8192.0, "completions/max_terminated_length": 4726.0, "completions/mean_length": 951.037109375, "completions/mean_terminated_length": 821.4771118164062, "completions/min_length": 134.0, "completions/min_terminated_length": 134.0, "entropy": 0.6564396917819977, "epoch": 0.43253968253968256, "frac_reward_zero_std": 0.0, "grad_norm": 0.035531267523765564, "learning_rate": 1e-05, "loss": 0.0056, "num_tokens": 43710535.0, "reward": 0.6739901900291443, "reward_std": 0.27842575311660767, "rewards/ngram_repetition2/mean": -0.022906571626663208, "rewards/ngram_repetition2/std": 0.11609412729740143, "rewards/ngram_repetition3/mean": -0.021929342299699783, "rewards/ngram_repetition3/std": 0.11581163108348846, "rewards/sentence_repetition/mean": -0.002434931229799986, "rewards/sentence_repetition/std": 0.036497294902801514, "rewards/symbolic_reward_accuracy/mean": 0.61328125, "rewards/symbolic_reward_accuracy/std": 0.48747459053993225, "rewards/symbolic_reward_partial_score/mean": 0.81982421875, "rewards/symbolic_reward_partial_score/std": 0.2665877938270569, "rewards/tag_count_reward/mean": -0.0078125, "rewards/tag_count_reward/std": 0.08812850713729858, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.1900908946990967, "sampling/importance_sampling_ratio/min": 0.0012011511716991663, "sampling/sampling_logp_difference/max": 6.724474906921387, "sampling/sampling_logp_difference/mean": 0.25281262397766113, "step": 109 }, { "clip_ratio/high_max": 0.125, "clip_ratio/high_mean": 0.0078125, "clip_ratio/low_mean": 0.21875, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.2265625, "entropy": 0.6186941266059875, "epoch": 0.4365079365079365, "grad_norm": 0.03890268877148628, "learning_rate": 1e-05, "loss": -0.0295, "step": 110 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.3125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.3125, "entropy": 0.6249259412288666, "epoch": 0.44047619047619047, "grad_norm": 0.044134460389614105, "learning_rate": 1e-05, "loss": 0.0311, "step": 111 }, { "clip_ratio/high_max": 0.125, "clip_ratio/high_mean": 0.0234375, "clip_ratio/low_mean": 0.28125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.3046875, "entropy": 0.6347298622131348, "epoch": 0.4444444444444444, "grad_norm": 0.044051025062799454, "learning_rate": 1e-05, "loss": 0.0145, "step": 112 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.01171875, "completions/max_length": 8192.0, "completions/max_terminated_length": 7436.0, "completions/mean_length": 971.494140625, "completions/mean_terminated_length": 885.8755493164062, "completions/min_length": 208.0, "completions/min_terminated_length": 208.0, "entropy": 0.6291132271289825, "epoch": 0.44841269841269843, "frac_reward_zero_std": 0.0, "grad_norm": 0.02774125710129738, "learning_rate": 1e-05, "loss": -0.0123, "num_tokens": 44566532.0, "reward": 0.7906748056411743, "reward_std": 0.23176395893096924, "rewards/ngram_repetition2/mean": -0.016727913171052933, "rewards/ngram_repetition2/std": 0.10360897332429886, "rewards/ngram_repetition3/mean": -0.0168988648802042, "rewards/ngram_repetition3/std": 0.10435084253549576, "rewards/sentence_repetition/mean": -0.0004538140201475471, "rewards/sentence_repetition/std": 0.008311084471642971, "rewards/symbolic_reward_accuracy/mean": 0.75, "rewards/symbolic_reward_accuracy/std": 0.43343618512153625, "rewards/symbolic_reward_partial_score/mean": 0.890625, "rewards/symbolic_reward_partial_score/std": 0.22393473982810974, "rewards/tag_count_reward/mean": -0.01171875, "rewards/tag_count_reward/std": 0.10772226005792618, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.2023460865020752, "sampling/importance_sampling_ratio/min": 0.002451813779771328, "sampling/sampling_logp_difference/max": 6.010927200317383, "sampling/sampling_logp_difference/mean": 0.26824408769607544, "step": 113 }, { "clip_ratio/high_max": 0.5, "clip_ratio/high_mean": 0.1171875, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.125, "entropy": 0.6809285581111908, "epoch": 0.4523809523809524, "grad_norm": 0.04288763552904129, "learning_rate": 1e-05, "loss": 0.0044, "step": 114 }, { "clip_ratio/high_max": 0.375, "clip_ratio/high_mean": 0.0859375, "clip_ratio/low_mean": 0.0703125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.15625, "entropy": 0.6826694011688232, "epoch": 0.45634920634920634, "grad_norm": 0.03629237413406372, "learning_rate": 1e-05, "loss": 0.0031, "step": 115 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.125, "entropy": 0.6445101499557495, "epoch": 0.4603174603174603, "grad_norm": 0.03722674772143364, "learning_rate": 1e-05, "loss": 0.0285, "step": 116 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0078125, "completions/max_length": 8192.0, "completions/max_terminated_length": 1157.0, "completions/mean_length": 672.791015625, "completions/mean_terminated_length": 613.5846557617188, "completions/min_length": 174.0, "completions/min_terminated_length": 174.0, "entropy": 0.6787378489971161, "epoch": 0.4642857142857143, "frac_reward_zero_std": 0.125, "grad_norm": 0.028261516243219376, "learning_rate": 1e-05, "loss": 0.0017, "num_tokens": 45254233.0, "reward": 0.8485878705978394, "reward_std": 0.1438256800174713, "rewards/ngram_repetition2/mean": -0.00943220965564251, "rewards/ngram_repetition2/std": 0.07950883358716965, "rewards/ngram_repetition3/mean": -0.00971515104174614, "rewards/ngram_repetition3/std": 0.07998541742563248, "rewards/sentence_repetition/mean": 0.0, "rewards/sentence_repetition/std": 0.0, "rewards/symbolic_reward_accuracy/mean": 0.82421875, "rewards/symbolic_reward_accuracy/std": 0.3810062110424042, "rewards/symbolic_reward_partial_score/mean": 0.91650390625, "rewards/symbolic_reward_partial_score/std": 0.22170855104923248, "rewards/tag_count_reward/mean": -0.03125, "rewards/tag_count_reward/std": 0.17416280508041382, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.2088536024093628, "sampling/importance_sampling_ratio/min": 0.0020660406444221735, "sampling/sampling_logp_difference/max": 6.182121276855469, "sampling/sampling_logp_difference/mean": 0.27693599462509155, "step": 117 }, { "clip_ratio/high_max": 0.125, "clip_ratio/high_mean": 0.015625, "clip_ratio/low_mean": 0.03125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.046875, "entropy": 0.6799056231975555, "epoch": 0.46825396825396826, "grad_norm": 0.03747940436005592, "learning_rate": 1e-05, "loss": 0.0055, "step": 118 }, { "clip_ratio/high_max": 0.25, "clip_ratio/high_mean": 0.0234375, "clip_ratio/low_mean": 0.1171875, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.140625, "entropy": 0.6918207406997681, "epoch": 0.4722222222222222, "grad_norm": 0.03874749690294266, "learning_rate": 1e-05, "loss": 0.0006, "step": 119 }, { "clip_ratio/high_max": 0.25, "clip_ratio/high_mean": 0.03125, "clip_ratio/low_mean": 0.1171875, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.1484375, "entropy": 0.6512312293052673, "epoch": 0.47619047619047616, "grad_norm": 0.026322009041905403, "learning_rate": 1e-05, "loss": 0.0036, "step": 120 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1116.0, "completions/max_terminated_length": 1116.0, "completions/mean_length": 555.556640625, "completions/mean_terminated_length": 555.556640625, "completions/min_length": 175.0, "completions/min_terminated_length": 175.0, "entropy": 0.7072266936302185, "epoch": 0.4801587301587302, "frac_reward_zero_std": 0.03125, "grad_norm": 0.03423263505101204, "learning_rate": 1e-05, "loss": -0.0023, "num_tokens": 45892918.0, "reward": 0.7657040357589722, "reward_std": 0.1510254442691803, "rewards/ngram_repetition2/mean": -0.0032717217691242695, "rewards/ngram_repetition2/std": 0.016812046989798546, "rewards/ngram_repetition3/mean": -0.0034686592407524586, "rewards/ngram_repetition3/std": 0.01725599728524685, "rewards/sentence_repetition/mean": 0.0, "rewards/sentence_repetition/std": 0.0, "rewards/symbolic_reward_accuracy/mean": 0.716796875, "rewards/symbolic_reward_accuracy/std": 0.4509948492050171, "rewards/symbolic_reward_partial_score/mean": 0.88720703125, "rewards/symbolic_reward_partial_score/std": 0.21295472979545593, "rewards/tag_count_reward/mean": -0.021484375, "rewards/tag_count_reward/std": 0.14513419568538666, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.2238849401474, "sampling/importance_sampling_ratio/min": 0.0022064188960939646, "sampling/sampling_logp_difference/max": 6.116384506225586, "sampling/sampling_logp_difference/mean": 0.2982763648033142, "step": 121 }, { "clip_ratio/high_max": 0.375, "clip_ratio/high_mean": 0.0859375, "clip_ratio/low_mean": 0.03125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.1171875, "entropy": 0.7036875784397125, "epoch": 0.48412698412698413, "grad_norm": 0.02480008825659752, "learning_rate": 1e-05, "loss": 0.003, "step": 122 }, { "clip_ratio/high_max": 0.5, "clip_ratio/high_mean": 0.09375, "clip_ratio/low_mean": 0.109375, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.203125, "entropy": 0.7029035091400146, "epoch": 0.4880952380952381, "grad_norm": 0.028795365244150162, "learning_rate": 1e-05, "loss": -0.0006, "step": 123 }, { "clip_ratio/high_max": 0.375, "clip_ratio/high_mean": 0.09375, "clip_ratio/low_mean": 0.15625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.25, "entropy": 0.7123904526233673, "epoch": 0.49206349206349204, "grad_norm": 0.02138606086373329, "learning_rate": 1e-05, "loss": 0.0021, "step": 124 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 986.0, "completions/max_terminated_length": 986.0, "completions/mean_length": 517.9765625, "completions/mean_terminated_length": 517.9765625, "completions/min_length": 191.0, "completions/min_terminated_length": 191.0, "entropy": 0.7109630107879639, "epoch": 0.49603174603174605, "frac_reward_zero_std": 0.1875, "grad_norm": 0.028875181451439857, "learning_rate": 1e-05, "loss": -0.0095, "num_tokens": 46497706.0, "reward": 0.8108024597167969, "reward_std": 0.1806805580854416, "rewards/ngram_repetition2/mean": -0.0018684344831854105, "rewards/ngram_repetition2/std": 0.01620560884475708, "rewards/ngram_repetition3/mean": -0.0018472126685082912, "rewards/ngram_repetition3/std": 0.015835756435990334, "rewards/sentence_repetition/mean": -1.939192952704616e-05, "rewards/sentence_repetition/std": 0.0004387892840895802, "rewards/symbolic_reward_accuracy/mean": 0.765625, "rewards/symbolic_reward_accuracy/std": 0.42402184009552, "rewards/symbolic_reward_partial_score/mean": 0.9208984375, "rewards/symbolic_reward_partial_score/std": 0.17994555830955505, "rewards/tag_count_reward/mean": -0.013671875, "rewards/tag_count_reward/std": 0.1162383034825325, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.2290794849395752, "sampling/importance_sampling_ratio/min": 0.0015681763179600239, "sampling/sampling_logp_difference/max": 6.457841873168945, "sampling/sampling_logp_difference/mean": 0.303376704454422, "step": 125 }, { "clip_ratio/high_max": 0.5, "clip_ratio/high_mean": 0.078125, "clip_ratio/low_mean": 0.03125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.109375, "entropy": 0.7282134890556335, "epoch": 0.5, "grad_norm": 0.02524893544614315, "learning_rate": 1e-05, "loss": -0.0004, "step": 126 }, { "clip_ratio/high_max": 0.375, "clip_ratio/high_mean": 0.046875, "clip_ratio/low_mean": 0.046875, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.09375, "entropy": 0.7219351232051849, "epoch": 0.503968253968254, "grad_norm": 0.03246616944670677, "learning_rate": 1e-05, "loss": 0.0049, "step": 127 }, { "clip_ratio/high_max": 0.375, "clip_ratio/high_mean": 0.0546875, "clip_ratio/low_mean": 0.1484375, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.203125, "entropy": 0.7251161336898804, "epoch": 0.5079365079365079, "grad_norm": 0.025919372215867043, "learning_rate": 1e-05, "loss": 0.0037, "step": 128 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 866.0, "completions/max_terminated_length": 866.0, "completions/mean_length": 455.146484375, "completions/mean_terminated_length": 455.146484375, "completions/min_length": 196.0, "completions/min_terminated_length": 196.0, "entropy": 0.7366768419742584, "epoch": 0.5119047619047619, "frac_reward_zero_std": 0.21875, "grad_norm": 0.024976996704936028, "learning_rate": 1e-05, "loss": -0.001, "num_tokens": 47082997.0, "reward": 0.7854200005531311, "reward_std": 0.1743355393409729, "rewards/ngram_repetition2/mean": -0.0014596395194530487, "rewards/ngram_repetition2/std": 0.008720839396119118, "rewards/ngram_repetition3/mean": -0.0014594721142202616, "rewards/ngram_repetition3/std": 0.008703470230102539, "rewards/sentence_repetition/mean": 0.0, "rewards/sentence_repetition/std": 0.0, "rewards/symbolic_reward_accuracy/mean": 0.7421875, "rewards/symbolic_reward_accuracy/std": 0.43785804510116577, "rewards/symbolic_reward_partial_score/mean": 0.8876953125, "rewards/symbolic_reward_partial_score/std": 0.2163451910018921, "rewards/tag_count_reward/mean": -0.00390625, "rewards/tag_count_reward/std": 0.06243881583213806, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.2310357093811035, "sampling/importance_sampling_ratio/min": 0.0033493756782263517, "sampling/sampling_logp_difference/max": 5.698981285095215, "sampling/sampling_logp_difference/mean": 0.3082762360572815, "step": 129 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0625, "entropy": 0.7423993349075317, "epoch": 0.5158730158730159, "grad_norm": 0.025209102779626846, "learning_rate": 1e-05, "loss": 0.0061, "step": 130 }, { "clip_ratio/high_max": 0.25, "clip_ratio/high_mean": 0.0234375, "clip_ratio/low_mean": 0.0703125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.09375, "entropy": 0.7399407625198364, "epoch": 0.5198412698412699, "grad_norm": 0.018009541556239128, "learning_rate": 1e-05, "loss": -0.0057, "step": 131 }, { "clip_ratio/high_max": 0.25, "clip_ratio/high_mean": 0.0546875, "clip_ratio/low_mean": 0.1015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.15625, "entropy": 0.731988787651062, "epoch": 0.5238095238095238, "grad_norm": 0.023537347093224525, "learning_rate": 1e-05, "loss": -0.0019, "step": 132 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1082.0, "completions/max_terminated_length": 1082.0, "completions/mean_length": 454.9453125, "completions/mean_terminated_length": 454.9453125, "completions/min_length": 183.0, "completions/min_terminated_length": 183.0, "entropy": 0.7710649967193604, "epoch": 0.5277777777777778, "frac_reward_zero_std": 0.3125, "grad_norm": 0.016300415620207787, "learning_rate": 1e-05, "loss": -0.0072, "num_tokens": 47653465.0, "reward": 0.8640967607498169, "reward_std": 0.13683994114398956, "rewards/ngram_repetition2/mean": -0.0007732085650786757, "rewards/ngram_repetition2/std": 0.005917973350733519, "rewards/ngram_repetition3/mean": -0.0006815695087425411, "rewards/ngram_repetition3/std": 0.0046678343787789345, "rewards/sentence_repetition/mean": 0.0, "rewards/sentence_repetition/std": 0.0, "rewards/symbolic_reward_accuracy/mean": 0.837890625, "rewards/symbolic_reward_accuracy/std": 0.3689115643501282, "rewards/symbolic_reward_partial_score/mean": 0.92529296875, "rewards/symbolic_reward_partial_score/std": 0.19449345767498016, "rewards/tag_count_reward/mean": 0.0, "rewards/tag_count_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.2421680688858032, "sampling/importance_sampling_ratio/min": 0.0021264441311359406, "sampling/sampling_logp_difference/max": 6.153304100036621, "sampling/sampling_logp_difference/mean": 0.31849098205566406, "step": 133 }, { "clip_ratio/high_max": 0.375, "clip_ratio/high_mean": 0.09375, "clip_ratio/low_mean": 0.0390625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.1328125, "entropy": 0.7506443560123444, "epoch": 0.5317460317460317, "grad_norm": 0.01342725194990635, "learning_rate": 1e-05, "loss": -0.0036, "step": 134 }, { "clip_ratio/high_max": 0.375, "clip_ratio/high_mean": 0.09375, "clip_ratio/low_mean": 0.078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.171875, "entropy": 0.7751710116863251, "epoch": 0.5357142857142857, "grad_norm": 0.01943376287817955, "learning_rate": 1e-05, "loss": 0.0041, "step": 135 }, { "clip_ratio/high_max": 0.625, "clip_ratio/high_mean": 0.1015625, "clip_ratio/low_mean": 0.109375, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.2109375, "entropy": 0.766922652721405, "epoch": 0.5396825396825397, "grad_norm": 0.014541191048920155, "learning_rate": 1e-05, "loss": 0.0058, "step": 136 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.001953125, "completions/max_length": 8192.0, "completions/max_terminated_length": 922.0, "completions/mean_length": 470.396484375, "completions/mean_terminated_length": 455.28570556640625, "completions/min_length": 184.0, "completions/min_terminated_length": 184.0, "entropy": 0.7778773009777069, "epoch": 0.5436507936507936, "frac_reward_zero_std": 0.15625, "grad_norm": 0.02924455888569355, "learning_rate": 1e-05, "loss": 0.0042, "num_tokens": 48257252.0, "reward": 0.7935994267463684, "reward_std": 0.21778659522533417, "rewards/ngram_repetition2/mean": -0.0026561322156339884, "rewards/ngram_repetition2/std": 0.03935694694519043, "rewards/ngram_repetition3/mean": -0.0026361742056906223, "rewards/ngram_repetition3/std": 0.039601780474185944, "rewards/sentence_repetition/mean": 0.0, "rewards/sentence_repetition/std": 0.0, "rewards/symbolic_reward_accuracy/mean": 0.75390625, "rewards/symbolic_reward_accuracy/std": 0.4311550557613373, "rewards/symbolic_reward_partial_score/mean": 0.8876953125, "rewards/symbolic_reward_partial_score/std": 0.2252091020345688, "rewards/tag_count_reward/mean": -0.00390625, "rewards/tag_count_reward/std": 0.06243881583213806, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.2430012226104736, "sampling/importance_sampling_ratio/min": 0.0027095244731754065, "sampling/sampling_logp_difference/max": 5.910982131958008, "sampling/sampling_logp_difference/mean": 0.31898462772369385, "step": 137 }, { "clip_ratio/high_max": 0.75, "clip_ratio/high_mean": 0.140625, "clip_ratio/low_mean": 0.0625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.203125, "entropy": 0.7596411108970642, "epoch": 0.5476190476190477, "grad_norm": 0.02992079220712185, "learning_rate": 1e-05, "loss": 0.004, "step": 138 }, { "clip_ratio/high_max": 0.5, "clip_ratio/high_mean": 0.125, "clip_ratio/low_mean": 0.046875, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.171875, "entropy": 0.7736047208309174, "epoch": 0.5515873015873016, "grad_norm": 0.01993100717663765, "learning_rate": 1e-05, "loss": -0.0029, "step": 139 }, { "clip_ratio/high_max": 0.375, "clip_ratio/high_mean": 0.109375, "clip_ratio/low_mean": 0.1015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.2109375, "entropy": 0.7778405547142029, "epoch": 0.5555555555555556, "grad_norm": 0.02159910649061203, "learning_rate": 1e-05, "loss": 0.0001, "step": 140 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 2494.0, "completions/max_terminated_length": 2494.0, "completions/mean_length": 426.638671875, "completions/mean_terminated_length": 426.638671875, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "entropy": 0.7608052790164948, "epoch": 0.5595238095238095, "frac_reward_zero_std": 0.1875, "grad_norm": 0.03284434601664543, "learning_rate": 1e-05, "loss": 0.003, "num_tokens": 48832619.0, "reward": 0.8041033744812012, "reward_std": 0.22923189401626587, "rewards/ngram_repetition2/mean": -0.002336258767172694, "rewards/ngram_repetition2/std": 0.03287862241268158, "rewards/ngram_repetition3/mean": -0.002363891340792179, "rewards/ngram_repetition3/std": 0.03287366405129433, "rewards/sentence_repetition/mean": 0.0, "rewards/sentence_repetition/std": 0.0, "rewards/symbolic_reward_accuracy/mean": 0.759765625, "rewards/symbolic_reward_accuracy/std": 0.4276435375213623, "rewards/symbolic_reward_partial_score/mean": 0.90966796875, "rewards/symbolic_reward_partial_score/std": 0.1935083568096161, "rewards/tag_count_reward/mean": -0.005859375, "rewards/tag_count_reward/std": 0.07639661431312561, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.23775315284729, "sampling/importance_sampling_ratio/min": 0.002077442826703191, "sampling/sampling_logp_difference/max": 6.176617622375488, "sampling/sampling_logp_difference/mean": 0.31676357984542847, "step": 141 }, { "clip_ratio/high_max": 0.25, "clip_ratio/high_mean": 0.0625, "clip_ratio/low_mean": 0.0625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.125, "entropy": 0.7492336928844452, "epoch": 0.5634920634920635, "grad_norm": 0.018240604549646378, "learning_rate": 1e-05, "loss": -0.0023, "step": 142 }, { "clip_ratio/high_max": 0.625, "clip_ratio/high_mean": 0.1328125, "clip_ratio/low_mean": 0.0546875, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.1875, "entropy": 0.7436127662658691, "epoch": 0.5674603174603174, "grad_norm": 0.029991867020726204, "learning_rate": 1e-05, "loss": 0.0007, "step": 143 }, { "clip_ratio/high_max": 0.375, "clip_ratio/high_mean": 0.0546875, "clip_ratio/low_mean": 0.109375, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.1640625, "entropy": 0.7479621469974518, "epoch": 0.5714285714285714, "grad_norm": 0.0196547694504261, "learning_rate": 1e-05, "loss": 0.0016, "step": 144 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 852.0, "completions/max_terminated_length": 852.0, "completions/mean_length": 418.68359375, "completions/mean_terminated_length": 418.68359375, "completions/min_length": 183.0, "completions/min_terminated_length": 183.0, "entropy": 0.7426201403141022, "epoch": 0.5753968253968254, "frac_reward_zero_std": 0.28125, "grad_norm": 0.03055027686059475, "learning_rate": 1e-05, "loss": 0.0021, "num_tokens": 49401225.0, "reward": 0.8642826080322266, "reward_std": 0.16769926249980927, "rewards/ngram_repetition2/mean": -0.0013196478830650449, "rewards/ngram_repetition2/std": 0.01366160623729229, "rewards/ngram_repetition3/mean": -0.0010644099675118923, "rewards/ngram_repetition3/std": 0.012863781303167343, "rewards/sentence_repetition/mean": -1.5174655345617794e-05, "rewards/sentence_repetition/std": 0.0003433632664382458, "rewards/symbolic_reward_accuracy/mean": 0.8359375, "rewards/symbolic_reward_accuracy/std": 0.37069445848464966, "rewards/symbolic_reward_partial_score/mean": 0.93115234375, "rewards/symbolic_reward_partial_score/std": 0.1837882399559021, "rewards/tag_count_reward/mean": -0.001953125, "rewards/tag_count_reward/std": 0.04419417306780815, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.2376279830932617, "sampling/importance_sampling_ratio/min": 0.0019788567442446947, "sampling/sampling_logp_difference/max": 6.225235939025879, "sampling/sampling_logp_difference/mean": 0.31502997875213623, "step": 145 }, { "clip_ratio/high_max": 0.25, "clip_ratio/high_mean": 0.0390625, "clip_ratio/low_mean": 0.046875, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0859375, "entropy": 0.7416156232357025, "epoch": 0.5793650793650794, "grad_norm": 0.024954255670309067, "learning_rate": 1e-05, "loss": 0.0013, "step": 146 }, { "clip_ratio/high_max": 0.375, "clip_ratio/high_mean": 0.0625, "clip_ratio/low_mean": 0.0234375, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0859375, "entropy": 0.7476767897605896, "epoch": 0.5833333333333334, "grad_norm": 0.02963467873632908, "learning_rate": 1e-05, "loss": -0.0061, "step": 147 }, { "clip_ratio/high_max": 0.5, "clip_ratio/high_mean": 0.109375, "clip_ratio/low_mean": 0.1015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.2109375, "entropy": 0.7358712553977966, "epoch": 0.5873015873015873, "grad_norm": 0.013027425855398178, "learning_rate": 1e-05, "loss": 0.0005, "step": 148 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1030.0, "completions/max_terminated_length": 1030.0, "completions/mean_length": 448.07421875, "completions/mean_terminated_length": 448.07421875, "completions/min_length": 218.0, "completions/min_terminated_length": 218.0, "entropy": 0.7424747943878174, "epoch": 0.5912698412698413, "frac_reward_zero_std": 0.21875, "grad_norm": 0.023708660155534744, "learning_rate": 1e-05, "loss": 0.0023, "num_tokens": 49989231.0, "reward": 0.8447631597518921, "reward_std": 0.17122790217399597, "rewards/ngram_repetition2/mean": -0.000716224021743983, "rewards/ngram_repetition2/std": 0.006064688786864281, "rewards/ngram_repetition3/mean": -0.0005084401345811784, "rewards/ngram_repetition3/std": 0.004929724149405956, "rewards/sentence_repetition/mean": 0.0, "rewards/sentence_repetition/std": 0.0, "rewards/symbolic_reward_accuracy/mean": 0.8046875, "rewards/symbolic_reward_accuracy/std": 0.3968288004398346, "rewards/symbolic_reward_partial_score/mean": 0.93896484375, "rewards/symbolic_reward_partial_score/std": 0.14857544004917145, "rewards/tag_count_reward/mean": -0.001953125, "rewards/tag_count_reward/std": 0.04419417306780815, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.2377886772155762, "sampling/importance_sampling_ratio/min": 0.0019743742886930704, "sampling/sampling_logp_difference/max": 6.227503776550293, "sampling/sampling_logp_difference/mean": 0.31468111276626587, "step": 149 }, { "clip_ratio/high_max": 0.375, "clip_ratio/high_mean": 0.078125, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.09375, "entropy": 0.7294594049453735, "epoch": 0.5952380952380952, "grad_norm": 0.02452780492603779, "learning_rate": 1e-05, "loss": -0.0049, "step": 150 }, { "clip_ratio/high_max": 0.25, "clip_ratio/high_mean": 0.078125, "clip_ratio/low_mean": 0.09375, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.171875, "entropy": 0.7370553314685822, "epoch": 0.5992063492063492, "grad_norm": 0.02377201057970524, "learning_rate": 1e-05, "loss": 0.0044, "step": 151 }, { "clip_ratio/high_max": 0.25, "clip_ratio/high_mean": 0.0703125, "clip_ratio/low_mean": 0.125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.1953125, "entropy": 0.746860921382904, "epoch": 0.6031746031746031, "grad_norm": 0.02668457105755806, "learning_rate": 1e-05, "loss": -0.0032, "step": 152 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 928.0, "completions/max_terminated_length": 928.0, "completions/mean_length": 421.001953125, "completions/mean_terminated_length": 421.001953125, "completions/min_length": 204.0, "completions/min_terminated_length": 204.0, "entropy": 0.7489960789680481, "epoch": 0.6071428571428571, "frac_reward_zero_std": 0.09375, "grad_norm": 0.0318756103515625, "learning_rate": 1e-05, "loss": 0.0054, "num_tokens": 50560368.0, "reward": 0.829513669013977, "reward_std": 0.14368176460266113, "rewards/ngram_repetition2/mean": -0.0015925484476611018, "rewards/ngram_repetition2/std": 0.012215675786137581, "rewards/ngram_repetition3/mean": -0.0011399569921195507, "rewards/ngram_repetition3/std": 0.01040260773152113, "rewards/sentence_repetition/mean": 0.0, "rewards/sentence_repetition/std": 0.0, "rewards/symbolic_reward_accuracy/mean": 0.791015625, "rewards/symbolic_reward_accuracy/std": 0.40698084235191345, "rewards/symbolic_reward_partial_score/mean": 0.91943359375, "rewards/symbolic_reward_partial_score/std": 0.18166513741016388, "rewards/tag_count_reward/mean": 0.0, "rewards/tag_count_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.2359821796417236, "sampling/importance_sampling_ratio/min": 0.0018309639999642968, "sampling/sampling_logp_difference/max": 6.302912712097168, "sampling/sampling_logp_difference/mean": 0.31529873609542847, "step": 153 }, { "clip_ratio/high_max": 0.375, "clip_ratio/high_mean": 0.1015625, "clip_ratio/low_mean": 0.046875, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.1484375, "entropy": 0.7310260534286499, "epoch": 0.6111111111111112, "grad_norm": 0.02926621586084366, "learning_rate": 1e-05, "loss": 0.0032, "step": 154 }, { "clip_ratio/high_max": 0.375, "clip_ratio/high_mean": 0.0703125, "clip_ratio/low_mean": 0.0703125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.140625, "entropy": 0.7456955909729004, "epoch": 0.6150793650793651, "grad_norm": 0.02747405879199505, "learning_rate": 1e-05, "loss": -0.0049, "step": 155 }, { "clip_ratio/high_max": 0.375, "clip_ratio/high_mean": 0.0703125, "clip_ratio/low_mean": 0.1015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.171875, "entropy": 0.7368719577789307, "epoch": 0.6190476190476191, "grad_norm": 0.013221199624240398, "learning_rate": 1e-05, "loss": -0.0002, "step": 156 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.001953125, "completions/max_length": 8192.0, "completions/max_terminated_length": 1005.0, "completions/mean_length": 420.419921875, "completions/mean_terminated_length": 405.2113342285156, "completions/min_length": 162.0, "completions/min_terminated_length": 162.0, "entropy": 0.753882646560669, "epoch": 0.623015873015873, "frac_reward_zero_std": 0.25, "grad_norm": 0.022960374131798744, "learning_rate": 1e-05, "loss": 0.0038, "num_tokens": 51126215.0, "reward": 0.8596706986427307, "reward_std": 0.13005268573760986, "rewards/ngram_repetition2/mean": -0.002401352394372225, "rewards/ngram_repetition2/std": 0.04045075923204422, "rewards/ngram_repetition3/mean": -0.002207773271948099, "rewards/ngram_repetition3/std": 0.04066954925656319, "rewards/sentence_repetition/mean": 0.0, "rewards/sentence_repetition/std": 0.0, "rewards/symbolic_reward_accuracy/mean": 0.83203125, "rewards/symbolic_reward_accuracy/std": 0.374204158782959, "rewards/symbolic_reward_partial_score/mean": 0.92431640625, "rewards/symbolic_reward_partial_score/std": 0.18705546855926514, "rewards/tag_count_reward/mean": 0.0, "rewards/tag_count_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.2354283332824707, "sampling/importance_sampling_ratio/min": 0.001049776910804212, "sampling/sampling_logp_difference/max": 6.859177589416504, "sampling/sampling_logp_difference/mean": 0.31415918469429016, "step": 157 }, { "clip_ratio/high_max": 0.5, "clip_ratio/high_mean": 0.1484375, "clip_ratio/low_mean": 0.0234375, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.171875, "entropy": 0.7279462516307831, "epoch": 0.626984126984127, "grad_norm": 0.016718048602342606, "learning_rate": 1e-05, "loss": 0.0071, "step": 158 }, { "clip_ratio/high_max": 0.5, "clip_ratio/high_mean": 0.140625, "clip_ratio/low_mean": 0.0390625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.1796875, "entropy": 0.7520553469657898, "epoch": 0.6309523809523809, "grad_norm": 0.022867700085043907, "learning_rate": 1e-05, "loss": -0.0021, "step": 159 }, { "clip_ratio/high_max": 0.375, "clip_ratio/high_mean": 0.09375, "clip_ratio/low_mean": 0.1015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.1953125, "entropy": 0.7489183247089386, "epoch": 0.6349206349206349, "grad_norm": 0.024869482964277267, "learning_rate": 1e-05, "loss": 0.0015, "step": 160 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.001953125, "completions/max_length": 8192.0, "completions/max_terminated_length": 7640.0, "completions/mean_length": 435.5, "completions/mean_terminated_length": 420.3209228515625, "completions/min_length": 104.0, "completions/min_terminated_length": 104.0, "entropy": 0.7565124034881592, "epoch": 0.6388888888888888, "frac_reward_zero_std": 0.21875, "grad_norm": 0.022382786497473717, "learning_rate": 1e-05, "loss": -0.0017, "num_tokens": 51700103.0, "reward": 0.8218352794647217, "reward_std": 0.12302250415086746, "rewards/ngram_repetition2/mean": -0.004329684656113386, "rewards/ngram_repetition2/std": 0.05752414092421532, "rewards/ngram_repetition3/mean": -0.004521804396063089, "rewards/ngram_repetition3/std": 0.05792729929089546, "rewards/sentence_repetition/mean": 0.0, "rewards/sentence_repetition/std": 0.0, "rewards/symbolic_reward_accuracy/mean": 0.7890625, "rewards/symbolic_reward_accuracy/std": 0.4083731174468994, "rewards/symbolic_reward_partial_score/mean": 0.89990234375, "rewards/symbolic_reward_partial_score/std": 0.20952138304710388, "rewards/tag_count_reward/mean": -0.00390625, "rewards/tag_count_reward/std": 0.06243881583213806, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.2375590801239014, "sampling/importance_sampling_ratio/min": 0.0024089612998068333, "sampling/sampling_logp_difference/max": 6.028559684753418, "sampling/sampling_logp_difference/mean": 0.3137196898460388, "step": 161 }, { "clip_ratio/high_max": 0.5, "clip_ratio/high_mean": 0.140625, "clip_ratio/low_mean": 0.0390625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.1796875, "entropy": 0.7605028748512268, "epoch": 0.6428571428571429, "grad_norm": 0.01553066074848175, "learning_rate": 1e-05, "loss": 0.0015, "step": 162 }, { "clip_ratio/high_max": 0.5, "clip_ratio/high_mean": 0.140625, "clip_ratio/low_mean": 0.0625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.203125, "entropy": 0.7431648671627045, "epoch": 0.6468253968253969, "grad_norm": 0.02609085664153099, "learning_rate": 1e-05, "loss": 0.0368, "step": 163 }, { "clip_ratio/high_max": 0.375, "clip_ratio/high_mean": 0.0859375, "clip_ratio/low_mean": 0.078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.1640625, "entropy": 0.7662903368473053, "epoch": 0.6507936507936508, "grad_norm": 0.022471001371741295, "learning_rate": 1e-05, "loss": -0.005, "step": 164 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 858.0, "completions/max_terminated_length": 858.0, "completions/mean_length": 371.578125, "completions/mean_terminated_length": 371.578125, "completions/min_length": 120.0, "completions/min_terminated_length": 120.0, "entropy": 0.7733668386936188, "epoch": 0.6547619047619048, "frac_reward_zero_std": 0.40625, "grad_norm": 0.022831041365861893, "learning_rate": 1e-05, "loss": 0.0015, "num_tokens": 52217263.0, "reward": 0.8468668460845947, "reward_std": 0.12018992751836777, "rewards/ngram_repetition2/mean": -0.0004056431935168803, "rewards/ngram_repetition2/std": 0.004367588087916374, "rewards/ngram_repetition3/mean": -0.00041355364373885095, "rewards/ngram_repetition3/std": 0.00433428306132555, "rewards/sentence_repetition/mean": 0.0, "rewards/sentence_repetition/std": 0.0, "rewards/symbolic_reward_accuracy/mean": 0.818359375, "rewards/symbolic_reward_accuracy/std": 0.38592514395713806, "rewards/symbolic_reward_partial_score/mean": 0.9140625, "rewards/symbolic_reward_partial_score/std": 0.20793341100215912, "rewards/tag_count_reward/mean": -0.001953125, "rewards/tag_count_reward/std": 0.04419417306780815, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.2520780563354492, "sampling/importance_sampling_ratio/min": 0.003758822102099657, "sampling/sampling_logp_difference/max": 5.583649635314941, "sampling/sampling_logp_difference/mean": 0.3351019620895386, "step": 165 }, { "clip_ratio/high_max": 0.375, "clip_ratio/high_mean": 0.109375, "clip_ratio/low_mean": 0.03125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.140625, "entropy": 0.7693372368812561, "epoch": 0.6587301587301587, "grad_norm": 0.012506059370934963, "learning_rate": 1e-05, "loss": -0.0052, "step": 166 }, { "clip_ratio/high_max": 0.25, "clip_ratio/high_mean": 0.078125, "clip_ratio/low_mean": 0.0546875, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.1328125, "entropy": 0.7789681553840637, "epoch": 0.6626984126984127, "grad_norm": 0.020651800557971, "learning_rate": 1e-05, "loss": 0.0029, "step": 167 }, { "clip_ratio/high_max": 0.375, "clip_ratio/high_mean": 0.0546875, "clip_ratio/low_mean": 0.109375, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.1640625, "entropy": 0.774013340473175, "epoch": 0.6666666666666666, "grad_norm": 0.01205294206738472, "learning_rate": 1e-05, "loss": -0.0017, "step": 168 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 833.0, "completions/max_terminated_length": 833.0, "completions/mean_length": 403.37890625, "completions/mean_terminated_length": 403.37890625, "completions/min_length": 73.0, "completions/min_terminated_length": 73.0, "entropy": 0.7953222095966339, "epoch": 0.6706349206349206, "frac_reward_zero_std": 0.09375, "grad_norm": 0.022628672420978546, "learning_rate": 1e-05, "loss": -0.0005, "num_tokens": 52790065.0, "reward": 0.8799871206283569, "reward_std": 0.14347109198570251, "rewards/ngram_repetition2/mean": -0.0017731175757944584, "rewards/ngram_repetition2/std": 0.011411396786570549, "rewards/ngram_repetition3/mean": -0.0024123992770910263, "rewards/ngram_repetition3/std": 0.013391369953751564, "rewards/sentence_repetition/mean": -2.9032831662334502e-05, "rewards/sentence_repetition/std": 0.0004716458497568965, "rewards/symbolic_reward_accuracy/mean": 0.853515625, "rewards/symbolic_reward_accuracy/std": 0.35393697023391724, "rewards/symbolic_reward_partial_score/mean": 0.94189453125, "rewards/symbolic_reward_partial_score/std": 0.1739315241575241, "rewards/tag_count_reward/mean": 0.0, "rewards/tag_count_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.264739751815796, "sampling/importance_sampling_ratio/min": 0.002341644372791052, "sampling/sampling_logp_difference/max": 6.056901931762695, "sampling/sampling_logp_difference/mean": 0.3465285897254944, "step": 169 }, { "clip_ratio/high_max": 0.375, "clip_ratio/high_mean": 0.1484375, "clip_ratio/low_mean": 0.0390625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.1875, "entropy": 0.7884858250617981, "epoch": 0.6746031746031746, "grad_norm": 0.02866140380501747, "learning_rate": 1e-05, "loss": 0.0067, "step": 170 }, { "clip_ratio/high_max": 0.5, "clip_ratio/high_mean": 0.1640625, "clip_ratio/low_mean": 0.0234375, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.1875, "entropy": 0.8134114146232605, "epoch": 0.6785714285714286, "grad_norm": 0.039607852697372437, "learning_rate": 1e-05, "loss": 0.0047, "step": 171 }, { "clip_ratio/high_max": 0.375, "clip_ratio/high_mean": 0.1171875, "clip_ratio/low_mean": 0.09375, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.2109375, "entropy": 0.7982254028320312, "epoch": 0.6825396825396826, "grad_norm": 0.024645311757922173, "learning_rate": 1e-05, "loss": -0.0062, "step": 172 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.001953125, "completions/max_length": 8192.0, "completions/max_terminated_length": 1048.0, "completions/mean_length": 444.85546875, "completions/mean_terminated_length": 429.6947021484375, "completions/min_length": 40.0, "completions/min_terminated_length": 40.0, "entropy": 0.7992468476295471, "epoch": 0.6865079365079365, "frac_reward_zero_std": 0.1875, "grad_norm": 0.023259693756699562, "learning_rate": 1e-05, "loss": -0.0033, "num_tokens": 53373415.0, "reward": 0.8464292287826538, "reward_std": 0.12976223230361938, "rewards/ngram_repetition2/mean": -0.004559888504445553, "rewards/ngram_repetition2/std": 0.03278464451432228, "rewards/ngram_repetition3/mean": -0.005261006765067577, "rewards/ngram_repetition3/std": 0.034283701330423355, "rewards/sentence_repetition/mean": -0.0005706384545192122, "rewards/sentence_repetition/std": 0.012872298248112202, "rewards/symbolic_reward_accuracy/mean": 0.81640625, "rewards/symbolic_reward_accuracy/std": 0.3875311613082886, "rewards/symbolic_reward_partial_score/mean": 0.91748046875, "rewards/symbolic_reward_partial_score/std": 0.20066450536251068, "rewards/tag_count_reward/mean": -0.001953125, "rewards/tag_count_reward/std": 0.04419417306780815, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.2592453956604004, "sampling/importance_sampling_ratio/min": 0.0008659582235850394, "sampling/sampling_logp_difference/max": 7.051673889160156, "sampling/sampling_logp_difference/mean": 0.3408927619457245, "step": 173 }, { "clip_ratio/high_max": 0.625, "clip_ratio/high_mean": 0.140625, "clip_ratio/low_mean": 0.0390625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.1796875, "entropy": 0.7798175513744354, "epoch": 0.6904761904761905, "grad_norm": 0.03253133222460747, "learning_rate": 1e-05, "loss": 0.0257, "step": 174 }, { "clip_ratio/high_max": 0.375, "clip_ratio/high_mean": 0.109375, "clip_ratio/low_mean": 0.0703125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.1796875, "entropy": 0.7998690605163574, "epoch": 0.6944444444444444, "grad_norm": 0.021749138832092285, "learning_rate": 1e-05, "loss": 0.0075, "step": 175 }, { "clip_ratio/high_max": 0.375, "clip_ratio/high_mean": 0.0390625, "clip_ratio/low_mean": 0.0625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.1015625, "entropy": 0.7944201827049255, "epoch": 0.6984126984126984, "grad_norm": 0.02608627825975418, "learning_rate": 1e-05, "loss": -0.0083, "step": 176 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 831.0, "completions/max_terminated_length": 831.0, "completions/mean_length": 389.10546875, "completions/mean_terminated_length": 389.10546875, "completions/min_length": 58.0, "completions/min_terminated_length": 58.0, "entropy": 0.7932335138320923, "epoch": 0.7023809523809523, "frac_reward_zero_std": 0.1875, "grad_norm": 0.022479817271232605, "learning_rate": 1e-05, "loss": 0.0028, "num_tokens": 53918557.0, "reward": 0.8576145768165588, "reward_std": 0.1379057765007019, "rewards/ngram_repetition2/mean": -0.002364100655540824, "rewards/ngram_repetition2/std": 0.023993264883756638, "rewards/ngram_repetition3/mean": -0.0027513415552675724, "rewards/ngram_repetition3/std": 0.024084746837615967, "rewards/sentence_repetition/mean": -2.9502600227715448e-05, "rewards/sentence_repetition/std": 0.0006675676559098065, "rewards/symbolic_reward_accuracy/mean": 0.822265625, "rewards/symbolic_reward_accuracy/std": 0.3826628625392914, "rewards/symbolic_reward_partial_score/mean": 0.94091796875, "rewards/symbolic_reward_partial_score/std": 0.1493643969297409, "rewards/tag_count_reward/mean": -0.001953125, "rewards/tag_count_reward/std": 0.04419417306780815, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.262540578842163, "sampling/importance_sampling_ratio/min": 0.003377706278115511, "sampling/sampling_logp_difference/max": 5.690558433532715, "sampling/sampling_logp_difference/mean": 0.3483034074306488, "step": 177 }, { "clip_ratio/high_max": 0.625, "clip_ratio/high_mean": 0.1328125, "clip_ratio/low_mean": 0.03125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.1640625, "entropy": 0.8167908787727356, "epoch": 0.7063492063492064, "grad_norm": 0.018045730888843536, "learning_rate": 1e-05, "loss": -0.0075, "step": 178 }, { "clip_ratio/high_max": 0.5, "clip_ratio/high_mean": 0.109375, "clip_ratio/low_mean": 0.0859375, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.1953125, "entropy": 0.8248741626739502, "epoch": 0.7103174603174603, "grad_norm": 0.027403375133872032, "learning_rate": 1e-05, "loss": 0.0069, "step": 179 }, { "clip_ratio/high_max": 0.375, "clip_ratio/high_mean": 0.09375, "clip_ratio/low_mean": 0.046875, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.140625, "entropy": 0.8203136920928955, "epoch": 0.7142857142857143, "grad_norm": 0.025887934491038322, "learning_rate": 1e-05, "loss": -0.0021, "step": 180 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.001953125, "completions/max_length": 8192.0, "completions/max_terminated_length": 753.0, "completions/mean_length": 368.279296875, "completions/mean_terminated_length": 352.96868896484375, "completions/min_length": 62.0, "completions/min_terminated_length": 62.0, "entropy": 0.7895427644252777, "epoch": 0.7182539682539683, "frac_reward_zero_std": 0.375, "grad_norm": 0.013407798483967781, "learning_rate": 1e-05, "loss": 0.0016, "num_tokens": 54457708.0, "reward": 0.8706079721450806, "reward_std": 0.0937468558549881, "rewards/ngram_repetition2/mean": -0.0021983219776302576, "rewards/ngram_repetition2/std": 0.036178067326545715, "rewards/ngram_repetition3/mean": -0.0023472520988434553, "rewards/ngram_repetition3/std": 0.03697047755122185, "rewards/sentence_repetition/mean": -8.527399768354371e-05, "rewards/sentence_repetition/std": 0.0019295302918180823, "rewards/symbolic_reward_accuracy/mean": 0.837890625, "rewards/symbolic_reward_accuracy/std": 0.3689115643501282, "rewards/symbolic_reward_partial_score/mean": 0.94775390625, "rewards/symbolic_reward_partial_score/std": 0.14698481559753418, "rewards/tag_count_reward/mean": -0.001953125, "rewards/tag_count_reward/std": 0.04419417306780815, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.2572288513183594, "sampling/importance_sampling_ratio/min": 0.003313617082312703, "sampling/sampling_logp_difference/max": 5.709714889526367, "sampling/sampling_logp_difference/mean": 0.33956456184387207, "step": 181 }, { "clip_ratio/high_max": 0.625, "clip_ratio/high_mean": 0.1171875, "clip_ratio/low_mean": 0.0390625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.15625, "entropy": 0.7903194725513458, "epoch": 0.7222222222222222, "grad_norm": 0.023882685229182243, "learning_rate": 1e-05, "loss": 0.0357, "step": 182 }, { "clip_ratio/high_max": 0.625, "clip_ratio/high_mean": 0.125, "clip_ratio/low_mean": 0.03125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.15625, "entropy": 0.8134073615074158, "epoch": 0.7261904761904762, "grad_norm": 0.020086700096726418, "learning_rate": 1e-05, "loss": -0.0047, "step": 183 }, { "clip_ratio/high_max": 0.5, "clip_ratio/high_mean": 0.1015625, "clip_ratio/low_mean": 0.0390625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.140625, "entropy": 0.8180882334709167, "epoch": 0.7301587301587301, "grad_norm": 0.011018384248018265, "learning_rate": 1e-05, "loss": -0.0044, "step": 184 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 916.0, "completions/max_terminated_length": 916.0, "completions/mean_length": 333.61328125, "completions/mean_terminated_length": 333.61328125, "completions/min_length": 50.0, "completions/min_terminated_length": 50.0, "entropy": 0.7859732508659363, "epoch": 0.7341269841269841, "frac_reward_zero_std": 0.375, "grad_norm": 0.026284197345376015, "learning_rate": 1e-05, "loss": -0.0035, "num_tokens": 54982438.0, "reward": 0.8855394124984741, "reward_std": 0.14925158023834229, "rewards/ngram_repetition2/mean": -0.00032174953958019614, "rewards/ngram_repetition2/std": 0.003948765806853771, "rewards/ngram_repetition3/mean": -0.0004272446967661381, "rewards/ngram_repetition3/std": 0.0041628675535321236, "rewards/sentence_repetition/mean": 0.0, "rewards/sentence_repetition/std": 0.0, "rewards/symbolic_reward_accuracy/mean": 0.857421875, "rewards/symbolic_reward_accuracy/std": 0.3499840497970581, "rewards/symbolic_reward_partial_score/mean": 0.953125, "rewards/symbolic_reward_partial_score/std": 0.15082940459251404, "rewards/tag_count_reward/mean": -0.005859375, "rewards/tag_count_reward/std": 0.07639661431312561, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.2548885345458984, "sampling/importance_sampling_ratio/min": 0.0013933131704106927, "sampling/sampling_logp_difference/max": 6.576070785522461, "sampling/sampling_logp_difference/mean": 0.33905351161956787, "step": 185 }, { "clip_ratio/high_max": 0.625, "clip_ratio/high_mean": 0.1171875, "clip_ratio/low_mean": 0.03125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.1484375, "entropy": 0.7900300920009613, "epoch": 0.7380952380952381, "grad_norm": 0.02144605666399002, "learning_rate": 1e-05, "loss": 0.0041, "step": 186 }, { "clip_ratio/high_max": 0.5, "clip_ratio/high_mean": 0.1328125, "clip_ratio/low_mean": 0.0625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.1953125, "entropy": 0.7845631837844849, "epoch": 0.7420634920634921, "grad_norm": 0.02783539891242981, "learning_rate": 1e-05, "loss": 0.0052, "step": 187 }, { "clip_ratio/high_max": 0.375, "clip_ratio/high_mean": 0.046875, "clip_ratio/low_mean": 0.078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.125, "entropy": 0.7857345938682556, "epoch": 0.746031746031746, "grad_norm": 0.009965131059288979, "learning_rate": 1e-05, "loss": -0.0023, "step": 188 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 682.0, "completions/max_terminated_length": 682.0, "completions/mean_length": 320.369140625, "completions/mean_terminated_length": 320.369140625, "completions/min_length": 39.0, "completions/min_terminated_length": 39.0, "entropy": 0.8152837157249451, "epoch": 0.75, "frac_reward_zero_std": 0.03125, "grad_norm": 0.021727463230490685, "learning_rate": 1e-05, "loss": -0.0047, "num_tokens": 55508067.0, "reward": 0.8853533267974854, "reward_std": 0.13714200258255005, "rewards/ngram_repetition2/mean": -0.0023568901233375072, "rewards/ngram_repetition2/std": 0.016553474590182304, "rewards/ngram_repetition3/mean": -0.002354577649384737, "rewards/ngram_repetition3/std": 0.01575542613863945, "rewards/sentence_repetition/mean": 0.0, "rewards/sentence_repetition/std": 0.0, "rewards/symbolic_reward_accuracy/mean": 0.86328125, "rewards/symbolic_reward_accuracy/std": 0.3438861668109894, "rewards/symbolic_reward_partial_score/mean": 0.94091796875, "rewards/symbolic_reward_partial_score/std": 0.17777857184410095, "rewards/tag_count_reward/mean": -0.01171875, "rewards/tag_count_reward/std": 0.10772226005792618, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.2657219171524048, "sampling/importance_sampling_ratio/min": 0.0016435382422059774, "sampling/sampling_logp_difference/max": 6.4109039306640625, "sampling/sampling_logp_difference/mean": 0.34767937660217285, "step": 189 }, { "clip_ratio/high_max": 0.75, "clip_ratio/high_mean": 0.296875, "clip_ratio/low_mean": 0.0234375, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.3203125, "entropy": 0.8162022233009338, "epoch": 0.753968253968254, "grad_norm": 0.015771852806210518, "learning_rate": 1e-05, "loss": -0.0056, "step": 190 }, { "clip_ratio/high_max": 0.625, "clip_ratio/high_mean": 0.1953125, "clip_ratio/low_mean": 0.046875, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.2421875, "entropy": 0.825575977563858, "epoch": 0.7579365079365079, "grad_norm": 0.012481063604354858, "learning_rate": 1e-05, "loss": -0.0017, "step": 191 }, { "clip_ratio/high_max": 0.625, "clip_ratio/high_mean": 0.1953125, "clip_ratio/low_mean": 0.078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.2734375, "entropy": 0.8164065778255463, "epoch": 0.7619047619047619, "grad_norm": 0.02995295636355877, "learning_rate": 1e-05, "loss": 0.0111, "step": 192 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 643.0, "completions/max_terminated_length": 643.0, "completions/mean_length": 314.73046875, "completions/mean_terminated_length": 314.73046875, "completions/min_length": 46.0, "completions/min_terminated_length": 46.0, "entropy": 0.8211237788200378, "epoch": 0.7658730158730159, "frac_reward_zero_std": 0.25, "grad_norm": 0.023150430992245674, "learning_rate": 1e-05, "loss": 0.0004, "num_tokens": 56027481.0, "reward": 0.7719883918762207, "reward_std": 0.13292020559310913, "rewards/ngram_repetition2/mean": -0.0016107636038213968, "rewards/ngram_repetition2/std": 0.01730688475072384, "rewards/ngram_repetition3/mean": -0.0015767120057716966, "rewards/ngram_repetition3/std": 0.01612774096429348, "rewards/sentence_repetition/mean": -0.00012316471838857979, "rewards/sentence_repetition/std": 0.0024990118108689785, "rewards/symbolic_reward_accuracy/mean": 0.724609375, "rewards/symbolic_reward_accuracy/std": 0.44714778661727905, "rewards/symbolic_reward_partial_score/mean": 0.89111328125, "rewards/symbolic_reward_partial_score/std": 0.21441252529621124, "rewards/tag_count_reward/mean": -0.025390625, "rewards/tag_count_reward/std": 0.15746226906776428, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.2719165086746216, "sampling/importance_sampling_ratio/min": 0.0007087697740644217, "sampling/sampling_logp_difference/max": 7.251979827880859, "sampling/sampling_logp_difference/mean": 0.3545856475830078, "step": 193 }, { "clip_ratio/high_max": 0.375, "clip_ratio/high_mean": 0.0859375, "clip_ratio/low_mean": 0.0859375, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.171875, "entropy": 0.8274082243442535, "epoch": 0.7698412698412699, "grad_norm": 0.022844254970550537, "learning_rate": 1e-05, "loss": 0.0029, "step": 194 }, { "clip_ratio/high_max": 0.25, "clip_ratio/high_mean": 0.0625, "clip_ratio/low_mean": 0.1171875, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.1796875, "entropy": 0.8443744480609894, "epoch": 0.7738095238095238, "grad_norm": 0.01150192879140377, "learning_rate": 1e-05, "loss": -0.0014, "step": 195 }, { "clip_ratio/high_max": 0.375, "clip_ratio/high_mean": 0.046875, "clip_ratio/low_mean": 0.1171875, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.1640625, "entropy": 0.8464749753475189, "epoch": 0.7777777777777778, "grad_norm": 0.013477575033903122, "learning_rate": 1e-05, "loss": -0.0, "step": 196 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.00390625, "completions/max_length": 8192.0, "completions/max_terminated_length": 705.0, "completions/mean_length": 359.345703125, "completions/mean_terminated_length": 328.6294250488281, "completions/min_length": 39.0, "completions/min_terminated_length": 39.0, "entropy": 0.8584817051887512, "epoch": 0.7817460317460317, "frac_reward_zero_std": 0.21875, "grad_norm": 0.03277435526251793, "learning_rate": 1e-05, "loss": 0.0314, "num_tokens": 56553034.0, "reward": 0.8151940107345581, "reward_std": 0.11151319742202759, "rewards/ngram_repetition2/mean": -0.0044229887425899506, "rewards/ngram_repetition2/std": 0.058326393365859985, "rewards/ngram_repetition3/mean": -0.004459192976355553, "rewards/ngram_repetition3/std": 0.05861266702413559, "rewards/sentence_repetition/mean": -3.294745329185389e-05, "rewards/sentence_repetition/std": 0.0007455156883224845, "rewards/symbolic_reward_accuracy/mean": 0.78125, "rewards/symbolic_reward_accuracy/std": 0.41380295157432556, "rewards/symbolic_reward_partial_score/mean": 0.89599609375, "rewards/symbolic_reward_partial_score/std": 0.23781166970729828, "rewards/tag_count_reward/mean": -0.00390625, "rewards/tag_count_reward/std": 0.06243881583213806, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.2720357179641724, "sampling/importance_sampling_ratio/min": 0.0010772914392873645, "sampling/sampling_logp_difference/max": 6.833305358886719, "sampling/sampling_logp_difference/mean": 0.35436925292015076, "step": 197 }, { "clip_ratio/high_max": 0.5, "clip_ratio/high_mean": 0.109375, "clip_ratio/low_mean": 0.03125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.140625, "entropy": 0.8918771743774414, "epoch": 0.7857142857142857, "grad_norm": 0.013567506335675716, "learning_rate": 1e-05, "loss": 0.0029, "step": 198 }, { "clip_ratio/high_max": 0.625, "clip_ratio/high_mean": 0.1796875, "clip_ratio/low_mean": 0.0546875, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.234375, "entropy": 0.862952321767807, "epoch": 0.7896825396825397, "grad_norm": 0.018849479034543037, "learning_rate": 1e-05, "loss": 0.024, "step": 199 }, { "clip_ratio/high_max": 0.625, "clip_ratio/high_mean": 0.140625, "clip_ratio/low_mean": 0.0546875, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.1953125, "entropy": 0.899914562702179, "epoch": 0.7936507936507936, "grad_norm": 0.02006298303604126, "learning_rate": 1e-05, "loss": -0.0018, "step": 200 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 513.0, "completions/max_terminated_length": 513.0, "completions/mean_length": 264.5703125, "completions/mean_terminated_length": 264.5703125, "completions/min_length": 35.0, "completions/min_terminated_length": 35.0, "entropy": 0.8936683535575867, "epoch": 0.7976190476190477, "frac_reward_zero_std": 0.59375, "grad_norm": 0.012529753148555756, "learning_rate": 1e-05, "loss": 0.0, "num_tokens": 57022702.0, "reward": 0.8934560418128967, "reward_std": 0.06837594509124756, "rewards/ngram_repetition2/mean": -6.041863889549859e-05, "rewards/ngram_repetition2/std": 0.0013671177439391613, "rewards/ngram_repetition3/mean": -4.2074119846802205e-05, "rewards/ngram_repetition3/std": 0.0006955123972147703, "rewards/sentence_repetition/mean": 0.0, "rewards/sentence_repetition/std": 0.0, "rewards/symbolic_reward_accuracy/mean": 0.869140625, "rewards/symbolic_reward_accuracy/std": 0.33757632970809937, "rewards/symbolic_reward_partial_score/mean": 0.9521484375, "rewards/symbolic_reward_partial_score/std": 0.15373775362968445, "rewards/tag_count_reward/mean": -0.005859375, "rewards/tag_count_reward/std": 0.07639661431312561, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.2860223054885864, "sampling/importance_sampling_ratio/min": 0.0012071708915755153, "sampling/sampling_logp_difference/max": 6.719475746154785, "sampling/sampling_logp_difference/mean": 0.37654852867126465, "step": 201 }, { "clip_ratio/high_max": 0.625, "clip_ratio/high_mean": 0.1484375, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.15625, "entropy": 0.8909170925617218, "epoch": 0.8015873015873016, "grad_norm": 0.01546390913426876, "learning_rate": 1e-05, "loss": 0.0004, "step": 202 }, { "clip_ratio/high_max": 0.875, "clip_ratio/high_mean": 0.1796875, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.1875, "entropy": 0.9000489115715027, "epoch": 0.8055555555555556, "grad_norm": 0.018938574939966202, "learning_rate": 1e-05, "loss": -0.0019, "step": 203 }, { "clip_ratio/high_max": 0.375, "clip_ratio/high_mean": 0.1015625, "clip_ratio/low_mean": 0.0390625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.140625, "entropy": 0.9011846482753754, "epoch": 0.8095238095238095, "grad_norm": 0.006644058506935835, "learning_rate": 1e-05, "loss": 0.0032, "step": 204 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 619.0, "completions/max_terminated_length": 619.0, "completions/mean_length": 239.53125, "completions/mean_terminated_length": 239.53125, "completions/min_length": 35.0, "completions/min_terminated_length": 35.0, "entropy": 0.9183432161808014, "epoch": 0.8134920634920635, "frac_reward_zero_std": 0.40625, "grad_norm": 0.028888095170259476, "learning_rate": 1e-05, "loss": 0.0035, "num_tokens": 57505278.0, "reward": 0.9636610150337219, "reward_std": 0.09627415239810944, "rewards/ngram_repetition2/mean": -0.0005565091269090772, "rewards/ngram_repetition2/std": 0.007866894826292992, "rewards/ngram_repetition3/mean": -0.0005293096182867885, "rewards/ngram_repetition3/std": 0.005413535982370377, "rewards/sentence_repetition/mean": 0.0, "rewards/sentence_repetition/std": 0.0, "rewards/symbolic_reward_accuracy/mean": 0.958984375, "rewards/symbolic_reward_accuracy/std": 0.19852031767368317, "rewards/symbolic_reward_partial_score/mean": 0.974609375, "rewards/symbolic_reward_partial_score/std": 0.14534473419189453, "rewards/tag_count_reward/mean": 0.0, "rewards/tag_count_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.2886309623718262, "sampling/importance_sampling_ratio/min": 0.0009261192171834409, "sampling/sampling_logp_difference/max": 6.9845075607299805, "sampling/sampling_logp_difference/mean": 0.37991994619369507, "step": 205 }, { "clip_ratio/high_max": 0.625, "clip_ratio/high_mean": 0.125, "clip_ratio/low_mean": 0.0390625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.1640625, "entropy": 0.8890846967697144, "epoch": 0.8174603174603174, "grad_norm": 0.01094829011708498, "learning_rate": 1e-05, "loss": -0.0013, "step": 206 }, { "clip_ratio/high_max": 0.25, "clip_ratio/high_mean": 0.0625, "clip_ratio/low_mean": 0.046875, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.109375, "entropy": 0.9073884189128876, "epoch": 0.8214285714285714, "grad_norm": 0.0091655682772398, "learning_rate": 1e-05, "loss": -0.002, "step": 207 }, { "clip_ratio/high_max": 0.375, "clip_ratio/high_mean": 0.0546875, "clip_ratio/low_mean": 0.0390625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.09375, "entropy": 0.8968927562236786, "epoch": 0.8253968253968254, "grad_norm": 0.014174257405102253, "learning_rate": 1e-05, "loss": -0.002, "step": 208 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.00390625, "completions/max_length": 8192.0, "completions/max_terminated_length": 739.0, "completions/mean_length": 303.826171875, "completions/mean_terminated_length": 272.8921813964844, "completions/min_length": 26.0, "completions/min_terminated_length": 26.0, "entropy": 0.9361216723918915, "epoch": 0.8293650793650794, "frac_reward_zero_std": 0.21875, "grad_norm": 0.01776828058063984, "learning_rate": 1e-05, "loss": -0.0028, "num_tokens": 58019109.0, "reward": 0.8948224782943726, "reward_std": 0.10340611636638641, "rewards/ngram_repetition2/mean": -0.0049706753343343735, "rewards/ngram_repetition2/std": 0.056981075555086136, "rewards/ngram_repetition3/mean": -0.0049719493836164474, "rewards/ngram_repetition3/std": 0.057049937546253204, "rewards/sentence_repetition/mean": 0.0, "rewards/sentence_repetition/std": 0.0, "rewards/symbolic_reward_accuracy/mean": 0.876953125, "rewards/symbolic_reward_accuracy/std": 0.32881227135658264, "rewards/symbolic_reward_partial_score/mean": 0.9375, "rewards/symbolic_reward_partial_score/std": 0.20332364737987518, "rewards/tag_count_reward/mean": -0.001953125, "rewards/tag_count_reward/std": 0.04419417306780815, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.2766987085342407, "sampling/importance_sampling_ratio/min": 0.0024687964469194412, "sampling/sampling_logp_difference/max": 6.004024505615234, "sampling/sampling_logp_difference/mean": 0.3604908585548401, "step": 209 }, { "clip_ratio/high_max": 0.75, "clip_ratio/high_mean": 0.28125, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.2890625, "entropy": 0.8996360898017883, "epoch": 0.8333333333333334, "grad_norm": 0.022127976641058922, "learning_rate": 1e-05, "loss": 0.0278, "step": 210 }, { "clip_ratio/high_max": 0.75, "clip_ratio/high_mean": 0.2734375, "clip_ratio/low_mean": 0.03125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.3046875, "entropy": 0.926760733127594, "epoch": 0.8373015873015873, "grad_norm": 0.02981238253414631, "learning_rate": 1e-05, "loss": 0.0028, "step": 211 }, { "clip_ratio/high_max": 0.5, "clip_ratio/high_mean": 0.15625, "clip_ratio/low_mean": 0.0546875, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.2109375, "entropy": 0.8881908655166626, "epoch": 0.8412698412698413, "grad_norm": 0.022208843380212784, "learning_rate": 1e-05, "loss": 0.0234, "step": 212 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 598.0, "completions/max_terminated_length": 598.0, "completions/mean_length": 285.84375, "completions/mean_terminated_length": 285.84375, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "entropy": 0.9268574714660645, "epoch": 0.8452380952380952, "frac_reward_zero_std": 0.3125, "grad_norm": 0.035097602754831314, "learning_rate": 1e-05, "loss": 0.0021, "num_tokens": 58530069.0, "reward": 0.8493503332138062, "reward_std": 0.11072182655334473, "rewards/ngram_repetition2/mean": -0.0008629187941551208, "rewards/ngram_repetition2/std": 0.0062627228908240795, "rewards/ngram_repetition3/mean": -0.0006272217724472284, "rewards/ngram_repetition3/std": 0.004538677167147398, "rewards/sentence_repetition/mean": 0.0, "rewards/sentence_repetition/std": 0.0, "rewards/symbolic_reward_accuracy/mean": 0.826171875, "rewards/symbolic_reward_accuracy/std": 0.3793322443962097, "rewards/symbolic_reward_partial_score/mean": 0.90673828125, "rewards/symbolic_reward_partial_score/std": 0.23138844966888428, "rewards/tag_count_reward/mean": -0.009765625, "rewards/tag_count_reward/std": 0.09843364357948303, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.296511173248291, "sampling/importance_sampling_ratio/min": 0.0015479788416996598, "sampling/sampling_logp_difference/max": 6.4708051681518555, "sampling/sampling_logp_difference/mean": 0.38745826482772827, "step": 213 }, { "clip_ratio/high_max": 0.25, "clip_ratio/high_mean": 0.03125, "clip_ratio/low_mean": 0.0703125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.1015625, "entropy": 0.9234279692173004, "epoch": 0.8492063492063492, "grad_norm": 0.02048969455063343, "learning_rate": 1e-05, "loss": -0.0012, "step": 214 }, { "clip_ratio/high_max": 0.25, "clip_ratio/high_mean": 0.03125, "clip_ratio/low_mean": 0.0390625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0703125, "entropy": 0.8959455490112305, "epoch": 0.8531746031746031, "grad_norm": 0.020006103441119194, "learning_rate": 1e-05, "loss": -0.0039, "step": 215 }, { "clip_ratio/high_max": 0.375, "clip_ratio/high_mean": 0.0390625, "clip_ratio/low_mean": 0.1015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.140625, "entropy": 0.9179676175117493, "epoch": 0.8571428571428571, "grad_norm": 0.014472209848463535, "learning_rate": 1e-05, "loss": 0.0031, "step": 216 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 553.0, "completions/max_terminated_length": 553.0, "completions/mean_length": 247.04296875, "completions/mean_terminated_length": 247.04296875, "completions/min_length": 26.0, "completions/min_terminated_length": 26.0, "entropy": 0.8857556581497192, "epoch": 0.8611111111111112, "frac_reward_zero_std": 0.28125, "grad_norm": 0.01699736714363098, "learning_rate": 1e-05, "loss": -0.0029, "num_tokens": 58999467.0, "reward": 0.8448055386543274, "reward_std": 0.11405006051063538, "rewards/ngram_repetition2/mean": -0.0010031338315457106, "rewards/ngram_repetition2/std": 0.007478474639356136, "rewards/ngram_repetition3/mean": -0.0008484901045449078, "rewards/ngram_repetition3/std": 0.005641194060444832, "rewards/sentence_repetition/mean": -1.6180860257009044e-05, "rewards/sentence_repetition/std": 0.0003661310765892267, "rewards/symbolic_reward_accuracy/mean": 0.81640625, "rewards/symbolic_reward_accuracy/std": 0.3875311613082886, "rewards/symbolic_reward_partial_score/mean": 0.9111328125, "rewards/symbolic_reward_partial_score/std": 0.22099465131759644, "rewards/tag_count_reward/mean": 0.0, "rewards/tag_count_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.2897417545318604, "sampling/importance_sampling_ratio/min": 0.002099553821608424, "sampling/sampling_logp_difference/max": 6.166030406951904, "sampling/sampling_logp_difference/mean": 0.3768787980079651, "step": 217 }, { "clip_ratio/high_max": 0.875, "clip_ratio/high_mean": 0.328125, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.34375, "entropy": 0.9093270599842072, "epoch": 0.8650793650793651, "grad_norm": 0.014405990950763226, "learning_rate": 1e-05, "loss": 0.0019, "step": 218 }, { "clip_ratio/high_max": 0.625, "clip_ratio/high_mean": 0.2734375, "clip_ratio/low_mean": 0.03125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.3046875, "entropy": 0.9079999327659607, "epoch": 0.8690476190476191, "grad_norm": 0.025361647829413414, "learning_rate": 1e-05, "loss": 0.0022, "step": 219 }, { "clip_ratio/high_max": 0.5, "clip_ratio/high_mean": 0.15625, "clip_ratio/low_mean": 0.0546875, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.2109375, "entropy": 0.9114420115947723, "epoch": 0.873015873015873, "grad_norm": 0.014707439579069614, "learning_rate": 1e-05, "loss": -0.0027, "step": 220 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.001953125, "completions/max_length": 8192.0, "completions/max_terminated_length": 580.0, "completions/mean_length": 246.443359375, "completions/mean_terminated_length": 230.89431762695312, "completions/min_length": 26.0, "completions/min_terminated_length": 26.0, "entropy": 0.9020665884017944, "epoch": 0.876984126984127, "frac_reward_zero_std": 0.03125, "grad_norm": 0.015136524103581905, "learning_rate": 1e-05, "loss": -0.0019, "num_tokens": 59477902.0, "reward": 0.8252206444740295, "reward_std": 0.0666418969631195, "rewards/ngram_repetition2/mean": -0.006709674373269081, "rewards/ngram_repetition2/std": 0.04598354548215866, "rewards/ngram_repetition3/mean": -0.00523712020367384, "rewards/ngram_repetition3/std": 0.044061921536922455, "rewards/sentence_repetition/mean": -0.00016643170965835452, "rewards/sentence_repetition/std": 0.0037659197114408016, "rewards/symbolic_reward_accuracy/mean": 0.78515625, "rewards/symbolic_reward_accuracy/std": 0.4111155867576599, "rewards/symbolic_reward_partial_score/mean": 0.92041015625, "rewards/symbolic_reward_partial_score/std": 0.17802028357982635, "rewards/tag_count_reward/mean": -0.00390625, "rewards/tag_count_reward/std": 0.06243881583213806, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.28969144821167, "sampling/importance_sampling_ratio/min": 0.0022876879666000605, "sampling/sampling_logp_difference/max": 6.08021354675293, "sampling/sampling_logp_difference/mean": 0.3743869662284851, "step": 221 }, { "clip_ratio/high_max": 0.875, "clip_ratio/high_mean": 0.3984375, "clip_ratio/low_mean": 0.0078125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.40625, "entropy": 0.8915797173976898, "epoch": 0.8809523809523809, "grad_norm": 0.034528739750385284, "learning_rate": 1e-05, "loss": 0.031, "step": 222 }, { "clip_ratio/high_max": 0.75, "clip_ratio/high_mean": 0.375, "clip_ratio/low_mean": 0.0234375, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.3984375, "entropy": 0.9305501878261566, "epoch": 0.8849206349206349, "grad_norm": 0.015339905396103859, "learning_rate": 1e-05, "loss": -0.005, "step": 223 }, { "clip_ratio/high_max": 0.75, "clip_ratio/high_mean": 0.2265625, "clip_ratio/low_mean": 0.0703125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.296875, "entropy": 0.9145832061767578, "epoch": 0.8888888888888888, "grad_norm": 0.012737227603793144, "learning_rate": 1e-05, "loss": 0.0016, "step": 224 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 437.0, "completions/max_terminated_length": 437.0, "completions/mean_length": 191.08984375, "completions/mean_terminated_length": 191.08984375, "completions/min_length": 26.0, "completions/min_terminated_length": 26.0, "entropy": 0.8822877407073975, "epoch": 0.8928571428571429, "frac_reward_zero_std": 0.0625, "grad_norm": 0.0251311082392931, "learning_rate": 1e-05, "loss": 0.0027, "num_tokens": 59918652.0, "reward": 0.9493194222450256, "reward_std": 0.1005542054772377, "rewards/ngram_repetition2/mean": -0.007611474953591824, "rewards/ngram_repetition2/std": 0.02910919487476349, "rewards/ngram_repetition3/mean": -0.006581652909517288, "rewards/ngram_repetition3/std": 0.025445405393838882, "rewards/sentence_repetition/mean": -0.00015292500029318035, "rewards/sentence_repetition/std": 0.0032785949297249317, "rewards/symbolic_reward_accuracy/mean": 0.935546875, "rewards/symbolic_reward_accuracy/std": 0.24579854309558868, "rewards/symbolic_reward_partial_score/mean": 0.98193359375, "rewards/symbolic_reward_partial_score/std": 0.09534555673599243, "rewards/tag_count_reward/mean": 0.0, "rewards/tag_count_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.2905950546264648, "sampling/importance_sampling_ratio/min": 0.003204482840374112, "sampling/sampling_logp_difference/max": 5.743204593658447, "sampling/sampling_logp_difference/mean": 0.3775941729545593, "step": 225 }, { "clip_ratio/high_max": 0.375, "clip_ratio/high_mean": 0.125, "clip_ratio/low_mean": 0.0703125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.1953125, "entropy": 0.8780872523784637, "epoch": 0.8968253968253969, "grad_norm": 0.019083455204963684, "learning_rate": 1e-05, "loss": -0.0007, "step": 226 }, { "clip_ratio/high_max": 0.5, "clip_ratio/high_mean": 0.140625, "clip_ratio/low_mean": 0.0859375, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.2265625, "entropy": 0.8845895230770111, "epoch": 0.9007936507936508, "grad_norm": 0.0213555246591568, "learning_rate": 1e-05, "loss": -0.0007, "step": 227 }, { "clip_ratio/high_max": 0.5, "clip_ratio/high_mean": 0.1328125, "clip_ratio/low_mean": 0.1015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.234375, "entropy": 0.8883778154850006, "epoch": 0.9047619047619048, "grad_norm": 0.01759985089302063, "learning_rate": 1e-05, "loss": 0.0008, "step": 228 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 343.0, "completions/max_terminated_length": 343.0, "completions/mean_length": 149.88671875, "completions/mean_terminated_length": 149.88671875, "completions/min_length": 35.0, "completions/min_terminated_length": 35.0, "entropy": 0.8275955617427826, "epoch": 0.9087301587301587, "frac_reward_zero_std": 0.15625, "grad_norm": 0.020159702748060226, "learning_rate": 1e-05, "loss": 0.0018, "num_tokens": 60364674.0, "reward": 0.855030357837677, "reward_std": 0.0830850899219513, "rewards/ngram_repetition2/mean": -0.0022108093835413456, "rewards/ngram_repetition2/std": 0.014428721740841866, "rewards/ngram_repetition3/mean": -0.002462912118062377, "rewards/ngram_repetition3/std": 0.014186910353600979, "rewards/sentence_repetition/mean": -0.00010279605339746922, "rewards/sentence_repetition/std": 0.0023260090965777636, "rewards/symbolic_reward_accuracy/mean": 0.81640625, "rewards/symbolic_reward_accuracy/std": 0.3875311613082886, "rewards/symbolic_reward_partial_score/mean": 0.9453125, "rewards/symbolic_reward_partial_score/std": 0.14141270518302917, "rewards/tag_count_reward/mean": 0.0, "rewards/tag_count_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.278295636177063, "sampling/importance_sampling_ratio/min": 0.0029066430870443583, "sampling/sampling_logp_difference/max": 5.840756416320801, "sampling/sampling_logp_difference/mean": 0.3616718649864197, "step": 229 }, { "clip_ratio/high_max": 0.625, "clip_ratio/high_mean": 0.1484375, "clip_ratio/low_mean": 0.09375, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.2421875, "entropy": 0.831957995891571, "epoch": 0.9126984126984127, "grad_norm": 0.01329135987907648, "learning_rate": 1e-05, "loss": 0.002, "step": 230 }, { "clip_ratio/high_max": 0.625, "clip_ratio/high_mean": 0.1875, "clip_ratio/low_mean": 0.03125, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.21875, "entropy": 0.8492329716682434, "epoch": 0.9166666666666666, "grad_norm": 0.00935891643166542, "learning_rate": 1e-05, "loss": -0.0028, "step": 231 }, { "clip_ratio/high_max": 0.625, "clip_ratio/high_mean": 0.140625, "clip_ratio/low_mean": 0.0625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.203125, "entropy": 0.8568889796733856, "epoch": 0.9206349206349206, "grad_norm": 0.009763935580849648, "learning_rate": 1e-05, "loss": -0.0002, "step": 232 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 364.0, "completions/max_terminated_length": 364.0, "completions/mean_length": 138.36328125, "completions/mean_terminated_length": 138.36328125, "completions/min_length": 35.0, "completions/min_terminated_length": 35.0, "entropy": 0.8046996593475342, "epoch": 0.9246031746031746, "frac_reward_zero_std": 0.09375, "grad_norm": 0.02738122083246708, "learning_rate": 1e-05, "loss": 0.0012, "num_tokens": 60789116.0, "reward": 0.7827650904655457, "reward_std": 0.06004778668284416, "rewards/ngram_repetition2/mean": -0.002412165980786085, "rewards/ngram_repetition2/std": 0.014702088199555874, "rewards/ngram_repetition3/mean": -0.002328635659068823, "rewards/ngram_repetition3/std": 0.012698164209723473, "rewards/sentence_repetition/mean": 0.0, "rewards/sentence_repetition/std": 0.0, "rewards/symbolic_reward_accuracy/mean": 0.7265625, "rewards/symbolic_reward_accuracy/std": 0.4461594223976135, "rewards/symbolic_reward_partial_score/mean": 0.9140625, "rewards/symbolic_reward_partial_score/std": 0.16754092276096344, "rewards/tag_count_reward/mean": 0.0, "rewards/tag_count_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.2642192840576172, "sampling/importance_sampling_ratio/min": 0.005728540942072868, "sampling/sampling_logp_difference/max": 5.162294387817383, "sampling/sampling_logp_difference/mean": 0.3446877896785736, "step": 233 }, { "clip_ratio/high_max": 0.75, "clip_ratio/high_mean": 0.1953125, "clip_ratio/low_mean": 0.046875, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.2421875, "entropy": 0.8074749410152435, "epoch": 0.9285714285714286, "grad_norm": 0.01026584580540657, "learning_rate": 1e-05, "loss": -0.001, "step": 234 }, { "clip_ratio/high_max": 0.625, "clip_ratio/high_mean": 0.203125, "clip_ratio/low_mean": 0.046875, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.25, "entropy": 0.8206076622009277, "epoch": 0.9325396825396826, "grad_norm": 0.007924714125692844, "learning_rate": 1e-05, "loss": -0.0001, "step": 235 }, { "clip_ratio/high_max": 0.625, "clip_ratio/high_mean": 0.125, "clip_ratio/low_mean": 0.0546875, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.1796875, "entropy": 0.7938217222690582, "epoch": 0.9365079365079365, "grad_norm": 0.013463746756315231, "learning_rate": 1e-05, "loss": 0.0009, "step": 236 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 357.0, "completions/max_terminated_length": 357.0, "completions/mean_length": 128.09375, "completions/mean_terminated_length": 128.09375, "completions/min_length": 35.0, "completions/min_terminated_length": 35.0, "entropy": 0.773484855890274, "epoch": 0.9404761904761905, "frac_reward_zero_std": 0.0625, "grad_norm": 0.012788292951881886, "learning_rate": 1e-05, "loss": -0.0006, "num_tokens": 61196268.0, "reward": 0.8414074182510376, "reward_std": 0.08055445551872253, "rewards/ngram_repetition2/mean": -0.0020546617452055216, "rewards/ngram_repetition2/std": 0.010285764001309872, "rewards/ngram_repetition3/mean": -0.002683112397789955, "rewards/ngram_repetition3/std": 0.011383618228137493, "rewards/sentence_repetition/mean": -3.181782449246384e-05, "rewards/sentence_repetition/std": 0.0007199551328085363, "rewards/symbolic_reward_accuracy/mean": 0.8046875, "rewards/symbolic_reward_accuracy/std": 0.3968288004398346, "rewards/symbolic_reward_partial_score/mean": 0.92724609375, "rewards/symbolic_reward_partial_score/std": 0.17543581128120422, "rewards/tag_count_reward/mean": 0.0, "rewards/tag_count_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.2582154273986816, "sampling/importance_sampling_ratio/min": 0.0037787086330354214, "sampling/sampling_logp_difference/max": 5.578372955322266, "sampling/sampling_logp_difference/mean": 0.33972322940826416, "step": 237 }, { "clip_ratio/high_max": 0.75, "clip_ratio/high_mean": 0.3203125, "clip_ratio/low_mean": 0.046875, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.3671875, "entropy": 0.7769236266613007, "epoch": 0.9444444444444444, "grad_norm": 0.016493212431669235, "learning_rate": 1e-05, "loss": 0.0012, "step": 238 }, { "clip_ratio/high_max": 0.5, "clip_ratio/high_mean": 0.1953125, "clip_ratio/low_mean": 0.09375, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.2890625, "entropy": 0.7654410302639008, "epoch": 0.9484126984126984, "grad_norm": 0.005981722846627235, "learning_rate": 1e-05, "loss": -0.0001, "step": 239 }, { "clip_ratio/high_max": 0.625, "clip_ratio/high_mean": 0.2109375, "clip_ratio/low_mean": 0.1171875, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.328125, "entropy": 0.8085787892341614, "epoch": 0.9523809523809523, "grad_norm": 0.011367063969373703, "learning_rate": 1e-05, "loss": 0.0002, "step": 240 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 311.0, "completions/max_terminated_length": 311.0, "completions/mean_length": 130.71484375, "completions/mean_terminated_length": 130.71484375, "completions/min_length": 35.0, "completions/min_terminated_length": 35.0, "entropy": 0.8103255331516266, "epoch": 0.9563492063492064, "frac_reward_zero_std": 0.46875, "grad_norm": 0.008278525434434414, "learning_rate": 1e-05, "loss": -0.0007, "num_tokens": 61627802.0, "reward": 0.9113024473190308, "reward_std": 0.03824722021818161, "rewards/ngram_repetition2/mean": -0.001225421205163002, "rewards/ngram_repetition2/std": 0.009212133474647999, "rewards/ngram_repetition3/mean": -0.001272004097700119, "rewards/ngram_repetition3/std": 0.007827307097613811, "rewards/sentence_repetition/mean": -7.233796350192279e-05, "rewards/sentence_repetition/std": 0.001636821310967207, "rewards/symbolic_reward_accuracy/mean": 0.89453125, "rewards/symbolic_reward_accuracy/std": 0.3074568510055542, "rewards/symbolic_reward_partial_score/mean": 0.951171875, "rewards/symbolic_reward_partial_score/std": 0.15814046561717987, "rewards/tag_count_reward/mean": -0.001953125, "rewards/tag_count_reward/std": 0.04419417306780815, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.2602877616882324, "sampling/importance_sampling_ratio/min": 0.002351208357140422, "sampling/sampling_logp_difference/max": 6.052825927734375, "sampling/sampling_logp_difference/mean": 0.3385169506072998, "step": 241 }, { "clip_ratio/high_max": 0.875, "clip_ratio/high_mean": 0.203125, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.21875, "entropy": 0.7839066982269287, "epoch": 0.9603174603174603, "grad_norm": 0.015846652910113335, "learning_rate": 1e-05, "loss": 0.0001, "step": 242 }, { "clip_ratio/high_max": 0.75, "clip_ratio/high_mean": 0.171875, "clip_ratio/low_mean": 0.0234375, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.1953125, "entropy": 0.7661570310592651, "epoch": 0.9642857142857143, "grad_norm": 0.013043319806456566, "learning_rate": 1e-05, "loss": 0.0019, "step": 243 }, { "clip_ratio/high_max": 0.5, "clip_ratio/high_mean": 0.1328125, "clip_ratio/low_mean": 0.0234375, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.15625, "entropy": 0.7846739292144775, "epoch": 0.9682539682539683, "grad_norm": 0.009260822087526321, "learning_rate": 1e-05, "loss": 0.0004, "step": 244 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 282.0, "completions/max_terminated_length": 282.0, "completions/mean_length": 107.6484375, "completions/mean_terminated_length": 107.6484375, "completions/min_length": 33.0, "completions/min_terminated_length": 33.0, "entropy": 0.713769406080246, "epoch": 0.9722222222222222, "frac_reward_zero_std": 0.53125, "grad_norm": 0.00511564826592803, "learning_rate": 1e-05, "loss": -0.0015, "num_tokens": 62050534.0, "reward": 0.8531219959259033, "reward_std": 0.06551604717969894, "rewards/ngram_repetition2/mean": -5.918560782447457e-05, "rewards/ngram_repetition2/std": 0.0013392174150794744, "rewards/ngram_repetition3/mean": -0.0001948696153704077, "rewards/ngram_repetition3/std": 0.00246166018769145, "rewards/sentence_repetition/mean": -4.405545041663572e-05, "rewards/sentence_repetition/std": 0.0009968610247597098, "rewards/symbolic_reward_accuracy/mean": 0.8203125, "rewards/symbolic_reward_accuracy/std": 0.38430243730545044, "rewards/symbolic_reward_partial_score/mean": 0.9296875, "rewards/symbolic_reward_partial_score/std": 0.17538747191429138, "rewards/tag_count_reward/mean": 0.0, "rewards/tag_count_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.2439318895339966, "sampling/importance_sampling_ratio/min": 0.005103512201458216, "sampling/sampling_logp_difference/max": 5.277826309204102, "sampling/sampling_logp_difference/mean": 0.32376348972320557, "step": 245 }, { "clip_ratio/high_max": 0.375, "clip_ratio/high_mean": 0.125, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.140625, "entropy": 0.7411008477210999, "epoch": 0.9761904761904762, "grad_norm": 0.0064419591799378395, "learning_rate": 1e-05, "loss": 0.0002, "step": 246 }, { "clip_ratio/high_max": 0.625, "clip_ratio/high_mean": 0.140625, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.15625, "entropy": 0.7386358678340912, "epoch": 0.9801587301587301, "grad_norm": 0.00715871574357152, "learning_rate": 1e-05, "loss": 0.0, "step": 247 }, { "clip_ratio/high_max": 0.75, "clip_ratio/high_mean": 0.1328125, "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.1484375, "entropy": 0.7352103888988495, "epoch": 0.9841269841269841, "grad_norm": 0.014257557690143585, "learning_rate": 1e-05, "loss": 0.0016, "step": 248 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 236.0, "completions/max_terminated_length": 236.0, "completions/mean_length": 89.72265625, "completions/mean_terminated_length": 89.72265625, "completions/min_length": 35.0, "completions/min_terminated_length": 35.0, "entropy": 0.6654102802276611, "epoch": 0.9880952380952381, "frac_reward_zero_std": 0.625, "grad_norm": 0.003523236606270075, "learning_rate": 1e-05, "loss": -0.0004, "num_tokens": 62446040.0, "reward": 0.9151766300201416, "reward_std": 0.04738668352365494, "rewards/ngram_repetition2/mean": -0.00034740459523163736, "rewards/ngram_repetition2/std": 0.006474743131548166, "rewards/ngram_repetition3/mean": -0.0005435039638541639, "rewards/ngram_repetition3/std": 0.008969147689640522, "rewards/sentence_repetition/mean": 0.0, "rewards/sentence_repetition/std": 0.0, "rewards/symbolic_reward_accuracy/mean": 0.89453125, "rewards/symbolic_reward_accuracy/std": 0.3074568510055542, "rewards/symbolic_reward_partial_score/mean": 0.96337890625, "rewards/symbolic_reward_partial_score/std": 0.12706883251667023, "rewards/tag_count_reward/mean": 0.0, "rewards/tag_count_reward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.0, "sampling/importance_sampling_ratio/mean": 1.2245663404464722, "sampling/importance_sampling_ratio/min": 0.003168715862557292, "sampling/sampling_logp_difference/max": 5.754428863525391, "sampling/sampling_logp_difference/mean": 0.2955351769924164, "step": 249 }, { "clip_ratio/high_max": 0.375, "clip_ratio/high_mean": 0.046875, "clip_ratio/low_mean": 0.0234375, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0703125, "entropy": 0.6670631170272827, "epoch": 0.9920634920634921, "grad_norm": 0.010117210447788239, "learning_rate": 1e-05, "loss": -0.0006, "step": 250 }, { "clip_ratio/high_max": 0.25, "clip_ratio/high_mean": 0.046875, "clip_ratio/low_mean": 0.0234375, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0703125, "entropy": 0.6484058499336243, "epoch": 0.996031746031746, "grad_norm": 0.008764887228608131, "learning_rate": 1e-05, "loss": 0.0005, "step": 251 }, { "clip_ratio/high_max": 0.25, "clip_ratio/high_mean": 0.078125, "clip_ratio/low_mean": 0.0546875, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.1328125, "entropy": 0.6381716728210449, "epoch": 1.0, "grad_norm": 0.015260268934071064, "learning_rate": 1e-05, "loss": -0.0004, "step": 252 }, { "epoch": 1.0, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.0, "eval_completions/max_length": 198.94736842105263, "eval_completions/max_terminated_length": 198.94736842105263, "eval_completions/mean_length": 76.859375, "eval_completions/mean_terminated_length": 76.859375, "eval_completions/min_length": 33.473684210526315, "eval_completions/min_terminated_length": 33.473684210526315, "eval_entropy": 0.5669561373560053, "eval_frac_reward_zero_std": 0.618421052631579, "eval_loss": -4.168831947026774e-05, "eval_num_tokens": 62446040.0, "eval_reward": 0.7070848502610859, "eval_reward_std": 0.049364458867593816, "eval_rewards/ngram_repetition2/mean": -0.0003738509003596846, "eval_rewards/ngram_repetition2/std": 0.0042079086920957225, "eval_rewards/ngram_repetition3/mean": -0.00043408882830638166, "eval_rewards/ngram_repetition3/std": 0.004775826616280124, "eval_rewards/sentence_repetition/mean": 0.0, "eval_rewards/sentence_repetition/std": 0.0, "eval_rewards/symbolic_reward_accuracy/mean": 0.6558388157894737, "eval_rewards/symbolic_reward_accuracy/std": 0.4506407693812722, "eval_rewards/symbolic_reward_partial_score/mean": 0.8270970394736842, "eval_rewards/symbolic_reward_partial_score/std": 0.24187982003939779, "eval_rewards/tag_count_reward/mean": -0.0012335526315789473, "eval_rewards/tag_count_reward/std": 0.013956054652992048, "eval_runtime": 253.287, "eval_samples_per_second": 0.592, "eval_sampling/importance_sampling_ratio/max": 2.0, "eval_sampling/importance_sampling_ratio/mean": 1.1915642274053473, "eval_sampling/importance_sampling_ratio/min": 0.0065373204262486025, "eval_sampling/sampling_logp_difference/max": 5.2662710892526725, "eval_sampling/sampling_logp_difference/mean": 0.25037836401086105, "eval_steps_per_second": 0.008, "step": 252 }, { "epoch": 1.0, "step": 252, "total_flos": 0.0, "train_loss": 0.017275968565728463, "train_runtime": 4300.0312, "train_samples_per_second": 0.471, "train_steps_per_second": 0.059 } ], "logging_steps": 1, "max_steps": 252, "num_input_tokens_seen": 62446040, "num_train_epochs": 1, "save_steps": 500, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": true, "should_training_stop": true }, "attributes": {} } }, "total_flos": 0.0, "train_batch_size": 4, "trial_name": null, "trial_params": null }