{ "best_global_step": null, "best_metric": null, "best_model_checkpoint": null, "epoch": 0.1282051282051282, "eval_steps": 500, "global_step": 60, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 9194.0, "completions/max_terminated_length": 9194.0, "completions/mean_length": 5745.15625, "completions/mean_terminated_length": 6128.166666666667, "completions/min_length": 0.0, "completions/min_terminated_length": 3974.0, "epoch": 0.002136752136752137, "grad_norm": 0.06384662206039221, "kl": 0.0, "learning_rate": 0.0, "loss": 0.0002, "num_tokens": 345573.0, "reward": 0.3754020035266876, "reward_std": 0.64158034324646, "rewards/constraint_satisfaction_reward_func": 1.0, "rewards/successful_code_execution_reward_func": 0.6662575006484985, "rewards/successful_no_deal_reward_func": 0.0, "rewards/successful_parsing_reward_func": 0.9164310693740845, "rewards/utility_reward_func": 0.07276386767625809, "step": 1 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 10133.0, "completions/max_terminated_length": 10133.0, "completions/mean_length": 6297.9375, "completions/mean_terminated_length": 6297.9375, "completions/min_length": 2801.0, "completions/min_terminated_length": 2801.0, "epoch": 0.004273504273504274, "grad_norm": 0.06139459826368875, "kl": 0.0, "learning_rate": 5.000000000000001e-07, "loss": -0.0013, "num_tokens": 689634.0, "reward": 0.8823189735412598, "reward_std": 0.7824905514717102, "rewards/constraint_satisfaction_reward_func": 1.0, "rewards/successful_code_execution_reward_func": 0.8089037537574768, "rewards/successful_no_deal_reward_func": 0.0, "rewards/successful_parsing_reward_func": 0.9081597328186035, "rewards/utility_reward_func": 0.17412039637565613, "step": 2 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 9356.0, "completions/max_terminated_length": 9356.0, "completions/mean_length": 5507.4375, "completions/mean_terminated_length": 5874.6, "completions/min_length": 0.0, "completions/min_terminated_length": 2936.0, "epoch": 0.00641025641025641, "grad_norm": 0.16125877365096486, "kl": 0.00312042236328125, "learning_rate": 1.0000000000000002e-06, "loss": -0.0375, "num_tokens": 1020236.0, "reward": 0.7772555351257324, "reward_std": 1.0026202201843262, "rewards/constraint_satisfaction_reward_func": 1.0, "rewards/successful_code_execution_reward_func": 0.7187950611114502, "rewards/successful_no_deal_reward_func": 0.0, "rewards/successful_parsing_reward_func": 0.8661947250366211, "rewards/utility_reward_func": 0.15313410758972168, "step": 3 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 9044.0, "completions/max_terminated_length": 9044.0, "completions/mean_length": 5101.34375, "completions/mean_terminated_length": 5265.903225806452, "completions/min_length": 0.0, "completions/min_terminated_length": 3087.0, "epoch": 0.008547008547008548, "grad_norm": 0.1331600814415893, "kl": 0.003368377685546875, "learning_rate": 1.5e-06, "loss": -0.0146, "num_tokens": 1322850.0, "reward": 0.7779074907302856, "reward_std": 0.866315484046936, "rewards/constraint_satisfaction_reward_func": 1.0, "rewards/successful_code_execution_reward_func": 0.682944118976593, "rewards/successful_no_deal_reward_func": 0.0, "rewards/successful_parsing_reward_func": 0.8875845670700073, "rewards/utility_reward_func": 0.15326739847660065, "step": 4 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 9605.0, "completions/max_terminated_length": 9605.0, "completions/mean_length": 5633.96875, "completions/mean_terminated_length": 6009.566666666667, "completions/min_length": 0.0, "completions/min_terminated_length": 3806.0, "epoch": 0.010683760683760684, "grad_norm": 0.10043275978920585, "kl": 0.0031251907348632812, "learning_rate": 2.0000000000000003e-06, "loss": 0.0091, "num_tokens": 1667050.0, "reward": 0.5307067036628723, "reward_std": 0.6169852018356323, "rewards/constraint_satisfaction_reward_func": 1.0, "rewards/successful_code_execution_reward_func": 0.6832882165908813, "rewards/successful_no_deal_reward_func": 0.0, "rewards/successful_parsing_reward_func": 0.8644717335700989, "rewards/utility_reward_func": 0.10383178293704987, "step": 5 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 9829.0, "completions/max_terminated_length": 9829.0, "completions/mean_length": 6319.625, "completions/mean_terminated_length": 6319.625, "completions/min_length": 2801.0, "completions/min_terminated_length": 2801.0, "epoch": 0.01282051282051282, "grad_norm": 0.09138627686091379, "kl": 0.0031518936157226562, "learning_rate": 2.5e-06, "loss": -0.0105, "num_tokens": 1999042.0, "reward": 0.6561113595962524, "reward_std": 0.8523557782173157, "rewards/constraint_satisfaction_reward_func": 1.0, "rewards/successful_code_execution_reward_func": 0.7507439851760864, "rewards/successful_no_deal_reward_func": 0.0, "rewards/successful_parsing_reward_func": 0.9472808837890625, "rewards/utility_reward_func": 0.12888266146183014, "step": 6 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 10098.0, "completions/max_terminated_length": 10098.0, "completions/mean_length": 6040.21875, "completions/mean_terminated_length": 6040.21875, "completions/min_length": 2345.0, "completions/min_terminated_length": 2345.0, "epoch": 0.014957264957264958, "grad_norm": 0.09859665471586132, "kl": 0.0033817291259765625, "learning_rate": 3e-06, "loss": -0.0343, "num_tokens": 2313691.0, "reward": 1.0388803482055664, "reward_std": 0.9269991517066956, "rewards/constraint_satisfaction_reward_func": 0.984375, "rewards/successful_code_execution_reward_func": 0.6589781641960144, "rewards/successful_no_deal_reward_func": 0.0, "rewards/successful_parsing_reward_func": 0.9249492883682251, "rewards/utility_reward_func": 0.20549052953720093, "step": 7 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 8935.0, "completions/max_terminated_length": 8935.0, "completions/mean_length": 4837.3125, "completions/mean_terminated_length": 4993.354838709677, "completions/min_length": 0.0, "completions/min_terminated_length": 2192.0, "epoch": 0.017094017094017096, "grad_norm": 0.13927683938664606, "kl": 0.0035686492919921875, "learning_rate": 3.5e-06, "loss": -0.0218, "num_tokens": 2607775.0, "reward": 0.7573515176773071, "reward_std": 0.747157633304596, "rewards/constraint_satisfaction_reward_func": 1.0, "rewards/successful_code_execution_reward_func": 0.7197544574737549, "rewards/successful_no_deal_reward_func": 0.0, "rewards/successful_parsing_reward_func": 0.8988715410232544, "rewards/utility_reward_func": 0.14914658665657043, "step": 8 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 8868.0, "completions/max_terminated_length": 8868.0, "completions/mean_length": 5284.78125, "completions/mean_terminated_length": 5455.258064516129, "completions/min_length": 0.0, "completions/min_terminated_length": 2504.0, "epoch": 0.019230769230769232, "grad_norm": 0.11735526956696517, "kl": 0.0043468475341796875, "learning_rate": 4.000000000000001e-06, "loss": 0.0019, "num_tokens": 2915637.0, "reward": 0.365405797958374, "reward_std": 0.5554816722869873, "rewards/constraint_satisfaction_reward_func": 0.984375, "rewards/successful_code_execution_reward_func": 0.579179048538208, "rewards/successful_no_deal_reward_func": 0.0, "rewards/successful_parsing_reward_func": 0.9264745712280273, "rewards/utility_reward_func": 0.07081127166748047, "step": 9 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 9892.0, "completions/max_terminated_length": 9892.0, "completions/mean_length": 5311.21875, "completions/mean_terminated_length": 5482.548387096775, "completions/min_length": 0.0, "completions/min_terminated_length": 1337.0, "epoch": 0.021367521367521368, "grad_norm": 0.08243472782779405, "kl": 0.0046443939208984375, "learning_rate": 4.5e-06, "loss": -0.0052, "num_tokens": 3218433.0, "reward": 0.7362505197525024, "reward_std": 0.9180038571357727, "rewards/constraint_satisfaction_reward_func": 1.0, "rewards/successful_code_execution_reward_func": 0.6661200523376465, "rewards/successful_no_deal_reward_func": 0.0, "rewards/successful_parsing_reward_func": 0.9426835179328918, "rewards/utility_reward_func": 0.14492833614349365, "step": 10 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 8914.0, "completions/max_terminated_length": 8914.0, "completions/mean_length": 5811.75, "completions/mean_terminated_length": 5811.75, "completions/min_length": 2232.0, "completions/min_terminated_length": 2232.0, "epoch": 0.023504273504273504, "grad_norm": 0.09543276542415491, "kl": 0.00598907470703125, "learning_rate": 5e-06, "loss": -0.0049, "num_tokens": 3529160.0, "reward": 0.5771545171737671, "reward_std": 0.8459901809692383, "rewards/constraint_satisfaction_reward_func": 1.0, "rewards/successful_code_execution_reward_func": 0.5785590410232544, "rewards/successful_no_deal_reward_func": 0.0, "rewards/successful_parsing_reward_func": 0.9260044693946838, "rewards/utility_reward_func": 0.11312998831272125, "step": 11 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 8499.0, "completions/max_terminated_length": 8499.0, "completions/mean_length": 4962.53125, "completions/mean_terminated_length": 5122.612903225807, "completions/min_length": 0.0, "completions/min_terminated_length": 2565.0, "epoch": 0.02564102564102564, "grad_norm": 0.06256974070929786, "kl": 0.0071735382080078125, "learning_rate": 5.500000000000001e-06, "loss": 0.0059, "num_tokens": 3838787.0, "reward": 0.7496746778488159, "reward_std": 0.9168016314506531, "rewards/constraint_satisfaction_reward_func": 0.96875, "rewards/successful_code_execution_reward_func": 0.6537078619003296, "rewards/successful_no_deal_reward_func": 0.0, "rewards/successful_parsing_reward_func": 0.9083457589149475, "rewards/utility_reward_func": 0.14768503606319427, "step": 12 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 8518.0, "completions/max_terminated_length": 8518.0, "completions/mean_length": 5306.21875, "completions/mean_terminated_length": 5659.966666666666, "completions/min_length": 0.0, "completions/min_terminated_length": 2676.0, "epoch": 0.027777777777777776, "grad_norm": 0.06188426771907535, "kl": 0.011411666870117188, "learning_rate": 6e-06, "loss": 0.0037, "num_tokens": 4162259.0, "reward": 0.4114573001861572, "reward_std": 0.5327050685882568, "rewards/constraint_satisfaction_reward_func": 0.984375, "rewards/successful_code_execution_reward_func": 0.7068666219711304, "rewards/successful_no_deal_reward_func": 0.0, "rewards/successful_parsing_reward_func": 0.9097241163253784, "rewards/utility_reward_func": 0.0799993947148323, "step": 13 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 8677.0, "completions/max_terminated_length": 8677.0, "completions/mean_length": 5330.875, "completions/mean_terminated_length": 5330.875, "completions/min_length": 1230.0, "completions/min_terminated_length": 1230.0, "epoch": 0.029914529914529916, "grad_norm": 0.07195177317932434, "kl": 0.020092010498046875, "learning_rate": 6.5000000000000004e-06, "loss": -0.0026, "num_tokens": 4465194.0, "reward": 0.26478463411331177, "reward_std": 0.33203744888305664, "rewards/constraint_satisfaction_reward_func": 1.0, "rewards/successful_code_execution_reward_func": 0.4670758843421936, "rewards/successful_no_deal_reward_func": 0.0, "rewards/successful_parsing_reward_func": 0.9135281443595886, "rewards/utility_reward_func": 0.05068080872297287, "step": 14 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 9578.0, "completions/max_terminated_length": 9578.0, "completions/mean_length": 6203.96875, "completions/mean_terminated_length": 6404.096774193548, "completions/min_length": 0.0, "completions/min_terminated_length": 2333.0, "epoch": 0.03205128205128205, "grad_norm": 0.10026513317887245, "kl": 0.018894195556640625, "learning_rate": 7e-06, "loss": 0.0411, "num_tokens": 4803381.0, "reward": 0.3010755777359009, "reward_std": 0.5123584866523743, "rewards/constraint_satisfaction_reward_func": 0.96875, "rewards/successful_code_execution_reward_func": 0.738226056098938, "rewards/successful_no_deal_reward_func": 0.0, "rewards/successful_parsing_reward_func": 0.9183079600334167, "rewards/utility_reward_func": 0.05794630944728851, "step": 15 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 8891.0, "completions/max_terminated_length": 8891.0, "completions/mean_length": 6081.84375, "completions/mean_terminated_length": 6081.84375, "completions/min_length": 2926.0, "completions/min_terminated_length": 2926.0, "epoch": 0.03418803418803419, "grad_norm": 0.060400168503078416, "kl": 0.03104400634765625, "learning_rate": 7.500000000000001e-06, "loss": -0.0067, "num_tokens": 5128775.0, "reward": 0.5646129846572876, "reward_std": 0.7254551649093628, "rewards/constraint_satisfaction_reward_func": 1.0, "rewards/successful_code_execution_reward_func": 0.6118528842926025, "rewards/successful_no_deal_reward_func": 0.0, "rewards/successful_parsing_reward_func": 0.9297844171524048, "rewards/utility_reward_func": 0.11061426997184753, "step": 16 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 9484.0, "completions/max_terminated_length": 9484.0, "completions/mean_length": 5118.625, "completions/mean_terminated_length": 5118.625, "completions/min_length": 1630.0, "completions/min_terminated_length": 1630.0, "epoch": 0.03632478632478633, "grad_norm": 0.05120826577801321, "kl": 0.036815643310546875, "learning_rate": 8.000000000000001e-06, "loss": 0.0048, "num_tokens": 5421396.0, "reward": 0.5202240347862244, "reward_std": 0.5871657133102417, "rewards/constraint_satisfaction_reward_func": 1.0, "rewards/successful_code_execution_reward_func": 0.7249752283096313, "rewards/successful_no_deal_reward_func": 0.0, "rewards/successful_parsing_reward_func": 0.8982887268066406, "rewards/utility_reward_func": 0.10172015428543091, "step": 17 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 10314.0, "completions/max_terminated_length": 10314.0, "completions/mean_length": 5530.75, "completions/mean_terminated_length": 5709.1612903225805, "completions/min_length": 0.0, "completions/min_terminated_length": 3094.0, "epoch": 0.038461538461538464, "grad_norm": 0.8805793412616941, "kl": 0.22267913818359375, "learning_rate": 8.5e-06, "loss": -0.0131, "num_tokens": 5733847.0, "reward": 0.5374739170074463, "reward_std": 0.7667478322982788, "rewards/constraint_satisfaction_reward_func": 0.984375, "rewards/successful_code_execution_reward_func": 0.5694985389709473, "rewards/successful_no_deal_reward_func": 0.0, "rewards/successful_parsing_reward_func": 0.8748263716697693, "rewards/utility_reward_func": 0.10523717105388641, "step": 18 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 9009.0, "completions/max_terminated_length": 9009.0, "completions/mean_length": 4953.90625, "completions/mean_terminated_length": 5113.709677419355, "completions/min_length": 0.0, "completions/min_terminated_length": 1940.0, "epoch": 0.0405982905982906, "grad_norm": 0.05562338889561032, "kl": 0.05136871337890625, "learning_rate": 9e-06, "loss": -0.0207, "num_tokens": 6040038.0, "reward": 0.40647947788238525, "reward_std": 0.5132466554641724, "rewards/constraint_satisfaction_reward_func": 1.0, "rewards/successful_code_execution_reward_func": 0.6577438116073608, "rewards/successful_no_deal_reward_func": 0.0, "rewards/successful_parsing_reward_func": 0.9326636791229248, "rewards/utility_reward_func": 0.07897782325744629, "step": 19 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 9768.0, "completions/max_terminated_length": 9768.0, "completions/mean_length": 5553.15625, "completions/mean_terminated_length": 5732.290322580645, "completions/min_length": 0.0, "completions/min_terminated_length": 2971.0, "epoch": 0.042735042735042736, "grad_norm": 0.05415251247993026, "kl": 0.0604705810546875, "learning_rate": 9.5e-06, "loss": -0.0159, "num_tokens": 6356055.0, "reward": 0.6564894914627075, "reward_std": 0.9057918787002563, "rewards/constraint_satisfaction_reward_func": 0.984375, "rewards/successful_code_execution_reward_func": 0.6670262813568115, "rewards/successful_no_deal_reward_func": 0.0, "rewards/successful_parsing_reward_func": 0.9299107193946838, "rewards/utility_reward_func": 0.12900978326797485, "step": 20 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 8973.0, "completions/max_terminated_length": 8973.0, "completions/mean_length": 4999.53125, "completions/mean_terminated_length": 5160.806451612903, "completions/min_length": 0.0, "completions/min_terminated_length": 2484.0, "epoch": 0.04487179487179487, "grad_norm": 0.07150378564767108, "kl": 0.05039215087890625, "learning_rate": 1e-05, "loss": 0.0295, "num_tokens": 6650399.0, "reward": 0.3270576596260071, "reward_std": 0.48058632016181946, "rewards/constraint_satisfaction_reward_func": 1.0, "rewards/successful_code_execution_reward_func": 0.5520957112312317, "rewards/successful_no_deal_reward_func": 0.0, "rewards/successful_parsing_reward_func": 0.9284598231315613, "rewards/utility_reward_func": 0.0631154254078865, "step": 21 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 9046.0, "completions/max_terminated_length": 9046.0, "completions/mean_length": 5165.90625, "completions/mean_terminated_length": 5332.548387096775, "completions/min_length": 0.0, "completions/min_terminated_length": 3100.0, "epoch": 0.04700854700854701, "grad_norm": 0.06845908919994707, "kl": 0.0538330078125, "learning_rate": 9.986128001799077e-06, "loss": 0.0153, "num_tokens": 6954265.0, "reward": 0.4070006310939789, "reward_std": 0.5391647815704346, "rewards/constraint_satisfaction_reward_func": 1.0, "rewards/successful_code_execution_reward_func": 0.624727189540863, "rewards/successful_no_deal_reward_func": 0.0, "rewards/successful_parsing_reward_func": 0.887859582901001, "rewards/utility_reward_func": 0.07909761369228363, "step": 22 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 9026.0, "completions/max_terminated_length": 9026.0, "completions/mean_length": 4882.46875, "completions/mean_terminated_length": 5207.966666666666, "completions/min_length": 0.0, "completions/min_terminated_length": 2413.0, "epoch": 0.049145299145299144, "grad_norm": 0.0575950278172028, "kl": 0.06992721557617188, "learning_rate": 9.94459753267812e-06, "loss": -0.0033, "num_tokens": 7263856.0, "reward": 0.14356863498687744, "reward_std": 0.22449877858161926, "rewards/constraint_satisfaction_reward_func": 1.0, "rewards/successful_code_execution_reward_func": 0.5870039463043213, "rewards/successful_no_deal_reward_func": 0.0, "rewards/successful_parsing_reward_func": 0.8971620798110962, "rewards/utility_reward_func": 0.026416894048452377, "step": 23 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 8652.0, "completions/max_terminated_length": 8652.0, "completions/mean_length": 4892.875, "completions/mean_terminated_length": 5050.709677419355, "completions/min_length": 0.0, "completions/min_terminated_length": 1523.0, "epoch": 0.05128205128205128, "grad_norm": 0.04255156882342247, "kl": 0.0586090087890625, "learning_rate": 9.875664641789545e-06, "loss": 0.0203, "num_tokens": 7571389.0, "reward": 0.6160452365875244, "reward_std": 1.0084484815597534, "rewards/constraint_satisfaction_reward_func": 1.0, "rewards/successful_code_execution_reward_func": 0.7209370136260986, "rewards/successful_no_deal_reward_func": 0.0, "rewards/successful_parsing_reward_func": 0.9045882821083069, "rewards/utility_reward_func": 0.12088394165039062, "step": 24 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 9417.0, "completions/max_terminated_length": 9417.0, "completions/mean_length": 4552.21875, "completions/mean_terminated_length": 4855.7, "completions/min_length": 0.0, "completions/min_terminated_length": 2353.0, "epoch": 0.053418803418803416, "grad_norm": 0.033573562073913446, "kl": 0.0653076171875, "learning_rate": 9.779754323328192e-06, "loss": -0.0009, "num_tokens": 7861415.0, "reward": 0.7228190898895264, "reward_std": 0.5367074608802795, "rewards/constraint_satisfaction_reward_func": 1.0, "rewards/successful_code_execution_reward_func": 0.614422082901001, "rewards/successful_no_deal_reward_func": 0.0, "rewards/successful_parsing_reward_func": 0.9327523708343506, "rewards/utility_reward_func": 0.14225439727306366, "step": 25 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 9038.0, "completions/max_terminated_length": 9038.0, "completions/mean_length": 5255.71875, "completions/mean_terminated_length": 5255.71875, "completions/min_length": 2542.0, "completions/min_terminated_length": 2542.0, "epoch": 0.05555555555555555, "grad_norm": 0.047795685473935486, "kl": 0.05504608154296875, "learning_rate": 9.65745789630079e-06, "loss": -0.0161, "num_tokens": 8175576.0, "reward": 0.5497658848762512, "reward_std": 0.7453822493553162, "rewards/constraint_satisfaction_reward_func": 1.0, "rewards/successful_code_execution_reward_func": 0.5872892141342163, "rewards/successful_no_deal_reward_func": 0.0, "rewards/successful_parsing_reward_func": 0.8717837333679199, "rewards/utility_reward_func": 0.10766136646270752, "step": 26 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 9164.0, "completions/max_terminated_length": 9164.0, "completions/mean_length": 5544.21875, "completions/mean_terminated_length": 5544.21875, "completions/min_length": 1633.0, "completions/min_terminated_length": 1633.0, "epoch": 0.057692307692307696, "grad_norm": 0.09411711877281828, "kl": 0.0565032958984375, "learning_rate": 9.509529358847655e-06, "loss": 0.0178, "num_tokens": 8475111.0, "reward": 0.6063368320465088, "reward_std": 0.7418020963668823, "rewards/constraint_satisfaction_reward_func": 0.984375, "rewards/successful_code_execution_reward_func": 0.615506649017334, "rewards/successful_no_deal_reward_func": 0.0, "rewards/successful_parsing_reward_func": 0.9354557991027832, "rewards/utility_reward_func": 0.11898841708898544, "step": 27 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 8644.0, "completions/max_terminated_length": 8644.0, "completions/mean_length": 5628.8125, "completions/mean_terminated_length": 5628.8125, "completions/min_length": 2365.0, "completions/min_terminated_length": 2365.0, "epoch": 0.05982905982905983, "grad_norm": 0.07320321937335796, "kl": 0.1031036376953125, "learning_rate": 9.336880739593415e-06, "loss": 0.0119, "num_tokens": 8780720.0, "reward": 0.6854258179664612, "reward_std": 0.8022749423980713, "rewards/constraint_satisfaction_reward_func": 0.984375, "rewards/successful_code_execution_reward_func": 0.6049659252166748, "rewards/successful_no_deal_reward_func": 0.0, "rewards/successful_parsing_reward_func": 0.9334280490875244, "rewards/utility_reward_func": 0.1348087340593338, "step": 28 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 9075.0, "completions/max_terminated_length": 9075.0, "completions/mean_length": 5317.21875, "completions/mean_terminated_length": 5488.741935483871, "completions/min_length": 0.0, "completions/min_terminated_length": 1165.0, "epoch": 0.06196581196581197, "grad_norm": 0.04791948119741524, "kl": 0.07018280029296875, "learning_rate": 9.140576474687263e-06, "loss": -0.0028, "num_tokens": 9102714.0, "reward": 0.47284555435180664, "reward_std": 0.5441389679908752, "rewards/constraint_satisfaction_reward_func": 0.984375, "rewards/successful_code_execution_reward_func": 0.644345223903656, "rewards/successful_no_deal_reward_func": 0.0, "rewards/successful_parsing_reward_func": 0.9321033954620361, "rewards/utility_reward_func": 0.09228507429361343, "step": 29 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 8686.0, "completions/max_terminated_length": 8686.0, "completions/mean_length": 5109.75, "completions/mean_terminated_length": 5274.580645161291, "completions/min_length": 0.0, "completions/min_terminated_length": 2237.0, "epoch": 0.0641025641025641, "grad_norm": 0.08100156568534053, "kl": 0.0840911865234375, "learning_rate": 8.92182684520014e-06, "loss": 0.0168, "num_tokens": 9403599.0, "reward": 0.2536351680755615, "reward_std": 0.4151955246925354, "rewards/constraint_satisfaction_reward_func": 1.0, "rewards/successful_code_execution_reward_func": 0.6711061000823975, "rewards/successful_no_deal_reward_func": 0.0, "rewards/successful_parsing_reward_func": 0.9160714149475098, "rewards/utility_reward_func": 0.048409603536129, "step": 30 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 8613.0, "completions/max_terminated_length": 8613.0, "completions/mean_length": 5094.875, "completions/mean_terminated_length": 5259.225806451613, "completions/min_length": 0.0, "completions/min_terminated_length": 1385.0, "epoch": 0.06623931623931624, "grad_norm": 0.05053782868196005, "kl": 0.10784912109375, "learning_rate": 8.681980515339464e-06, "loss": 0.0073, "num_tokens": 9713252.0, "reward": 0.2064223438501358, "reward_std": 0.29352596402168274, "rewards/constraint_satisfaction_reward_func": 0.984375, "rewards/successful_code_execution_reward_func": 0.7445651292800903, "rewards/successful_no_deal_reward_func": 0.0, "rewards/successful_parsing_reward_func": 0.926922082901001, "rewards/utility_reward_func": 0.038981422781944275, "step": 31 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 8011.0, "completions/max_terminated_length": 8011.0, "completions/mean_length": 5101.9375, "completions/mean_terminated_length": 5101.9375, "completions/min_length": 2572.0, "completions/min_terminated_length": 2572.0, "epoch": 0.06837606837606838, "grad_norm": 0.05517114109543934, "kl": 0.153045654296875, "learning_rate": 8.422516217485826e-06, "loss": -0.0196, "num_tokens": 10013403.0, "reward": 0.6316404938697815, "reward_std": 0.7787704467773438, "rewards/constraint_satisfaction_reward_func": 1.0, "rewards/successful_code_execution_reward_func": 0.6698119640350342, "rewards/successful_no_deal_reward_func": 0.0, "rewards/successful_parsing_reward_func": 0.8996039628982544, "rewards/utility_reward_func": 0.12401421368122101, "step": 32 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 9270.0, "completions/max_terminated_length": 9270.0, "completions/mean_length": 5491.46875, "completions/mean_terminated_length": 5491.46875, "completions/min_length": 2332.0, "completions/min_terminated_length": 2332.0, "epoch": 0.07051282051282051, "grad_norm": 0.20802919593498223, "kl": 0.469573974609375, "learning_rate": 8.14503363531613e-06, "loss": -0.0119, "num_tokens": 10312689.0, "reward": 0.9118640422821045, "reward_std": 1.0636224746704102, "rewards/constraint_satisfaction_reward_func": 1.0, "rewards/successful_code_execution_reward_func": 0.5969223976135254, "rewards/successful_no_deal_reward_func": 0.0, "rewards/successful_parsing_reward_func": 0.9264756441116333, "rewards/utility_reward_func": 0.18006812036037445, "step": 33 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 9056.0, "completions/max_terminated_length": 9056.0, "completions/mean_length": 5139.9375, "completions/mean_terminated_length": 5305.741935483871, "completions/min_length": 0.0, "completions/min_terminated_length": 2597.0, "epoch": 0.07264957264957266, "grad_norm": 0.8404594204961415, "kl": 0.9882049560546875, "learning_rate": 7.85124354122177e-06, "loss": -0.0162, "num_tokens": 10607350.0, "reward": 0.8799901008605957, "reward_std": 1.1067553758621216, "rewards/constraint_satisfaction_reward_func": 1.0, "rewards/successful_code_execution_reward_func": 0.46376484632492065, "rewards/successful_no_deal_reward_func": 0.0, "rewards/successful_parsing_reward_func": 0.910142719745636, "rewards/utility_reward_func": 0.1737232506275177, "step": 34 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 9108.0, "completions/max_terminated_length": 9108.0, "completions/mean_length": 5446.0625, "completions/mean_terminated_length": 5621.741935483871, "completions/min_length": 0.0, "completions/min_terminated_length": 2769.0, "epoch": 0.07478632478632478, "grad_norm": 0.10678545617196525, "kl": 0.215911865234375, "learning_rate": 7.5429572488279615e-06, "loss": 0.0023, "num_tokens": 10928816.0, "reward": 1.17676842212677, "reward_std": 1.2874033451080322, "rewards/constraint_satisfaction_reward_func": 1.0, "rewards/successful_code_execution_reward_func": 0.7114211320877075, "rewards/successful_no_deal_reward_func": 0.0, "rewards/successful_parsing_reward_func": 0.8751848936080933, "rewards/utility_reward_func": 0.2330363690853119, "step": 35 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 9427.0, "completions/max_terminated_length": 9427.0, "completions/mean_length": 5561.625, "completions/mean_terminated_length": 5561.625, "completions/min_length": 2739.0, "completions/min_terminated_length": 2739.0, "epoch": 0.07692307692307693, "grad_norm": 0.13555129034265773, "kl": 0.295166015625, "learning_rate": 7.222075445642904e-06, "loss": 0.0035, "num_tokens": 11225979.0, "reward": 1.0547581911087036, "reward_std": 0.7715986967086792, "rewards/constraint_satisfaction_reward_func": 1.0, "rewards/successful_code_execution_reward_func": 0.6989527344703674, "rewards/successful_no_deal_reward_func": 0.0, "rewards/successful_parsing_reward_func": 0.9386740326881409, "rewards/utility_reward_func": 0.20862412452697754, "step": 36 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 9042.0, "completions/max_terminated_length": 9042.0, "completions/mean_length": 4831.125, "completions/mean_terminated_length": 4986.967741935484, "completions/min_length": 0.0, "completions/min_terminated_length": 2146.0, "epoch": 0.07905982905982906, "grad_norm": 0.24559297239149214, "kl": 0.448211669921875, "learning_rate": 6.890576474687264e-06, "loss": -0.0071, "num_tokens": 11511340.0, "reward": 0.502151370048523, "reward_std": 0.784339427947998, "rewards/constraint_satisfaction_reward_func": 1.0, "rewards/successful_code_execution_reward_func": 0.6361043453216553, "rewards/successful_no_deal_reward_func": 0.0, "rewards/successful_parsing_reward_func": 0.9133680462837219, "rewards/utility_reward_func": 0.09812036901712418, "step": 37 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 7620.0, "completions/max_terminated_length": 7620.0, "completions/mean_length": 4775.96875, "completions/mean_terminated_length": 4775.96875, "completions/min_length": 2299.0, "completions/min_terminated_length": 2299.0, "epoch": 0.0811965811965812, "grad_norm": 0.09317535164809768, "kl": 0.346588134765625, "learning_rate": 6.550504137351576e-06, "loss": 0.0011, "num_tokens": 11786543.0, "reward": 0.3758894205093384, "reward_std": 0.5644346475601196, "rewards/constraint_satisfaction_reward_func": 1.0, "rewards/successful_code_execution_reward_func": 0.6146777868270874, "rewards/successful_no_deal_reward_func": 0.0, "rewards/successful_parsing_reward_func": 0.9283109903335571, "rewards/utility_reward_func": 0.07286929339170456, "step": 38 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 8610.0, "completions/max_terminated_length": 8610.0, "completions/mean_length": 5071.4375, "completions/mean_terminated_length": 5235.032258064516, "completions/min_length": 0.0, "completions/min_terminated_length": 2368.0, "epoch": 0.08333333333333333, "grad_norm": 0.11546537555277975, "kl": 0.41107177734375, "learning_rate": 6.20395509268104e-06, "loss": 0.0004, "num_tokens": 12090692.0, "reward": 0.3179802894592285, "reward_std": 0.403035968542099, "rewards/constraint_satisfaction_reward_func": 1.0, "rewards/successful_code_execution_reward_func": 0.4927416443824768, "rewards/successful_no_deal_reward_func": 0.0, "rewards/successful_parsing_reward_func": 0.9184781312942505, "rewards/utility_reward_func": 0.06131381541490555, "step": 39 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 9147.0, "completions/max_terminated_length": 9147.0, "completions/mean_length": 4912.9375, "completions/mean_terminated_length": 4912.9375, "completions/min_length": 1763.0, "completions/min_terminated_length": 1763.0, "epoch": 0.08547008547008547, "grad_norm": 0.12462480268811632, "kl": 0.5187835693359375, "learning_rate": 5.8530659307753034e-06, "loss": -0.0116, "num_tokens": 12375359.0, "reward": 0.31748560070991516, "reward_std": 0.4341599941253662, "rewards/constraint_satisfaction_reward_func": 0.984375, "rewards/successful_code_execution_reward_func": 0.4765625, "rewards/successful_no_deal_reward_func": 0.0, "rewards/successful_parsing_reward_func": 0.9282118082046509, "rewards/utility_reward_func": 0.061247408390045166, "step": 40 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 9041.0, "completions/max_terminated_length": 9041.0, "completions/mean_length": 5569.53125, "completions/mean_terminated_length": 5569.53125, "completions/min_length": 1926.0, "completions/min_terminated_length": 1926.0, "epoch": 0.0876068376068376, "grad_norm": 0.5440516549354221, "kl": 0.73486328125, "learning_rate": 5.500000000000001e-06, "loss": 0.0102, "num_tokens": 12677098.0, "reward": 0.4059770703315735, "reward_std": 0.4924428462982178, "rewards/constraint_satisfaction_reward_func": 1.0, "rewards/successful_code_execution_reward_func": 0.6707386374473572, "rewards/successful_no_deal_reward_func": 0.0, "rewards/successful_parsing_reward_func": 0.9652777910232544, "rewards/utility_reward_func": 0.07886820286512375, "step": 41 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 8277.0, "completions/max_terminated_length": 8277.0, "completions/mean_length": 5186.34375, "completions/mean_terminated_length": 5186.34375, "completions/min_length": 2468.0, "completions/min_terminated_length": 2468.0, "epoch": 0.08974358974358974, "grad_norm": 0.07323387926836804, "kl": 0.26275634765625, "learning_rate": 5.1469340692247e-06, "loss": -0.0073, "num_tokens": 12967501.0, "reward": 0.36183637380599976, "reward_std": 0.38023996353149414, "rewards/constraint_satisfaction_reward_func": 1.0, "rewards/successful_code_execution_reward_func": 0.577970564365387, "rewards/successful_no_deal_reward_func": 0.0, "rewards/successful_parsing_reward_func": 0.9075149297714233, "rewards/utility_reward_func": 0.07007017731666565, "step": 42 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 8859.0, "completions/max_terminated_length": 8859.0, "completions/mean_length": 4840.65625, "completions/mean_terminated_length": 5163.366666666667, "completions/min_length": 0.0, "completions/min_terminated_length": 2604.0, "epoch": 0.09188034188034189, "grad_norm": 0.13037129951366888, "kl": 0.36981201171875, "learning_rate": 4.79604490731896e-06, "loss": 0.0331, "num_tokens": 13268952.0, "reward": 0.6934996843338013, "reward_std": 0.9891730546951294, "rewards/constraint_satisfaction_reward_func": 1.0, "rewards/successful_code_execution_reward_func": 0.6411582231521606, "rewards/successful_no_deal_reward_func": 0.0, "rewards/successful_parsing_reward_func": 0.9228783249855042, "rewards/utility_reward_func": 0.13638713955879211, "step": 43 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 8947.0, "completions/max_terminated_length": 8947.0, "completions/mean_length": 4978.1875, "completions/mean_terminated_length": 5138.774193548387, "completions/min_length": 0.0, "completions/min_terminated_length": 2443.0, "epoch": 0.09401709401709402, "grad_norm": 0.11949859363278825, "kl": 0.3361663818359375, "learning_rate": 4.449495862648427e-06, "loss": -0.0085, "num_tokens": 13556932.0, "reward": 0.3446043133735657, "reward_std": 0.4400365352630615, "rewards/constraint_satisfaction_reward_func": 1.0, "rewards/successful_code_execution_reward_func": 0.6666632890701294, "rewards/successful_no_deal_reward_func": 0.0, "rewards/successful_parsing_reward_func": 0.92415452003479, "rewards/utility_reward_func": 0.06660269945859909, "step": 44 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 9695.0, "completions/max_terminated_length": 9695.0, "completions/mean_length": 5289.65625, "completions/mean_terminated_length": 5642.3, "completions/min_length": 0.0, "completions/min_terminated_length": 2270.0, "epoch": 0.09615384615384616, "grad_norm": 0.10187792105154207, "kl": 0.206329345703125, "learning_rate": 4.109423525312737e-06, "loss": -0.014, "num_tokens": 13878344.0, "reward": 1.069939136505127, "reward_std": 1.3795127868652344, "rewards/constraint_satisfaction_reward_func": 1.0, "rewards/successful_code_execution_reward_func": 0.6028566956520081, "rewards/successful_no_deal_reward_func": 0.0, "rewards/successful_parsing_reward_func": 0.9103050231933594, "rewards/utility_reward_func": 0.2116851955652237, "step": 45 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 9234.0, "completions/max_terminated_length": 9234.0, "completions/mean_length": 5689.28125, "completions/mean_terminated_length": 5689.28125, "completions/min_length": 2629.0, "completions/min_terminated_length": 2629.0, "epoch": 0.09829059829059829, "grad_norm": 0.09907432590761146, "kl": 0.252288818359375, "learning_rate": 3.777924554357096e-06, "loss": 0.0125, "num_tokens": 14191060.0, "reward": 0.8821263313293457, "reward_std": 0.9199132323265076, "rewards/constraint_satisfaction_reward_func": 1.0, "rewards/successful_code_execution_reward_func": 0.6297247409820557, "rewards/successful_no_deal_reward_func": 0.0, "rewards/successful_parsing_reward_func": 0.8932163715362549, "rewards/utility_reward_func": 0.17412066459655762, "step": 46 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 9690.0, "completions/max_terminated_length": 9690.0, "completions/mean_length": 5127.65625, "completions/mean_terminated_length": 5293.064516129032, "completions/min_length": 0.0, "completions/min_terminated_length": 3001.0, "epoch": 0.10042735042735043, "grad_norm": 0.117442853699414, "kl": 0.20562744140625, "learning_rate": 3.45704275117204e-06, "loss": -0.0347, "num_tokens": 14487972.0, "reward": 0.9619556069374084, "reward_std": 1.1848845481872559, "rewards/constraint_satisfaction_reward_func": 0.96875, "rewards/successful_code_execution_reward_func": 0.7243213057518005, "rewards/successful_no_deal_reward_func": 0.0, "rewards/successful_parsing_reward_func": 0.8971354365348816, "rewards/utility_reward_func": 0.19012930989265442, "step": 47 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 9927.0, "completions/max_terminated_length": 9927.0, "completions/mean_length": 5571.90625, "completions/mean_terminated_length": 5571.90625, "completions/min_length": 2299.0, "completions/min_terminated_length": 2299.0, "epoch": 0.10256410256410256, "grad_norm": 0.11263502888805668, "kl": 0.3785247802734375, "learning_rate": 3.1487564587782306e-06, "loss": -0.0166, "num_tokens": 14796817.0, "reward": 0.6865731477737427, "reward_std": 0.9901103973388672, "rewards/constraint_satisfaction_reward_func": 1.0, "rewards/successful_code_execution_reward_func": 0.7390385866165161, "rewards/successful_no_deal_reward_func": 0.0, "rewards/successful_parsing_reward_func": 0.9030382037162781, "rewards/utility_reward_func": 0.13498620688915253, "step": 48 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 8879.0, "completions/max_terminated_length": 8879.0, "completions/mean_length": 5262.96875, "completions/mean_terminated_length": 5432.741935483871, "completions/min_length": 0.0, "completions/min_terminated_length": 2690.0, "epoch": 0.1047008547008547, "grad_norm": 0.07011556216519438, "kl": 0.29779052734375, "learning_rate": 2.8549663646838717e-06, "loss": -0.0141, "num_tokens": 15126585.0, "reward": 0.11910636723041534, "reward_std": 0.14076979458332062, "rewards/constraint_satisfaction_reward_func": 1.0, "rewards/successful_code_execution_reward_func": 0.5717679262161255, "rewards/successful_no_deal_reward_func": 0.0, "rewards/successful_parsing_reward_func": 0.896297812461853, "rewards/utility_reward_func": 0.02152765914797783, "step": 49 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 8900.0, "completions/max_terminated_length": 8900.0, "completions/mean_length": 5700.8125, "completions/mean_terminated_length": 5700.8125, "completions/min_length": 2535.0, "completions/min_terminated_length": 2535.0, "epoch": 0.10683760683760683, "grad_norm": 0.41419598815650255, "kl": 0.519134521484375, "learning_rate": 2.577483782514174e-06, "loss": 0.0138, "num_tokens": 15442066.0, "reward": 0.22905027866363525, "reward_std": 0.3577722907066345, "rewards/constraint_satisfaction_reward_func": 0.984375, "rewards/successful_code_execution_reward_func": 0.7121212482452393, "rewards/successful_no_deal_reward_func": 0.0, "rewards/successful_parsing_reward_func": 0.9120686650276184, "rewards/utility_reward_func": 0.04351647198200226, "step": 50 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 10034.0, "completions/max_terminated_length": 10034.0, "completions/mean_length": 5278.96875, "completions/mean_terminated_length": 5278.96875, "completions/min_length": 1832.0, "completions/min_terminated_length": 1832.0, "epoch": 0.10897435897435898, "grad_norm": 0.09133923773827608, "kl": 0.3924560546875, "learning_rate": 2.3180194846605362e-06, "loss": 0.0272, "num_tokens": 15741043.0, "reward": 0.27375099062919617, "reward_std": 0.4570543169975281, "rewards/constraint_satisfaction_reward_func": 1.0, "rewards/successful_code_execution_reward_func": 0.5625715255737305, "rewards/successful_no_deal_reward_func": 0.0, "rewards/successful_parsing_reward_func": 0.9071673154830933, "rewards/utility_reward_func": 0.05245624855160713, "step": 51 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 7426.0, "completions/max_terminated_length": 7426.0, "completions/mean_length": 4905.0625, "completions/mean_terminated_length": 5063.290322580645, "completions/min_length": 0.0, "completions/min_terminated_length": 2637.0, "epoch": 0.1111111111111111, "grad_norm": 0.06136048782432445, "kl": 0.271026611328125, "learning_rate": 2.078173154799861e-06, "loss": -0.0061, "num_tokens": 16037534.0, "reward": 0.24373172223567963, "reward_std": 0.32430875301361084, "rewards/constraint_satisfaction_reward_func": 1.0, "rewards/successful_code_execution_reward_func": 0.5638384222984314, "rewards/successful_no_deal_reward_func": 0.0, "rewards/successful_parsing_reward_func": 0.924727201461792, "rewards/utility_reward_func": 0.04644863307476044, "step": 52 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 9527.0, "completions/max_terminated_length": 9527.0, "completions/mean_length": 4619.46875, "completions/mean_terminated_length": 4927.433333333333, "completions/min_length": 0.0, "completions/min_terminated_length": 1581.0, "epoch": 0.11324786324786325, "grad_norm": 0.05808359717947583, "kl": 0.22265625, "learning_rate": 1.859423525312737e-06, "loss": 0.0079, "num_tokens": 16335450.0, "reward": 0.21148943901062012, "reward_std": 0.2547459900379181, "rewards/constraint_satisfaction_reward_func": 1.0, "rewards/successful_code_execution_reward_func": 0.6628144979476929, "rewards/successful_no_deal_reward_func": 0.0, "rewards/successful_parsing_reward_func": 0.9208818674087524, "rewards/utility_reward_func": 0.03998114913702011, "step": 53 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 9463.0, "completions/max_terminated_length": 9463.0, "completions/mean_length": 5099.0, "completions/mean_terminated_length": 5263.4838709677415, "completions/min_length": 0.0, "completions/min_terminated_length": 2053.0, "epoch": 0.11538461538461539, "grad_norm": 0.08558113646292538, "kl": 0.288360595703125, "learning_rate": 1.6631192604065856e-06, "loss": 0.0064, "num_tokens": 16636420.0, "reward": 0.28765812516212463, "reward_std": 0.46562910079956055, "rewards/constraint_satisfaction_reward_func": 1.0, "rewards/successful_code_execution_reward_func": 0.6401413679122925, "rewards/successful_no_deal_reward_func": 0.0, "rewards/successful_parsing_reward_func": 0.9321180582046509, "rewards/utility_reward_func": 0.05521716922521591, "step": 54 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 8165.0, "completions/max_terminated_length": 8165.0, "completions/mean_length": 4702.5, "completions/mean_terminated_length": 4854.193548387097, "completions/min_length": 0.0, "completions/min_terminated_length": 1435.0, "epoch": 0.11752136752136752, "grad_norm": 0.06964602043928762, "kl": 0.302154541015625, "learning_rate": 1.490470641152345e-06, "loss": -0.0101, "num_tokens": 16923891.0, "reward": 0.1673395037651062, "reward_std": 0.2552884519100189, "rewards/constraint_satisfaction_reward_func": 1.0, "rewards/successful_code_execution_reward_func": 0.5235062837600708, "rewards/successful_no_deal_reward_func": 0.0, "rewards/successful_parsing_reward_func": 0.9171998500823975, "rewards/utility_reward_func": 0.0311797596514225, "step": 55 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 8940.0, "completions/max_terminated_length": 8940.0, "completions/mean_length": 5183.78125, "completions/mean_terminated_length": 5351.0, "completions/min_length": 0.0, "completions/min_terminated_length": 2205.0, "epoch": 0.11965811965811966, "grad_norm": 0.06968922813852535, "kl": 0.2431640625, "learning_rate": 1.3425421036992098e-06, "loss": 0.002, "num_tokens": 17234992.0, "reward": 0.3152891993522644, "reward_std": 0.4457589387893677, "rewards/constraint_satisfaction_reward_func": 1.0, "rewards/successful_code_execution_reward_func": 0.6660599708557129, "rewards/successful_no_deal_reward_func": 0.0, "rewards/successful_parsing_reward_func": 0.8508894443511963, "rewards/utility_reward_func": 0.0607544481754303, "step": 56 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 7843.0, "completions/max_terminated_length": 7843.0, "completions/mean_length": 4938.625, "completions/mean_terminated_length": 4938.625, "completions/min_length": 2325.0, "completions/min_terminated_length": 2325.0, "epoch": 0.12179487179487179, "grad_norm": 0.10517713287333766, "kl": 0.155487060546875, "learning_rate": 1.2202456766718092e-06, "loss": -0.0089, "num_tokens": 17528965.0, "reward": 0.6346352100372314, "reward_std": 0.614031195640564, "rewards/constraint_satisfaction_reward_func": 1.0, "rewards/successful_code_execution_reward_func": 0.7036257982254028, "rewards/successful_no_deal_reward_func": 0.0, "rewards/successful_parsing_reward_func": 0.8766368627548218, "rewards/utility_reward_func": 0.12461098283529282, "step": 57 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 9878.0, "completions/max_terminated_length": 9878.0, "completions/mean_length": 5030.96875, "completions/mean_terminated_length": 5030.96875, "completions/min_length": 2257.0, "completions/min_terminated_length": 2257.0, "epoch": 0.12393162393162394, "grad_norm": 0.11120189846420504, "kl": 0.26873779296875, "learning_rate": 1.1243353582104554e-06, "loss": -0.0108, "num_tokens": 17819650.0, "reward": 0.22769173979759216, "reward_std": 0.24561473727226257, "rewards/constraint_satisfaction_reward_func": 1.0, "rewards/successful_code_execution_reward_func": 0.6461749076843262, "rewards/successful_no_deal_reward_func": 0.0, "rewards/successful_parsing_reward_func": 0.923301100730896, "rewards/utility_reward_func": 0.043224453926086426, "step": 58 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09375, "completions/max_length": 9634.0, "completions/max_terminated_length": 9634.0, "completions/mean_length": 4724.53125, "completions/mean_terminated_length": 5213.275862068966, "completions/min_length": 0.0, "completions/min_terminated_length": 2819.0, "epoch": 0.12606837606837606, "grad_norm": 0.07749116939560954, "kl": 0.22208786010742188, "learning_rate": 1.0554024673218806e-06, "loss": 0.0051, "num_tokens": 18126661.0, "reward": 0.3350598216056824, "reward_std": 0.4074792265892029, "rewards/constraint_satisfaction_reward_func": 1.0, "rewards/successful_code_execution_reward_func": 0.5758612751960754, "rewards/successful_no_deal_reward_func": 0.0, "rewards/successful_parsing_reward_func": 0.8875744342803955, "rewards/utility_reward_func": 0.06471927464008331, "step": 59 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 9403.0, "completions/max_terminated_length": 9403.0, "completions/mean_length": 5518.96875, "completions/mean_terminated_length": 5518.96875, "completions/min_length": 2955.0, "completions/min_terminated_length": 2955.0, "epoch": 0.1282051282051282, "grad_norm": 0.11648518986862501, "kl": 0.1890869140625, "learning_rate": 1.013871998200924e-06, "loss": 0.0471, "num_tokens": 18432399.0, "reward": 0.4429653584957123, "reward_std": 0.6282458305358887, "rewards/constraint_satisfaction_reward_func": 1.0, "rewards/successful_code_execution_reward_func": 0.6058531999588013, "rewards/successful_no_deal_reward_func": 0.0, "rewards/successful_parsing_reward_func": 0.9258173108100891, "rewards/utility_reward_func": 0.08628673851490021, "step": 60 } ], "logging_steps": 1, "max_steps": 60, "num_input_tokens_seen": 18432399, "num_train_epochs": 1, "save_steps": 30, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": true, "should_training_stop": true }, "attributes": {} } }, "total_flos": 0.0, "train_batch_size": 1, "trial_name": null, "trial_params": null }