{ "best_global_step": null, "best_metric": null, "best_model_checkpoint": null, "epoch": 1.1896431070678797, "eval_steps": 500, "global_step": 3400, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "clip_ratio": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 235.0, "completions/max_terminated_length": 235.0, "completions/mean_length": 146.0, "completions/mean_terminated_length": 146.0, "completions/min_length": 106.0, "completions/min_terminated_length": 106.0, "epoch": 0.0003498950314905528, "grad_norm": 0.13926473185400237, "kl": 0.00019741058349609375, "learning_rate": 0.0, "loss": -0.0027, "num_tokens": 18568.0, "reward": 1.0466493368148804, "reward_std": 0.04344530776143074, "rewards/check_gptzero_func/mean": 0.0, "rewards/check_gptzero_func/std": 0.0, "rewards/check_winston_local_func/mean": 0.050220753997564316, "rewards/check_winston_local_func/std": 0.08986040204763412, "rewards/sentence_count_match_reward_logic/mean": 0.9964285492897034, "rewards/sentence_count_match_reward_logic/std": 0.026726122945547104, "step": 1 }, { "clip_ratio": 0.0, "epoch": 0.0006997900629811056, "grad_norm": 0.13896232880064227, "kl": 0.00019741058349609375, "learning_rate": 1.7482517482517483e-08, "loss": -0.0027, "step": 2 }, { "clip_ratio": 0.0006617679027840495, "epoch": 0.0010496850944716584, "grad_norm": 0.13569449449757717, "kl": 0.00015926361083984375, "learning_rate": 3.4965034965034967e-08, "loss": -0.0027, "step": 3 }, { "clip_ratio": 0.0008874954073689878, "epoch": 0.0013995801259622112, "grad_norm": 0.1379188184403024, "kl": 0.0001735687255859375, "learning_rate": 5.244755244755245e-08, "loss": -0.0026, "step": 4 }, { "clip_ratio": 0.001115030492655933, "completions/clipped_ratio": 0.1428571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 202.0, "completions/mean_length": 174.57144165039062, "completions/mean_terminated_length": 161.0, "completions/min_length": 97.0, "completions/min_terminated_length": 97.0, "epoch": 0.0017494751574527643, "grad_norm": 0.12369089975717877, "kl": 0.00016117095947265625, "learning_rate": 6.993006993006993e-08, "loss": 0.001, "num_tokens": 41352.0, "reward": 0.9870626330375671, "reward_std": 0.024708224460482597, "rewards/check_gptzero_func/mean": 0.0, "rewards/check_gptzero_func/std": 0.0, "rewards/check_winston_local_func/mean": 0.023005839437246323, "rewards/check_winston_local_func/std": 0.03662038594484329, "rewards/sentence_count_match_reward_logic/mean": 0.9640567898750305, "rewards/sentence_count_match_reward_logic/std": 0.08456307649612427, "step": 5 }, { "clip_ratio": 0.0006518490263260901, "epoch": 0.002099370188943317, "grad_norm": 0.1297535922716751, "kl": 0.00016307830810546875, "learning_rate": 8.741258741258742e-08, "loss": 0.0012, "step": 6 }, { "clip_ratio": 0.0005000568926334381, "epoch": 0.00244926522043387, "grad_norm": 0.1289022388310105, "kl": 0.00016117095947265625, "learning_rate": 1.048951048951049e-07, "loss": 0.0012, "step": 7 }, { "clip_ratio": 0.001084880088455975, "epoch": 0.0027991602519244225, "grad_norm": 0.12489888528060963, "kl": 0.0001583099365234375, "learning_rate": 1.223776223776224e-07, "loss": 0.001, "step": 8 }, { "clip_ratio": 0.0014356564497575164, "completions/clipped_ratio": 0.0, "completions/max_length": 225.0, "completions/max_terminated_length": 225.0, "completions/mean_length": 138.30357360839844, "completions/mean_terminated_length": 138.30357360839844, "completions/min_length": 79.0, "completions/min_terminated_length": 79.0, "epoch": 0.0031490552834149755, "grad_norm": 0.11963195616239433, "kl": 0.000152587890625, "learning_rate": 1.3986013986013987e-07, "loss": -0.0031, "num_tokens": 59033.0, "reward": 1.1912063360214233, "reward_std": 0.11973810195922852, "rewards/check_gptzero_func/mean": 0.1964285671710968, "rewards/check_gptzero_func/std": 0.40089187026023865, "rewards/check_winston_local_func/mean": 0.01135932095348835, "rewards/check_winston_local_func/std": 0.00917299184948206, "rewards/sentence_count_match_reward_logic/mean": 0.983418345451355, "rewards/sentence_count_match_reward_logic/std": 0.04846002906560898, "step": 9 }, { "clip_ratio": 0.0013433565618470311, "epoch": 0.0034989503149055285, "grad_norm": 0.1136509266232329, "kl": 0.00011444091796875, "learning_rate": 1.5734265734265737e-07, "loss": -0.0032, "step": 10 }, { "clip_ratio": 0.0014313423307612538, "epoch": 0.003848845346396081, "grad_norm": 0.1147228896766, "kl": 0.0001544952392578125, "learning_rate": 1.7482517482517484e-07, "loss": -0.0032, "step": 11 }, { "clip_ratio": 0.0018479650607332587, "epoch": 0.004198740377886634, "grad_norm": 0.11325768399172727, "kl": 0.0001544952392578125, "learning_rate": 1.9230769230769234e-07, "loss": -0.0031, "step": 12 }, { "clip_ratio": 0.0004998444928787649, "completions/clipped_ratio": 0.0, "completions/max_length": 181.0, "completions/max_terminated_length": 181.0, "completions/mean_length": 148.23214721679688, "completions/mean_terminated_length": 148.23214721679688, "completions/min_length": 113.0, "completions/min_terminated_length": 113.0, "epoch": 0.004548635409377187, "grad_norm": 0.10705127925306658, "kl": 0.00017833709716796875, "learning_rate": 2.097902097902098e-07, "loss": 0.0021, "num_tokens": 78246.0, "reward": 1.2323626279830933, "reward_std": 0.14597664773464203, "rewards/check_gptzero_func/mean": 0.2142857164144516, "rewards/check_gptzero_func/std": 0.4140393137931824, "rewards/check_winston_local_func/mean": 0.02615508809685707, "rewards/check_winston_local_func/std": 0.057357173413038254, "rewards/sentence_count_match_reward_logic/mean": 0.9919217824935913, "rewards/sentence_count_match_reward_logic/std": 0.04385015740990639, "step": 13 }, { "clip_ratio": 0.0005970150232315063, "epoch": 0.00489853044086774, "grad_norm": 0.10731767304001945, "kl": 0.00016498565673828125, "learning_rate": 2.2727272727272729e-07, "loss": 0.0021, "step": 14 }, { "clip_ratio": 0.0009528238442726433, "epoch": 0.005248425472358292, "grad_norm": 0.10731321754663682, "kl": 0.00020122528076171875, "learning_rate": 2.447552447552448e-07, "loss": 0.002, "step": 15 }, { "clip_ratio": 0.00038168515311554074, "epoch": 0.005598320503848845, "grad_norm": 0.1057284056435777, "kl": 0.0002460479736328125, "learning_rate": 2.622377622377623e-07, "loss": 0.0022, "step": 16 }, { "clip_ratio": 0.00038828590186312795, "completions/clipped_ratio": 0.0, "completions/max_length": 210.0, "completions/max_terminated_length": 210.0, "completions/mean_length": 136.5, "completions/mean_terminated_length": 136.5, "completions/min_length": 98.0, "completions/min_terminated_length": 98.0, "epoch": 0.005948215535339398, "grad_norm": 0.08937689014427173, "kl": 9.489059448242188e-05, "learning_rate": 2.7972027972027973e-07, "loss": 0.0003, "num_tokens": 95650.0, "reward": 1.1084154844284058, "reward_std": 0.07177835702896118, "rewards/check_gptzero_func/mean": 0.1071428582072258, "rewards/check_gptzero_func/std": 0.3120938837528229, "rewards/check_winston_local_func/mean": 0.029843999072909355, "rewards/check_winston_local_func/std": 0.056215979158878326, "rewards/sentence_count_match_reward_logic/mean": 0.9714285731315613, "rewards/sentence_count_match_reward_logic/std": 0.07061877846717834, "step": 17 }, { "clip_ratio": 0.0008194794063456357, "epoch": 0.006298110566829951, "grad_norm": 0.08814966203670069, "kl": 0.00014400482177734375, "learning_rate": 2.9720279720279723e-07, "loss": 0.0002, "step": 18 }, { "clip_ratio": 0.0005247841472737491, "epoch": 0.006648005598320504, "grad_norm": 0.08937208167298635, "kl": 0.0001277923583984375, "learning_rate": 3.1468531468531473e-07, "loss": 0.0004, "step": 19 }, { "clip_ratio": 0.00043661746894940734, "epoch": 0.006997900629811057, "grad_norm": 0.09015200651542393, "kl": 0.00010251998901367188, "learning_rate": 3.321678321678322e-07, "loss": 0.0003, "step": 20 }, { "clip_ratio": 0.0012135835131630301, "completions/clipped_ratio": 0.0, "completions/max_length": 147.0, "completions/max_terminated_length": 147.0, "completions/mean_length": 110.6964340209961, "completions/mean_terminated_length": 110.6964340209961, "completions/min_length": 48.0, "completions/min_terminated_length": 48.0, "epoch": 0.00734779566130161, "grad_norm": 0.10880687630045917, "kl": 0.00017261505126953125, "learning_rate": 3.496503496503497e-07, "loss": -0.0002, "num_tokens": 110321.0, "reward": 1.064292073249817, "reward_std": 0.07793872803449631, "rewards/check_gptzero_func/mean": 0.0535714291036129, "rewards/check_gptzero_func/std": 0.22720779478549957, "rewards/check_winston_local_func/mean": 0.010720552876591682, "rewards/check_winston_local_func/std": 0.00994865596294403, "rewards/sentence_count_match_reward_logic/mean": 1.0, "rewards/sentence_count_match_reward_logic/std": 0.0, "step": 21 }, { "clip_ratio": 0.000693286769092083, "epoch": 0.007697690692792162, "grad_norm": 0.10963598547024328, "kl": 0.00017642974853515625, "learning_rate": 3.6713286713286713e-07, "loss": -0.0004, "step": 22 }, { "clip_ratio": 0.0012160154292359948, "epoch": 0.008047585724282715, "grad_norm": 0.1072283459695525, "kl": 0.00018978118896484375, "learning_rate": 3.846153846153847e-07, "loss": -0.0004, "step": 23 }, { "clip_ratio": 0.0006280947127379477, "epoch": 0.008397480755773267, "grad_norm": 0.10895100844400454, "kl": 0.0001964569091796875, "learning_rate": 4.020979020979021e-07, "loss": -0.0003, "step": 24 }, { "clip_ratio": 0.0004313708923291415, "completions/clipped_ratio": 0.0, "completions/max_length": 219.0, "completions/max_terminated_length": 219.0, "completions/mean_length": 165.71429443359375, "completions/mean_terminated_length": 165.71429443359375, "completions/min_length": 113.0, "completions/min_terminated_length": 113.0, "epoch": 0.00874737578726382, "grad_norm": 0.13224848593851027, "kl": 0.00020313262939453125, "learning_rate": 4.195804195804196e-07, "loss": -0.0039, "num_tokens": 131633.0, "reward": 1.0131828784942627, "reward_std": 0.08222130686044693, "rewards/check_gptzero_func/mean": 0.01785714365541935, "rewards/check_gptzero_func/std": 0.13363061845302582, "rewards/check_winston_local_func/mean": 0.01913519576191902, "rewards/check_winston_local_func/std": 0.03970739617943764, "rewards/sentence_count_match_reward_logic/mean": 0.976190447807312, "rewards/sentence_count_match_reward_logic/std": 0.04600436985492706, "step": 25 }, { "clip_ratio": 0.00039483033469878137, "epoch": 0.009097270818754374, "grad_norm": 0.13269351880664515, "kl": 0.0002460479736328125, "learning_rate": 4.3706293706293707e-07, "loss": -0.0036, "step": 26 }, { "clip_ratio": 0.0010684655280783772, "epoch": 0.009447165850244927, "grad_norm": 0.12881418460344454, "kl": 0.00020885467529296875, "learning_rate": 4.5454545454545457e-07, "loss": -0.0036, "step": 27 }, { "clip_ratio": 0.0008756893803365529, "epoch": 0.00979706088173548, "grad_norm": 0.1305510245028988, "kl": 0.00023365020751953125, "learning_rate": 4.72027972027972e-07, "loss": -0.0038, "step": 28 }, { "clip_ratio": 0.0005462879198603332, "completions/clipped_ratio": 0.0, "completions/max_length": 230.0, "completions/max_terminated_length": 230.0, "completions/mean_length": 147.73214721679688, "completions/mean_terminated_length": 147.73214721679688, "completions/min_length": 75.0, "completions/min_terminated_length": 75.0, "epoch": 0.010146955913226032, "grad_norm": 0.1236066356147624, "kl": 0.00018024444580078125, "learning_rate": 4.895104895104896e-07, "loss": -0.0007, "num_tokens": 150346.0, "reward": 1.2165313959121704, "reward_std": 0.08650125563144684, "rewards/check_gptzero_func/mean": 0.2142857164144516, "rewards/check_gptzero_func/std": 0.41403928399086, "rewards/check_winston_local_func/mean": 0.008942089043557644, "rewards/check_winston_local_func/std": 0.0029591810889542103, "rewards/sentence_count_match_reward_logic/mean": 0.9933035969734192, "rewards/sentence_count_match_reward_logic/std": 0.028400972485542297, "step": 29 }, { "clip_ratio": 0.0007844219799153507, "epoch": 0.010496850944716585, "grad_norm": 0.12246770673449538, "kl": 0.00016117095947265625, "learning_rate": 5.06993006993007e-07, "loss": -0.0007, "step": 30 }, { "clip_ratio": 0.0009301375248469412, "epoch": 0.010846745976207137, "grad_norm": 0.12525894793151712, "kl": 0.00019550323486328125, "learning_rate": 5.244755244755246e-07, "loss": -0.0008, "step": 31 }, { "clip_ratio": 0.000925201631616801, "epoch": 0.01119664100769769, "grad_norm": 0.12346775884552169, "kl": 0.00017547607421875, "learning_rate": 5.41958041958042e-07, "loss": -0.0007, "step": 32 }, { "clip_ratio": 0.0007329060463234782, "completions/clipped_ratio": 0.0535714285714286, "completions/max_length": 256.0, "completions/max_terminated_length": 256.0, "completions/mean_length": 196.19644165039062, "completions/mean_terminated_length": 192.8113250732422, "completions/min_length": 128.0, "completions/min_terminated_length": 128.0, "epoch": 0.011546536039188244, "grad_norm": 0.12145054223570449, "kl": 0.0001544952392578125, "learning_rate": 5.594405594405595e-07, "loss": 0.0007, "num_tokens": 174421.0, "reward": 1.1842153072357178, "reward_std": 0.1431759148836136, "rewards/check_gptzero_func/mean": 0.1964285671710968, "rewards/check_gptzero_func/std": 0.40089187026023865, "rewards/check_winston_local_func/mean": 0.015762856230139732, "rewards/check_winston_local_func/std": 0.022408926859498024, "rewards/sentence_count_match_reward_logic/mean": 0.9720238447189331, "rewards/sentence_count_match_reward_logic/std": 0.06475339084863663, "step": 33 }, { "clip_ratio": 0.0003239391080569476, "epoch": 0.011896431070678797, "grad_norm": 0.12061274926842597, "kl": 0.00014400482177734375, "learning_rate": 5.76923076923077e-07, "loss": 0.0005, "step": 34 }, { "clip_ratio": 0.0005788104608654976, "epoch": 0.01224632610216935, "grad_norm": 0.12147165700855542, "kl": 0.00016689300537109375, "learning_rate": 5.944055944055945e-07, "loss": 0.0007, "step": 35 }, { "clip_ratio": 0.0006216546171344817, "epoch": 0.012596221133659902, "grad_norm": 0.12105014769924233, "kl": 0.00014591217041015625, "learning_rate": 6.118881118881119e-07, "loss": 0.0006, "step": 36 }, { "clip_ratio": 0.0006173049914650619, "completions/clipped_ratio": 0.0, "completions/max_length": 230.0, "completions/max_terminated_length": 230.0, "completions/mean_length": 155.5357208251953, "completions/mean_terminated_length": 155.5357208251953, "completions/min_length": 89.0, "completions/min_terminated_length": 89.0, "epoch": 0.012946116165150455, "grad_norm": 0.1086043394380577, "kl": 0.00015544891357421875, "learning_rate": 6.293706293706295e-07, "loss": 0.0029, "num_tokens": 193675.0, "reward": 1.044965386390686, "reward_std": 0.06759525835514069, "rewards/check_gptzero_func/mean": 0.0357142873108387, "rewards/check_gptzero_func/std": 0.1872563362121582, "rewards/check_winston_local_func/mean": 0.009251018054783344, "rewards/check_winston_local_func/std": 0.0025284173898398876, "rewards/sentence_count_match_reward_logic/mean": 1.0, "rewards/sentence_count_match_reward_logic/std": 0.0, "step": 37 }, { "clip_ratio": 0.0006024344475008547, "epoch": 0.013296011196641007, "grad_norm": 0.10824813170011442, "kl": 0.0001926422119140625, "learning_rate": 6.468531468531469e-07, "loss": 0.0028, "step": 38 }, { "clip_ratio": 0.0011489762691780925, "epoch": 0.01364590622813156, "grad_norm": 0.10515208654936264, "kl": 0.00019168853759765625, "learning_rate": 6.643356643356644e-07, "loss": 0.003, "step": 39 }, { "clip_ratio": 0.0003609401173889637, "epoch": 0.013995801259622114, "grad_norm": 0.10802760740396175, "kl": 0.0001811981201171875, "learning_rate": 6.818181818181818e-07, "loss": 0.0029, "step": 40 }, { "clip_ratio": 0.0010470313718542457, "completions/clipped_ratio": 0.0, "completions/max_length": 248.0, "completions/max_terminated_length": 248.0, "completions/mean_length": 163.9107208251953, "completions/mean_terminated_length": 163.9107208251953, "completions/min_length": 108.0, "completions/min_terminated_length": 108.0, "epoch": 0.014345696291112667, "grad_norm": 0.11686108479081263, "kl": 0.0001544952392578125, "learning_rate": 6.993006993006994e-07, "loss": -0.0047, "num_tokens": 214518.0, "reward": 1.1534308195114136, "reward_std": 0.08829309791326523, "rewards/check_gptzero_func/mean": 0.1964285671710968, "rewards/check_gptzero_func/std": 0.40089187026023865, "rewards/check_winston_local_func/mean": 0.011317678727209568, "rewards/check_winston_local_func/std": 0.009559924714267254, "rewards/sentence_count_match_reward_logic/mean": 0.9456844925880432, "rewards/sentence_count_match_reward_logic/std": 0.07700175791978836, "step": 41 }, { "clip_ratio": 0.0002398944488959387, "epoch": 0.01469559132260322, "grad_norm": 0.1159449811202, "kl": 0.0001964569091796875, "learning_rate": 7.167832167832168e-07, "loss": -0.0044, "step": 42 }, { "clip_ratio": 0.00042812275933101773, "epoch": 0.015045486354093772, "grad_norm": 0.11702733849206226, "kl": 0.00014209747314453125, "learning_rate": 7.342657342657343e-07, "loss": -0.0044, "step": 43 }, { "clip_ratio": 0.00024830043548718095, "epoch": 0.015395381385584325, "grad_norm": 0.11769727618946345, "kl": 0.00016880035400390625, "learning_rate": 7.517482517482517e-07, "loss": -0.0044, "step": 44 }, { "clip_ratio": 0.0004099614161532372, "completions/clipped_ratio": 0.0, "completions/max_length": 202.0, "completions/max_terminated_length": 202.0, "completions/mean_length": 149.25, "completions/mean_terminated_length": 149.25, "completions/min_length": 84.0, "completions/min_terminated_length": 84.0, "epoch": 0.015745276417074877, "grad_norm": 0.11421717570798745, "kl": 0.00012683868408203125, "learning_rate": 7.692307692307694e-07, "loss": -0.0016, "num_tokens": 233572.0, "reward": 0.9959798455238342, "reward_std": 0.020449798554182053, "rewards/check_gptzero_func/mean": 0.0, "rewards/check_gptzero_func/std": 0.0, "rewards/check_winston_local_func/mean": 0.009160107932984829, "rewards/check_winston_local_func/std": 0.0033154112752527, "rewards/sentence_count_match_reward_logic/mean": 0.9868197441101074, "rewards/sentence_count_match_reward_logic/std": 0.04257231950759888, "step": 45 }, { "clip_ratio": 0.0009000140125863254, "epoch": 0.01609517144856543, "grad_norm": 0.11584043389710226, "kl": 0.00011730194091796875, "learning_rate": 7.867132867132868e-07, "loss": -0.0018, "step": 46 }, { "clip_ratio": 0.0008447869913652539, "epoch": 0.016445066480055982, "grad_norm": 0.11379011597145802, "kl": 0.00011110305786132812, "learning_rate": 8.041958041958043e-07, "loss": -0.0017, "step": 47 }, { "clip_ratio": 0.0009082485921680927, "epoch": 0.016794961511546535, "grad_norm": 0.11118190428286075, "kl": 0.00015735626220703125, "learning_rate": 8.216783216783217e-07, "loss": -0.0017, "step": 48 }, { "clip_ratio": 8.705492655280977e-05, "completions/clipped_ratio": 0.0, "completions/max_length": 212.0, "completions/max_terminated_length": 212.0, "completions/mean_length": 164.73214721679688, "completions/mean_terminated_length": 164.73214721679688, "completions/min_length": 106.0, "completions/min_terminated_length": 106.0, "epoch": 0.017144856543037088, "grad_norm": 0.11939149230249552, "kl": 0.0001220703125, "learning_rate": 8.391608391608393e-07, "loss": 0.0013, "num_tokens": 254405.0, "reward": 1.1011784076690674, "reward_std": 0.18991562724113464, "rewards/check_gptzero_func/mean": 0.1071428582072258, "rewards/check_gptzero_func/std": 0.3120938837528229, "rewards/check_winston_local_func/mean": 0.009341643191874027, "rewards/check_winston_local_func/std": 0.003626402700319886, "rewards/sentence_count_match_reward_logic/mean": 0.9846939444541931, "rewards/sentence_count_match_reward_logic/std": 0.044584836810827255, "step": 49 }, { "clip_ratio": 0.00020206102635711432, "epoch": 0.01749475157452764, "grad_norm": 0.12131501347454206, "kl": 0.00010347366333007812, "learning_rate": 8.566433566433567e-07, "loss": 0.001, "step": 50 }, { "clip_ratio": 0.000300360843539238, "epoch": 0.017844646606018196, "grad_norm": 0.11853471062287785, "kl": 0.00011968612670898438, "learning_rate": 8.741258741258741e-07, "loss": 0.0011, "step": 51 }, { "clip_ratio": 0.0004939218633808196, "epoch": 0.01819454163750875, "grad_norm": 0.11936343065949043, "kl": 0.0001220703125, "learning_rate": 8.916083916083917e-07, "loss": 0.0012, "step": 52 }, { "clip_ratio": 0.0009658340713940561, "completions/clipped_ratio": 0.125, "completions/max_length": 256.0, "completions/max_terminated_length": 251.0, "completions/mean_length": 163.125, "completions/mean_terminated_length": 149.85714721679688, "completions/min_length": 111.0, "completions/min_terminated_length": 111.0, "epoch": 0.0185444366689993, "grad_norm": 0.13564678749722267, "kl": 0.00016689300537109375, "learning_rate": 9.090909090909091e-07, "loss": -0.0032, "num_tokens": 275260.0, "reward": 1.1651623249053955, "reward_std": 0.08778183162212372, "rewards/check_gptzero_func/mean": 0.1964285671710968, "rewards/check_gptzero_func/std": 0.40089187026023865, "rewards/check_winston_local_func/mean": 0.009549999609589577, "rewards/check_winston_local_func/std": 0.004860693588852882, "rewards/sentence_count_match_reward_logic/mean": 0.9591836929321289, "rewards/sentence_count_match_reward_logic/std": 0.08199158310890198, "step": 53 }, { "clip_ratio": 0.0009339131647720933, "epoch": 0.018894331700489854, "grad_norm": 0.1394080358336403, "kl": 0.00019168853759765625, "learning_rate": 9.265734265734266e-07, "loss": -0.0031, "step": 54 }, { "clip_ratio": 0.0010491813300177455, "epoch": 0.019244226731980407, "grad_norm": 0.13724513478333206, "kl": 0.00016498565673828125, "learning_rate": 9.44055944055944e-07, "loss": -0.0031, "step": 55 }, { "clip_ratio": 0.0009760420070961118, "epoch": 0.01959412176347096, "grad_norm": 0.14088717710229104, "kl": 0.00017261505126953125, "learning_rate": 9.615384615384617e-07, "loss": -0.0033, "step": 56 }, { "clip_ratio": 0.0009269608999602497, "completions/clipped_ratio": 0.0, "completions/max_length": 201.0, "completions/max_terminated_length": 201.0, "completions/mean_length": 151.57144165039062, "completions/mean_terminated_length": 151.57144165039062, "completions/min_length": 117.0, "completions/min_terminated_length": 117.0, "epoch": 0.01994401679496151, "grad_norm": 0.11541143315084512, "kl": 0.000255584716796875, "learning_rate": 9.790209790209791e-07, "loss": -0.0037, "num_tokens": 294692.0, "reward": 1.2376203536987305, "reward_std": 0.11196891218423843, "rewards/check_gptzero_func/mean": 0.1964285671710968, "rewards/check_gptzero_func/std": 0.40089187026023865, "rewards/check_winston_local_func/mean": 0.06517131626605988, "rewards/check_winston_local_func/std": 0.14439889788627625, "rewards/sentence_count_match_reward_logic/mean": 0.9760203957557678, "rewards/sentence_count_match_reward_logic/std": 0.055769674479961395, "step": 57 }, { "clip_ratio": 0.0009491803939454257, "epoch": 0.020293911826452064, "grad_norm": 0.11382103622532667, "kl": 0.00017452239990234375, "learning_rate": 9.965034965034966e-07, "loss": -0.0038, "step": 58 }, { "clip_ratio": 0.0015787631273269653, "epoch": 0.020643806857942617, "grad_norm": 0.11419229484947102, "kl": 0.0002231597900390625, "learning_rate": 1.013986013986014e-06, "loss": -0.0038, "step": 59 }, { "clip_ratio": 0.0009065744234248996, "epoch": 0.02099370188943317, "grad_norm": 0.11945855963633359, "kl": 0.00020885467529296875, "learning_rate": 1.0314685314685317e-06, "loss": -0.0038, "step": 60 }, { "clip_ratio": 0.0006790471961721778, "completions/clipped_ratio": 0.0, "completions/max_length": 201.0, "completions/max_terminated_length": 201.0, "completions/mean_length": 161.7678680419922, "completions/mean_terminated_length": 161.7678680419922, "completions/min_length": 100.0, "completions/min_terminated_length": 100.0, "epoch": 0.021343596920923722, "grad_norm": 0.11961682697881691, "kl": 0.00019931793212890625, "learning_rate": 1.0489510489510491e-06, "loss": -0.0031, "num_tokens": 315279.0, "reward": 1.11587393283844, "reward_std": 0.08950456231832504, "rewards/check_gptzero_func/mean": 0.0892857164144516, "rewards/check_gptzero_func/std": 0.28773635625839233, "rewards/check_winston_local_func/mean": 0.026588287204504013, "rewards/check_winston_local_func/std": 0.039137572050094604, "rewards/sentence_count_match_reward_logic/mean": 1.0, "rewards/sentence_count_match_reward_logic/std": 0.0, "step": 61 }, { "clip_ratio": 0.0006481995806097984, "epoch": 0.021693491952414275, "grad_norm": 0.11772376671213997, "kl": 0.00012969970703125, "learning_rate": 1.0664335664335666e-06, "loss": -0.0031, "step": 62 }, { "clip_ratio": 0.0011078041279688478, "epoch": 0.022043386983904827, "grad_norm": 0.11928252428416161, "kl": 0.00017452239990234375, "learning_rate": 1.083916083916084e-06, "loss": -0.0031, "step": 63 }, { "clip_ratio": 0.0007510888972319663, "epoch": 0.02239328201539538, "grad_norm": 0.11859222316582982, "kl": 0.00014495849609375, "learning_rate": 1.1013986013986015e-06, "loss": -0.0031, "step": 64 }, { "clip_ratio": 0.0010351062519475818, "completions/clipped_ratio": 0.1071428571428571, "completions/max_length": 256.0, "completions/max_terminated_length": 256.0, "completions/mean_length": 168.4107208251953, "completions/mean_terminated_length": 157.89999389648438, "completions/min_length": 100.0, "completions/min_terminated_length": 100.0, "epoch": 0.022743177046885936, "grad_norm": 0.11636914503104306, "kl": 0.00020694732666015625, "learning_rate": 1.118881118881119e-06, "loss": 0.001, "num_tokens": 336486.0, "reward": 1.0091066360473633, "reward_std": 0.01701197773218155, "rewards/check_gptzero_func/mean": 0.0, "rewards/check_gptzero_func/std": 0.0, "rewards/check_winston_local_func/mean": 0.015058928169310093, "rewards/check_winston_local_func/std": 0.025575408712029457, "rewards/sentence_count_match_reward_logic/mean": 0.9940475821495056, "rewards/sentence_count_match_reward_logic/std": 0.031209392473101616, "step": 65 }, { "clip_ratio": 0.0012604560470208526, "epoch": 0.02309307207837649, "grad_norm": 0.10578319320486446, "kl": 0.000213623046875, "learning_rate": 1.1363636363636364e-06, "loss": 0.0012, "step": 66 }, { "clip_ratio": 0.0008662527543492615, "epoch": 0.02344296710986704, "grad_norm": 0.11268209879460758, "kl": 0.00020313262939453125, "learning_rate": 1.153846153846154e-06, "loss": 0.0011, "step": 67 }, { "clip_ratio": 0.001680804300121963, "epoch": 0.023792862141357594, "grad_norm": 0.11787325272051905, "kl": 0.0002384185791015625, "learning_rate": 1.1713286713286715e-06, "loss": 0.0012, "step": 68 }, { "clip_ratio": 0.0005422039539553225, "completions/clipped_ratio": 0.0, "completions/max_length": 218.0, "completions/max_terminated_length": 218.0, "completions/mean_length": 134.25, "completions/mean_terminated_length": 134.25, "completions/min_length": 66.0, "completions/min_terminated_length": 66.0, "epoch": 0.024142757172848146, "grad_norm": 0.12432143237730255, "kl": 0.0001983642578125, "learning_rate": 1.188811188811189e-06, "loss": 0.0011, "num_tokens": 353652.0, "reward": 0.9878070950508118, "reward_std": 0.017213769257068634, "rewards/check_gptzero_func/mean": 0.0, "rewards/check_gptzero_func/std": 0.0, "rewards/check_winston_local_func/mean": 0.015584892593324184, "rewards/check_winston_local_func/std": 0.019516510888934135, "rewards/sentence_count_match_reward_logic/mean": 0.9722222089767456, "rewards/sentence_count_match_reward_logic/std": 0.06085806339979172, "step": 69 }, { "clip_ratio": 0.00044731609523296356, "epoch": 0.0244926522043387, "grad_norm": 0.12369009469347342, "kl": 0.00020122528076171875, "learning_rate": 1.2062937062937064e-06, "loss": 0.0011, "step": 70 }, { "clip_ratio": 0.0008453303598798811, "epoch": 0.02484254723582925, "grad_norm": 0.1228161369121329, "kl": 0.000247955322265625, "learning_rate": 1.2237762237762238e-06, "loss": 0.0008, "step": 71 }, { "clip_ratio": 0.0006443153251893818, "epoch": 0.025192442267319804, "grad_norm": 0.1223488806810559, "kl": 0.00023365020751953125, "learning_rate": 1.2412587412587413e-06, "loss": 0.0008, "step": 72 }, { "clip_ratio": 0.0003851877117995173, "completions/clipped_ratio": 0.0, "completions/max_length": 239.0, "completions/max_terminated_length": 239.0, "completions/mean_length": 169.55357360839844, "completions/mean_terminated_length": 169.55357360839844, "completions/min_length": 79.0, "completions/min_terminated_length": 79.0, "epoch": 0.025542337298810357, "grad_norm": 0.10254138619302688, "kl": 0.000152587890625, "learning_rate": 1.258741258741259e-06, "loss": 0.0017, "num_tokens": 374635.0, "reward": 1.107696294784546, "reward_std": 0.08249878883361816, "rewards/check_gptzero_func/mean": 0.1071428582072258, "rewards/check_gptzero_func/std": 0.3120938837528229, "rewards/check_winston_local_func/mean": 0.009481946937739849, "rewards/check_winston_local_func/std": 0.0027094038669019938, "rewards/sentence_count_match_reward_logic/mean": 0.9910714030265808, "rewards/sentence_count_match_reward_logic/std": 0.04029114916920662, "step": 73 }, { "clip_ratio": 0.0007471967255696654, "epoch": 0.02589223233030091, "grad_norm": 0.10184534815414635, "kl": 0.0001506805419921875, "learning_rate": 1.2762237762237764e-06, "loss": 0.0017, "step": 74 }, { "clip_ratio": 0.0007018879405222833, "epoch": 0.026242127361791462, "grad_norm": 0.10267580972022489, "kl": 0.0001659393310546875, "learning_rate": 1.2937062937062938e-06, "loss": 0.0015, "step": 75 }, { "clip_ratio": 0.0008300124318338931, "epoch": 0.026592022393282014, "grad_norm": 0.10404022641056011, "kl": 0.0001811981201171875, "learning_rate": 1.3111888111888113e-06, "loss": 0.0015, "step": 76 }, { "clip_ratio": 0.0008495976799167693, "completions/clipped_ratio": 0.1428571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 193.0, "completions/mean_length": 149.69644165039062, "completions/mean_terminated_length": 131.9791717529297, "completions/min_length": 86.0, "completions/min_terminated_length": 86.0, "epoch": 0.026941917424772567, "grad_norm": 0.11900011131283979, "kl": 0.00021076202392578125, "learning_rate": 1.3286713286713287e-06, "loss": 0.0007, "num_tokens": 393778.0, "reward": 0.9858284592628479, "reward_std": 0.014893691055476665, "rewards/check_gptzero_func/mean": 0.0, "rewards/check_gptzero_func/std": 0.0, "rewards/check_winston_local_func/mean": 0.011019749566912651, "rewards/check_winston_local_func/std": 0.004641581326723099, "rewards/sentence_count_match_reward_logic/mean": 0.9748086929321289, "rewards/sentence_count_match_reward_logic/std": 0.060988374054431915, "step": 77 }, { "clip_ratio": 0.00053420290350914, "epoch": 0.02729181245626312, "grad_norm": 0.11691422308152354, "kl": 0.0002498626708984375, "learning_rate": 1.3461538461538462e-06, "loss": 0.0006, "step": 78 }, { "clip_ratio": 0.0006275806808844209, "epoch": 0.027641707487753672, "grad_norm": 0.11698089822452107, "kl": 0.00022220611572265625, "learning_rate": 1.3636363636363636e-06, "loss": 0.001, "step": 79 }, { "clip_ratio": 0.0007402468472719193, "epoch": 0.02799160251924423, "grad_norm": 0.11754543703242941, "kl": 0.00022125244140625, "learning_rate": 1.381118881118881e-06, "loss": 0.0007, "step": 80 }, { "clip_ratio": 0.0004531976592261344, "completions/clipped_ratio": 0.0, "completions/max_length": 250.0, "completions/max_terminated_length": 250.0, "completions/mean_length": 161.6428680419922, "completions/mean_terminated_length": 161.6428680419922, "completions/min_length": 102.0, "completions/min_terminated_length": 102.0, "epoch": 0.02834149755073478, "grad_norm": 0.12179562992890718, "kl": 0.00016689300537109375, "learning_rate": 1.3986013986013987e-06, "loss": 0.0023, "num_tokens": 414454.0, "reward": 0.9975681900978088, "reward_std": 0.03566206619143486, "rewards/check_gptzero_func/mean": 0.0, "rewards/check_gptzero_func/std": 0.0, "rewards/check_winston_local_func/mean": 0.030603840947151184, "rewards/check_winston_local_func/std": 0.11377153545618057, "rewards/sentence_count_match_reward_logic/mean": 0.9669643044471741, "rewards/sentence_count_match_reward_logic/std": 0.07644176483154297, "step": 81 }, { "clip_ratio": 0.0004885850939899683, "epoch": 0.028691392582225334, "grad_norm": 0.12964947841866312, "kl": 0.00017642974853515625, "learning_rate": 1.4160839160839162e-06, "loss": 0.0021, "step": 82 }, { "clip_ratio": 0.000530487101059407, "epoch": 0.029041287613715886, "grad_norm": 0.13014776501617656, "kl": 0.00017452239990234375, "learning_rate": 1.4335664335664336e-06, "loss": 0.002, "step": 83 }, { "clip_ratio": 0.000868549628648907, "epoch": 0.02939118264520644, "grad_norm": 0.12073185650382014, "kl": 0.00018596649169921875, "learning_rate": 1.451048951048951e-06, "loss": 0.0023, "step": 84 }, { "clip_ratio": 0.0002567764313425869, "completions/clipped_ratio": 0.0, "completions/max_length": 205.0, "completions/max_terminated_length": 205.0, "completions/mean_length": 138.92857360839844, "completions/mean_terminated_length": 138.92857360839844, "completions/min_length": 74.0, "completions/min_terminated_length": 74.0, "epoch": 0.02974107767669699, "grad_norm": 0.11603304219686804, "kl": 0.0003261566162109375, "learning_rate": 1.4685314685314685e-06, "loss": -0.0038, "num_tokens": 432514.0, "reward": 1.066003441810608, "reward_std": 0.10648027807474136, "rewards/check_gptzero_func/mean": 0.0357142873108387, "rewards/check_gptzero_func/std": 0.1872563362121582, "rewards/check_winston_local_func/mean": 0.04191039130091667, "rewards/check_winston_local_func/std": 0.0907791405916214, "rewards/sentence_count_match_reward_logic/mean": 0.988378643989563, "rewards/sentence_count_match_reward_logic/std": 0.03774381801486015, "step": 85 }, { "clip_ratio": 0.0008500944823026657, "epoch": 0.030090972708187544, "grad_norm": 0.11796925052183806, "kl": 0.0003681182861328125, "learning_rate": 1.486013986013986e-06, "loss": -0.0039, "step": 86 }, { "clip_ratio": 0.0009460473083890975, "epoch": 0.030440867739678096, "grad_norm": 0.11960372518472216, "kl": 0.000335693359375, "learning_rate": 1.5034965034965034e-06, "loss": -0.004, "step": 87 }, { "clip_ratio": 0.00046199397183954716, "epoch": 0.03079076277116865, "grad_norm": 0.11699738690527163, "kl": 0.0004215240478515625, "learning_rate": 1.5209790209790213e-06, "loss": -0.0036, "step": 88 }, { "clip_ratio": 0.0005682393093593419, "completions/clipped_ratio": 0.0, "completions/max_length": 192.0, "completions/max_terminated_length": 192.0, "completions/mean_length": 113.42857360839844, "completions/mean_terminated_length": 113.42857360839844, "completions/min_length": 61.0, "completions/min_terminated_length": 61.0, "epoch": 0.0311406578026592, "grad_norm": 0.12134161533745676, "kl": 0.000576019287109375, "learning_rate": 1.5384615384615387e-06, "loss": -0.0011, "num_tokens": 447426.0, "reward": 1.2269538640975952, "reward_std": 0.06828977912664413, "rewards/check_gptzero_func/mean": 0.1428571492433548, "rewards/check_gptzero_func/std": 0.3530939221382141, "rewards/check_winston_local_func/mean": 0.10790622979402542, "rewards/check_winston_local_func/std": 0.2077721506357193, "rewards/sentence_count_match_reward_logic/mean": 0.976190447807312, "rewards/sentence_count_match_reward_logic/std": 0.058848995715379715, "step": 89 }, { "clip_ratio": 0.0011104935547336936, "epoch": 0.031490552834149754, "grad_norm": 0.1230795395773899, "kl": 0.00057220458984375, "learning_rate": 1.5559440559440562e-06, "loss": -0.0011, "step": 90 }, { "clip_ratio": 0.0012868117773905396, "epoch": 0.03184044786564031, "grad_norm": 0.11978619283297685, "kl": 0.000720977783203125, "learning_rate": 1.5734265734265736e-06, "loss": -0.0013, "step": 91 }, { "clip_ratio": 0.000921164290048182, "epoch": 0.03219034289713086, "grad_norm": 0.12179554201939195, "kl": 0.000789642333984375, "learning_rate": 1.590909090909091e-06, "loss": -0.0014, "step": 92 }, { "clip_ratio": 0.0009944516932591796, "completions/clipped_ratio": 0.0, "completions/max_length": 212.0, "completions/max_terminated_length": 212.0, "completions/mean_length": 157.8928680419922, "completions/mean_terminated_length": 157.8928680419922, "completions/min_length": 59.0, "completions/min_terminated_length": 59.0, "epoch": 0.032540237928621416, "grad_norm": 0.10701353802168172, "kl": 0.000751495361328125, "learning_rate": 1.6083916083916085e-06, "loss": 0.002, "num_tokens": 467244.0, "reward": 1.0189404487609863, "reward_std": 0.01859230361878872, "rewards/check_gptzero_func/mean": 0.0, "rewards/check_gptzero_func/std": 0.0, "rewards/check_winston_local_func/mean": 0.02117251791059971, "rewards/check_winston_local_func/std": 0.04275018349289894, "rewards/sentence_count_match_reward_logic/mean": 0.9977678656578064, "rewards/sentence_count_match_reward_logic/std": 0.016703827306628227, "step": 93 }, { "clip_ratio": 0.0009145128424279392, "epoch": 0.032890132960111965, "grad_norm": 0.10963514794387642, "kl": 0.000896453857421875, "learning_rate": 1.625874125874126e-06, "loss": 0.0018, "step": 94 }, { "clip_ratio": 0.0008585512987338006, "epoch": 0.03324002799160252, "grad_norm": 0.10687644455839151, "kl": 0.00106048583984375, "learning_rate": 1.6433566433566434e-06, "loss": 0.0019, "step": 95 }, { "clip_ratio": 0.0014031510800123215, "epoch": 0.03358992302309307, "grad_norm": 0.10587642312106371, "kl": 0.00119781494140625, "learning_rate": 1.660839160839161e-06, "loss": 0.0019, "step": 96 }, { "clip_ratio": 0.00046559475595131516, "completions/clipped_ratio": 0.0, "completions/max_length": 250.0, "completions/max_terminated_length": 250.0, "completions/mean_length": 162.17857360839844, "completions/mean_terminated_length": 162.17857360839844, "completions/min_length": 117.0, "completions/min_terminated_length": 117.0, "epoch": 0.033939818054583626, "grad_norm": 0.13409068317479314, "kl": 0.000640869140625, "learning_rate": 1.6783216783216785e-06, "loss": 0.0004, "num_tokens": 487742.0, "reward": 1.1995136737823486, "reward_std": 0.16836346685886383, "rewards/check_gptzero_func/mean": 0.1607142835855484, "rewards/check_gptzero_func/std": 0.3705909848213196, "rewards/check_winston_local_func/mean": 0.03879930451512337, "rewards/check_winston_local_func/std": 0.0803212895989418, "rewards/sentence_count_match_reward_logic/mean": 1.0, "rewards/sentence_count_match_reward_logic/std": 0.0, "step": 97 }, { "clip_ratio": 0.0008512325002811849, "epoch": 0.034289713086074175, "grad_norm": 0.13239822353976427, "kl": 0.00063323974609375, "learning_rate": 1.695804195804196e-06, "loss": 0.0006, "step": 98 }, { "clip_ratio": 0.0006009258213452995, "epoch": 0.03463960811756473, "grad_norm": 0.13269428802460786, "kl": 0.000759124755859375, "learning_rate": 1.7132867132867134e-06, "loss": 0.0005, "step": 99 }, { "clip_ratio": 0.000875029363669455, "epoch": 0.03498950314905528, "grad_norm": 0.134949865164823, "kl": 0.00079345703125, "learning_rate": 1.7307692307692308e-06, "loss": 0.0003, "step": 100 }, { "clip_ratio": 0.001501179882325232, "completions/clipped_ratio": 0.0, "completions/max_length": 248.0, "completions/max_terminated_length": 248.0, "completions/mean_length": 146.5357208251953, "completions/mean_terminated_length": 146.5357208251953, "completions/min_length": 63.0, "completions/min_terminated_length": 63.0, "epoch": 0.035339398180545836, "grad_norm": 0.13409461447536003, "kl": 0.00177001953125, "learning_rate": 1.7482517482517483e-06, "loss": 0.0025, "num_tokens": 506748.0, "reward": 1.3041856288909912, "reward_std": 0.21378019452095032, "rewards/check_gptzero_func/mean": 0.25, "rewards/check_gptzero_func/std": 0.43693143129348755, "rewards/check_winston_local_func/mean": 0.06906656920909882, "rewards/check_winston_local_func/std": 0.08903421461582184, "rewards/sentence_count_match_reward_logic/mean": 0.9851189851760864, "rewards/sentence_count_match_reward_logic/std": 0.043829165399074554, "step": 101 }, { "clip_ratio": 0.001914474181830883, "epoch": 0.03568929321203639, "grad_norm": 0.13229925015009564, "kl": 0.0020751953125, "learning_rate": 1.7657342657342657e-06, "loss": 0.0022, "step": 102 }, { "clip_ratio": 0.0016294092638418078, "epoch": 0.03603918824352694, "grad_norm": 0.1368081382077121, "kl": 0.002227783203125, "learning_rate": 1.7832167832167834e-06, "loss": 0.0022, "step": 103 }, { "clip_ratio": 0.0019209239399060607, "epoch": 0.0363890832750175, "grad_norm": 0.12943039905158935, "kl": 0.002655029296875, "learning_rate": 1.8006993006993008e-06, "loss": 0.0022, "step": 104 }, { "clip_ratio": 0.0004996333154849708, "completions/clipped_ratio": 0.1428571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 196.0, "completions/mean_length": 153.96429443359375, "completions/mean_terminated_length": 136.95834350585938, "completions/min_length": 83.0, "completions/min_terminated_length": 83.0, "epoch": 0.03673897830650805, "grad_norm": 0.1193553028936385, "kl": 0.003631591796875, "learning_rate": 1.8181818181818183e-06, "loss": -0.0014, "num_tokens": 526426.0, "reward": 1.0289571285247803, "reward_std": 0.0857250913977623, "rewards/check_gptzero_func/mean": 0.0357142873108387, "rewards/check_gptzero_func/std": 0.1872563362121582, "rewards/check_winston_local_func/mean": 0.021516624838113785, "rewards/check_winston_local_func/std": 0.04109776020050049, "rewards/sentence_count_match_reward_logic/mean": 0.97172611951828, "rewards/sentence_count_match_reward_logic/std": 0.06311922520399094, "step": 105 }, { "clip_ratio": 0.0007029459229670465, "epoch": 0.0370888733379986, "grad_norm": 0.12253455228780971, "kl": 0.0035247802734375, "learning_rate": 1.8356643356643357e-06, "loss": -0.0015, "step": 106 }, { "clip_ratio": 0.0007953400490805507, "epoch": 0.03743876836948915, "grad_norm": 0.10760058645283356, "kl": 0.003387451171875, "learning_rate": 1.8531468531468532e-06, "loss": -0.0015, "step": 107 }, { "clip_ratio": 0.0011392123997211456, "epoch": 0.03778866340097971, "grad_norm": 0.10821426033852834, "kl": 0.0035552978515625, "learning_rate": 1.8706293706293706e-06, "loss": -0.0014, "step": 108 }, { "clip_ratio": 0.0009404453448951244, "completions/clipped_ratio": 0.0357142857142857, "completions/max_length": 256.0, "completions/max_terminated_length": 248.0, "completions/mean_length": 156.6607208251953, "completions/mean_terminated_length": 152.98147583007812, "completions/min_length": 86.0, "completions/min_terminated_length": 86.0, "epoch": 0.03813855843247026, "grad_norm": 0.13782198038196666, "kl": 0.0032501220703125, "learning_rate": 1.888111888111888e-06, "loss": 0.0016, "num_tokens": 546543.0, "reward": 1.088487982749939, "reward_std": 0.06418253481388092, "rewards/check_gptzero_func/mean": 0.0, "rewards/check_gptzero_func/std": 0.0, "rewards/check_winston_local_func/mean": 0.09642434120178223, "rewards/check_winston_local_func/std": 0.18789586424827576, "rewards/sentence_count_match_reward_logic/mean": 0.9920635223388672, "rewards/sentence_count_match_reward_logic/std": 0.028874453157186508, "step": 109 }, { "clip_ratio": 0.0010657277889549732, "epoch": 0.03848845346396081, "grad_norm": 0.15274310437900615, "kl": 0.0033721923828125, "learning_rate": 1.9055944055944055e-06, "loss": 0.0015, "step": 110 }, { "clip_ratio": 0.0011859153164550662, "epoch": 0.03883834849545136, "grad_norm": 0.14430360409724557, "kl": 0.003173828125, "learning_rate": 1.9230769230769234e-06, "loss": 0.0014, "step": 111 }, { "clip_ratio": 0.0010776224080473185, "epoch": 0.03918824352694192, "grad_norm": 0.14097864391122877, "kl": 0.0031890869140625, "learning_rate": 1.9405594405594406e-06, "loss": 0.0014, "step": 112 }, { "clip_ratio": 0.000348119530826807, "completions/clipped_ratio": 0.0, "completions/max_length": 221.0, "completions/max_terminated_length": 221.0, "completions/mean_length": 139.46429443359375, "completions/mean_terminated_length": 139.46429443359375, "completions/min_length": 81.0, "completions/min_terminated_length": 81.0, "epoch": 0.03953813855843247, "grad_norm": 0.12522818264625665, "kl": 0.0019683837890625, "learning_rate": 1.9580419580419583e-06, "loss": 0.0006, "num_tokens": 564513.0, "reward": 1.0199968814849854, "reward_std": 0.013820764608681202, "rewards/check_gptzero_func/mean": 0.0, "rewards/check_gptzero_func/std": 0.0, "rewards/check_winston_local_func/mean": 0.01999678649008274, "rewards/check_winston_local_func/std": 0.03707566112279892, "rewards/sentence_count_match_reward_logic/mean": 1.0, "rewards/sentence_count_match_reward_logic/std": 0.0, "step": 113 }, { "clip_ratio": 0.00014214641123544425, "epoch": 0.03988803358992302, "grad_norm": 0.1245629059936155, "kl": 0.0021820068359375, "learning_rate": 1.9755244755244755e-06, "loss": 0.0003, "step": 114 }, { "clip_ratio": 0.00011865211126860231, "epoch": 0.04023792862141357, "grad_norm": 0.12629252461902274, "kl": 0.001953125, "learning_rate": 1.993006993006993e-06, "loss": 0.0003, "step": 115 }, { "clip_ratio": 0.0008119372650980949, "epoch": 0.04058782365290413, "grad_norm": 0.1247356588995937, "kl": 0.0019683837890625, "learning_rate": 2.0104895104895104e-06, "loss": 0.0005, "step": 116 }, { "clip_ratio": 0.0007098448695614934, "completions/clipped_ratio": 0.0892857142857143, "completions/max_length": 256.0, "completions/max_terminated_length": 256.0, "completions/mean_length": 178.33929443359375, "completions/mean_terminated_length": 170.72549438476562, "completions/min_length": 119.0, "completions/min_terminated_length": 119.0, "epoch": 0.040937718684394685, "grad_norm": 0.12395489790640825, "kl": 0.00135040283203125, "learning_rate": 2.027972027972028e-06, "loss": -0.0026, "num_tokens": 586828.0, "reward": 1.1953665018081665, "reward_std": 0.1535467952489853, "rewards/check_gptzero_func/mean": 0.1785714328289032, "rewards/check_gptzero_func/std": 0.3864591121673584, "rewards/check_winston_local_func/mean": 0.030329661443829536, "rewards/check_winston_local_func/std": 0.05158039927482605, "rewards/sentence_count_match_reward_logic/mean": 0.9864654541015625, "rewards/sentence_count_match_reward_logic/std": 0.03965746983885765, "step": 117 }, { "clip_ratio": 0.0007177282823249698, "epoch": 0.041287613715885234, "grad_norm": 0.12456193230514237, "kl": 0.00138092041015625, "learning_rate": 2.0454545454545457e-06, "loss": -0.0029, "step": 118 }, { "clip_ratio": 0.0007906201644800603, "epoch": 0.04163750874737579, "grad_norm": 0.12493823926791649, "kl": 0.00145721435546875, "learning_rate": 2.0629370629370634e-06, "loss": -0.0027, "step": 119 }, { "clip_ratio": 0.0009159811888821423, "epoch": 0.04198740377886634, "grad_norm": 0.125853694174646, "kl": 0.00150299072265625, "learning_rate": 2.0804195804195806e-06, "loss": -0.0029, "step": 120 }, { "clip_ratio": 0.0006258777575567365, "completions/clipped_ratio": 0.0, "completions/max_length": 210.0, "completions/max_terminated_length": 210.0, "completions/mean_length": 152.67857360839844, "completions/mean_terminated_length": 152.67857360839844, "completions/min_length": 89.0, "completions/min_terminated_length": 89.0, "epoch": 0.042337298810356895, "grad_norm": 0.12106167138891491, "kl": 0.002532958984375, "learning_rate": 2.0979020979020983e-06, "loss": -0.0013, "num_tokens": 606906.0, "reward": 0.9927490949630737, "reward_std": 0.014727780595421791, "rewards/check_gptzero_func/mean": 0.0, "rewards/check_gptzero_func/std": 0.0, "rewards/check_winston_local_func/mean": 0.010606267489492893, "rewards/check_winston_local_func/std": 0.009034112095832825, "rewards/sentence_count_match_reward_logic/mean": 0.9821428656578064, "rewards/sentence_count_match_reward_logic/std": 0.05201565474271774, "step": 121 }, { "clip_ratio": 0.0004782965697813779, "epoch": 0.042687193841847444, "grad_norm": 0.12222724431216128, "kl": 0.0025787353515625, "learning_rate": 2.1153846153846155e-06, "loss": -0.0015, "step": 122 }, { "clip_ratio": 0.0006577796302735806, "epoch": 0.043037088873338, "grad_norm": 0.12135142530927646, "kl": 0.002716064453125, "learning_rate": 2.132867132867133e-06, "loss": -0.0016, "step": 123 }, { "clip_ratio": 0.0012292936444282532, "epoch": 0.04338698390482855, "grad_norm": 0.11992113904725807, "kl": 0.0030364990234375, "learning_rate": 2.1503496503496504e-06, "loss": -0.0016, "step": 124 }, { "clip_ratio": 0.0004256329848431051, "completions/clipped_ratio": 0.017857142857142905, "completions/max_length": 256.0, "completions/max_terminated_length": 256.0, "completions/mean_length": 152.96429443359375, "completions/mean_terminated_length": 151.09091186523438, "completions/min_length": 107.0, "completions/min_terminated_length": 107.0, "epoch": 0.043736878936319105, "grad_norm": 0.1253898253728491, "kl": 0.0021209716796875, "learning_rate": 2.167832167832168e-06, "loss": -0.0014, "num_tokens": 626568.0, "reward": 1.1008204221725464, "reward_std": 0.15488740801811218, "rewards/check_gptzero_func/mean": 0.0892857164144516, "rewards/check_gptzero_func/std": 0.28773635625839233, "rewards/check_winston_local_func/mean": 0.0395108237862587, "rewards/check_winston_local_func/std": 0.087964728474617, "rewards/sentence_count_match_reward_logic/mean": 0.9720238447189331, "rewards/sentence_count_match_reward_logic/std": 0.06927525997161865, "step": 125 }, { "clip_ratio": 0.0009575130534358323, "epoch": 0.044086773967809655, "grad_norm": 0.12725413419690731, "kl": 0.0021514892578125, "learning_rate": 2.1853146853146857e-06, "loss": -0.0014, "step": 126 }, { "clip_ratio": 0.0003450095246080309, "epoch": 0.04443666899930021, "grad_norm": 0.12767981690157268, "kl": 0.0022125244140625, "learning_rate": 2.202797202797203e-06, "loss": -0.0017, "step": 127 }, { "clip_ratio": 0.0008429319714196026, "epoch": 0.04478656403079076, "grad_norm": 0.12767248435738385, "kl": 0.002227783203125, "learning_rate": 2.2202797202797206e-06, "loss": -0.0016, "step": 128 }, { "clip_ratio": 0.0010564322583377361, "completions/clipped_ratio": 0.0, "completions/max_length": 193.0, "completions/max_terminated_length": 193.0, "completions/mean_length": 134.17857360839844, "completions/mean_terminated_length": 134.17857360839844, "completions/min_length": 89.0, "completions/min_terminated_length": 89.0, "epoch": 0.045136459062281316, "grad_norm": 0.14239428214083977, "kl": 0.00396728515625, "learning_rate": 2.237762237762238e-06, "loss": -0.0037, "num_tokens": 643962.0, "reward": 0.9932838082313538, "reward_std": 0.021540360525250435, "rewards/check_gptzero_func/mean": 0.0, "rewards/check_gptzero_func/std": 0.0, "rewards/check_winston_local_func/mean": 0.008589820936322212, "rewards/check_winston_local_func/std": 0.0019994585309177637, "rewards/sentence_count_match_reward_logic/mean": 0.9846938848495483, "rewards/sentence_count_match_reward_logic/std": 0.044584840536117554, "step": 129 }, { "clip_ratio": 0.0011695045977830887, "epoch": 0.04548635409377187, "grad_norm": 0.14112740228377085, "kl": 0.004058837890625, "learning_rate": 2.2552447552447555e-06, "loss": -0.0037, "step": 130 }, { "clip_ratio": 0.0008528234320692718, "epoch": 0.04583624912526242, "grad_norm": 0.14517180597515014, "kl": 0.00421142578125, "learning_rate": 2.2727272727272728e-06, "loss": -0.0038, "step": 131 }, { "clip_ratio": 0.0009448612690903246, "epoch": 0.04618614415675298, "grad_norm": 0.13240336586752155, "kl": 0.004608154296875, "learning_rate": 2.2902097902097904e-06, "loss": -0.0039, "step": 132 }, { "clip_ratio": 0.0008797069313004613, "completions/clipped_ratio": 0.0, "completions/max_length": 201.0, "completions/max_terminated_length": 201.0, "completions/mean_length": 139.6607208251953, "completions/mean_terminated_length": 139.6607208251953, "completions/min_length": 72.0, "completions/min_terminated_length": 72.0, "epoch": 0.046536039188243526, "grad_norm": 0.10791241004244197, "kl": 0.003173828125, "learning_rate": 2.307692307692308e-06, "loss": -0.0005, "num_tokens": 661903.0, "reward": 1.0770456790924072, "reward_std": 0.0958779975771904, "rewards/check_gptzero_func/mean": 0.01785714365541935, "rewards/check_gptzero_func/std": 0.13363061845302582, "rewards/check_winston_local_func/mean": 0.07615270465612411, "rewards/check_winston_local_func/std": 0.17876580357551575, "rewards/sentence_count_match_reward_logic/mean": 0.983035683631897, "rewards/sentence_count_match_reward_logic/std": 0.06199471279978752, "step": 133 }, { "clip_ratio": 0.0014511224580928683, "epoch": 0.04688593421973408, "grad_norm": 0.10446511211326356, "kl": 0.0034332275390625, "learning_rate": 2.3251748251748253e-06, "loss": -0.0007, "step": 134 }, { "clip_ratio": 0.001020337687805295, "epoch": 0.04723582925122463, "grad_norm": 0.10561370888253528, "kl": 0.0034942626953125, "learning_rate": 2.342657342657343e-06, "loss": -0.0009, "step": 135 }, { "clip_ratio": 0.0013692511711269617, "epoch": 0.04758572428271519, "grad_norm": 0.10488484110308828, "kl": 0.003753662109375, "learning_rate": 2.36013986013986e-06, "loss": -0.0006, "step": 136 }, { "clip_ratio": 0.0007398722809739411, "completions/clipped_ratio": 0.0, "completions/max_length": 250.0, "completions/max_terminated_length": 250.0, "completions/mean_length": 140.55357360839844, "completions/mean_terminated_length": 140.55357360839844, "completions/min_length": 60.0, "completions/min_terminated_length": 60.0, "epoch": 0.04793561931420574, "grad_norm": 0.11266609544613967, "kl": 0.004302978515625, "learning_rate": 2.377622377622378e-06, "loss": -0.0011, "num_tokens": 679902.0, "reward": 1.0216115713119507, "reward_std": 0.014239765703678131, "rewards/check_gptzero_func/mean": 0.0, "rewards/check_gptzero_func/std": 0.0, "rewards/check_winston_local_func/mean": 0.03459857031702995, "rewards/check_winston_local_func/std": 0.05158328637480736, "rewards/sentence_count_match_reward_logic/mean": 0.9870129823684692, "rewards/sentence_count_match_reward_logic/std": 0.03209943696856499, "step": 137 }, { "clip_ratio": 0.0005360156646929681, "epoch": 0.04828551434569629, "grad_norm": 0.11459799076692208, "kl": 0.004241943359375, "learning_rate": 2.395104895104895e-06, "loss": -0.0012, "step": 138 }, { "clip_ratio": 0.0005465717404149473, "epoch": 0.04863540937718684, "grad_norm": 0.1143231756314304, "kl": 0.00469970703125, "learning_rate": 2.4125874125874128e-06, "loss": -0.0012, "step": 139 }, { "clip_ratio": 0.0007871190900914371, "epoch": 0.0489853044086774, "grad_norm": 0.11307226225165584, "kl": 0.00469970703125, "learning_rate": 2.43006993006993e-06, "loss": -0.0015, "step": 140 }, { "clip_ratio": 0.0007199059473350644, "completions/clipped_ratio": 0.0, "completions/max_length": 211.0, "completions/max_terminated_length": 211.0, "completions/mean_length": 144.25, "completions/mean_terminated_length": 144.25, "completions/min_length": 86.0, "completions/min_terminated_length": 86.0, "epoch": 0.04933519944016795, "grad_norm": 0.13778252243683942, "kl": 0.004852294921875, "learning_rate": 2.4475524475524477e-06, "loss": -0.0008, "num_tokens": 698236.0, "reward": 1.272670030593872, "reward_std": 0.1746688038110733, "rewards/check_gptzero_func/mean": 0.25, "rewards/check_gptzero_func/std": 0.43693143129348755, "rewards/check_winston_local_func/mean": 0.026241431012749672, "rewards/check_winston_local_func/std": 0.051847297698259354, "rewards/sentence_count_match_reward_logic/mean": 0.9964285492897034, "rewards/sentence_count_match_reward_logic/std": 0.026726121082901955, "step": 141 }, { "clip_ratio": 0.0011318778852000833, "epoch": 0.0496850944716585, "grad_norm": 0.13449670677099435, "kl": 0.00506591796875, "learning_rate": 2.4650349650349653e-06, "loss": -0.0005, "step": 142 }, { "clip_ratio": 0.0009914974216371775, "epoch": 0.05003498950314905, "grad_norm": 0.13139277057898557, "kl": 0.00537109375, "learning_rate": 2.4825174825174825e-06, "loss": -0.0007, "step": 143 }, { "clip_ratio": 0.0008875980856828392, "epoch": 0.05038488453463961, "grad_norm": 0.1340674783914633, "kl": 0.005340576171875, "learning_rate": 2.5e-06, "loss": -0.0009, "step": 144 }, { "clip_ratio": 0.0005821470404043794, "completions/clipped_ratio": 0.0, "completions/max_length": 229.0, "completions/max_terminated_length": 229.0, "completions/mean_length": 156.05357360839844, "completions/mean_terminated_length": 156.05357360839844, "completions/min_length": 102.0, "completions/min_terminated_length": 102.0, "epoch": 0.050734779566130164, "grad_norm": 0.11835979382206845, "kl": 0.00848388671875, "learning_rate": 2.517482517482518e-06, "loss": -0.002, "num_tokens": 718279.0, "reward": 1.01095712184906, "reward_std": 0.007470408454537392, "rewards/check_gptzero_func/mean": 0.0, "rewards/check_gptzero_func/std": 0.0, "rewards/check_winston_local_func/mean": 0.010957053862512112, "rewards/check_winston_local_func/std": 0.019402220845222473, "rewards/sentence_count_match_reward_logic/mean": 1.0, "rewards/sentence_count_match_reward_logic/std": 0.0, "step": 145 }, { "clip_ratio": 0.0011486097937449813, "epoch": 0.05108467459762071, "grad_norm": 0.11753901653413973, "kl": 0.0089111328125, "learning_rate": 2.534965034965035e-06, "loss": -0.0023, "step": 146 }, { "clip_ratio": 0.0006538996240124106, "epoch": 0.05143456962911127, "grad_norm": 0.11778478068677324, "kl": 0.0091552734375, "learning_rate": 2.5524475524475528e-06, "loss": -0.0023, "step": 147 }, { "clip_ratio": 0.0021357533987611532, "epoch": 0.05178446466060182, "grad_norm": 0.1172206071917601, "kl": 0.00946044921875, "learning_rate": 2.56993006993007e-06, "loss": -0.0024, "step": 148 }, { "clip_ratio": 0.0006477880524471402, "completions/clipped_ratio": 0.0, "completions/max_length": 227.0, "completions/max_terminated_length": 227.0, "completions/mean_length": 173.57144165039062, "completions/mean_terminated_length": 173.57144165039062, "completions/min_length": 115.0, "completions/min_terminated_length": 115.0, "epoch": 0.052134359692092375, "grad_norm": 0.12624198333451608, "kl": 0.005218505859375, "learning_rate": 2.5874125874125877e-06, "loss": 0.0018, "num_tokens": 739815.0, "reward": 1.160845398902893, "reward_std": 0.21960385143756866, "rewards/check_gptzero_func/mean": 0.1428571492433548, "rewards/check_gptzero_func/std": 0.3530939221382141, "rewards/check_winston_local_func/mean": 0.030885126441717148, "rewards/check_winston_local_func/std": 0.06393590569496155, "rewards/sentence_count_match_reward_logic/mean": 0.9871031641960144, "rewards/sentence_count_match_reward_logic/std": 0.03762904554605484, "step": 149 }, { "clip_ratio": 0.0007642790442332625, "epoch": 0.052484254723582924, "grad_norm": 0.12625816848763516, "kl": 0.00537109375, "learning_rate": 2.604895104895105e-06, "loss": 0.0018, "step": 150 }, { "clip_ratio": 0.000610198883805424, "epoch": 0.05283414975507348, "grad_norm": 0.12725761453530898, "kl": 0.00567626953125, "learning_rate": 2.6223776223776225e-06, "loss": 0.0019, "step": 151 }, { "clip_ratio": 0.0012525006895884871, "epoch": 0.05318404478656403, "grad_norm": 0.12937167218704948, "kl": 0.006134033203125, "learning_rate": 2.63986013986014e-06, "loss": 0.0014, "step": 152 }, { "clip_ratio": 0.00016572754248045385, "completions/clipped_ratio": 0.1428571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 191.0, "completions/mean_length": 154.7857208251953, "completions/mean_terminated_length": 137.9166717529297, "completions/min_length": 70.0, "completions/min_terminated_length": 70.0, "epoch": 0.053533939818054585, "grad_norm": 0.11461815224422436, "kl": 0.0089111328125, "learning_rate": 2.6573426573426574e-06, "loss": 0.0002, "num_tokens": 759555.0, "reward": 1.1002349853515625, "reward_std": 0.0774240642786026, "rewards/check_gptzero_func/mean": 0.0892857164144516, "rewards/check_gptzero_func/std": 0.28773635625839233, "rewards/check_winston_local_func/mean": 0.010949249379336834, "rewards/check_winston_local_func/std": 0.010598714463412762, "rewards/sentence_count_match_reward_logic/mean": 1.0, "rewards/sentence_count_match_reward_logic/std": 0.0, "step": 153 }, { "clip_ratio": 0.00028437963919714093, "epoch": 0.053883834849545134, "grad_norm": 0.11669638589966294, "kl": 0.00921630859375, "learning_rate": 2.674825174825175e-06, "loss": 0.0001, "step": 154 }, { "clip_ratio": 0.000531310448423028, "epoch": 0.05423372988103569, "grad_norm": 0.11438494018829032, "kl": 0.00958251953125, "learning_rate": 2.6923076923076923e-06, "loss": -0.0002, "step": 155 }, { "clip_ratio": 0.0010012250859290361, "epoch": 0.05458362491252624, "grad_norm": 0.11080645408057087, "kl": 0.00933837890625, "learning_rate": 2.70979020979021e-06, "loss": 0.0001, "step": 156 }, { "clip_ratio": 0.000515090418048203, "completions/clipped_ratio": 0.0, "completions/max_length": 185.0, "completions/max_terminated_length": 185.0, "completions/mean_length": 125.6964340209961, "completions/mean_terminated_length": 125.6964340209961, "completions/min_length": 70.0, "completions/min_terminated_length": 70.0, "epoch": 0.054933519944016795, "grad_norm": 0.1396584377485103, "kl": 0.0081787109375, "learning_rate": 2.7272727272727272e-06, "loss": -0.0014, "num_tokens": 775674.0, "reward": 1.0591048002243042, "reward_std": 0.05839777737855911, "rewards/check_gptzero_func/mean": 0.0, "rewards/check_gptzero_func/std": 0.0, "rewards/check_winston_local_func/mean": 0.05910468101501465, "rewards/check_winston_local_func/std": 0.1297863870859146, "rewards/sentence_count_match_reward_logic/mean": 1.0, "rewards/sentence_count_match_reward_logic/std": 0.0, "step": 157 }, { "clip_ratio": 0.0008127451874315739, "epoch": 0.055283414975507345, "grad_norm": 0.13897864978283256, "kl": 0.0087890625, "learning_rate": 2.744755244755245e-06, "loss": -0.0013, "step": 158 }, { "clip_ratio": 0.0012381842825561762, "epoch": 0.0556333100069979, "grad_norm": 0.14909963471029336, "kl": 0.00921630859375, "learning_rate": 2.762237762237762e-06, "loss": -0.0014, "step": 159 }, { "clip_ratio": 0.0013186910655349493, "epoch": 0.05598320503848846, "grad_norm": 0.12395671119848438, "kl": 0.0106201171875, "learning_rate": 2.7797202797202798e-06, "loss": -0.0015, "step": 160 }, { "clip_ratio": 0.001140271662734449, "completions/clipped_ratio": 0.0, "completions/max_length": 235.0, "completions/max_terminated_length": 235.0, "completions/mean_length": 137.57144165039062, "completions/mean_terminated_length": 137.57144165039062, "completions/min_length": 65.0, "completions/min_terminated_length": 65.0, "epoch": 0.056333100069979006, "grad_norm": 0.19324930115828348, "kl": 0.0157470703125, "learning_rate": 2.7972027972027974e-06, "loss": 0.001, "num_tokens": 793170.0, "reward": 1.4608160257339478, "reward_std": 0.2410106360912323, "rewards/check_gptzero_func/mean": 0.4107142984867096, "rewards/check_gptzero_func/std": 0.49641573429107666, "rewards/check_winston_local_func/mean": 0.052652694284915924, "rewards/check_winston_local_func/std": 0.10858624428510666, "rewards/sentence_count_match_reward_logic/mean": 0.9974489808082581, "rewards/sentence_count_match_reward_logic/std": 0.019090088084340096, "step": 161 }, { "clip_ratio": 0.0014388071140274405, "epoch": 0.05668299510146956, "grad_norm": 0.17497137414605582, "kl": 0.0172119140625, "learning_rate": 2.8146853146853147e-06, "loss": 0.0008, "step": 162 }, { "clip_ratio": 0.00210485584102571, "epoch": 0.05703289013296011, "grad_norm": 0.15342873650833766, "kl": 0.0186767578125, "learning_rate": 2.8321678321678323e-06, "loss": 0.0007, "step": 163 }, { "clip_ratio": 0.007764071226119995, "epoch": 0.05738278516445067, "grad_norm": 0.1389556022068963, "kl": 0.021728515625, "learning_rate": 2.8496503496503496e-06, "loss": 0.0006, "step": 164 }, { "clip_ratio": 0.0007517566555179656, "completions/clipped_ratio": 0.0, "completions/max_length": 226.0, "completions/max_terminated_length": 226.0, "completions/mean_length": 134.25, "completions/mean_terminated_length": 134.25, "completions/min_length": 80.0, "completions/min_terminated_length": 80.0, "epoch": 0.057732680195941216, "grad_norm": 0.132768313742277, "kl": 0.0135498046875, "learning_rate": 2.8671328671328672e-06, "loss": 0.0109, "num_tokens": 810304.0, "reward": 1.0892752408981323, "reward_std": 0.10255607962608337, "rewards/check_gptzero_func/mean": 0.0892857164144516, "rewards/check_gptzero_func/std": 0.28773635625839233, "rewards/check_winston_local_func/mean": 0.023798927664756775, "rewards/check_winston_local_func/std": 0.054797690361738205, "rewards/sentence_count_match_reward_logic/mean": 0.9761905074119568, "rewards/sentence_count_match_reward_logic/std": 0.12483756244182587, "step": 165 }, { "clip_ratio": 0.0018981721950694919, "epoch": 0.05808257522743177, "grad_norm": 0.13117652243538638, "kl": 0.0147705078125, "learning_rate": 2.8846153846153845e-06, "loss": 0.0107, "step": 166 }, { "clip_ratio": 0.002603230532258749, "epoch": 0.05843247025892232, "grad_norm": 0.13572978638577132, "kl": 0.01544189453125, "learning_rate": 2.902097902097902e-06, "loss": 0.0109, "step": 167 }, { "clip_ratio": 0.005056026857346296, "epoch": 0.05878236529041288, "grad_norm": 0.13289048467507814, "kl": 0.0177001953125, "learning_rate": 2.91958041958042e-06, "loss": 0.0107, "step": 168 }, { "clip_ratio": 0.0013041617348790169, "completions/clipped_ratio": 0.0, "completions/max_length": 225.0, "completions/max_terminated_length": 225.0, "completions/mean_length": 155.6428680419922, "completions/mean_terminated_length": 155.6428680419922, "completions/min_length": 103.0, "completions/min_terminated_length": 103.0, "epoch": 0.059132260321903427, "grad_norm": 0.14048231585756651, "kl": 0.0242919921875, "learning_rate": 2.937062937062937e-06, "loss": -0.0033, "num_tokens": 830100.0, "reward": 1.0021851062774658, "reward_std": 0.011832266114652157, "rewards/check_gptzero_func/mean": 0.0, "rewards/check_gptzero_func/std": 0.0, "rewards/check_winston_local_func/mean": 0.008137410506606102, "rewards/check_winston_local_func/std": 0.0014836116461083293, "rewards/sentence_count_match_reward_logic/mean": 0.9940475821495056, "rewards/sentence_count_match_reward_logic/std": 0.031209392473101616, "step": 169 }, { "clip_ratio": 0.0018199168844148517, "epoch": 0.05948215535339398, "grad_norm": 0.12026626313113882, "kl": 0.0267333984375, "learning_rate": 2.954545454545455e-06, "loss": -0.0032, "step": 170 }, { "clip_ratio": 0.0018916603876277804, "epoch": 0.05983205038488453, "grad_norm": 0.11659011316707112, "kl": 0.0299072265625, "learning_rate": 2.972027972027972e-06, "loss": -0.0033, "step": 171 }, { "clip_ratio": 0.003944566939026117, "epoch": 0.06018194541637509, "grad_norm": 0.11915140907991849, "kl": 0.032958984375, "learning_rate": 2.98951048951049e-06, "loss": -0.0034, "step": 172 }, { "clip_ratio": 0.0009639568743295968, "completions/clipped_ratio": 0.0714285714285714, "completions/max_length": 256.0, "completions/max_terminated_length": 254.0, "completions/mean_length": 168.05357360839844, "completions/mean_terminated_length": 161.28846740722656, "completions/min_length": 117.0, "completions/min_terminated_length": 117.0, "epoch": 0.06053184044786564, "grad_norm": 0.18355285373413174, "kl": 0.0247802734375, "learning_rate": 3.006993006993007e-06, "loss": -0.0024, "num_tokens": 851399.0, "reward": 1.0206879377365112, "reward_std": 0.08080258220434189, "rewards/check_gptzero_func/mean": 0.01785714365541935, "rewards/check_gptzero_func/std": 0.13363061845302582, "rewards/check_winston_local_func/mean": 0.03216744586825371, "rewards/check_winston_local_func/std": 0.07536119222640991, "rewards/sentence_count_match_reward_logic/mean": 0.9706632494926453, "rewards/sentence_count_match_reward_logic/std": 0.05478374660015106, "step": 173 }, { "clip_ratio": 0.0008050219039432704, "epoch": 0.06088173547935619, "grad_norm": 0.1748937962794138, "kl": 0.0272216796875, "learning_rate": 3.024475524475525e-06, "loss": -0.0024, "step": 174 }, { "clip_ratio": 0.0021379876416176558, "epoch": 0.06123163051084675, "grad_norm": 0.16204527437706362, "kl": 0.0299072265625, "learning_rate": 3.0419580419580425e-06, "loss": -0.0024, "step": 175 }, { "clip_ratio": 0.007139333989471197, "epoch": 0.0615815255423373, "grad_norm": 0.1424554593841472, "kl": 0.03369140625, "learning_rate": 3.0594405594405598e-06, "loss": -0.0029, "step": 176 }, { "clip_ratio": 0.0007079481147229671, "completions/clipped_ratio": 0.0, "completions/max_length": 246.0, "completions/max_terminated_length": 246.0, "completions/mean_length": 165.10714721679688, "completions/mean_terminated_length": 165.10714721679688, "completions/min_length": 107.0, "completions/min_terminated_length": 107.0, "epoch": 0.061931420573827854, "grad_norm": 0.1490411661840889, "kl": 0.0274658203125, "learning_rate": 3.0769230769230774e-06, "loss": 0.0025, "num_tokens": 871925.0, "reward": 1.2302898168563843, "reward_std": 0.07825194299221039, "rewards/check_gptzero_func/mean": 0.1785714328289032, "rewards/check_gptzero_func/std": 0.3864591419696808, "rewards/check_winston_local_func/mean": 0.06957541406154633, "rewards/check_winston_local_func/std": 0.14850835502147675, "rewards/sentence_count_match_reward_logic/mean": 0.9821428656578064, "rewards/sentence_count_match_reward_logic/std": 0.05754726752638817, "step": 177 }, { "clip_ratio": 0.0011978240218013525, "epoch": 0.0622813156053184, "grad_norm": 0.1571157661053595, "kl": 0.0311279296875, "learning_rate": 3.0944055944055947e-06, "loss": 0.0028, "step": 178 }, { "clip_ratio": 0.002103519858792424, "epoch": 0.06263121063680896, "grad_norm": 0.15196359017397065, "kl": 0.03271484375, "learning_rate": 3.1118881118881123e-06, "loss": 0.0026, "step": 179 }, { "clip_ratio": 0.0036522436421364546, "epoch": 0.06298110566829951, "grad_norm": 0.15311375056248816, "kl": 0.03466796875, "learning_rate": 3.1293706293706296e-06, "loss": 0.0025, "step": 180 }, { "clip_ratio": 0.0006816100212745368, "completions/clipped_ratio": 0.1428571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 248.0, "completions/mean_length": 181.21429443359375, "completions/mean_terminated_length": 168.75, "completions/min_length": 94.0, "completions/min_terminated_length": 94.0, "epoch": 0.06333100069979006, "grad_norm": 0.20744979453056567, "kl": 0.055908203125, "learning_rate": 3.1468531468531472e-06, "loss": -0.0021, "num_tokens": 894681.0, "reward": 0.9750702977180481, "reward_std": 0.04895690828561783, "rewards/check_gptzero_func/mean": 0.0, "rewards/check_gptzero_func/std": 0.0, "rewards/check_winston_local_func/mean": 0.039270929992198944, "rewards/check_winston_local_func/std": 0.08283208310604095, "rewards/sentence_count_match_reward_logic/mean": 0.9357993006706238, "rewards/sentence_count_match_reward_logic/std": 0.15890520811080933, "step": 181 }, { "clip_ratio": 0.0006810487248003483, "epoch": 0.06368089573128062, "grad_norm": 0.19410194515004586, "kl": 0.056640625, "learning_rate": 3.164335664335665e-06, "loss": -0.0019, "step": 182 }, { "clip_ratio": 0.002175950910896063, "epoch": 0.06403079076277117, "grad_norm": 0.16427804542410424, "kl": 0.056396484375, "learning_rate": 3.181818181818182e-06, "loss": -0.0022, "step": 183 }, { "clip_ratio": 0.004616398364305496, "epoch": 0.06438068579426172, "grad_norm": 0.15807749905364918, "kl": 0.05517578125, "learning_rate": 3.1993006993006998e-06, "loss": -0.0023, "step": 184 }, { "clip_ratio": 0.001249321736395359, "completions/clipped_ratio": 0.0, "completions/max_length": 255.0, "completions/max_terminated_length": 255.0, "completions/mean_length": 180.21429443359375, "completions/mean_terminated_length": 180.21429443359375, "completions/min_length": 111.0, "completions/min_terminated_length": 111.0, "epoch": 0.06473058082575227, "grad_norm": 0.11765729371984533, "kl": 0.031494140625, "learning_rate": 3.216783216783217e-06, "loss": -0.0004, "num_tokens": 916957.0, "reward": 1.0341953039169312, "reward_std": 0.055656228214502335, "rewards/check_gptzero_func/mean": 0.01785714365541935, "rewards/check_gptzero_func/std": 0.13363061845302582, "rewards/check_winston_local_func/mean": 0.016338160261511803, "rewards/check_winston_local_func/std": 0.016797292977571487, "rewards/sentence_count_match_reward_logic/mean": 1.0, "rewards/sentence_count_match_reward_logic/std": 0.0, "step": 185 }, { "clip_ratio": 0.0014023665571585298, "epoch": 0.06508047585724283, "grad_norm": 0.12434699643629528, "kl": 0.0302734375, "learning_rate": 3.2342657342657347e-06, "loss": -0.0004, "step": 186 }, { "clip_ratio": 0.0012795067159458995, "epoch": 0.06543037088873338, "grad_norm": 0.12237846019291358, "kl": 0.0302734375, "learning_rate": 3.251748251748252e-06, "loss": -0.0004, "step": 187 }, { "clip_ratio": 0.0021061107981950045, "epoch": 0.06578026592022393, "grad_norm": 0.12294248147951199, "kl": 0.0311279296875, "learning_rate": 3.2692307692307696e-06, "loss": -0.0007, "step": 188 }, { "clip_ratio": 0.0005266699590720236, "completions/clipped_ratio": 0.2857142857142857, "completions/max_length": 256.0, "completions/max_terminated_length": 236.0, "completions/mean_length": 187.10714721679688, "completions/mean_terminated_length": 159.5500030517578, "completions/min_length": 90.0, "completions/min_terminated_length": 90.0, "epoch": 0.06613016095171449, "grad_norm": 0.13682123293569748, "kl": 0.039306640625, "learning_rate": 3.286713286713287e-06, "loss": 0.0019, "num_tokens": 940651.0, "reward": 1.0345672369003296, "reward_std": 0.06559489667415619, "rewards/check_gptzero_func/mean": 0.01785714365541935, "rewards/check_gptzero_func/std": 0.13363061845302582, "rewards/check_winston_local_func/mean": 0.03059898130595684, "rewards/check_winston_local_func/std": 0.05861172825098038, "rewards/sentence_count_match_reward_logic/mean": 0.9861111044883728, "rewards/sentence_count_match_reward_logic/std": 0.0370790995657444, "step": 189 }, { "clip_ratio": 0.0006569987745024264, "epoch": 0.06648005598320504, "grad_norm": 0.13533892371891912, "kl": 0.040283203125, "learning_rate": 3.3041958041958045e-06, "loss": 0.0017, "step": 190 }, { "clip_ratio": 0.0015332151670008898, "epoch": 0.06682995101469559, "grad_norm": 0.13055791269098996, "kl": 0.040771484375, "learning_rate": 3.321678321678322e-06, "loss": 0.0018, "step": 191 }, { "clip_ratio": 0.002679276978597045, "epoch": 0.06717984604618614, "grad_norm": 0.1313279104070725, "kl": 0.041259765625, "learning_rate": 3.3391608391608394e-06, "loss": 0.0013, "step": 192 }, { "clip_ratio": 0.0007733047241345048, "completions/clipped_ratio": 0.1428571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 213.0, "completions/mean_length": 146.0178680419922, "completions/mean_terminated_length": 127.6875, "completions/min_length": 59.0, "completions/min_terminated_length": 59.0, "epoch": 0.0675297410776767, "grad_norm": 0.11998890731030845, "kl": 0.047119140625, "learning_rate": 3.356643356643357e-06, "loss": -0.0005, "num_tokens": 959188.0, "reward": 1.0401893854141235, "reward_std": 0.10633274167776108, "rewards/check_gptzero_func/mean": 0.0357142873108387, "rewards/check_gptzero_func/std": 0.1872563362121582, "rewards/check_winston_local_func/mean": 0.03423694521188736, "rewards/check_winston_local_func/std": 0.10150615125894547, "rewards/sentence_count_match_reward_logic/mean": 0.9702381491661072, "rewards/sentence_count_match_reward_logic/std": 0.09822458028793335, "step": 193 }, { "clip_ratio": 0.001374699524603784, "epoch": 0.06787963610916725, "grad_norm": 0.11987505827152634, "kl": 0.049560546875, "learning_rate": 3.3741258741258742e-06, "loss": -0.0007, "step": 194 }, { "clip_ratio": 0.0029294344130903482, "epoch": 0.0682295311406578, "grad_norm": 0.11929702605053781, "kl": 0.0625, "learning_rate": 3.391608391608392e-06, "loss": -0.0009, "step": 195 }, { "clip_ratio": 0.004617974162101746, "epoch": 0.06857942617214835, "grad_norm": 0.11766355020110128, "kl": 0.062255859375, "learning_rate": 3.409090909090909e-06, "loss": -0.0011, "step": 196 }, { "clip_ratio": 0.0010515264002606273, "completions/clipped_ratio": 0.0, "completions/max_length": 203.0, "completions/max_terminated_length": 203.0, "completions/mean_length": 121.17857360839844, "completions/mean_terminated_length": 121.17857360839844, "completions/min_length": 66.0, "completions/min_terminated_length": 66.0, "epoch": 0.06892932120363891, "grad_norm": 0.13738734403595593, "kl": 0.06201171875, "learning_rate": 3.426573426573427e-06, "loss": -0.0003, "num_tokens": 974806.0, "reward": 1.0669376850128174, "reward_std": 0.08011036366224289, "rewards/check_gptzero_func/mean": 0.0535714291036129, "rewards/check_gptzero_func/std": 0.22720779478549957, "rewards/check_winston_local_func/mean": 0.013366213999688625, "rewards/check_winston_local_func/std": 0.008986729197204113, "rewards/sentence_count_match_reward_logic/mean": 1.0, "rewards/sentence_count_match_reward_logic/std": 0.0, "step": 197 }, { "clip_ratio": 0.0023423973470926285, "epoch": 0.06927921623512946, "grad_norm": 0.13691667495351678, "kl": 0.0673828125, "learning_rate": 3.4440559440559445e-06, "loss": -0.0005, "step": 198 }, { "clip_ratio": 0.00471023004502058, "epoch": 0.06962911126662001, "grad_norm": 0.12377304725545424, "kl": 0.07373046875, "learning_rate": 3.4615384615384617e-06, "loss": -0.0007, "step": 199 }, { "clip_ratio": 0.009433700703084469, "epoch": 0.06997900629811056, "grad_norm": 0.12230318681164248, "kl": 0.07763671875, "learning_rate": 3.4790209790209793e-06, "loss": -0.0008, "step": 200 }, { "clip_ratio": 0.0010318687418475747, "completions/clipped_ratio": 0.0, "completions/max_length": 232.0, "completions/max_terminated_length": 232.0, "completions/mean_length": 148.3928680419922, "completions/mean_terminated_length": 148.3928680419922, "completions/min_length": 88.0, "completions/min_terminated_length": 88.0, "epoch": 0.07032890132960112, "grad_norm": 0.13414753068703836, "kl": 0.09326171875, "learning_rate": 3.4965034965034966e-06, "loss": -0.0025, "num_tokens": 993612.0, "reward": 1.1341443061828613, "reward_std": 0.060026366263628006, "rewards/check_gptzero_func/mean": 0.125, "rewards/check_gptzero_func/std": 0.3337118923664093, "rewards/check_winston_local_func/mean": 0.011695231311023235, "rewards/check_winston_local_func/std": 0.006628900766372681, "rewards/sentence_count_match_reward_logic/mean": 0.9974489808082581, "rewards/sentence_count_match_reward_logic/std": 0.019090088084340096, "step": 201 }, { "clip_ratio": 0.0006603718502447009, "epoch": 0.07067879636109167, "grad_norm": 0.13423522506795213, "kl": 0.0966796875, "learning_rate": 3.5139860139860142e-06, "loss": -0.0028, "step": 202 }, { "clip_ratio": 0.0017517702654004097, "epoch": 0.07102869139258222, "grad_norm": 0.1330223515034695, "kl": 0.1005859375, "learning_rate": 3.5314685314685315e-06, "loss": -0.0027, "step": 203 }, { "clip_ratio": 0.004209080711007118, "epoch": 0.07137858642407278, "grad_norm": 0.13520696892435385, "kl": 0.10693359375, "learning_rate": 3.548951048951049e-06, "loss": -0.0031, "step": 204 }, { "clip_ratio": 0.0008172184461727738, "completions/clipped_ratio": 0.0, "completions/max_length": 215.0, "completions/max_terminated_length": 215.0, "completions/mean_length": 154.6428680419922, "completions/mean_terminated_length": 154.6428680419922, "completions/min_length": 87.0, "completions/min_terminated_length": 87.0, "epoch": 0.07172848145556333, "grad_norm": 0.15106737020313316, "kl": 0.06201171875, "learning_rate": 3.566433566433567e-06, "loss": 0.0001, "num_tokens": 1013192.0, "reward": 1.106567621231079, "reward_std": 0.1508132517337799, "rewards/check_gptzero_func/mean": 0.0892857164144516, "rewards/check_gptzero_func/std": 0.28773635625839233, "rewards/check_winston_local_func/mean": 0.019513947889208794, "rewards/check_winston_local_func/std": 0.03471992909908295, "rewards/sentence_count_match_reward_logic/mean": 0.9977678656578064, "rewards/sentence_count_match_reward_logic/std": 0.016703827306628227, "step": 205 }, { "clip_ratio": 0.0016171481693163514, "epoch": 0.07207837648705388, "grad_norm": 0.15351224625252852, "kl": 0.06396484375, "learning_rate": 3.583916083916084e-06, "loss": 0.0001, "step": 206 }, { "clip_ratio": 0.0024266631808131933, "epoch": 0.07242827151854443, "grad_norm": 0.13747922851507843, "kl": 0.06591796875, "learning_rate": 3.6013986013986017e-06, "loss": -0.0001, "step": 207 }, { "clip_ratio": 0.0056519112549722195, "epoch": 0.072778166550035, "grad_norm": 0.1515043704778549, "kl": 0.0654296875, "learning_rate": 3.618881118881119e-06, "loss": -0.0007, "step": 208 }, { "clip_ratio": 0.0021090623922646046, "completions/clipped_ratio": 0.017857142857142905, "completions/max_length": 256.0, "completions/max_terminated_length": 250.0, "completions/mean_length": 177.21429443359375, "completions/mean_terminated_length": 175.7818145751953, "completions/min_length": 116.0, "completions/min_terminated_length": 116.0, "epoch": 0.07312806158152554, "grad_norm": 0.15332229891366775, "kl": 0.078125, "learning_rate": 3.6363636363636366e-06, "loss": 0.0009, "num_tokens": 1035236.0, "reward": 1.248361587524414, "reward_std": 0.08066682517528534, "rewards/check_gptzero_func/mean": 0.2321428507566452, "rewards/check_gptzero_func/std": 0.4260205328464508, "rewards/check_winston_local_func/mean": 0.01621871441602707, "rewards/check_winston_local_func/std": 0.01999880000948906, "rewards/sentence_count_match_reward_logic/mean": 1.0, "rewards/sentence_count_match_reward_logic/std": 0.0, "step": 209 }, { "clip_ratio": 0.0012915466213598847, "epoch": 0.0734779566130161, "grad_norm": 0.1365062065655474, "kl": 0.0810546875, "learning_rate": 3.653846153846154e-06, "loss": 0.0006, "step": 210 }, { "clip_ratio": 0.0033342258539050817, "epoch": 0.07382785164450664, "grad_norm": 0.1444987280071572, "kl": 0.0830078125, "learning_rate": 3.6713286713286715e-06, "loss": 0.0007, "step": 211 }, { "clip_ratio": 0.004936265293508768, "epoch": 0.0741777466759972, "grad_norm": 0.13367491625513186, "kl": 0.08544921875, "learning_rate": 3.6888111888111896e-06, "loss": 0.0004, "step": 212 }, { "clip_ratio": 0.0013585982378572226, "completions/clipped_ratio": 0.0, "completions/max_length": 252.0, "completions/max_terminated_length": 252.0, "completions/mean_length": 153.17857360839844, "completions/mean_terminated_length": 153.17857360839844, "completions/min_length": 86.0, "completions/min_terminated_length": 86.0, "epoch": 0.07452764170748775, "grad_norm": 0.1380098198065624, "kl": 0.09228515625, "learning_rate": 3.7062937062937064e-06, "loss": 0.0008, "num_tokens": 1054670.0, "reward": 1.2996104955673218, "reward_std": 0.1075432300567627, "rewards/check_gptzero_func/mean": 0.2857142984867096, "rewards/check_gptzero_func/std": 0.4558423161506653, "rewards/check_winston_local_func/mean": 0.013896160759031773, "rewards/check_winston_local_func/std": 0.022084053605794907, "rewards/sentence_count_match_reward_logic/mean": 1.0, "rewards/sentence_count_match_reward_logic/std": 0.0, "step": 213 }, { "clip_ratio": 0.001276697381399572, "epoch": 0.0748775367389783, "grad_norm": 0.1424621199545182, "kl": 0.09716796875, "learning_rate": 3.7237762237762245e-06, "loss": 0.0006, "step": 214 }, { "clip_ratio": 0.0031235532369464636, "epoch": 0.07522743177046885, "grad_norm": 0.1366894002120923, "kl": 0.1015625, "learning_rate": 3.7412587412587413e-06, "loss": 0.0006, "step": 215 }, { "clip_ratio": 0.0035247975029051304, "epoch": 0.07557732680195942, "grad_norm": 0.1392087166842338, "kl": 0.1025390625, "learning_rate": 3.7587412587412593e-06, "loss": 0.0004, "step": 216 }, { "clip_ratio": 0.001789732021279633, "completions/clipped_ratio": 0.0, "completions/max_length": 240.0, "completions/max_terminated_length": 240.0, "completions/mean_length": 148.08929443359375, "completions/mean_terminated_length": 148.08929443359375, "completions/min_length": 95.0, "completions/min_terminated_length": 95.0, "epoch": 0.07592722183344996, "grad_norm": 0.13804988421054654, "kl": 0.10595703125, "learning_rate": 3.776223776223776e-06, "loss": 0.0002, "num_tokens": 1073563.0, "reward": 1.1395840644836426, "reward_std": 0.011591251008212566, "rewards/check_gptzero_func/mean": 0.1428571492433548, "rewards/check_gptzero_func/std": 0.3530939221382141, "rewards/check_winston_local_func/mean": 0.00863167829811573, "rewards/check_winston_local_func/std": 0.001269258907996118, "rewards/sentence_count_match_reward_logic/mean": 0.988095223903656, "rewards/sentence_count_match_reward_logic/std": 0.04029115289449692, "step": 217 }, { "clip_ratio": 0.00032592096249572933, "epoch": 0.07627711686494051, "grad_norm": 0.1442467875448282, "kl": 0.1083984375, "learning_rate": 3.7937062937062942e-06, "loss": 0.0002, "step": 218 }, { "clip_ratio": 0.0016076882602646947, "epoch": 0.07662701189643108, "grad_norm": 0.13835569263087064, "kl": 0.10986328125, "learning_rate": 3.811188811188811e-06, "loss": 0.0, "step": 219 }, { "clip_ratio": 0.0022299750708043575, "epoch": 0.07697690692792163, "grad_norm": 0.13368973572947832, "kl": 0.1162109375, "learning_rate": 3.828671328671329e-06, "loss": -0.0003, "step": 220 }, { "clip_ratio": 0.001560531323775649, "completions/clipped_ratio": 0.2857142857142857, "completions/max_length": 256.0, "completions/max_terminated_length": 172.0, "completions/mean_length": 162.2678680419922, "completions/mean_terminated_length": 124.7750015258789, "completions/min_length": 82.0, "completions/min_terminated_length": 82.0, "epoch": 0.07732680195941218, "grad_norm": 0.16747192612421535, "kl": 0.1767578125, "learning_rate": 3.846153846153847e-06, "loss": 0.0022, "num_tokens": 1094546.0, "reward": 1.112830400466919, "reward_std": 0.09652649611234665, "rewards/check_gptzero_func/mean": 0.1071428582072258, "rewards/check_gptzero_func/std": 0.3120938837528229, "rewards/check_winston_local_func/mean": 0.03112032078206539, "rewards/check_winston_local_func/std": 0.06431093066930771, "rewards/sentence_count_match_reward_logic/mean": 0.9745670557022095, "rewards/sentence_count_match_reward_logic/std": 0.05943125858902931, "step": 221 }, { "clip_ratio": 0.0023317174054682255, "epoch": 0.07767669699090272, "grad_norm": 0.1577355547411498, "kl": 0.1875, "learning_rate": 3.863636363636364e-06, "loss": 0.0022, "step": 222 }, { "clip_ratio": 0.0029285538475960493, "epoch": 0.07802659202239329, "grad_norm": 0.1483240032991762, "kl": 0.1904296875, "learning_rate": 3.881118881118881e-06, "loss": 0.0021, "step": 223 }, { "clip_ratio": 0.003131802659481764, "epoch": 0.07837648705388384, "grad_norm": 0.14696736997818233, "kl": 0.1826171875, "learning_rate": 3.898601398601399e-06, "loss": 0.0017, "step": 224 }, { "clip_ratio": 0.0011412432650104165, "completions/clipped_ratio": 0.1428571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 243.0, "completions/mean_length": 178.8928680419922, "completions/mean_terminated_length": 166.0416717529297, "completions/min_length": 107.0, "completions/min_terminated_length": 107.0, "epoch": 0.07872638208537439, "grad_norm": 0.15673590485268435, "kl": 0.1123046875, "learning_rate": 3.916083916083917e-06, "loss": -0.0014, "num_tokens": 1117332.0, "reward": 1.269762635231018, "reward_std": 0.10914304107427597, "rewards/check_gptzero_func/mean": 0.25, "rewards/check_gptzero_func/std": 0.43693143129348755, "rewards/check_winston_local_func/mean": 0.01976262591779232, "rewards/check_winston_local_func/std": 0.021939096972346306, "rewards/sentence_count_match_reward_logic/mean": 1.0, "rewards/sentence_count_match_reward_logic/std": 0.0, "step": 225 }, { "clip_ratio": 0.0011999926064163446, "epoch": 0.07907627711686493, "grad_norm": 0.1523103797650259, "kl": 0.10888671875, "learning_rate": 3.933566433566433e-06, "loss": -0.0013, "step": 226 }, { "clip_ratio": 0.0028939521871507168, "epoch": 0.0794261721483555, "grad_norm": 0.1462745673399363, "kl": 0.10595703125, "learning_rate": 3.951048951048951e-06, "loss": -0.0019, "step": 227 }, { "clip_ratio": 0.005597718991339207, "epoch": 0.07977606717984605, "grad_norm": 0.14118311287915733, "kl": 0.10400390625, "learning_rate": 3.968531468531469e-06, "loss": -0.0018, "step": 228 }, { "clip_ratio": 0.0009402756695635617, "completions/clipped_ratio": 0.0, "completions/max_length": 225.0, "completions/max_terminated_length": 225.0, "completions/mean_length": 140.19644165039062, "completions/mean_terminated_length": 140.19644165039062, "completions/min_length": 102.0, "completions/min_terminated_length": 102.0, "epoch": 0.0801259622113366, "grad_norm": 0.1252301292855151, "kl": 0.1201171875, "learning_rate": 3.986013986013986e-06, "loss": 0.0032, "num_tokens": 1135287.0, "reward": 1.315366268157959, "reward_std": 0.022618545219302177, "rewards/check_gptzero_func/mean": 0.2857142984867096, "rewards/check_gptzero_func/std": 0.4558423161506653, "rewards/check_winston_local_func/mean": 0.029651926830410957, "rewards/check_winston_local_func/std": 0.06586192548274994, "rewards/sentence_count_match_reward_logic/mean": 1.0, "rewards/sentence_count_match_reward_logic/std": 0.0, "step": 229 }, { "clip_ratio": 0.0008755471208132803, "epoch": 0.08047585724282715, "grad_norm": 0.12491650494403206, "kl": 0.1201171875, "learning_rate": 4.003496503496504e-06, "loss": 0.0029, "step": 230 }, { "clip_ratio": 0.002537158550694585, "epoch": 0.08082575227431771, "grad_norm": 0.12189624824943901, "kl": 0.1162109375, "learning_rate": 4.020979020979021e-06, "loss": 0.0029, "step": 231 }, { "clip_ratio": 0.003726217895746231, "epoch": 0.08117564730580826, "grad_norm": 0.11895020051403522, "kl": 0.1142578125, "learning_rate": 4.0384615384615385e-06, "loss": 0.0026, "step": 232 }, { "clip_ratio": 0.0012891425285488367, "completions/clipped_ratio": 0.1428571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 233.0, "completions/mean_length": 147.7857208251953, "completions/mean_terminated_length": 129.75, "completions/min_length": 82.0, "completions/min_terminated_length": 82.0, "epoch": 0.0815255423372988, "grad_norm": 0.1578520628812955, "kl": 0.1240234375, "learning_rate": 4.055944055944056e-06, "loss": -0.0005, "num_tokens": 1154555.0, "reward": 1.0183244943618774, "reward_std": 0.08909717202186584, "rewards/check_gptzero_func/mean": 0.0357142873108387, "rewards/check_gptzero_func/std": 0.1872563362121582, "rewards/check_winston_local_func/mean": 0.02427680417895317, "rewards/check_winston_local_func/std": 0.05610140040516853, "rewards/sentence_count_match_reward_logic/mean": 0.9583333134651184, "rewards/sentence_count_match_reward_logic/std": 0.06552045047283173, "step": 233 }, { "clip_ratio": 0.0016302053118124604, "epoch": 0.08187543736878937, "grad_norm": 0.16054034918171145, "kl": 0.1181640625, "learning_rate": 4.073426573426574e-06, "loss": -0.0007, "step": 234 }, { "clip_ratio": 0.003354746615514159, "epoch": 0.08222533240027992, "grad_norm": 0.13697868787551884, "kl": 0.10986328125, "learning_rate": 4.0909090909090915e-06, "loss": -0.001, "step": 235 }, { "clip_ratio": 0.006563612259924412, "epoch": 0.08257522743177047, "grad_norm": 0.13859311950098765, "kl": 0.10205078125, "learning_rate": 4.108391608391608e-06, "loss": -0.0012, "step": 236 }, { "clip_ratio": 0.0006972505943849683, "completions/clipped_ratio": 0.017857142857142905, "completions/max_length": 256.0, "completions/max_terminated_length": 248.0, "completions/mean_length": 178.12501525878906, "completions/mean_terminated_length": 176.70909118652344, "completions/min_length": 128.0, "completions/min_terminated_length": 128.0, "epoch": 0.08292512246326102, "grad_norm": 0.17802557012422685, "kl": 0.07470703125, "learning_rate": 4.125874125874127e-06, "loss": -0.0018, "num_tokens": 1177186.0, "reward": 1.157689094543457, "reward_std": 0.09392265230417252, "rewards/check_gptzero_func/mean": 0.1785714328289032, "rewards/check_gptzero_func/std": 0.3864591419696808, "rewards/check_winston_local_func/mean": 0.018020659685134888, "rewards/check_winston_local_func/std": 0.028779305517673492, "rewards/sentence_count_match_reward_logic/mean": 0.9610968828201294, "rewards/sentence_count_match_reward_logic/std": 0.07367467880249023, "step": 237 }, { "clip_ratio": 0.0009236847399733961, "epoch": 0.08327501749475158, "grad_norm": 0.17624178925343878, "kl": 0.07275390625, "learning_rate": 4.143356643356644e-06, "loss": -0.0024, "step": 238 }, { "clip_ratio": 0.0016791063826531172, "epoch": 0.08362491252624213, "grad_norm": 0.17085697145815398, "kl": 0.07177734375, "learning_rate": 4.160839160839161e-06, "loss": -0.0026, "step": 239 }, { "clip_ratio": 0.0032592525240033865, "epoch": 0.08397480755773268, "grad_norm": 0.18033250831529088, "kl": 0.07275390625, "learning_rate": 4.178321678321678e-06, "loss": -0.003, "step": 240 }, { "clip_ratio": 0.00046174367889761925, "completions/clipped_ratio": 0.0, "completions/max_length": 192.0, "completions/max_terminated_length": 192.0, "completions/mean_length": 138.125, "completions/mean_terminated_length": 138.125, "completions/min_length": 69.0, "completions/min_terminated_length": 69.0, "epoch": 0.08432470258922323, "grad_norm": 0.16437361420226285, "kl": 0.0986328125, "learning_rate": 4.195804195804197e-06, "loss": -0.0002, "num_tokens": 1195169.0, "reward": 1.0828746557235718, "reward_std": 0.16202867031097412, "rewards/check_gptzero_func/mean": 0.0714285746216774, "rewards/check_gptzero_func/std": 0.25987008213996887, "rewards/check_winston_local_func/mean": 0.03015357255935669, "rewards/check_winston_local_func/std": 0.06849774718284607, "rewards/sentence_count_match_reward_logic/mean": 0.9812925457954407, "rewards/sentence_count_match_reward_logic/std": 0.057019032537937164, "step": 241 }, { "clip_ratio": 0.0017396664479747415, "epoch": 0.08467459762071379, "grad_norm": 0.1598223306815377, "kl": 0.1005859375, "learning_rate": 4.213286713286714e-06, "loss": -0.0003, "step": 242 }, { "clip_ratio": 0.0023850146681070328, "epoch": 0.08502449265220434, "grad_norm": 0.15993716027574043, "kl": 0.10302734375, "learning_rate": 4.230769230769231e-06, "loss": -0.001, "step": 243 }, { "clip_ratio": 0.006157177034765482, "epoch": 0.08537438768369489, "grad_norm": 0.14553336985898327, "kl": 0.10498046875, "learning_rate": 4.248251748251749e-06, "loss": -0.0013, "step": 244 }, { "clip_ratio": 0.001388163072988391, "completions/clipped_ratio": 0.0, "completions/max_length": 207.0, "completions/max_terminated_length": 207.0, "completions/mean_length": 163.82144165039062, "completions/mean_terminated_length": 163.82144165039062, "completions/min_length": 136.0, "completions/min_terminated_length": 136.0, "epoch": 0.08572428271518544, "grad_norm": 0.16539449312731686, "kl": 0.09375, "learning_rate": 4.265734265734266e-06, "loss": -0.0013, "num_tokens": 1215847.0, "reward": 1.3211240768432617, "reward_std": 0.030137138441205025, "rewards/check_gptzero_func/mean": 0.2857142984867096, "rewards/check_gptzero_func/std": 0.4558423161506653, "rewards/check_winston_local_func/mean": 0.03540985658764839, "rewards/check_winston_local_func/std": 0.06747763603925705, "rewards/sentence_count_match_reward_logic/mean": 1.0, "rewards/sentence_count_match_reward_logic/std": 0.0, "step": 245 }, { "clip_ratio": 0.002036431571468711, "epoch": 0.086074177746676, "grad_norm": 0.16007593159387773, "kl": 0.0927734375, "learning_rate": 4.283216783216784e-06, "loss": -0.0015, "step": 246 }, { "clip_ratio": 0.003406030358746648, "epoch": 0.08642407277816655, "grad_norm": 0.1543401601410168, "kl": 0.09423828125, "learning_rate": 4.300699300699301e-06, "loss": -0.0019, "step": 247 }, { "clip_ratio": 0.004411115311086178, "epoch": 0.0867739678096571, "grad_norm": 0.14528834545969715, "kl": 0.09375, "learning_rate": 4.3181818181818185e-06, "loss": -0.0026, "step": 248 }, { "clip_ratio": 0.0013637760421261191, "completions/clipped_ratio": 0.0, "completions/max_length": 245.0, "completions/max_terminated_length": 245.0, "completions/mean_length": 179.67857360839844, "completions/mean_terminated_length": 179.67857360839844, "completions/min_length": 125.0, "completions/min_terminated_length": 125.0, "epoch": 0.08712386284114766, "grad_norm": 0.1779924322658719, "kl": 0.07275390625, "learning_rate": 4.335664335664336e-06, "loss": -0.0024, "num_tokens": 1238253.0, "reward": 1.2416980266571045, "reward_std": 0.13668301701545715, "rewards/check_gptzero_func/mean": 0.2142857164144516, "rewards/check_gptzero_func/std": 0.41403928399086, "rewards/check_winston_local_func/mean": 0.03138057142496109, "rewards/check_winston_local_func/std": 0.04326331615447998, "rewards/sentence_count_match_reward_logic/mean": 0.9960317611694336, "rewards/sentence_count_match_reward_logic/std": 0.02080625854432583, "step": 249 }, { "clip_ratio": 0.001642137998715043, "epoch": 0.08747375787263821, "grad_norm": 0.17886301280780018, "kl": 0.07421875, "learning_rate": 4.353146853146854e-06, "loss": -0.0025, "step": 250 }, { "clip_ratio": 0.0019916254095733166, "epoch": 0.08782365290412876, "grad_norm": 0.17260265432562205, "kl": 0.07568359375, "learning_rate": 4.3706293706293715e-06, "loss": -0.0028, "step": 251 }, { "clip_ratio": 0.0034346848260611296, "epoch": 0.08817354793561931, "grad_norm": 0.1703263503814706, "kl": 0.07861328125, "learning_rate": 4.388111888111888e-06, "loss": -0.0037, "step": 252 }, { "clip_ratio": 0.0022678503300994635, "completions/clipped_ratio": 0.0, "completions/max_length": 222.0, "completions/max_terminated_length": 222.0, "completions/mean_length": 136.6428680419922, "completions/mean_terminated_length": 136.6428680419922, "completions/min_length": 75.0, "completions/min_terminated_length": 75.0, "epoch": 0.08852344296710987, "grad_norm": 0.1918655830844664, "kl": 0.130859375, "learning_rate": 4.405594405594406e-06, "loss": -0.0025, "num_tokens": 1255961.0, "reward": 1.0771400928497314, "reward_std": 0.15622936189174652, "rewards/check_gptzero_func/mean": 0.0535714291036129, "rewards/check_gptzero_func/std": 0.22720779478549957, "rewards/check_winston_local_func/mean": 0.0592828206717968, "rewards/check_winston_local_func/std": 0.14022715389728546, "rewards/sentence_count_match_reward_logic/mean": 0.9642857313156128, "rewards/sentence_count_match_reward_logic/std": 0.10403129458427429, "step": 253 }, { "clip_ratio": 0.0020531495101749897, "epoch": 0.08887333799860042, "grad_norm": 0.1731390489458585, "kl": 0.1376953125, "learning_rate": 4.423076923076924e-06, "loss": -0.0026, "step": 254 }, { "clip_ratio": 0.006719258148223162, "epoch": 0.08922323303009097, "grad_norm": 0.15282415974876712, "kl": 0.1435546875, "learning_rate": 4.440559440559441e-06, "loss": -0.0027, "step": 255 }, { "clip_ratio": 0.01157472375780344, "epoch": 0.08957312806158152, "grad_norm": 0.1653189303007201, "kl": 0.1494140625, "learning_rate": 4.458041958041958e-06, "loss": -0.0033, "step": 256 }, { "clip_ratio": 0.0012768071610480547, "completions/clipped_ratio": 0.0, "completions/max_length": 196.0, "completions/max_terminated_length": 196.0, "completions/mean_length": 139.625, "completions/mean_terminated_length": 139.625, "completions/min_length": 65.0, "completions/min_terminated_length": 65.0, "epoch": 0.08992302309307208, "grad_norm": 0.14573605010331347, "kl": 0.12353515625, "learning_rate": 4.475524475524476e-06, "loss": 0.0014, "num_tokens": 1273828.0, "reward": 1.0078840255737305, "reward_std": 0.0005795403267256916, "rewards/check_gptzero_func/mean": 0.0, "rewards/check_gptzero_func/std": 0.0, "rewards/check_winston_local_func/mean": 0.007884000428020954, "rewards/check_winston_local_func/std": 0.0008539072005078197, "rewards/sentence_count_match_reward_logic/mean": 1.0, "rewards/sentence_count_match_reward_logic/std": 0.0, "step": 257 }, { "clip_ratio": 0.0015460728900507092, "epoch": 0.09027291812456263, "grad_norm": 0.13581153128773546, "kl": 0.1337890625, "learning_rate": 4.493006993006993e-06, "loss": 0.0011, "step": 258 }, { "clip_ratio": 0.002190938452258706, "epoch": 0.09062281315605318, "grad_norm": 0.1289865088517383, "kl": 0.130859375, "learning_rate": 4.510489510489511e-06, "loss": 0.0008, "step": 259 }, { "clip_ratio": 0.002858331659808755, "epoch": 0.09097270818754374, "grad_norm": 0.12980395271262996, "kl": 0.1328125, "learning_rate": 4.527972027972029e-06, "loss": 0.0007, "step": 260 }, { "clip_ratio": 0.0011120263952761889, "completions/clipped_ratio": 0.0, "completions/max_length": 205.0, "completions/max_terminated_length": 205.0, "completions/mean_length": 127.76786041259766, "completions/mean_terminated_length": 127.76786041259766, "completions/min_length": 81.0, "completions/min_terminated_length": 81.0, "epoch": 0.09132260321903429, "grad_norm": 0.19179133712646018, "kl": 0.1357421875, "learning_rate": 4.5454545454545455e-06, "loss": 0.0003, "num_tokens": 1290279.0, "reward": 1.0243053436279297, "reward_std": 0.010790552943944931, "rewards/check_gptzero_func/mean": 0.0, "rewards/check_gptzero_func/std": 0.0, "rewards/check_winston_local_func/mean": 0.02430528588593006, "rewards/check_winston_local_func/std": 0.0322561115026474, "rewards/sentence_count_match_reward_logic/mean": 1.0, "rewards/sentence_count_match_reward_logic/std": 0.0, "step": 261 }, { "clip_ratio": 0.0013144657714292407, "epoch": 0.09167249825052484, "grad_norm": 0.19644398551752554, "kl": 0.13671875, "learning_rate": 4.562937062937063e-06, "loss": -0.0002, "step": 262 }, { "clip_ratio": 0.005317430477589369, "epoch": 0.09202239328201539, "grad_norm": 0.1601176552457988, "kl": 0.134765625, "learning_rate": 4.580419580419581e-06, "loss": -0.0008, "step": 263 }, { "clip_ratio": 0.01402143482118845, "epoch": 0.09237228831350595, "grad_norm": 0.16190106665319043, "kl": 0.1337890625, "learning_rate": 4.5979020979020985e-06, "loss": -0.001, "step": 264 }, { "clip_ratio": 0.0011223562760278583, "completions/clipped_ratio": 0.1428571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 206.0, "completions/mean_length": 151.17857360839844, "completions/mean_terminated_length": 133.70834350585938, "completions/min_length": 60.0, "completions/min_terminated_length": 60.0, "epoch": 0.0927221833449965, "grad_norm": 0.2153458654040487, "kl": 0.09326171875, "learning_rate": 4.615384615384616e-06, "loss": -0.0042, "num_tokens": 1309537.0, "reward": 1.1527012586593628, "reward_std": 0.18382686376571655, "rewards/check_gptzero_func/mean": 0.125, "rewards/check_gptzero_func/std": 0.3337118923664093, "rewards/check_winston_local_func/mean": 0.05066044256091118, "rewards/check_winston_local_func/std": 0.11411377787590027, "rewards/sentence_count_match_reward_logic/mean": 0.9770408272743225, "rewards/sentence_count_match_reward_logic/std": 0.0538424514234066, "step": 265 }, { "clip_ratio": 0.0025029671378433704, "epoch": 0.09307207837648705, "grad_norm": 0.19919104760458184, "kl": 0.08740234375, "learning_rate": 4.632867132867133e-06, "loss": -0.0045, "step": 266 }, { "clip_ratio": 0.0066896160133183, "epoch": 0.0934219734079776, "grad_norm": 0.18008434373326893, "kl": 0.09130859375, "learning_rate": 4.650349650349651e-06, "loss": -0.0048, "step": 267 }, { "clip_ratio": 0.009237267076969147, "epoch": 0.09377186843946816, "grad_norm": 0.19413957219775768, "kl": 0.09619140625, "learning_rate": 4.667832167832168e-06, "loss": -0.0052, "step": 268 }, { "clip_ratio": 0.001115252380259335, "completions/clipped_ratio": 0.0, "completions/max_length": 219.0, "completions/max_terminated_length": 219.0, "completions/mean_length": 158.98214721679688, "completions/mean_terminated_length": 158.98214721679688, "completions/min_length": 105.0, "completions/min_terminated_length": 105.0, "epoch": 0.09412176347095871, "grad_norm": 0.1621871274507171, "kl": 0.07666015625, "learning_rate": 4.685314685314686e-06, "loss": 0.0013, "num_tokens": 1329664.0, "reward": 1.0981379747390747, "reward_std": 0.10825291275978088, "rewards/check_gptzero_func/mean": 0.0357142873108387, "rewards/check_gptzero_func/std": 0.1872563362121582, "rewards/check_winston_local_func/mean": 0.07007674872875214, "rewards/check_winston_local_func/std": 0.1409245878458023, "rewards/sentence_count_match_reward_logic/mean": 0.9923468828201294, "rewards/sentence_count_match_reward_logic/std": 0.03245825320482254, "step": 269 }, { "clip_ratio": 0.0010561853414401412, "epoch": 0.09447165850244926, "grad_norm": 0.15790868014093226, "kl": 0.078125, "learning_rate": 4.702797202797203e-06, "loss": 0.0011, "step": 270 }, { "clip_ratio": 0.001953608589246869, "epoch": 0.09482155353393981, "grad_norm": 0.16385398695105444, "kl": 0.07958984375, "learning_rate": 4.72027972027972e-06, "loss": 0.0006, "step": 271 }, { "clip_ratio": 0.0038181941490620375, "epoch": 0.09517144856543037, "grad_norm": 0.1523215678474823, "kl": 0.08154296875, "learning_rate": 4.737762237762238e-06, "loss": 0.0, "step": 272 }, { "clip_ratio": 0.0013836461585015059, "completions/clipped_ratio": 0.0, "completions/max_length": 180.0, "completions/max_terminated_length": 180.0, "completions/mean_length": 139.1607208251953, "completions/mean_terminated_length": 139.1607208251953, "completions/min_length": 71.0, "completions/min_terminated_length": 71.0, "epoch": 0.09552134359692092, "grad_norm": 0.20776052486211166, "kl": 0.09130859375, "learning_rate": 4.755244755244756e-06, "loss": 0.0019, "num_tokens": 1347457.0, "reward": 1.1259037256240845, "reward_std": 0.09171999245882034, "rewards/check_gptzero_func/mean": 0.0892857164144516, "rewards/check_gptzero_func/std": 0.28773635625839233, "rewards/check_winston_local_func/mean": 0.07233219593763351, "rewards/check_winston_local_func/std": 0.19942350685596466, "rewards/sentence_count_match_reward_logic/mean": 0.9642857313156128, "rewards/sentence_count_match_reward_logic/std": 0.15147185325622559, "step": 273 }, { "clip_ratio": 0.0017606535693630576, "epoch": 0.09587123862841147, "grad_norm": 0.18315876605687728, "kl": 0.09619140625, "learning_rate": 4.772727272727273e-06, "loss": 0.0015, "step": 274 }, { "clip_ratio": 0.0067878589034080505, "epoch": 0.09622113365990204, "grad_norm": 0.16121968536966996, "kl": 0.1044921875, "learning_rate": 4.79020979020979e-06, "loss": 0.001, "step": 275 }, { "clip_ratio": 0.012204698286950588, "epoch": 0.09657102869139259, "grad_norm": 0.16291705329625758, "kl": 0.11181640625, "learning_rate": 4.807692307692308e-06, "loss": 0.0005, "step": 276 }, { "clip_ratio": 0.0009149397374130785, "completions/clipped_ratio": 0.1428571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 253.0, "completions/mean_length": 199.0178680419922, "completions/mean_terminated_length": 189.52084350585938, "completions/min_length": 95.0, "completions/min_terminated_length": 95.0, "epoch": 0.09692092372288313, "grad_norm": 0.1658503213988515, "kl": 0.1279296875, "learning_rate": 4.8251748251748255e-06, "loss": -0.0018, "num_tokens": 1372210.0, "reward": 1.1607154607772827, "reward_std": 0.16219405829906464, "rewards/check_gptzero_func/mean": 0.1964285671710968, "rewards/check_gptzero_func/std": 0.40089187026023865, "rewards/check_winston_local_func/mean": 0.016441090032458305, "rewards/check_winston_local_func/std": 0.03034871816635132, "rewards/sentence_count_match_reward_logic/mean": 0.9478458166122437, "rewards/sentence_count_match_reward_logic/std": 0.07167437672615051, "step": 277 }, { "clip_ratio": 0.0009133369894698262, "epoch": 0.09727081875437368, "grad_norm": 0.19976214668970954, "kl": 0.1435546875, "learning_rate": 4.842657342657343e-06, "loss": -0.002, "step": 278 }, { "clip_ratio": 0.0022801051381975412, "epoch": 0.09762071378586425, "grad_norm": 0.15747681522061327, "kl": 0.11669921875, "learning_rate": 4.86013986013986e-06, "loss": -0.0022, "step": 279 }, { "clip_ratio": 0.004417723976075649, "epoch": 0.0979706088173548, "grad_norm": 0.2026516956840227, "kl": 0.10595703125, "learning_rate": 4.877622377622378e-06, "loss": -0.0026, "step": 280 }, { "clip_ratio": 0.0018030249048024416, "completions/clipped_ratio": 0.1428571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 218.0, "completions/mean_length": 171.6607208251953, "completions/mean_terminated_length": 157.6041717529297, "completions/min_length": 96.0, "completions/min_terminated_length": 96.0, "epoch": 0.09832050384884534, "grad_norm": 0.20204806402856126, "kl": 0.1298828125, "learning_rate": 4.895104895104895e-06, "loss": 0.0008, "num_tokens": 1393591.0, "reward": 1.37916100025177, "reward_std": 0.14674268662929535, "rewards/check_gptzero_func/mean": 0.3571428656578064, "rewards/check_gptzero_func/std": 0.48349377512931824, "rewards/check_winston_local_func/mean": 0.02916085347533226, "rewards/check_winston_local_func/std": 0.062200333923101425, "rewards/sentence_count_match_reward_logic/mean": 0.9928570985794067, "rewards/sentence_count_match_reward_logic/std": 0.03745126724243164, "step": 281 }, { "clip_ratio": 0.0033392966724932194, "epoch": 0.0986703988803359, "grad_norm": 0.1825861789463742, "kl": 0.12255859375, "learning_rate": 4.912587412587413e-06, "loss": 0.0004, "step": 282 }, { "clip_ratio": 0.005870327353477478, "epoch": 0.09902029391182646, "grad_norm": 0.24556589748665372, "kl": 0.1201171875, "learning_rate": 4.930069930069931e-06, "loss": 0.0001, "step": 283 }, { "clip_ratio": 0.008429295383393764, "epoch": 0.099370188943317, "grad_norm": 0.17355913787256907, "kl": 0.12060546875, "learning_rate": 4.9475524475524474e-06, "loss": -0.0008, "step": 284 }, { "clip_ratio": 0.0011515930527821183, "completions/clipped_ratio": 0.125, "completions/max_length": 256.0, "completions/max_terminated_length": 256.0, "completions/mean_length": 171.62501525878906, "completions/mean_terminated_length": 159.57142639160156, "completions/min_length": 109.0, "completions/min_terminated_length": 109.0, "epoch": 0.09972008397480756, "grad_norm": 0.21092765659169738, "kl": 0.11083984375, "learning_rate": 4.965034965034965e-06, "loss": 0.002, "num_tokens": 1415082.0, "reward": 1.3636823892593384, "reward_std": 0.18934746086597443, "rewards/check_gptzero_func/mean": 0.3214285671710968, "rewards/check_gptzero_func/std": 0.4712514281272888, "rewards/check_winston_local_func/mean": 0.042253732681274414, "rewards/check_winston_local_func/std": 0.049309246242046356, "rewards/sentence_count_match_reward_logic/mean": 1.0, "rewards/sentence_count_match_reward_logic/std": 0.0, "step": 285 }, { "clip_ratio": 0.0023888596333563328, "epoch": 0.1000699790062981, "grad_norm": 0.19419976078287962, "kl": 0.11181640625, "learning_rate": 4.982517482517483e-06, "loss": 0.0018, "step": 286 }, { "clip_ratio": 0.006591304205358028, "epoch": 0.10041987403778867, "grad_norm": 0.2019512953039041, "kl": 0.1123046875, "learning_rate": 5e-06, "loss": 0.0009, "step": 287 }, { "clip_ratio": 0.009994546882808208, "epoch": 0.10076976906927922, "grad_norm": 0.19257156900546984, "kl": 0.11279296875, "learning_rate": 5.017482517482518e-06, "loss": 0.0002, "step": 288 }, { "clip_ratio": 0.0011042110854759812, "completions/clipped_ratio": 0.0, "completions/max_length": 249.0, "completions/max_terminated_length": 249.0, "completions/mean_length": 157.32144165039062, "completions/mean_terminated_length": 157.32144165039062, "completions/min_length": 60.0, "completions/min_terminated_length": 60.0, "epoch": 0.10111966410076977, "grad_norm": 0.22048554068083484, "kl": 0.1171875, "learning_rate": 5.034965034965036e-06, "loss": 0.0323, "num_tokens": 1434756.0, "reward": 1.2107988595962524, "reward_std": 0.09393064677715302, "rewards/check_gptzero_func/mean": 0.125, "rewards/check_gptzero_func/std": 0.3337118923664093, "rewards/check_winston_local_func/mean": 0.09855394810438156, "rewards/check_winston_local_func/std": 0.18421082198619843, "rewards/sentence_count_match_reward_logic/mean": 0.9872449040412903, "rewards/sentence_count_match_reward_logic/std": 0.09545043855905533, "step": 289 }, { "clip_ratio": 0.0031990648712962866, "epoch": 0.10146955913226033, "grad_norm": 0.1976636395592892, "kl": 0.1259765625, "learning_rate": 5.0524475524475525e-06, "loss": 0.032, "step": 290 }, { "clip_ratio": 0.00647585466504097, "epoch": 0.10181945416375088, "grad_norm": 0.17267027972825488, "kl": 0.1337890625, "learning_rate": 5.06993006993007e-06, "loss": 0.0314, "step": 291 }, { "clip_ratio": 0.01600334420800209, "epoch": 0.10216934919524143, "grad_norm": 0.1468943270855828, "kl": 0.1494140625, "learning_rate": 5.087412587412588e-06, "loss": 0.031, "step": 292 }, { "clip_ratio": 0.0019415343413129449, "completions/clipped_ratio": 0.2321428571428571, "completions/max_length": 256.0, "completions/max_terminated_length": 256.0, "completions/mean_length": 171.5178680419922, "completions/mean_terminated_length": 145.97674560546875, "completions/min_length": 91.0, "completions/min_terminated_length": 91.0, "epoch": 0.10251924422673198, "grad_norm": 0.18021581133138637, "kl": 0.1279296875, "learning_rate": 5.1048951048951055e-06, "loss": -0.0008, "num_tokens": 1456329.0, "reward": 1.2655681371688843, "reward_std": 0.1362372487783432, "rewards/check_gptzero_func/mean": 0.2142857164144516, "rewards/check_gptzero_func/std": 0.41403934359550476, "rewards/check_winston_local_func/mean": 0.051282431930303574, "rewards/check_winston_local_func/std": 0.09000397473573685, "rewards/sentence_count_match_reward_logic/mean": 1.0, "rewards/sentence_count_match_reward_logic/std": 0.0, "step": 293 }, { "clip_ratio": 0.002540014451369643, "epoch": 0.10286913925822254, "grad_norm": 0.18814042786929577, "kl": 0.1416015625, "learning_rate": 5.122377622377622e-06, "loss": -0.0017, "step": 294 }, { "clip_ratio": 0.004364078864455223, "epoch": 0.10321903428971309, "grad_norm": 0.19286402385094423, "kl": 0.1484375, "learning_rate": 5.13986013986014e-06, "loss": -0.0018, "step": 295 }, { "clip_ratio": 0.007896633818745613, "epoch": 0.10356892932120364, "grad_norm": 0.2125778051107411, "kl": 0.146484375, "learning_rate": 5.157342657342658e-06, "loss": -0.0025, "step": 296 }, { "clip_ratio": 0.0011130113853141665, "completions/clipped_ratio": 0.2857142857142857, "completions/max_length": 256.0, "completions/max_terminated_length": 183.0, "completions/mean_length": 171.17857360839844, "completions/mean_terminated_length": 137.25, "completions/min_length": 99.0, "completions/min_terminated_length": 99.0, "epoch": 0.10391882435269419, "grad_norm": 0.22068614440718354, "kl": 0.1103515625, "learning_rate": 5.174825174825175e-06, "loss": 0.0003, "num_tokens": 1477667.0, "reward": 1.0596853494644165, "reward_std": 0.08659543097019196, "rewards/check_gptzero_func/mean": 0.0357142873108387, "rewards/check_gptzero_func/std": 0.1872563362121582, "rewards/check_winston_local_func/mean": 0.025955161079764366, "rewards/check_winston_local_func/std": 0.0474109873175621, "rewards/sentence_count_match_reward_logic/mean": 0.9980158805847168, "rewards/sentence_count_match_reward_logic/std": 0.014847845770418644, "step": 297 }, { "clip_ratio": 0.002831217600032687, "epoch": 0.10426871938418475, "grad_norm": 0.2173374285029113, "kl": 0.1123046875, "learning_rate": 5.192307692307693e-06, "loss": -0.0002, "step": 298 }, { "clip_ratio": 0.005880268756300211, "epoch": 0.1046186144156753, "grad_norm": 0.20339778253017707, "kl": 0.1083984375, "learning_rate": 5.20979020979021e-06, "loss": -0.0006, "step": 299 }, { "clip_ratio": 0.011315790936350822, "epoch": 0.10496850944716585, "grad_norm": 0.18852547389424443, "kl": 0.109375, "learning_rate": 5.2272727272727274e-06, "loss": -0.0014, "step": 300 }, { "clip_ratio": 0.0028740817215293646, "completions/clipped_ratio": 0.0, "completions/max_length": 235.0, "completions/max_terminated_length": 235.0, "completions/mean_length": 138.5357208251953, "completions/mean_terminated_length": 138.5357208251953, "completions/min_length": 73.0, "completions/min_terminated_length": 73.0, "epoch": 0.1053184044786564, "grad_norm": 0.2151739991212639, "kl": 0.1923828125, "learning_rate": 5.244755244755245e-06, "loss": 0.0013, "num_tokens": 1495569.0, "reward": 1.132407784461975, "reward_std": 0.0864190012216568, "rewards/check_gptzero_func/mean": 0.125, "rewards/check_gptzero_func/std": 0.3337118923664093, "rewards/check_winston_local_func/mean": 0.043122000992298126, "rewards/check_winston_local_func/std": 0.08123642951250076, "rewards/sentence_count_match_reward_logic/mean": 0.9642857313156128, "rewards/sentence_count_match_reward_logic/std": 0.08827348798513412, "step": 301 }, { "clip_ratio": 0.006993815768510103, "epoch": 0.10566829951014696, "grad_norm": 0.17122453755375508, "kl": 0.2177734375, "learning_rate": 5.262237762237763e-06, "loss": 0.0008, "step": 302 }, { "clip_ratio": 0.018803460523486137, "epoch": 0.10601819454163751, "grad_norm": 0.23502873394608306, "kl": 0.255859375, "learning_rate": 5.27972027972028e-06, "loss": 0.0004, "step": 303 }, { "clip_ratio": 0.02051079273223877, "epoch": 0.10636808957312806, "grad_norm": 0.22250344268081387, "kl": 0.25, "learning_rate": 5.297202797202797e-06, "loss": -0.0002, "step": 304 }, { "clip_ratio": 0.0017106694867834449, "completions/clipped_ratio": 0.0, "completions/max_length": 202.0, "completions/max_terminated_length": 202.0, "completions/mean_length": 159.98214721679688, "completions/mean_terminated_length": 159.98214721679688, "completions/min_length": 113.0, "completions/min_terminated_length": 113.0, "epoch": 0.10671798460461862, "grad_norm": 0.22567099461085455, "kl": 0.142578125, "learning_rate": 5.314685314685315e-06, "loss": 0.0003, "num_tokens": 1515752.0, "reward": 1.0658248662948608, "reward_std": 0.05233468487858772, "rewards/check_gptzero_func/mean": 0.0, "rewards/check_gptzero_func/std": 0.0, "rewards/check_winston_local_func/mean": 0.06582483649253845, "rewards/check_winston_local_func/std": 0.10365913808345795, "rewards/sentence_count_match_reward_logic/mean": 1.0, "rewards/sentence_count_match_reward_logic/std": 0.0, "step": 305 }, { "clip_ratio": 0.005860524717718363, "epoch": 0.10706787963610917, "grad_norm": 0.21671881699881737, "kl": 0.1376953125, "learning_rate": 5.3321678321678325e-06, "loss": -0.0, "step": 306 }, { "clip_ratio": 0.015205634757876396, "epoch": 0.10741777466759972, "grad_norm": 0.3339146808073341, "kl": 0.140625, "learning_rate": 5.34965034965035e-06, "loss": -0.0002, "step": 307 }, { "clip_ratio": 0.012430930510163307, "epoch": 0.10776766969909027, "grad_norm": 0.19799541252726013, "kl": 0.1552734375, "learning_rate": 5.367132867132867e-06, "loss": -0.0013, "step": 308 }, { "clip_ratio": 0.0010512897279113531, "completions/clipped_ratio": 0.0892857142857143, "completions/max_length": 256.0, "completions/max_terminated_length": 247.0, "completions/mean_length": 200.12501525878906, "completions/mean_terminated_length": 194.64706420898438, "completions/min_length": 134.0, "completions/min_terminated_length": 134.0, "epoch": 0.10811756473058083, "grad_norm": 0.61280768631283, "kl": 0.271484375, "learning_rate": 5.384615384615385e-06, "loss": -0.0024, "num_tokens": 1540647.0, "reward": 1.5637649297714233, "reward_std": 0.16765080392360687, "rewards/check_gptzero_func/mean": 0.4464285671710968, "rewards/check_gptzero_func/std": 0.5016207695007324, "rewards/check_winston_local_func/mean": 0.11956846714019775, "rewards/check_winston_local_func/std": 0.2062736451625824, "rewards/sentence_count_match_reward_logic/mean": 0.9977678656578064, "rewards/sentence_count_match_reward_logic/std": 0.016703827306628227, "step": 309 }, { "clip_ratio": 0.009277991019189358, "epoch": 0.10846745976207138, "grad_norm": 0.47435019094261094, "kl": 0.12109375, "learning_rate": 5.402097902097902e-06, "loss": -0.0029, "step": 310 }, { "clip_ratio": 0.011535934172570705, "epoch": 0.10881735479356193, "grad_norm": 0.6440526091116034, "kl": 0.12060546875, "learning_rate": 5.41958041958042e-06, "loss": -0.0028, "step": 311 }, { "clip_ratio": 0.005284632556140423, "epoch": 0.10916724982505248, "grad_norm": 0.2022478735545499, "kl": 0.1396484375, "learning_rate": 5.437062937062938e-06, "loss": -0.005, "step": 312 }, { "clip_ratio": 0.0006729075685143471, "completions/clipped_ratio": 0.1607142857142857, "completions/max_length": 256.0, "completions/max_terminated_length": 254.0, "completions/mean_length": 183.85714721679688, "completions/mean_terminated_length": 170.04254150390625, "completions/min_length": 109.0, "completions/min_terminated_length": 109.0, "epoch": 0.10951714485654304, "grad_norm": 0.2163910875374486, "kl": 0.1474609375, "learning_rate": 5.4545454545454545e-06, "loss": -0.0005, "num_tokens": 1564255.0, "reward": 1.0444849729537964, "reward_std": 0.12014703452587128, "rewards/check_gptzero_func/mean": 0.0357142873108387, "rewards/check_gptzero_func/std": 0.1872563362121582, "rewards/check_winston_local_func/mean": 0.021525785326957703, "rewards/check_winston_local_func/std": 0.025998668745160103, "rewards/sentence_count_match_reward_logic/mean": 0.9872449040412903, "rewards/sentence_count_match_reward_logic/std": 0.04110519215464592, "step": 313 }, { "clip_ratio": 0.004981005098670721, "epoch": 0.10986703988803359, "grad_norm": 0.25226387284353985, "kl": 0.1591796875, "learning_rate": 5.472027972027972e-06, "loss": -0.0004, "step": 314 }, { "clip_ratio": 0.013797113671898842, "epoch": 0.11021693491952414, "grad_norm": 0.2969083280992991, "kl": 0.1787109375, "learning_rate": 5.48951048951049e-06, "loss": -0.0007, "step": 315 }, { "clip_ratio": 0.02525135688483715, "epoch": 0.11056682995101469, "grad_norm": 0.2550259305928417, "kl": 0.2001953125, "learning_rate": 5.5069930069930074e-06, "loss": -0.0016, "step": 316 }, { "clip_ratio": 0.003102579852566123, "completions/clipped_ratio": 0.1428571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 176.0, "completions/mean_length": 143.30357360839844, "completions/mean_terminated_length": 124.52083587646484, "completions/min_length": 67.0, "completions/min_terminated_length": 67.0, "epoch": 0.11091672498250525, "grad_norm": 0.21293387992839513, "kl": 0.2392578125, "learning_rate": 5.524475524475524e-06, "loss": 0.0037, "num_tokens": 1583160.0, "reward": 1.1557663679122925, "reward_std": 0.1653430461883545, "rewards/check_gptzero_func/mean": 0.1071428582072258, "rewards/check_gptzero_func/std": 0.31209391355514526, "rewards/check_winston_local_func/mean": 0.08093641698360443, "rewards/check_winston_local_func/std": 0.1306265890598297, "rewards/sentence_count_match_reward_logic/mean": 0.9676870703697205, "rewards/sentence_count_match_reward_logic/std": 0.06265423446893692, "step": 317 }, { "clip_ratio": 0.006775357760488987, "epoch": 0.1112666200139958, "grad_norm": 0.22026764217144548, "kl": 0.255859375, "learning_rate": 5.541958041958042e-06, "loss": 0.0035, "step": 318 }, { "clip_ratio": 0.010312951169908047, "epoch": 0.11161651504548635, "grad_norm": 0.24827415878030532, "kl": 0.267578125, "learning_rate": 5.5594405594405596e-06, "loss": 0.0031, "step": 319 }, { "clip_ratio": 0.01228560321033001, "epoch": 0.11196641007697691, "grad_norm": 0.21382128238334785, "kl": 0.26953125, "learning_rate": 5.576923076923077e-06, "loss": 0.0024, "step": 320 }, { "clip_ratio": 0.003071809886023402, "completions/clipped_ratio": 0.0, "completions/max_length": 213.0, "completions/max_terminated_length": 213.0, "completions/mean_length": 145.82144165039062, "completions/mean_terminated_length": 145.82144165039062, "completions/min_length": 110.0, "completions/min_terminated_length": 110.0, "epoch": 0.11231630510846746, "grad_norm": 0.23448814375477112, "kl": 0.275390625, "learning_rate": 5.594405594405595e-06, "loss": 0.0054, "num_tokens": 1601982.0, "reward": 1.2631809711456299, "reward_std": 0.14928074181079865, "rewards/check_gptzero_func/mean": 0.1964285671710968, "rewards/check_gptzero_func/std": 0.40089187026023865, "rewards/check_winston_local_func/mean": 0.06675225496292114, "rewards/check_winston_local_func/std": 0.10018416494131088, "rewards/sentence_count_match_reward_logic/mean": 1.0, "rewards/sentence_count_match_reward_logic/std": 0.0, "step": 321 }, { "clip_ratio": 0.0031621006783097982, "epoch": 0.11266620013995801, "grad_norm": 0.29407902711048234, "kl": 0.267578125, "learning_rate": 5.611888111888112e-06, "loss": 0.005, "step": 322 }, { "clip_ratio": 0.005902358330786228, "epoch": 0.11301609517144856, "grad_norm": 0.20793127329458697, "kl": 0.27734375, "learning_rate": 5.629370629370629e-06, "loss": 0.0042, "step": 323 }, { "clip_ratio": 0.011960254982113838, "epoch": 0.11336599020293912, "grad_norm": 0.17422688008930504, "kl": 0.283203125, "learning_rate": 5.646853146853147e-06, "loss": 0.0034, "step": 324 }, { "clip_ratio": 0.0023432052694261074, "completions/clipped_ratio": 0.0, "completions/max_length": 249.0, "completions/max_terminated_length": 249.0, "completions/mean_length": 159.2678680419922, "completions/mean_terminated_length": 159.2678680419922, "completions/min_length": 90.0, "completions/min_terminated_length": 90.0, "epoch": 0.11371588523442967, "grad_norm": 0.19747082866155408, "kl": 0.2080078125, "learning_rate": 5.664335664335665e-06, "loss": -0.0012, "num_tokens": 1622253.0, "reward": 1.1814132928848267, "reward_std": 0.05678678676486015, "rewards/check_gptzero_func/mean": 0.1607142835855484, "rewards/check_gptzero_func/std": 0.3705909848213196, "rewards/check_winston_local_func/mean": 0.02427033893764019, "rewards/check_winston_local_func/std": 0.028806565329432487, "rewards/sentence_count_match_reward_logic/mean": 0.9964285492897034, "rewards/sentence_count_match_reward_logic/std": 0.026726121082901955, "step": 325 }, { "clip_ratio": 0.0016205792780965567, "epoch": 0.11406578026592022, "grad_norm": 0.20310082335166518, "kl": 0.208984375, "learning_rate": 5.681818181818183e-06, "loss": -0.0015, "step": 326 }, { "clip_ratio": 0.006426213774830103, "epoch": 0.11441567529741077, "grad_norm": 0.2111875235410987, "kl": 0.21875, "learning_rate": 5.699300699300699e-06, "loss": -0.0016, "step": 327 }, { "clip_ratio": 0.00977273192256689, "epoch": 0.11476557032890133, "grad_norm": 0.19885985514410262, "kl": 0.224609375, "learning_rate": 5.716783216783217e-06, "loss": -0.0028, "step": 328 }, { "clip_ratio": 0.0010586668504402041, "completions/clipped_ratio": 0.0, "completions/max_length": 188.0, "completions/max_terminated_length": 188.0, "completions/mean_length": 119.67857360839844, "completions/mean_terminated_length": 119.67857360839844, "completions/min_length": 53.0, "completions/min_terminated_length": 53.0, "epoch": 0.11511546536039188, "grad_norm": 0.21129664891449262, "kl": 0.193359375, "learning_rate": 5.7342657342657345e-06, "loss": -0.0027, "num_tokens": 1637747.0, "reward": 1.1048681735992432, "reward_std": 0.07437149435281754, "rewards/check_gptzero_func/mean": 0.01785714365541935, "rewards/check_gptzero_func/std": 0.13363061845302582, "rewards/check_winston_local_func/mean": 0.08701099455356598, "rewards/check_winston_local_func/std": 0.18749867379665375, "rewards/sentence_count_match_reward_logic/mean": 1.0, "rewards/sentence_count_match_reward_logic/std": 0.0, "step": 329 }, { "clip_ratio": 0.005627453327178955, "epoch": 0.11546536039188243, "grad_norm": 0.1464287510310564, "kl": 0.1962890625, "learning_rate": 5.751748251748253e-06, "loss": -0.0031, "step": 330 }, { "clip_ratio": 0.011728744022548199, "epoch": 0.11581525542337298, "grad_norm": 0.20338150745482228, "kl": 0.203125, "learning_rate": 5.769230769230769e-06, "loss": -0.0034, "step": 331 }, { "clip_ratio": 0.015728794038295746, "epoch": 0.11616515045486354, "grad_norm": 0.1424808258534998, "kl": 0.2138671875, "learning_rate": 5.786713286713287e-06, "loss": -0.0038, "step": 332 }, { "clip_ratio": 0.0012551115360110998, "completions/clipped_ratio": 0.0, "completions/max_length": 206.0, "completions/max_terminated_length": 206.0, "completions/mean_length": 149.08929443359375, "completions/mean_terminated_length": 149.08929443359375, "completions/min_length": 87.0, "completions/min_terminated_length": 87.0, "epoch": 0.1165150454863541, "grad_norm": 0.22130028684773448, "kl": 0.1865234375, "learning_rate": 5.804195804195804e-06, "loss": 0.0005, "num_tokens": 1656712.0, "reward": 1.1089284420013428, "reward_std": 0.10445155948400497, "rewards/check_gptzero_func/mean": 0.01785714365541935, "rewards/check_gptzero_func/std": 0.13363061845302582, "rewards/check_winston_local_func/mean": 0.09107126295566559, "rewards/check_winston_local_func/std": 0.1355779469013214, "rewards/sentence_count_match_reward_logic/mean": 1.0, "rewards/sentence_count_match_reward_logic/std": 0.0, "step": 333 }, { "clip_ratio": 0.00508350832387805, "epoch": 0.11686494051784464, "grad_norm": 0.21291420886942627, "kl": 0.1962890625, "learning_rate": 5.821678321678323e-06, "loss": -0.0, "step": 334 }, { "clip_ratio": 0.010267363861203194, "epoch": 0.1172148355493352, "grad_norm": 0.23385601123974445, "kl": 0.2021484375, "learning_rate": 5.83916083916084e-06, "loss": -0.0009, "step": 335 }, { "clip_ratio": 0.013112110085785389, "epoch": 0.11756473058082575, "grad_norm": 0.193752273036307, "kl": 0.203125, "learning_rate": 5.856643356643356e-06, "loss": -0.0016, "step": 336 }, { "clip_ratio": 0.001886657322756946, "completions/clipped_ratio": 0.1428571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 218.0, "completions/mean_length": 184.42857360839844, "completions/mean_terminated_length": 172.5, "completions/min_length": 105.0, "completions/min_terminated_length": 105.0, "epoch": 0.1179146256123163, "grad_norm": 0.23726652252273223, "kl": 0.224609375, "learning_rate": 5.874125874125874e-06, "loss": -0.0024, "num_tokens": 1680136.0, "reward": 1.4584593772888184, "reward_std": 0.13366219401359558, "rewards/check_gptzero_func/mean": 0.4285714328289032, "rewards/check_gptzero_func/std": 0.4993502199649811, "rewards/check_winston_local_func/mean": 0.059649642556905746, "rewards/check_winston_local_func/std": 0.09396229684352875, "rewards/sentence_count_match_reward_logic/mean": 0.9702381491661072, "rewards/sentence_count_match_reward_logic/std": 0.0748843252658844, "step": 337 }, { "clip_ratio": 0.003353315172716975, "epoch": 0.11826452064380685, "grad_norm": 0.24641465370236298, "kl": 0.228515625, "learning_rate": 5.8916083916083925e-06, "loss": -0.003, "step": 338 }, { "clip_ratio": 0.005755515303462744, "epoch": 0.11861441567529742, "grad_norm": 0.1969658149100851, "kl": 0.224609375, "learning_rate": 5.90909090909091e-06, "loss": -0.0037, "step": 339 }, { "clip_ratio": 0.011797568760812283, "epoch": 0.11896431070678797, "grad_norm": 0.20857822483859134, "kl": 0.2255859375, "learning_rate": 5.926573426573428e-06, "loss": -0.0049, "step": 340 }, { "clip_ratio": 0.001479465514421463, "completions/clipped_ratio": 0.1428571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 201.0, "completions/mean_length": 168.57144165039062, "completions/mean_terminated_length": 154.0, "completions/min_length": 109.0, "completions/min_terminated_length": 109.0, "epoch": 0.11931420573827851, "grad_norm": 0.21343058201530876, "kl": 0.203125, "learning_rate": 5.944055944055944e-06, "loss": -0.0012, "num_tokens": 1701456.0, "reward": 1.4312164783477783, "reward_std": 0.22158053517341614, "rewards/check_gptzero_func/mean": 0.25, "rewards/check_gptzero_func/std": 0.43693143129348755, "rewards/check_winston_local_func/mean": 0.18568073213100433, "rewards/check_winston_local_func/std": 0.31614944338798523, "rewards/sentence_count_match_reward_logic/mean": 0.9955357313156128, "rewards/sentence_count_match_reward_logic/std": 0.023407040163874626, "step": 341 }, { "clip_ratio": 0.001956225372850895, "epoch": 0.11966410076976906, "grad_norm": 0.2045821677521692, "kl": 0.2041015625, "learning_rate": 5.961538461538462e-06, "loss": -0.0019, "step": 342 }, { "clip_ratio": 0.004985008854418993, "epoch": 0.12001399580125963, "grad_norm": 0.20227812151120275, "kl": 0.2080078125, "learning_rate": 5.97902097902098e-06, "loss": -0.0024, "step": 343 }, { "clip_ratio": 0.013344677165150642, "epoch": 0.12036389083275018, "grad_norm": 0.24360260814279044, "kl": 0.2099609375, "learning_rate": 5.996503496503498e-06, "loss": -0.0032, "step": 344 }, { "clip_ratio": 0.001797290169633925, "completions/clipped_ratio": 0.2857142857142857, "completions/max_length": 256.0, "completions/max_terminated_length": 245.0, "completions/mean_length": 203.21429443359375, "completions/mean_terminated_length": 182.10000610351562, "completions/min_length": 152.0, "completions/min_terminated_length": 152.0, "epoch": 0.12071378586424072, "grad_norm": 0.24574082274406014, "kl": 0.2451171875, "learning_rate": 6.013986013986014e-06, "loss": -0.0015, "num_tokens": 1726820.0, "reward": 1.5624874830245972, "reward_std": 0.15373721718788147, "rewards/check_gptzero_func/mean": 0.4464285671710968, "rewards/check_gptzero_func/std": 0.5016207695007324, "rewards/check_winston_local_func/mean": 0.14945411682128906, "rewards/check_winston_local_func/std": 0.3087153136730194, "rewards/sentence_count_match_reward_logic/mean": 0.9666048288345337, "rewards/sentence_count_match_reward_logic/std": 0.05509126931428909, "step": 345 }, { "clip_ratio": 0.002899815095588565, "epoch": 0.12106368089573127, "grad_norm": 0.24357609082010764, "kl": 0.25390625, "learning_rate": 6.031468531468532e-06, "loss": -0.0022, "step": 346 }, { "clip_ratio": 0.007019499782472849, "epoch": 0.12141357592722184, "grad_norm": 0.28046320025672455, "kl": 0.263671875, "learning_rate": 6.04895104895105e-06, "loss": -0.0034, "step": 347 }, { "clip_ratio": 0.011716331355273724, "epoch": 0.12176347095871239, "grad_norm": 0.22382696823456738, "kl": 0.26953125, "learning_rate": 6.0664335664335674e-06, "loss": -0.0044, "step": 348 }, { "clip_ratio": 0.0021677904296666384, "completions/clipped_ratio": 0.0, "completions/max_length": 216.0, "completions/max_terminated_length": 216.0, "completions/mean_length": 136.25, "completions/mean_terminated_length": 136.25, "completions/min_length": 78.0, "completions/min_terminated_length": 78.0, "epoch": 0.12211336599020294, "grad_norm": 0.22758917617674107, "kl": 0.298828125, "learning_rate": 6.083916083916085e-06, "loss": -0.0006, "num_tokens": 1744042.0, "reward": 1.4274382591247559, "reward_std": 0.13005150854587555, "rewards/check_gptzero_func/mean": 0.3392857015132904, "rewards/check_gptzero_func/std": 0.477751761674881, "rewards/check_winston_local_func/mean": 0.0988667830824852, "rewards/check_winston_local_func/std": 0.18401487171649933, "rewards/sentence_count_match_reward_logic/mean": 0.9892857670783997, "rewards/sentence_count_match_reward_logic/std": 0.08017836511135101, "step": 349 }, { "clip_ratio": 0.004017199389636517, "epoch": 0.1224632610216935, "grad_norm": 0.21857411965668455, "kl": 0.298828125, "learning_rate": 6.101398601398602e-06, "loss": -0.0014, "step": 350 }, { "clip_ratio": 0.011743930168449879, "epoch": 0.12281315605318405, "grad_norm": 0.19093148158110662, "kl": 0.3125, "learning_rate": 6.1188811188811196e-06, "loss": -0.002, "step": 351 }, { "clip_ratio": 0.02450181357562542, "epoch": 0.1231630510846746, "grad_norm": 0.23111420801469493, "kl": 0.333984375, "learning_rate": 6.136363636363637e-06, "loss": -0.0027, "step": 352 }, { "clip_ratio": 0.0021311279851943254, "completions/clipped_ratio": 0.1428571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 187.0, "completions/mean_length": 152.44644165039062, "completions/mean_terminated_length": 135.1875, "completions/min_length": 82.0, "completions/min_terminated_length": 82.0, "epoch": 0.12351294611616515, "grad_norm": 0.5140059295919319, "kl": 0.412109375, "learning_rate": 6.153846153846155e-06, "loss": 0.0007, "num_tokens": 1763531.0, "reward": 1.5523687601089478, "reward_std": 0.0930856242775917, "rewards/check_gptzero_func/mean": 0.4285714328289032, "rewards/check_gptzero_func/std": 0.4993502199649811, "rewards/check_winston_local_func/mean": 0.1476067453622818, "rewards/check_winston_local_func/std": 0.22036640346050262, "rewards/sentence_count_match_reward_logic/mean": 0.976190447807312, "rewards/sentence_count_match_reward_logic/std": 0.058848995715379715, "step": 353 }, { "clip_ratio": 0.008525061421096325, "epoch": 0.12386284114765571, "grad_norm": 0.4594913631725265, "kl": 0.26953125, "learning_rate": 6.171328671328672e-06, "loss": 0.0001, "step": 354 }, { "clip_ratio": 0.013916070573031902, "epoch": 0.12421273617914626, "grad_norm": 0.48520891032768154, "kl": 0.27734375, "learning_rate": 6.188811188811189e-06, "loss": -0.0007, "step": 355 }, { "clip_ratio": 0.0168914832174778, "epoch": 0.1245626312106368, "grad_norm": 0.4756642802693391, "kl": 0.44140625, "learning_rate": 6.206293706293707e-06, "loss": -0.0014, "step": 356 }, { "clip_ratio": 0.002881354885175824, "completions/clipped_ratio": 0.0535714285714286, "completions/max_length": 256.0, "completions/max_terminated_length": 255.0, "completions/mean_length": 183.30357360839844, "completions/mean_terminated_length": 179.1886749267578, "completions/min_length": 116.0, "completions/min_terminated_length": 116.0, "epoch": 0.12491252624212736, "grad_norm": 0.3264192480594978, "kl": 0.234375, "learning_rate": 6.223776223776225e-06, "loss": -0.0001, "num_tokens": 1785972.0, "reward": 1.1166620254516602, "reward_std": 0.05386506766080856, "rewards/check_gptzero_func/mean": 0.0, "rewards/check_gptzero_func/std": 0.0, "rewards/check_winston_local_func/mean": 0.11666198819875717, "rewards/check_winston_local_func/std": 0.16129028797149658, "rewards/sentence_count_match_reward_logic/mean": 1.0, "rewards/sentence_count_match_reward_logic/std": 0.0, "step": 357 }, { "clip_ratio": 0.0073357499204576015, "epoch": 0.12526242127361792, "grad_norm": 0.2132417760826214, "kl": 0.2578125, "learning_rate": 6.241258741258742e-06, "loss": -0.0009, "step": 358 }, { "clip_ratio": 0.01750616915524006, "epoch": 0.12561231630510847, "grad_norm": 0.2816675802864827, "kl": 0.294921875, "learning_rate": 6.258741258741259e-06, "loss": -0.0012, "step": 359 }, { "clip_ratio": 0.022547069936990738, "epoch": 0.12596221133659902, "grad_norm": 0.3291733642875763, "kl": 0.302734375, "learning_rate": 6.276223776223777e-06, "loss": -0.002, "step": 360 }, { "clip_ratio": 0.0022626311983913183, "completions/clipped_ratio": 0.0, "completions/max_length": 227.0, "completions/max_terminated_length": 227.0, "completions/mean_length": 172.5357208251953, "completions/mean_terminated_length": 172.5357208251953, "completions/min_length": 127.0, "completions/min_terminated_length": 127.0, "epoch": 0.12631210636808957, "grad_norm": 0.3211332869073257, "kl": 0.33203125, "learning_rate": 6.2937062937062944e-06, "loss": -0.0091, "num_tokens": 1807378.0, "reward": 1.1064554452896118, "reward_std": 0.08471377193927765, "rewards/check_gptzero_func/mean": 0.0, "rewards/check_gptzero_func/std": 0.0, "rewards/check_winston_local_func/mean": 0.10645540803670883, "rewards/check_winston_local_func/std": 0.1561792641878128, "rewards/sentence_count_match_reward_logic/mean": 1.0, "rewards/sentence_count_match_reward_logic/std": 0.0, "step": 361 }, { "clip_ratio": 0.0029495805501937866, "epoch": 0.12666200139958012, "grad_norm": 0.2775407642206096, "kl": 0.3125, "learning_rate": 6.311188811188812e-06, "loss": -0.01, "step": 362 }, { "clip_ratio": 0.0069335210137069225, "epoch": 0.1270118964310707, "grad_norm": 0.2682759264448839, "kl": 0.322265625, "learning_rate": 6.32867132867133e-06, "loss": -0.0108, "step": 363 }, { "clip_ratio": 0.013484792783856392, "epoch": 0.12736179146256124, "grad_norm": 0.23580690689981726, "kl": 0.337890625, "learning_rate": 6.3461538461538466e-06, "loss": -0.0119, "step": 364 }, { "clip_ratio": 0.0031740639824420214, "completions/clipped_ratio": 0.0, "completions/max_length": 188.0, "completions/max_terminated_length": 188.0, "completions/mean_length": 130.60714721679688, "completions/mean_terminated_length": 130.60714721679688, "completions/min_length": 77.0, "completions/min_terminated_length": 77.0, "epoch": 0.1277116864940518, "grad_norm": 0.26157745566662144, "kl": 0.380859375, "learning_rate": 6.363636363636364e-06, "loss": -0.0014, "num_tokens": 1823836.0, "reward": 1.1674407720565796, "reward_std": 0.16359707713127136, "rewards/check_gptzero_func/mean": 0.0535714291036129, "rewards/check_gptzero_func/std": 0.22720779478549957, "rewards/check_winston_local_func/mean": 0.11386922746896744, "rewards/check_winston_local_func/std": 0.1806529015302658, "rewards/sentence_count_match_reward_logic/mean": 1.0, "rewards/sentence_count_match_reward_logic/std": 0.0, "step": 365 }, { "clip_ratio": 0.005468177143484354, "epoch": 0.12806158152554234, "grad_norm": 0.25767256798050214, "kl": 0.390625, "learning_rate": 6.381118881118882e-06, "loss": -0.0018, "step": 366 }, { "clip_ratio": 0.013521556742489338, "epoch": 0.1284114765570329, "grad_norm": 0.2451499889150475, "kl": 0.376953125, "learning_rate": 6.3986013986013996e-06, "loss": -0.0032, "step": 367 }, { "clip_ratio": 0.023130128160119057, "epoch": 0.12876137158852344, "grad_norm": 0.24314991458491345, "kl": 0.37109375, "learning_rate": 6.416083916083916e-06, "loss": -0.0041, "step": 368 }, { "clip_ratio": 0.0012382750865072012, "completions/clipped_ratio": 0.4285714285714286, "completions/max_length": 256.0, "completions/max_terminated_length": 192.0, "completions/mean_length": 210.85714721679688, "completions/mean_terminated_length": 177.0, "completions/min_length": 144.0, "completions/min_terminated_length": 144.0, "epoch": 0.129111266620014, "grad_norm": 0.2593722769457948, "kl": 0.2060546875, "learning_rate": 6.433566433566434e-06, "loss": -0.0005, "num_tokens": 1852380.0, "reward": 1.7306464910507202, "reward_std": 0.17789988219738007, "rewards/check_gptzero_func/mean": 0.6428571343421936, "rewards/check_gptzero_func/std": 0.48349377512931824, "rewards/check_winston_local_func/mean": 0.17912986874580383, "rewards/check_winston_local_func/std": 0.22781342267990112, "rewards/sentence_count_match_reward_logic/mean": 0.9086592793464661, "rewards/sentence_count_match_reward_logic/std": 0.13458386063575745, "step": 369 }, { "clip_ratio": 0.0027914943639189005, "epoch": 0.12946116165150454, "grad_norm": 0.28812747820201745, "kl": 0.203125, "learning_rate": 6.451048951048952e-06, "loss": -0.0011, "step": 370 }, { "clip_ratio": 0.005053988192230463, "epoch": 0.1298110566829951, "grad_norm": 0.21158787475113494, "kl": 0.2021484375, "learning_rate": 6.468531468531469e-06, "loss": -0.002, "step": 371 }, { "clip_ratio": 0.013785302639007568, "epoch": 0.13016095171448566, "grad_norm": 0.36951164458083546, "kl": 0.201171875, "learning_rate": 6.486013986013987e-06, "loss": -0.0024, "step": 372 }, { "clip_ratio": 0.0010094314347952604, "completions/clipped_ratio": 0.2857142857142857, "completions/max_length": 256.0, "completions/max_terminated_length": 215.0, "completions/mean_length": 193.98214721679688, "completions/mean_terminated_length": 169.1750030517578, "completions/min_length": 102.0, "completions/min_terminated_length": 102.0, "epoch": 0.1305108467459762, "grad_norm": 0.30808881791820236, "kl": 0.2734375, "learning_rate": 6.503496503496504e-06, "loss": 0.0023, "num_tokens": 1877739.0, "reward": 1.3523743152618408, "reward_std": 0.09998296201229095, "rewards/check_gptzero_func/mean": 0.1607142835855484, "rewards/check_gptzero_func/std": 0.3705909848213196, "rewards/check_winston_local_func/mean": 0.2551519274711609, "rewards/check_winston_local_func/std": 0.3321893513202667, "rewards/sentence_count_match_reward_logic/mean": 0.9365078806877136, "rewards/sentence_count_match_reward_logic/std": 0.15693062543869019, "step": 373 }, { "clip_ratio": 0.008852927014231682, "epoch": 0.13086074177746676, "grad_norm": 0.32245729373015364, "kl": 0.294921875, "learning_rate": 6.5209790209790215e-06, "loss": 0.0017, "step": 374 }, { "clip_ratio": 0.01349488366395235, "epoch": 0.1312106368089573, "grad_norm": 0.3864568245480622, "kl": 0.310546875, "learning_rate": 6.538461538461539e-06, "loss": 0.0008, "step": 375 }, { "clip_ratio": 0.013566027395427227, "epoch": 0.13156053184044786, "grad_norm": 0.20412817796509725, "kl": 0.29296875, "learning_rate": 6.555944055944057e-06, "loss": -0.0005, "step": 376 }, { "clip_ratio": 0.0019930994603782892, "completions/clipped_ratio": 0.1428571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 186.0, "completions/mean_length": 174.00001525878906, "completions/mean_terminated_length": 160.33334350585938, "completions/min_length": 106.0, "completions/min_terminated_length": 106.0, "epoch": 0.1319104268719384, "grad_norm": 0.3610618002556271, "kl": 0.2734375, "learning_rate": 6.573426573426574e-06, "loss": 0.0002, "num_tokens": 1899707.0, "reward": 1.3076162338256836, "reward_std": 0.1718134582042694, "rewards/check_gptzero_func/mean": 0.2321428507566452, "rewards/check_gptzero_func/std": 0.4260205328464508, "rewards/check_winston_local_func/mean": 0.0754733607172966, "rewards/check_winston_local_func/std": 0.1425040066242218, "rewards/sentence_count_match_reward_logic/mean": 1.0, "rewards/sentence_count_match_reward_logic/std": 0.0, "step": 377 }, { "clip_ratio": 0.00464494526386261, "epoch": 0.13226032190342898, "grad_norm": 0.25080261229120476, "kl": 0.259765625, "learning_rate": 6.590909090909091e-06, "loss": -0.0003, "step": 378 }, { "clip_ratio": 0.008967523463070393, "epoch": 0.13261021693491953, "grad_norm": 0.19901427866101595, "kl": 0.251953125, "learning_rate": 6.608391608391609e-06, "loss": -0.0016, "step": 379 }, { "clip_ratio": 0.01597750000655651, "epoch": 0.13296011196641008, "grad_norm": 0.1830865696337312, "kl": 0.251953125, "learning_rate": 6.6258741258741266e-06, "loss": -0.0025, "step": 380 }, { "clip_ratio": 0.0024607661180198193, "completions/clipped_ratio": 0.0, "completions/max_length": 217.0, "completions/max_terminated_length": 217.0, "completions/mean_length": 160.125, "completions/mean_terminated_length": 160.125, "completions/min_length": 96.0, "completions/min_terminated_length": 96.0, "epoch": 0.13331000699790063, "grad_norm": 0.260428045710222, "kl": 0.400390625, "learning_rate": 6.643356643356644e-06, "loss": -0.0084, "num_tokens": 1919642.0, "reward": 1.496222972869873, "reward_std": 0.1605362743139267, "rewards/check_gptzero_func/mean": 0.2678571343421936, "rewards/check_gptzero_func/std": 0.44685041904449463, "rewards/check_winston_local_func/mean": 0.261954128742218, "rewards/check_winston_local_func/std": 0.3064861297607422, "rewards/sentence_count_match_reward_logic/mean": 0.9664115309715271, "rewards/sentence_count_match_reward_logic/std": 0.16146980226039886, "step": 381 }, { "clip_ratio": 0.0040736086666584015, "epoch": 0.13365990202939118, "grad_norm": 0.24115835396666516, "kl": 0.392578125, "learning_rate": 6.660839160839161e-06, "loss": -0.0096, "step": 382 }, { "clip_ratio": 0.007449435535818338, "epoch": 0.13400979706088173, "grad_norm": 0.2270058748612584, "kl": 0.392578125, "learning_rate": 6.678321678321679e-06, "loss": -0.0107, "step": 383 }, { "clip_ratio": 0.01476029772311449, "epoch": 0.13435969209237228, "grad_norm": 0.19624832309413984, "kl": 0.388671875, "learning_rate": 6.695804195804196e-06, "loss": -0.0122, "step": 384 }, { "clip_ratio": 0.001635270775295794, "completions/clipped_ratio": 0.0, "completions/max_length": 196.0, "completions/max_terminated_length": 196.0, "completions/mean_length": 140.44644165039062, "completions/mean_terminated_length": 140.44644165039062, "completions/min_length": 102.0, "completions/min_terminated_length": 102.0, "epoch": 0.13470958712386283, "grad_norm": 0.24143660076884643, "kl": 0.341796875, "learning_rate": 6.713286713286714e-06, "loss": -0.0001, "num_tokens": 1937131.0, "reward": 1.9716657400131226, "reward_std": 0.173959881067276, "rewards/check_gptzero_func/mean": 0.6607142686843872, "rewards/check_gptzero_func/std": 0.477751761674881, "rewards/check_winston_local_func/mean": 0.3109513819217682, "rewards/check_winston_local_func/std": 0.32934409379959106, "rewards/sentence_count_match_reward_logic/mean": 1.0, "rewards/sentence_count_match_reward_logic/std": 0.0, "step": 385 }, { "clip_ratio": 0.0029957874212414026, "epoch": 0.1350594821553534, "grad_norm": 0.2354296014870618, "kl": 0.34375, "learning_rate": 6.730769230769232e-06, "loss": -0.001, "step": 386 }, { "clip_ratio": 0.008635490201413631, "epoch": 0.13540937718684395, "grad_norm": 0.214649144759786, "kl": 0.34765625, "learning_rate": 6.7482517482517485e-06, "loss": -0.002, "step": 387 }, { "clip_ratio": 0.017329266294836998, "epoch": 0.1357592722183345, "grad_norm": 0.24045837567894393, "kl": 0.361328125, "learning_rate": 6.765734265734266e-06, "loss": -0.0027, "step": 388 }, { "clip_ratio": 0.0015457982663065195, "completions/clipped_ratio": 0.0, "completions/max_length": 238.0, "completions/max_terminated_length": 238.0, "completions/mean_length": 157.10714721679688, "completions/mean_terminated_length": 157.10714721679688, "completions/min_length": 120.0, "completions/min_terminated_length": 120.0, "epoch": 0.13610916724982505, "grad_norm": 0.24833668595155353, "kl": 0.30859375, "learning_rate": 6.783216783216784e-06, "loss": 0.0017, "num_tokens": 1956521.0, "reward": 1.431527853012085, "reward_std": 0.1642507165670395, "rewards/check_gptzero_func/mean": 0.2678571343421936, "rewards/check_gptzero_func/std": 0.44685041904449463, "rewards/check_winston_local_func/mean": 0.16367056965827942, "rewards/check_winston_local_func/std": 0.20147056877613068, "rewards/sentence_count_match_reward_logic/mean": 1.0, "rewards/sentence_count_match_reward_logic/std": 0.0, "step": 389 }, { "clip_ratio": 0.007765950169414282, "epoch": 0.1364590622813156, "grad_norm": 0.2967598831260165, "kl": 0.3125, "learning_rate": 6.8006993006993015e-06, "loss": 0.0011, "step": 390 }, { "clip_ratio": 0.01029274333268404, "epoch": 0.13680895731280615, "grad_norm": 0.18895853305456725, "kl": 0.32421875, "learning_rate": 6.818181818181818e-06, "loss": 0.0002, "step": 391 }, { "clip_ratio": 0.01520638819783926, "epoch": 0.1371588523442967, "grad_norm": 0.21384742440385113, "kl": 0.33203125, "learning_rate": 6.835664335664336e-06, "loss": -0.0005, "step": 392 }, { "clip_ratio": 0.0013057651231065392, "completions/clipped_ratio": 0.1428571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 242.0, "completions/mean_length": 168.3928680419922, "completions/mean_terminated_length": 153.7916717529297, "completions/min_length": 102.0, "completions/min_terminated_length": 102.0, "epoch": 0.13750874737578728, "grad_norm": 0.2836921527065075, "kl": 0.26953125, "learning_rate": 6.853146853146854e-06, "loss": 0.0003, "num_tokens": 1977471.0, "reward": 1.7786555290222168, "reward_std": 0.1952591985464096, "rewards/check_gptzero_func/mean": 0.4464285671710968, "rewards/check_gptzero_func/std": 0.5016207695007324, "rewards/check_winston_local_func/mean": 0.3322269022464752, "rewards/check_winston_local_func/std": 0.3119673728942871, "rewards/sentence_count_match_reward_logic/mean": 1.0, "rewards/sentence_count_match_reward_logic/std": 0.0, "step": 393 }, { "clip_ratio": 0.002897304017096758, "epoch": 0.13785864240727783, "grad_norm": 0.24802098017440607, "kl": 0.271484375, "learning_rate": 6.870629370629371e-06, "loss": -0.0003, "step": 394 }, { "clip_ratio": 0.008783673867583275, "epoch": 0.13820853743876838, "grad_norm": 0.26397672533634053, "kl": 0.279296875, "learning_rate": 6.888111888111889e-06, "loss": -0.0011, "step": 395 }, { "clip_ratio": 0.013899356126785278, "epoch": 0.13855843247025892, "grad_norm": 0.20098508168778137, "kl": 0.29296875, "learning_rate": 6.905594405594406e-06, "loss": -0.0016, "step": 396 }, { "clip_ratio": 0.0011869054287672043, "completions/clipped_ratio": 0.0, "completions/max_length": 211.0, "completions/max_terminated_length": 211.0, "completions/mean_length": 140.0357208251953, "completions/mean_terminated_length": 140.0357208251953, "completions/min_length": 78.0, "completions/min_terminated_length": 78.0, "epoch": 0.13890832750174947, "grad_norm": 0.34837376907434564, "kl": 0.333984375, "learning_rate": 6.923076923076923e-06, "loss": 0.0034, "num_tokens": 1995089.0, "reward": 1.8162022829055786, "reward_std": 0.09236056357622147, "rewards/check_gptzero_func/mean": 0.4285714328289032, "rewards/check_gptzero_func/std": 0.4993502199649811, "rewards/check_winston_local_func/mean": 0.38763073086738586, "rewards/check_winston_local_func/std": 0.3814632296562195, "rewards/sentence_count_match_reward_logic/mean": 1.0, "rewards/sentence_count_match_reward_logic/std": 0.0, "step": 397 }, { "clip_ratio": 0.015551012940704823, "epoch": 0.13925822253324002, "grad_norm": 0.35441893248966344, "kl": 0.36328125, "learning_rate": 6.940559440559441e-06, "loss": 0.0026, "step": 398 }, { "clip_ratio": 0.023791054263710976, "epoch": 0.13960811756473057, "grad_norm": 0.4085952377929552, "kl": 0.361328125, "learning_rate": 6.958041958041959e-06, "loss": 0.0019, "step": 399 }, { "clip_ratio": 0.021492736414074898, "epoch": 0.13995801259622112, "grad_norm": 0.29455238320863475, "kl": 0.333984375, "learning_rate": 6.975524475524476e-06, "loss": 0.001, "step": 400 }, { "clip_ratio": 0.0013321159640327096, "completions/clipped_ratio": 0.0, "completions/max_length": 208.0, "completions/max_terminated_length": 208.0, "completions/mean_length": 166.21429443359375, "completions/mean_terminated_length": 166.21429443359375, "completions/min_length": 82.0, "completions/min_terminated_length": 82.0, "epoch": 0.1403079076277117, "grad_norm": 0.2471934930585158, "kl": 0.306640625, "learning_rate": 6.993006993006993e-06, "loss": 0.0006, "num_tokens": 2015677.0, "reward": 1.563424825668335, "reward_std": 0.027971439063549042, "rewards/check_gptzero_func/mean": 0.4285714328289032, "rewards/check_gptzero_func/std": 0.4993502199649811, "rewards/check_winston_local_func/mean": 0.13485345244407654, "rewards/check_winston_local_func/std": 0.17693433165550232, "rewards/sentence_count_match_reward_logic/mean": 1.0, "rewards/sentence_count_match_reward_logic/std": 0.0, "step": 401 }, { "clip_ratio": 0.0030935178510844707, "epoch": 0.14065780265920225, "grad_norm": 0.2176156239276549, "kl": 0.3125, "learning_rate": 7.010489510489511e-06, "loss": 0.0, "step": 402 }, { "clip_ratio": 0.007586360909044743, "epoch": 0.1410076976906928, "grad_norm": 0.24085764120233322, "kl": 0.32421875, "learning_rate": 7.0279720279720285e-06, "loss": -0.0007, "step": 403 }, { "clip_ratio": 0.012538720853626728, "epoch": 0.14135759272218335, "grad_norm": 0.18578209474477383, "kl": 0.330078125, "learning_rate": 7.045454545454546e-06, "loss": -0.0015, "step": 404 }, { "clip_ratio": 0.000557621824555099, "completions/clipped_ratio": 0.1428571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 214.0, "completions/mean_length": 189.9107208251953, "completions/mean_terminated_length": 178.89584350585938, "completions/min_length": 119.0, "completions/min_terminated_length": 119.0, "epoch": 0.1417074877536739, "grad_norm": 0.2933599302638457, "kl": 0.212890625, "learning_rate": 7.062937062937063e-06, "loss": -0.0013, "num_tokens": 2038792.0, "reward": 1.409575343132019, "reward_std": 0.20927050709724426, "rewards/check_gptzero_func/mean": 0.3392857015132904, "rewards/check_gptzero_func/std": 0.477751761674881, "rewards/check_winston_local_func/mean": 0.07028958946466446, "rewards/check_winston_local_func/std": 0.12407641112804413, "rewards/sentence_count_match_reward_logic/mean": 1.0, "rewards/sentence_count_match_reward_logic/std": 0.0, "step": 405 }, { "clip_ratio": 0.006114542484283447, "epoch": 0.14205738278516444, "grad_norm": 0.25608671062056904, "kl": 0.220703125, "learning_rate": 7.080419580419581e-06, "loss": -0.0024, "step": 406 }, { "clip_ratio": 0.014371895231306553, "epoch": 0.142407277816655, "grad_norm": 0.3396273808046611, "kl": 0.2255859375, "learning_rate": 7.097902097902098e-06, "loss": -0.003, "step": 407 }, { "clip_ratio": 0.016640661284327507, "epoch": 0.14275717284814557, "grad_norm": 0.26646861820433504, "kl": 0.224609375, "learning_rate": 7.115384615384616e-06, "loss": -0.0045, "step": 408 }, { "clip_ratio": 0.0015078979777172208, "completions/clipped_ratio": 0.0, "completions/max_length": 252.0, "completions/max_terminated_length": 252.0, "completions/mean_length": 153.60714721679688, "completions/mean_terminated_length": 153.60714721679688, "completions/min_length": 70.0, "completions/min_terminated_length": 70.0, "epoch": 0.14310706787963612, "grad_norm": 0.26052973291604614, "kl": 0.38671875, "learning_rate": 7.132867132867134e-06, "loss": 0.0029, "num_tokens": 2057674.0, "reward": 1.7425328493118286, "reward_std": 0.21579310297966003, "rewards/check_gptzero_func/mean": 0.2857142984867096, "rewards/check_gptzero_func/std": 0.4558423161506653, "rewards/check_winston_local_func/mean": 0.4568184018135071, "rewards/check_winston_local_func/std": 0.38294315338134766, "rewards/sentence_count_match_reward_logic/mean": 1.0, "rewards/sentence_count_match_reward_logic/std": 0.0, "step": 409 }, { "clip_ratio": 0.0062240371480584145, "epoch": 0.14345696291112667, "grad_norm": 0.30627486420055916, "kl": 0.39453125, "learning_rate": 7.15034965034965e-06, "loss": 0.0024, "step": 410 }, { "clip_ratio": 0.011509239673614502, "epoch": 0.14380685794261722, "grad_norm": 0.2849495622916079, "kl": 0.400390625, "learning_rate": 7.167832167832168e-06, "loss": 0.0014, "step": 411 }, { "clip_ratio": 0.01830577477812767, "epoch": 0.14415675297410777, "grad_norm": 0.20092965074118121, "kl": 0.408203125, "learning_rate": 7.185314685314686e-06, "loss": 0.0, "step": 412 }, { "clip_ratio": 0.0018688318086788058, "completions/clipped_ratio": 0.0, "completions/max_length": 232.0, "completions/max_terminated_length": 232.0, "completions/mean_length": 173.94644165039062, "completions/mean_terminated_length": 173.94644165039062, "completions/min_length": 85.0, "completions/min_terminated_length": 85.0, "epoch": 0.14450664800559831, "grad_norm": 0.27836492607473023, "kl": 0.310546875, "learning_rate": 7.202797202797203e-06, "loss": 0.0007, "num_tokens": 2078887.0, "reward": 1.566281795501709, "reward_std": 0.07921759784221649, "rewards/check_gptzero_func/mean": 0.2857142984867096, "rewards/check_gptzero_func/std": 0.4558423161506653, "rewards/check_winston_local_func/mean": 0.2805674970149994, "rewards/check_winston_local_func/std": 0.2964022755622864, "rewards/sentence_count_match_reward_logic/mean": 1.0, "rewards/sentence_count_match_reward_logic/std": 0.0, "step": 413 }, { "clip_ratio": 0.00754125090315938, "epoch": 0.14485654303708886, "grad_norm": 0.3079509555316775, "kl": 0.330078125, "learning_rate": 7.22027972027972e-06, "loss": 0.0003, "step": 414 }, { "clip_ratio": 0.013883129693567753, "epoch": 0.1452064380685794, "grad_norm": 0.37315060341012535, "kl": 0.328125, "learning_rate": 7.237762237762238e-06, "loss": -0.0004, "step": 415 }, { "clip_ratio": 0.01384143065661192, "epoch": 0.14555633310007, "grad_norm": 0.1712397730453394, "kl": 0.318359375, "learning_rate": 7.2552447552447555e-06, "loss": -0.0016, "step": 416 }, { "clip_ratio": 0.002083007711917162, "completions/clipped_ratio": 0.2321428571428571, "completions/max_length": 256.0, "completions/max_terminated_length": 256.0, "completions/mean_length": 186.7678680419922, "completions/mean_terminated_length": 165.8372039794922, "completions/min_length": 124.0, "completions/min_terminated_length": 124.0, "epoch": 0.14590622813156054, "grad_norm": 0.2545182788260153, "kl": 0.28125, "learning_rate": 7.272727272727273e-06, "loss": 0.0007, "num_tokens": 2101698.0, "reward": 1.5783374309539795, "reward_std": 0.21811677515506744, "rewards/check_gptzero_func/mean": 0.4464285671710968, "rewards/check_gptzero_func/std": 0.5016207695007324, "rewards/check_winston_local_func/mean": 0.13190877437591553, "rewards/check_winston_local_func/std": 0.13942861557006836, "rewards/sentence_count_match_reward_logic/mean": 1.0, "rewards/sentence_count_match_reward_logic/std": 0.0, "step": 417 }, { "clip_ratio": 0.004520895890891552, "epoch": 0.1462561231630511, "grad_norm": 0.2787355208969411, "kl": 0.279296875, "learning_rate": 7.290209790209791e-06, "loss": 0.0002, "step": 418 }, { "clip_ratio": 0.010786348022520542, "epoch": 0.14660601819454164, "grad_norm": 0.2590229862064906, "kl": 0.29296875, "learning_rate": 7.307692307692308e-06, "loss": -0.0007, "step": 419 }, { "clip_ratio": 0.016586506739258766, "epoch": 0.1469559132260322, "grad_norm": 0.2358676027416836, "kl": 0.298828125, "learning_rate": 7.325174825174825e-06, "loss": -0.0018, "step": 420 }, { "clip_ratio": 0.0026480332016944885, "completions/clipped_ratio": 0.0, "completions/max_length": 253.0, "completions/max_terminated_length": 253.0, "completions/mean_length": 164.0357208251953, "completions/mean_terminated_length": 164.0357208251953, "completions/min_length": 98.0, "completions/min_terminated_length": 98.0, "epoch": 0.14730580825752274, "grad_norm": 0.30720003357984127, "kl": 0.431640625, "learning_rate": 7.342657342657343e-06, "loss": -0.0004, "num_tokens": 2121700.0, "reward": 1.598415493965149, "reward_std": 0.29570960998535156, "rewards/check_gptzero_func/mean": 0.1785714328289032, "rewards/check_gptzero_func/std": 0.3864591121673584, "rewards/check_winston_local_func/mean": 0.4198439419269562, "rewards/check_winston_local_func/std": 0.3352963328361511, "rewards/sentence_count_match_reward_logic/mean": 1.0, "rewards/sentence_count_match_reward_logic/std": 0.0, "step": 421 }, { "clip_ratio": 0.0036178831942379475, "epoch": 0.14765570328901328, "grad_norm": 0.2846594213331004, "kl": 0.435546875, "learning_rate": 7.360139860139861e-06, "loss": -0.0013, "step": 422 }, { "clip_ratio": 0.008796660229563713, "epoch": 0.14800559832050386, "grad_norm": 0.2863800505503501, "kl": 0.451171875, "learning_rate": 7.377622377622379e-06, "loss": -0.0027, "step": 423 }, { "clip_ratio": 0.021984100341796875, "epoch": 0.1483554933519944, "grad_norm": 0.2555121308830602, "kl": 0.455078125, "learning_rate": 7.395104895104895e-06, "loss": -0.0044, "step": 424 }, { "clip_ratio": 0.0012805163860321045, "completions/clipped_ratio": 0.1428571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 250.0, "completions/mean_length": 199.05357360839844, "completions/mean_terminated_length": 189.5625, "completions/min_length": 123.0, "completions/min_terminated_length": 123.0, "epoch": 0.14870538838348496, "grad_norm": 0.34194350249917865, "kl": 0.314453125, "learning_rate": 7.412587412587413e-06, "loss": 0.0005, "num_tokens": 2145839.0, "reward": 1.511635661125183, "reward_std": 0.16619546711444855, "rewards/check_gptzero_func/mean": 0.3035714328289032, "rewards/check_gptzero_func/std": 0.4639608860015869, "rewards/check_winston_local_func/mean": 0.20806418359279633, "rewards/check_winston_local_func/std": 0.2967940866947174, "rewards/sentence_count_match_reward_logic/mean": 1.0, "rewards/sentence_count_match_reward_logic/std": 0.0, "step": 425 }, { "clip_ratio": 0.0061330413445830345, "epoch": 0.1490552834149755, "grad_norm": 0.29072454714716717, "kl": 0.326171875, "learning_rate": 7.43006993006993e-06, "loss": -0.0008, "step": 426 }, { "clip_ratio": 0.015007817186415195, "epoch": 0.14940517844646606, "grad_norm": 0.28926895379914497, "kl": 0.337890625, "learning_rate": 7.447552447552449e-06, "loss": -0.002, "step": 427 }, { "clip_ratio": 0.022604120895266533, "epoch": 0.1497550734779566, "grad_norm": 0.2857770812579358, "kl": 0.337890625, "learning_rate": 7.465034965034965e-06, "loss": -0.0034, "step": 428 }, { "clip_ratio": 0.0015564641216769814, "completions/clipped_ratio": 0.1428571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 242.0, "completions/mean_length": 180.55357360839844, "completions/mean_terminated_length": 167.9791717529297, "completions/min_length": 117.0, "completions/min_terminated_length": 117.0, "epoch": 0.15010496850944716, "grad_norm": 0.32301623324595324, "kl": 0.330078125, "learning_rate": 7.4825174825174825e-06, "loss": 0.0001, "num_tokens": 2168142.0, "reward": 1.5288002490997314, "reward_std": 0.11731768399477005, "rewards/check_gptzero_func/mean": 0.3214285671710968, "rewards/check_gptzero_func/std": 0.4712514281272888, "rewards/check_winston_local_func/mean": 0.2232445925474167, "rewards/check_winston_local_func/std": 0.32961133122444153, "rewards/sentence_count_match_reward_logic/mean": 0.9841269850730896, "rewards/sentence_count_match_reward_logic/std": 0.039232656359672546, "step": 429 }, { "clip_ratio": 0.004885607864707708, "epoch": 0.1504548635409377, "grad_norm": 0.28479696937038446, "kl": 0.33203125, "learning_rate": 7.500000000000001e-06, "loss": -0.0008, "step": 430 }, { "clip_ratio": 0.014452706091105938, "epoch": 0.15080475857242828, "grad_norm": 0.26869405802474833, "kl": 0.34375, "learning_rate": 7.517482517482519e-06, "loss": -0.0018, "step": 431 }, { "clip_ratio": 0.02680354006588459, "epoch": 0.15115465360391883, "grad_norm": 0.23103457671420982, "kl": 0.365234375, "learning_rate": 7.534965034965036e-06, "loss": -0.0026, "step": 432 }, { "clip_ratio": 0.002486415673047304, "completions/clipped_ratio": 0.1428571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 169.0, "completions/mean_length": 160.30357360839844, "completions/mean_terminated_length": 144.3541717529297, "completions/min_length": 111.0, "completions/min_terminated_length": 111.0, "epoch": 0.15150454863540938, "grad_norm": 0.35160577608847376, "kl": 0.400390625, "learning_rate": 7.552447552447552e-06, "loss": 0.0021, "num_tokens": 2188519.0, "reward": 1.3543288707733154, "reward_std": 0.14017397165298462, "rewards/check_gptzero_func/mean": 0.1785714328289032, "rewards/check_gptzero_func/std": 0.3864591419696808, "rewards/check_winston_local_func/mean": 0.20173154771327972, "rewards/check_winston_local_func/std": 0.2691003978252411, "rewards/sentence_count_match_reward_logic/mean": 0.9740260243415833, "rewards/sentence_count_match_reward_logic/std": 0.06419889628887177, "step": 433 }, { "clip_ratio": 0.005476833321154118, "epoch": 0.15185444366689993, "grad_norm": 0.2654789667085697, "kl": 0.423828125, "learning_rate": 7.569930069930071e-06, "loss": 0.0012, "step": 434 }, { "clip_ratio": 0.013552045449614525, "epoch": 0.15220433869839048, "grad_norm": 0.23882280788069749, "kl": 0.423828125, "learning_rate": 7.5874125874125885e-06, "loss": 0.0004, "step": 435 }, { "clip_ratio": 0.02174517884850502, "epoch": 0.15255423372988103, "grad_norm": 0.234597193501421, "kl": 0.408203125, "learning_rate": 7.604895104895106e-06, "loss": -0.0002, "step": 436 }, { "clip_ratio": 0.0033845878206193447, "completions/clipped_ratio": 0.1428571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 244.0, "completions/mean_length": 171.33929443359375, "completions/mean_terminated_length": 157.2291717529297, "completions/min_length": 72.0, "completions/min_terminated_length": 72.0, "epoch": 0.15290412876137158, "grad_norm": 0.3532584781419529, "kl": 0.455078125, "learning_rate": 7.622377622377622e-06, "loss": 0.0054, "num_tokens": 2209682.0, "reward": 1.910186529159546, "reward_std": 0.2905838191509247, "rewards/check_gptzero_func/mean": 0.3928571343421936, "rewards/check_gptzero_func/std": 0.4928053617477417, "rewards/check_winston_local_func/mean": 0.5402230620384216, "rewards/check_winston_local_func/std": 0.2783198952674866, "rewards/sentence_count_match_reward_logic/mean": 0.9771062135696411, "rewards/sentence_count_match_reward_logic/std": 0.04844817519187927, "step": 437 }, { "clip_ratio": 0.006196768023073673, "epoch": 0.15325402379286215, "grad_norm": 0.30178769005061595, "kl": 0.458984375, "learning_rate": 7.63986013986014e-06, "loss": 0.0043, "step": 438 }, { "clip_ratio": 0.015474257990717888, "epoch": 0.1536039188243527, "grad_norm": 0.2725775912421194, "kl": 0.462890625, "learning_rate": 7.657342657342658e-06, "loss": 0.0028, "step": 439 }, { "clip_ratio": 0.024890949949622154, "epoch": 0.15395381385584325, "grad_norm": 0.22796149207741565, "kl": 0.462890625, "learning_rate": 7.674825174825176e-06, "loss": 0.002, "step": 440 }, { "clip_ratio": 0.002534298226237297, "completions/clipped_ratio": 0.1428571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 204.0, "completions/mean_length": 146.6428680419922, "completions/mean_terminated_length": 128.4166717529297, "completions/min_length": 69.0, "completions/min_terminated_length": 69.0, "epoch": 0.1543037088873338, "grad_norm": 0.39104597093023535, "kl": 0.39453125, "learning_rate": 7.692307692307694e-06, "loss": 0.0, "num_tokens": 2228062.0, "reward": 1.6101306676864624, "reward_std": 0.16989201307296753, "rewards/check_gptzero_func/mean": 0.2857142984867096, "rewards/check_gptzero_func/std": 0.4558423161506653, "rewards/check_winston_local_func/mean": 0.3374033272266388, "rewards/check_winston_local_func/std": 0.24404221773147583, "rewards/sentence_count_match_reward_logic/mean": 0.9870129823684692, "rewards/sentence_count_match_reward_logic/std": 0.03209943696856499, "step": 441 }, { "clip_ratio": 0.00732621131464839, "epoch": 0.15465360391882435, "grad_norm": 0.2948704028394016, "kl": 0.40625, "learning_rate": 7.70979020979021e-06, "loss": -0.0011, "step": 442 }, { "clip_ratio": 0.02522905543446541, "epoch": 0.1550034989503149, "grad_norm": 0.3413858626362577, "kl": 0.42578125, "learning_rate": 7.727272727272727e-06, "loss": -0.0023, "step": 443 }, { "clip_ratio": 0.03583642095327377, "epoch": 0.15535339398180545, "grad_norm": 0.23930279592605674, "kl": 0.423828125, "learning_rate": 7.744755244755245e-06, "loss": -0.0033, "step": 444 }, { "clip_ratio": 0.001314126537181437, "completions/clipped_ratio": 0.0, "completions/max_length": 216.0, "completions/max_terminated_length": 216.0, "completions/mean_length": 163.94644165039062, "completions/mean_terminated_length": 163.94644165039062, "completions/min_length": 125.0, "completions/min_terminated_length": 125.0, "epoch": 0.155703289013296, "grad_norm": 0.2907006313093877, "kl": 0.396484375, "learning_rate": 7.762237762237763e-06, "loss": 0.0014, "num_tokens": 2248251.0, "reward": 1.7949892282485962, "reward_std": 0.17679433524608612, "rewards/check_gptzero_func/mean": 0.4285714328289032, "rewards/check_gptzero_func/std": 0.4993502199649811, "rewards/check_winston_local_func/mean": 0.3664177358150482, "rewards/check_winston_local_func/std": 0.2660636901855469, "rewards/sentence_count_match_reward_logic/mean": 1.0, "rewards/sentence_count_match_reward_logic/std": 0.0, "step": 445 }, { "clip_ratio": 0.0040738522075116634, "epoch": 0.15605318404478657, "grad_norm": 0.3010602195747919, "kl": 0.390625, "learning_rate": 7.77972027972028e-06, "loss": 0.0006, "step": 446 }, { "clip_ratio": 0.009368784725666046, "epoch": 0.15640307907627712, "grad_norm": 0.2634248791172037, "kl": 0.388671875, "learning_rate": 7.797202797202798e-06, "loss": -0.0007, "step": 447 }, { "clip_ratio": 0.02144339308142662, "epoch": 0.15675297410776767, "grad_norm": 0.22856962756682944, "kl": 0.404296875, "learning_rate": 7.814685314685316e-06, "loss": -0.002, "step": 448 }, { "clip_ratio": 0.0026887166313827038, "completions/clipped_ratio": 0.0, "completions/max_length": 249.0, "completions/max_terminated_length": 249.0, "completions/mean_length": 171.8928680419922, "completions/mean_terminated_length": 171.8928680419922, "completions/min_length": 87.0, "completions/min_terminated_length": 87.0, "epoch": 0.15710286913925822, "grad_norm": 0.3689375507897709, "kl": 0.458984375, "learning_rate": 7.832167832167833e-06, "loss": 0.0048, "num_tokens": 2269093.0, "reward": 1.9781241416931152, "reward_std": 0.24992763996124268, "rewards/check_gptzero_func/mean": 0.5535714030265808, "rewards/check_gptzero_func/std": 0.5016207695007324, "rewards/check_winston_local_func/mean": 0.43526691198349, "rewards/check_winston_local_func/std": 0.36228349804878235, "rewards/sentence_count_match_reward_logic/mean": 0.9892857670783997, "rewards/sentence_count_match_reward_logic/std": 0.045441556721925735, "step": 449 }, { "clip_ratio": 0.009120641276240349, "epoch": 0.15745276417074877, "grad_norm": 0.2937618912741811, "kl": 0.486328125, "learning_rate": 7.84965034965035e-06, "loss": 0.0039, "step": 450 }, { "clip_ratio": 0.0207231305539608, "epoch": 0.15780265920223932, "grad_norm": 0.4157827281167139, "kl": 0.50390625, "learning_rate": 7.867132867132867e-06, "loss": 0.0032, "step": 451 }, { "clip_ratio": 0.023313628509640694, "epoch": 0.15815255423372987, "grad_norm": 0.32644339484666607, "kl": 0.50390625, "learning_rate": 7.884615384615384e-06, "loss": 0.0018, "step": 452 }, { "clip_ratio": 0.0033437241800129414, "completions/clipped_ratio": 0.0714285714285714, "completions/max_length": 256.0, "completions/max_terminated_length": 256.0, "completions/mean_length": 141.875, "completions/mean_terminated_length": 133.09616088867188, "completions/min_length": 71.0, "completions/min_terminated_length": 71.0, "epoch": 0.15850244926522045, "grad_norm": 0.33729450206960615, "kl": 0.48046875, "learning_rate": 7.902097902097902e-06, "loss": 0.003, "num_tokens": 2286902.0, "reward": 2.009737491607666, "reward_std": 0.3233366012573242, "rewards/check_gptzero_func/mean": 0.5, "rewards/check_gptzero_func/std": 0.5045249462127686, "rewards/check_winston_local_func/mean": 0.5097372531890869, "rewards/check_winston_local_func/std": 0.39070695638656616, "rewards/sentence_count_match_reward_logic/mean": 1.0, "rewards/sentence_count_match_reward_logic/std": 0.0, "step": 453 }, { "clip_ratio": 0.007453261874616146, "epoch": 0.158852344296711, "grad_norm": 0.27947706346562434, "kl": 0.48046875, "learning_rate": 7.91958041958042e-06, "loss": 0.0018, "step": 454 }, { "clip_ratio": 0.02038760669529438, "epoch": 0.15920223932820154, "grad_norm": 0.39955921023260754, "kl": 0.494140625, "learning_rate": 7.937062937062937e-06, "loss": 0.0009, "step": 455 }, { "clip_ratio": 0.024210628122091293, "epoch": 0.1595521343596921, "grad_norm": 0.19561317452428248, "kl": 0.51953125, "learning_rate": 7.954545454545455e-06, "loss": -0.0004, "step": 456 }, { "clip_ratio": 0.0019444540375843644, "completions/clipped_ratio": 0.0892857142857143, "completions/max_length": 256.0, "completions/max_terminated_length": 255.0, "completions/mean_length": 209.92857360839844, "completions/mean_terminated_length": 205.41177368164062, "completions/min_length": 162.0, "completions/min_terminated_length": 162.0, "epoch": 0.15990202939118264, "grad_norm": 0.3382004286253432, "kl": 0.296875, "learning_rate": 7.972027972027973e-06, "loss": 0.002, "num_tokens": 2312170.0, "reward": 1.4092880487442017, "reward_std": 0.12459789216518402, "rewards/check_gptzero_func/mean": 0.25, "rewards/check_gptzero_func/std": 0.43693143129348755, "rewards/check_winston_local_func/mean": 0.1592879295349121, "rewards/check_winston_local_func/std": 0.1978483647108078, "rewards/sentence_count_match_reward_logic/mean": 1.0, "rewards/sentence_count_match_reward_logic/std": 0.0, "step": 457 }, { "clip_ratio": 0.004722057841718197, "epoch": 0.1602519244226732, "grad_norm": 0.3073006079508227, "kl": 0.298828125, "learning_rate": 7.98951048951049e-06, "loss": 0.0008, "step": 458 }, { "clip_ratio": 0.014721943065524101, "epoch": 0.16060181945416374, "grad_norm": 0.29217915747289397, "kl": 0.314453125, "learning_rate": 8.006993006993008e-06, "loss": -0.0, "step": 459 }, { "clip_ratio": 0.02053391933441162, "epoch": 0.1609517144856543, "grad_norm": 0.2678705401363171, "kl": 0.314453125, "learning_rate": 8.024475524475524e-06, "loss": -0.0009, "step": 460 }, { "clip_ratio": 0.002114697592332959, "completions/clipped_ratio": 0.0, "completions/max_length": 230.0, "completions/max_terminated_length": 230.0, "completions/mean_length": 175.55357360839844, "completions/mean_terminated_length": 175.55357360839844, "completions/min_length": 134.0, "completions/min_terminated_length": 134.0, "epoch": 0.16130160951714487, "grad_norm": 0.3119136225569698, "kl": 0.37109375, "learning_rate": 8.041958041958042e-06, "loss": 0.0009, "num_tokens": 2333569.0, "reward": 1.5116980075836182, "reward_std": 0.195057213306427, "rewards/check_gptzero_func/mean": 0.0892857164144516, "rewards/check_gptzero_func/std": 0.28773635625839233, "rewards/check_winston_local_func/mean": 0.42241212725639343, "rewards/check_winston_local_func/std": 0.3077186644077301, "rewards/sentence_count_match_reward_logic/mean": 1.0, "rewards/sentence_count_match_reward_logic/std": 0.0, "step": 461 }, { "clip_ratio": 0.004161514807492495, "epoch": 0.16165150454863542, "grad_norm": 0.30002661620782034, "kl": 0.37109375, "learning_rate": 8.05944055944056e-06, "loss": 0.0, "step": 462 }, { "clip_ratio": 0.012802811339497566, "epoch": 0.16200139958012597, "grad_norm": 0.3190051749243094, "kl": 0.380859375, "learning_rate": 8.076923076923077e-06, "loss": -0.001, "step": 463 }, { "clip_ratio": 0.02071589231491089, "epoch": 0.16235129461161651, "grad_norm": 0.20252739607352946, "kl": 0.388671875, "learning_rate": 8.094405594405595e-06, "loss": -0.0021, "step": 464 }, { "clip_ratio": 0.0028850622475147247, "completions/clipped_ratio": 0.0, "completions/max_length": 253.0, "completions/max_terminated_length": 253.0, "completions/mean_length": 161.1607208251953, "completions/mean_terminated_length": 161.1607208251953, "completions/min_length": 109.0, "completions/min_terminated_length": 109.0, "epoch": 0.16270118964310706, "grad_norm": 0.3484417742161421, "kl": 0.443359375, "learning_rate": 8.111888111888112e-06, "loss": 0.0031, "num_tokens": 2353258.0, "reward": 1.2427427768707275, "reward_std": 0.12674756348133087, "rewards/check_gptzero_func/mean": 0.0, "rewards/check_gptzero_func/std": 0.0, "rewards/check_winston_local_func/mean": 0.24274270236492157, "rewards/check_winston_local_func/std": 0.21333779394626617, "rewards/sentence_count_match_reward_logic/mean": 1.0, "rewards/sentence_count_match_reward_logic/std": 0.0, "step": 465 }, { "clip_ratio": 0.007440138142555952, "epoch": 0.1630510846745976, "grad_norm": 0.30496705728421397, "kl": 0.455078125, "learning_rate": 8.12937062937063e-06, "loss": 0.0018, "step": 466 }, { "clip_ratio": 0.018080251291394234, "epoch": 0.16340097970608816, "grad_norm": 0.28468719819033517, "kl": 0.470703125, "learning_rate": 8.146853146853148e-06, "loss": 0.0007, "step": 467 }, { "clip_ratio": 0.031660046428442, "epoch": 0.16375087473757874, "grad_norm": 0.24163008921716025, "kl": 0.48046875, "learning_rate": 8.164335664335665e-06, "loss": -0.0008, "step": 468 }, { "clip_ratio": 0.003314686706289649, "completions/clipped_ratio": 0.0, "completions/max_length": 224.0, "completions/max_terminated_length": 224.0, "completions/mean_length": 168.25, "completions/mean_terminated_length": 168.25, "completions/min_length": 104.0, "completions/min_terminated_length": 104.0, "epoch": 0.1641007697690693, "grad_norm": 0.38857370277334874, "kl": 0.47265625, "learning_rate": 8.181818181818183e-06, "loss": 0.0031, "num_tokens": 2373720.0, "reward": 1.7488372325897217, "reward_std": 0.24989056587219238, "rewards/check_gptzero_func/mean": 0.3571428656578064, "rewards/check_gptzero_func/std": 0.48349374532699585, "rewards/check_winston_local_func/mean": 0.40700048208236694, "rewards/check_winston_local_func/std": 0.33777353167533875, "rewards/sentence_count_match_reward_logic/mean": 0.9846938848495483, "rewards/sentence_count_match_reward_logic/std": 0.044584840536117554, "step": 469 }, { "clip_ratio": 0.01585053838789463, "epoch": 0.16445066480055984, "grad_norm": 0.4916826561992214, "kl": 0.484375, "learning_rate": 8.199300699300699e-06, "loss": 0.0023, "step": 470 }, { "clip_ratio": 0.01804908737540245, "epoch": 0.1648005598320504, "grad_norm": 0.265728848827889, "kl": 0.455078125, "learning_rate": 8.216783216783217e-06, "loss": 0.001, "step": 471 }, { "clip_ratio": 0.024978479370474815, "epoch": 0.16515045486354094, "grad_norm": 0.3665948976690427, "kl": 0.455078125, "learning_rate": 8.234265734265734e-06, "loss": -0.0001, "step": 472 }, { "clip_ratio": 0.0011671276297420263, "completions/clipped_ratio": 0.0, "completions/max_length": 228.0, "completions/max_terminated_length": 228.0, "completions/mean_length": 188.85714721679688, "completions/mean_terminated_length": 188.85714721679688, "completions/min_length": 125.0, "completions/min_terminated_length": 125.0, "epoch": 0.16550034989503148, "grad_norm": 0.42550815540823816, "kl": 0.408203125, "learning_rate": 8.251748251748254e-06, "loss": 0.0033, "num_tokens": 2396640.0, "reward": 1.588765025138855, "reward_std": 0.10541228204965591, "rewards/check_gptzero_func/mean": 0.2678571343421936, "rewards/check_gptzero_func/std": 0.44685041904449463, "rewards/check_winston_local_func/mean": 0.320907860994339, "rewards/check_winston_local_func/std": 0.3647613823413849, "rewards/sentence_count_match_reward_logic/mean": 1.0, "rewards/sentence_count_match_reward_logic/std": 0.0, "step": 473 }, { "clip_ratio": 0.016155285760760307, "epoch": 0.16585024492652203, "grad_norm": 0.5664178098976281, "kl": 0.423828125, "learning_rate": 8.26923076923077e-06, "loss": 0.0019, "step": 474 }, { "clip_ratio": 0.01407536305487156, "epoch": 0.16620013995801258, "grad_norm": 0.2348971329198549, "kl": 0.42578125, "learning_rate": 8.286713286713287e-06, "loss": 0.0004, "step": 475 }, { "clip_ratio": 0.02166922576725483, "epoch": 0.16655003498950316, "grad_norm": 0.4843177887295129, "kl": 0.42578125, "learning_rate": 8.304195804195805e-06, "loss": -0.0006, "step": 476 }, { "clip_ratio": 0.001625568955205381, "completions/clipped_ratio": 0.0714285714285714, "completions/max_length": 256.0, "completions/max_terminated_length": 251.0, "completions/mean_length": 194.73214721679688, "completions/mean_terminated_length": 190.0192413330078, "completions/min_length": 119.0, "completions/min_terminated_length": 119.0, "epoch": 0.1668999300209937, "grad_norm": 0.30309705181069496, "kl": 0.34765625, "learning_rate": 8.321678321678323e-06, "loss": 0.0043, "num_tokens": 2420073.0, "reward": 1.1887184381484985, "reward_std": 0.17804189026355743, "rewards/check_gptzero_func/mean": 0.1071428582072258, "rewards/check_gptzero_func/std": 0.3120938837528229, "rewards/check_winston_local_func/mean": 0.08157553523778915, "rewards/check_winston_local_func/std": 0.11442459374666214, "rewards/sentence_count_match_reward_logic/mean": 1.0, "rewards/sentence_count_match_reward_logic/std": 0.0, "step": 477 }, { "clip_ratio": 0.009027406573295593, "epoch": 0.16724982505248426, "grad_norm": 0.4809112480453129, "kl": 0.349609375, "learning_rate": 8.33916083916084e-06, "loss": 0.0032, "step": 478 }, { "clip_ratio": 0.009488689713180065, "epoch": 0.1675997200839748, "grad_norm": 0.2993516242069882, "kl": 0.35546875, "learning_rate": 8.356643356643356e-06, "loss": 0.0017, "step": 479 }, { "clip_ratio": 0.01604805700480938, "epoch": 0.16794961511546536, "grad_norm": 0.5126614338436082, "kl": 0.357421875, "learning_rate": 8.374125874125874e-06, "loss": 0.0008, "step": 480 }, { "clip_ratio": 0.0020686278585344553, "completions/clipped_ratio": 0.125, "completions/max_length": 256.0, "completions/max_terminated_length": 256.0, "completions/mean_length": 181.35714721679688, "completions/mean_terminated_length": 170.69387817382812, "completions/min_length": 104.0, "completions/min_terminated_length": 104.0, "epoch": 0.1682995101469559, "grad_norm": 0.34314311922360313, "kl": 0.423828125, "learning_rate": 8.391608391608393e-06, "loss": 0.0029, "num_tokens": 2442005.0, "reward": 2.0099587440490723, "reward_std": 0.1696201115846634, "rewards/check_gptzero_func/mean": 0.5178571343421936, "rewards/check_gptzero_func/std": 0.5042031407356262, "rewards/check_winston_local_func/mean": 0.49210163950920105, "rewards/check_winston_local_func/std": 0.39190492033958435, "rewards/sentence_count_match_reward_logic/mean": 1.0, "rewards/sentence_count_match_reward_logic/std": 0.0, "step": 481 }, { "clip_ratio": 0.012048572301864624, "epoch": 0.16864940517844645, "grad_norm": 0.5595177895601937, "kl": 0.42578125, "learning_rate": 8.40909090909091e-06, "loss": 0.0024, "step": 482 }, { "clip_ratio": 0.019190585240721703, "epoch": 0.16899930020993703, "grad_norm": 0.31017042962933483, "kl": 0.443359375, "learning_rate": 8.426573426573428e-06, "loss": 0.0011, "step": 483 }, { "clip_ratio": 0.02366839535534382, "epoch": 0.16934919524142758, "grad_norm": 0.3361760148858441, "kl": 0.45703125, "learning_rate": 8.444055944055944e-06, "loss": 0.0001, "step": 484 }, { "clip_ratio": 0.0024166235234588385, "completions/clipped_ratio": 0.0, "completions/max_length": 206.0, "completions/max_terminated_length": 206.0, "completions/mean_length": 154.83929443359375, "completions/mean_terminated_length": 154.83929443359375, "completions/min_length": 81.0, "completions/min_terminated_length": 81.0, "epoch": 0.16969909027291813, "grad_norm": 0.3449551946555126, "kl": 0.48828125, "learning_rate": 8.461538461538462e-06, "loss": 0.0007, "num_tokens": 2461092.0, "reward": 1.3444124460220337, "reward_std": 0.18194104731082916, "rewards/check_gptzero_func/mean": 0.125, "rewards/check_gptzero_func/std": 0.3337118923664093, "rewards/check_winston_local_func/mean": 0.21941246092319489, "rewards/check_winston_local_func/std": 0.18975071609020233, "rewards/sentence_count_match_reward_logic/mean": 1.0, "rewards/sentence_count_match_reward_logic/std": 0.0, "step": 485 }, { "clip_ratio": 0.008729660883545876, "epoch": 0.17004898530440868, "grad_norm": 0.33816269276599986, "kl": 0.5, "learning_rate": 8.47902097902098e-06, "loss": -0.0001, "step": 486 }, { "clip_ratio": 0.018869876861572266, "epoch": 0.17039888033589923, "grad_norm": 0.3673047792825678, "kl": 0.48828125, "learning_rate": 8.496503496503497e-06, "loss": -0.0014, "step": 487 }, { "clip_ratio": 0.030215127393603325, "epoch": 0.17074877536738978, "grad_norm": 0.23929203828061588, "kl": 0.5078125, "learning_rate": 8.513986013986013e-06, "loss": -0.0023, "step": 488 }, { "clip_ratio": 0.0017747401725500822, "completions/clipped_ratio": 0.0, "completions/max_length": 250.0, "completions/max_terminated_length": 250.0, "completions/mean_length": 190.30357360839844, "completions/mean_terminated_length": 190.30357360839844, "completions/min_length": 117.0, "completions/min_terminated_length": 117.0, "epoch": 0.17109867039888033, "grad_norm": 0.31509850052968436, "kl": 0.384765625, "learning_rate": 8.531468531468533e-06, "loss": 0.0015, "num_tokens": 2484085.0, "reward": 1.7836031913757324, "reward_std": 0.1947409212589264, "rewards/check_gptzero_func/mean": 0.375, "rewards/check_gptzero_func/std": 0.48850420117378235, "rewards/check_winston_local_func/mean": 0.40860313177108765, "rewards/check_winston_local_func/std": 0.2514760494232178, "rewards/sentence_count_match_reward_logic/mean": 1.0, "rewards/sentence_count_match_reward_logic/std": 0.0, "step": 489 }, { "clip_ratio": 0.006396306212991476, "epoch": 0.17144856543037088, "grad_norm": 0.34592448875890475, "kl": 0.38671875, "learning_rate": 8.54895104895105e-06, "loss": 0.0004, "step": 490 }, { "clip_ratio": 0.01266641728579998, "epoch": 0.17179846046186145, "grad_norm": 0.2876968771860765, "kl": 0.390625, "learning_rate": 8.566433566433568e-06, "loss": -0.0009, "step": 491 }, { "clip_ratio": 0.021901670843362808, "epoch": 0.172148355493352, "grad_norm": 0.5243611789872008, "kl": 0.388671875, "learning_rate": 8.583916083916086e-06, "loss": -0.0017, "step": 492 }, { "clip_ratio": 0.0029558849055320024, "completions/clipped_ratio": 0.1428571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 254.0, "completions/mean_length": 187.57144165039062, "completions/mean_terminated_length": 176.1666717529297, "completions/min_length": 123.0, "completions/min_terminated_length": 123.0, "epoch": 0.17249825052484255, "grad_norm": 0.3339814704370877, "kl": 0.40234375, "learning_rate": 8.601398601398602e-06, "loss": 0.0018, "num_tokens": 2507053.0, "reward": 1.3852427005767822, "reward_std": 0.12461994588375092, "rewards/check_gptzero_func/mean": 0.1964285671710968, "rewards/check_gptzero_func/std": 0.40089187026023865, "rewards/check_winston_local_func/mean": 0.18881405889987946, "rewards/check_winston_local_func/std": 0.22928322851657867, "rewards/sentence_count_match_reward_logic/mean": 1.0, "rewards/sentence_count_match_reward_logic/std": 0.0, "step": 493 }, { "clip_ratio": 0.010974119417369366, "epoch": 0.1728481455563331, "grad_norm": 0.5977058150498048, "kl": 0.408203125, "learning_rate": 8.61888111888112e-06, "loss": 0.0013, "step": 494 }, { "clip_ratio": 0.013151595368981361, "epoch": 0.17319804058782365, "grad_norm": 0.2874419588135076, "kl": 0.404296875, "learning_rate": 8.636363636363637e-06, "loss": -0.0008, "step": 495 }, { "clip_ratio": 0.02555953897535801, "epoch": 0.1735479356193142, "grad_norm": 0.38444355670503044, "kl": 0.41015625, "learning_rate": 8.653846153846155e-06, "loss": -0.0016, "step": 496 }, { "clip_ratio": 0.0030345325358211994, "completions/clipped_ratio": 0.0, "completions/max_length": 245.0, "completions/max_terminated_length": 245.0, "completions/mean_length": 149.57144165039062, "completions/mean_terminated_length": 149.57144165039062, "completions/min_length": 89.0, "completions/min_terminated_length": 89.0, "epoch": 0.17389783065080475, "grad_norm": 0.3635433428874623, "kl": 0.474609375, "learning_rate": 8.671328671328672e-06, "loss": -0.0009, "num_tokens": 2525453.0, "reward": 1.532361388206482, "reward_std": 0.07404016703367233, "rewards/check_gptzero_func/mean": 0.1428571492433548, "rewards/check_gptzero_func/std": 0.3530939221382141, "rewards/check_winston_local_func/mean": 0.3895041346549988, "rewards/check_winston_local_func/std": 0.3913831114768982, "rewards/sentence_count_match_reward_logic/mean": 1.0, "rewards/sentence_count_match_reward_logic/std": 0.0, "step": 497 }, { "clip_ratio": 0.016416609287261963, "epoch": 0.17424772568229532, "grad_norm": 0.5866410149457207, "kl": 0.54296875, "learning_rate": 8.68881118881119e-06, "loss": -0.0013, "step": 498 }, { "clip_ratio": 0.018947849050164223, "epoch": 0.17459762071378587, "grad_norm": 0.2650596998359, "kl": 0.51171875, "learning_rate": 8.706293706293708e-06, "loss": -0.0028, "step": 499 }, { "clip_ratio": 0.027842286974191666, "epoch": 0.17494751574527642, "grad_norm": 0.38085187505763957, "kl": 0.51171875, "learning_rate": 8.723776223776225e-06, "loss": -0.0033, "step": 500 }, { "clip_ratio": 0.0018209757981821895, "completions/clipped_ratio": 0.0, "completions/max_length": 200.0, "completions/max_terminated_length": 200.0, "completions/mean_length": 147.2678680419922, "completions/mean_terminated_length": 147.2678680419922, "completions/min_length": 115.0, "completions/min_terminated_length": 115.0, "epoch": 0.17529741077676697, "grad_norm": 0.3697608689392851, "kl": 0.54296875, "learning_rate": 8.741258741258743e-06, "loss": -0.0005, "num_tokens": 2543644.0, "reward": 1.7672983407974243, "reward_std": 0.2623600959777832, "rewards/check_gptzero_func/mean": 0.2857142984867096, "rewards/check_gptzero_func/std": 0.4558423161506653, "rewards/check_winston_local_func/mean": 0.48158401250839233, "rewards/check_winston_local_func/std": 0.30237624049186707, "rewards/sentence_count_match_reward_logic/mean": 1.0, "rewards/sentence_count_match_reward_logic/std": 0.0, "step": 501 }, { "clip_ratio": 0.010607264004647732, "epoch": 0.17564730580825752, "grad_norm": 0.41867182965528976, "kl": 0.5625, "learning_rate": 8.758741258741259e-06, "loss": -0.0013, "step": 502 }, { "clip_ratio": 0.01620391756296158, "epoch": 0.17599720083974807, "grad_norm": 0.23037632334894964, "kl": 0.57421875, "learning_rate": 8.776223776223777e-06, "loss": -0.0027, "step": 503 }, { "clip_ratio": 0.027134869247674942, "epoch": 0.17634709587123862, "grad_norm": 0.2785901178516869, "kl": 0.57421875, "learning_rate": 8.793706293706294e-06, "loss": -0.0038, "step": 504 }, { "clip_ratio": 0.002535139909014106, "completions/clipped_ratio": 0.0, "completions/max_length": 216.0, "completions/max_terminated_length": 216.0, "completions/mean_length": 161.08929443359375, "completions/mean_terminated_length": 161.08929443359375, "completions/min_length": 110.0, "completions/min_terminated_length": 110.0, "epoch": 0.17669699090272917, "grad_norm": 0.2911134047847343, "kl": 0.453125, "learning_rate": 8.811188811188812e-06, "loss": 0.0025, "num_tokens": 2563385.0, "reward": 1.8466441631317139, "reward_std": 0.15541155636310577, "rewards/check_gptzero_func/mean": 0.4285714328289032, "rewards/check_gptzero_func/std": 0.4993502199649811, "rewards/check_winston_local_func/mean": 0.4180726408958435, "rewards/check_winston_local_func/std": 0.344923734664917, "rewards/sentence_count_match_reward_logic/mean": 1.0, "rewards/sentence_count_match_reward_logic/std": 0.0, "step": 505 }, { "clip_ratio": 0.005856961943209171, "epoch": 0.17704688593421974, "grad_norm": 0.2910753182535158, "kl": 0.458984375, "learning_rate": 8.82867132867133e-06, "loss": 0.0015, "step": 506 }, { "clip_ratio": 0.015433662571012974, "epoch": 0.1773967809657103, "grad_norm": 0.28400685904470285, "kl": 0.466796875, "learning_rate": 8.846153846153847e-06, "loss": 0.0005, "step": 507 }, { "clip_ratio": 0.02246575430035591, "epoch": 0.17774667599720084, "grad_norm": 0.20581495294363472, "kl": 0.474609375, "learning_rate": 8.863636363636365e-06, "loss": -0.0003, "step": 508 }, { "clip_ratio": 0.002999447053298354, "completions/clipped_ratio": 0.1428571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 246.0, "completions/mean_length": 166.4107208251953, "completions/mean_terminated_length": 151.4791717529297, "completions/min_length": 65.0, "completions/min_terminated_length": 65.0, "epoch": 0.1780965710286914, "grad_norm": 0.3889910593054323, "kl": 0.58203125, "learning_rate": 8.881118881118883e-06, "loss": 0.0029, "num_tokens": 2584080.0, "reward": 1.5861401557922363, "reward_std": 0.16534475982189178, "rewards/check_gptzero_func/mean": 0.3035714328289032, "rewards/check_gptzero_func/std": 0.4639608860015869, "rewards/check_winston_local_func/mean": 0.28256872296333313, "rewards/check_winston_local_func/std": 0.2864142954349518, "rewards/sentence_count_match_reward_logic/mean": 1.0, "rewards/sentence_count_match_reward_logic/std": 0.0, "step": 509 }, { "clip_ratio": 0.01457700040191412, "epoch": 0.17844646606018194, "grad_norm": 0.5328681420404116, "kl": 0.5859375, "learning_rate": 8.8986013986014e-06, "loss": 0.0021, "step": 510 }, { "clip_ratio": 0.01978486403822899, "epoch": 0.1787963610916725, "grad_norm": 0.33759797895590665, "kl": 0.55859375, "learning_rate": 8.916083916083916e-06, "loss": 0.0004, "step": 511 }, { "clip_ratio": 0.030716633424162865, "epoch": 0.17914625612316304, "grad_norm": 0.3090373707542759, "kl": 0.55859375, "learning_rate": 8.933566433566434e-06, "loss": -0.0004, "step": 512 }, { "clip_ratio": 0.002454178174957633, "completions/clipped_ratio": 0.0, "completions/max_length": 222.0, "completions/max_terminated_length": 222.0, "completions/mean_length": 166.0357208251953, "completions/mean_terminated_length": 166.0357208251953, "completions/min_length": 98.0, "completions/min_terminated_length": 98.0, "epoch": 0.17949615115465362, "grad_norm": 0.4896429320344099, "kl": 0.498046875, "learning_rate": 8.951048951048951e-06, "loss": 0.0006, "num_tokens": 2604114.0, "reward": 2.2359702587127686, "reward_std": 0.21801330149173737, "rewards/check_gptzero_func/mean": 0.6071428656578064, "rewards/check_gptzero_func/std": 0.4928053617477417, "rewards/check_winston_local_func/mean": 0.6288270950317383, "rewards/check_winston_local_func/std": 0.22519780695438385, "rewards/sentence_count_match_reward_logic/mean": 1.0, "rewards/sentence_count_match_reward_logic/std": 0.0, "step": 513 }, { "clip_ratio": 0.0225469209253788, "epoch": 0.17984604618614417, "grad_norm": 0.6836754639631413, "kl": 0.51171875, "learning_rate": 8.968531468531469e-06, "loss": 0.0004, "step": 514 }, { "clip_ratio": 0.022309375926852226, "epoch": 0.18019594121763471, "grad_norm": 0.4160429320340005, "kl": 0.5078125, "learning_rate": 8.986013986013987e-06, "loss": -0.0012, "step": 515 }, { "clip_ratio": 0.025239741429686546, "epoch": 0.18054583624912526, "grad_norm": 0.6602029534874106, "kl": 0.48828125, "learning_rate": 9.003496503496504e-06, "loss": -0.0014, "step": 516 }, { "clip_ratio": 0.002277107210829854, "completions/clipped_ratio": 0.0, "completions/max_length": 252.0, "completions/max_terminated_length": 252.0, "completions/mean_length": 179.67857360839844, "completions/mean_terminated_length": 179.67857360839844, "completions/min_length": 69.0, "completions/min_terminated_length": 69.0, "epoch": 0.1808957312806158, "grad_norm": 0.43394241233656095, "kl": 0.48046875, "learning_rate": 9.020979020979022e-06, "loss": 0.0016, "num_tokens": 2625872.0, "reward": 1.6724772453308105, "reward_std": 0.06239153817296028, "rewards/check_gptzero_func/mean": 0.4285714328289032, "rewards/check_gptzero_func/std": 0.4993502199649811, "rewards/check_winston_local_func/mean": 0.24390578269958496, "rewards/check_winston_local_func/std": 0.3079458177089691, "rewards/sentence_count_match_reward_logic/mean": 1.0, "rewards/sentence_count_match_reward_logic/std": 0.0, "step": 517 }, { "clip_ratio": 0.01570487581193447, "epoch": 0.18124562631210636, "grad_norm": 0.4926300519072286, "kl": 0.51171875, "learning_rate": 9.03846153846154e-06, "loss": 0.001, "step": 518 }, { "clip_ratio": 0.02380753494799137, "epoch": 0.1815955213435969, "grad_norm": 0.41601419215137886, "kl": 0.5234375, "learning_rate": 9.055944055944057e-06, "loss": -0.0006, "step": 519 }, { "clip_ratio": 0.030563652515411377, "epoch": 0.1819454163750875, "grad_norm": 0.5207939623029403, "kl": 0.49609375, "learning_rate": 9.073426573426573e-06, "loss": -0.0009, "step": 520 }, { "clip_ratio": 0.001674280851148069, "completions/clipped_ratio": 0.1071428571428571, "completions/max_length": 256.0, "completions/max_terminated_length": 256.0, "completions/mean_length": 171.6428680419922, "completions/mean_terminated_length": 161.51998901367188, "completions/min_length": 121.0, "completions/min_terminated_length": 121.0, "epoch": 0.18229531140657804, "grad_norm": 0.3210803203401672, "kl": 0.455078125, "learning_rate": 9.090909090909091e-06, "loss": 0.0039, "num_tokens": 2646796.0, "reward": 1.6289840936660767, "reward_std": 0.1878174990415573, "rewards/check_gptzero_func/mean": 0.25, "rewards/check_gptzero_func/std": 0.43693143129348755, "rewards/check_winston_local_func/mean": 0.3789839744567871, "rewards/check_winston_local_func/std": 0.395847886800766, "rewards/sentence_count_match_reward_logic/mean": 1.0, "rewards/sentence_count_match_reward_logic/std": 0.0, "step": 521 }, { "clip_ratio": 0.003796635428443551, "epoch": 0.18264520643806859, "grad_norm": 0.2883256404812445, "kl": 0.455078125, "learning_rate": 9.108391608391609e-06, "loss": 0.0028, "step": 522 }, { "clip_ratio": 0.014686674810945988, "epoch": 0.18299510146955913, "grad_norm": 0.28300572614676744, "kl": 0.453125, "learning_rate": 9.125874125874126e-06, "loss": 0.0021, "step": 523 }, { "clip_ratio": 0.020579392090439796, "epoch": 0.18334499650104968, "grad_norm": 0.172745116613981, "kl": 0.453125, "learning_rate": 9.143356643356644e-06, "loss": 0.0013, "step": 524 }, { "clip_ratio": 0.0030248595867305994, "completions/clipped_ratio": 0.1607142857142857, "completions/max_length": 256.0, "completions/max_terminated_length": 256.0, "completions/mean_length": 174.71429443359375, "completions/mean_terminated_length": 159.14892578125, "completions/min_length": 73.0, "completions/min_terminated_length": 73.0, "epoch": 0.18369489153254023, "grad_norm": 0.3263918873643858, "kl": 0.48828125, "learning_rate": 9.160839160839162e-06, "loss": 0.0021, "num_tokens": 2668132.0, "reward": 1.8998308181762695, "reward_std": 0.1852719634771347, "rewards/check_gptzero_func/mean": 0.25, "rewards/check_gptzero_func/std": 0.43693143129348755, "rewards/check_winston_local_func/mean": 0.6605449318885803, "rewards/check_winston_local_func/std": 0.3763046860694885, "rewards/sentence_count_match_reward_logic/mean": 0.9892857670783997, "rewards/sentence_count_match_reward_logic/std": 0.045441556721925735, "step": 525 }, { "clip_ratio": 0.005325676407665014, "epoch": 0.18404478656403078, "grad_norm": 0.3172816113872408, "kl": 0.48828125, "learning_rate": 9.17832167832168e-06, "loss": 0.0009, "step": 526 }, { "clip_ratio": 0.016637369990348816, "epoch": 0.18439468159552133, "grad_norm": 0.2137740301066888, "kl": 0.490234375, "learning_rate": 9.195804195804197e-06, "loss": -0.0004, "step": 527 }, { "clip_ratio": 0.029917102307081223, "epoch": 0.1847445766270119, "grad_norm": 0.24122424203671466, "kl": 0.494140625, "learning_rate": 9.213286713286715e-06, "loss": -0.0012, "step": 528 }, { "clip_ratio": 0.0024585823994129896, "completions/clipped_ratio": 0.0, "completions/max_length": 236.0, "completions/max_terminated_length": 236.0, "completions/mean_length": 151.7678680419922, "completions/mean_terminated_length": 151.7678680419922, "completions/min_length": 83.0, "completions/min_terminated_length": 83.0, "epoch": 0.18509447165850246, "grad_norm": 0.3836129648480788, "kl": 0.67578125, "learning_rate": 9.230769230769232e-06, "loss": 0.0016, "num_tokens": 2686687.0, "reward": 1.9570097923278809, "reward_std": 0.411967009305954, "rewards/check_gptzero_func/mean": 0.3928571343421936, "rewards/check_gptzero_func/std": 0.4928053617477417, "rewards/check_winston_local_func/mean": 0.5661367177963257, "rewards/check_winston_local_func/std": 0.3584493100643158, "rewards/sentence_count_match_reward_logic/mean": 0.9980158805847168, "rewards/sentence_count_match_reward_logic/std": 0.014847845770418644, "step": 529 }, { "clip_ratio": 0.012967901304364204, "epoch": 0.185444366689993, "grad_norm": 0.38412697169303384, "kl": 0.6875, "learning_rate": 9.248251748251748e-06, "loss": 0.0004, "step": 530 }, { "clip_ratio": 0.01901688240468502, "epoch": 0.18579426172148356, "grad_norm": 0.3153259550743991, "kl": 0.68359375, "learning_rate": 9.265734265734266e-06, "loss": -0.0005, "step": 531 }, { "clip_ratio": 0.0272157434374094, "epoch": 0.1861441567529741, "grad_norm": 0.2674336865082817, "kl": 0.67578125, "learning_rate": 9.283216783216784e-06, "loss": -0.0015, "step": 532 }, { "clip_ratio": 0.0026872451417148113, "completions/clipped_ratio": 0.0, "completions/max_length": 217.0, "completions/max_terminated_length": 217.0, "completions/mean_length": 153.46429443359375, "completions/mean_terminated_length": 153.46429443359375, "completions/min_length": 122.0, "completions/min_terminated_length": 122.0, "epoch": 0.18649405178446465, "grad_norm": 0.351892679529498, "kl": 0.6171875, "learning_rate": 9.300699300699301e-06, "loss": 0.0008, "num_tokens": 2705673.0, "reward": 1.9354158639907837, "reward_std": 0.16476771235466003, "rewards/check_gptzero_func/mean": 0.3214285671710968, "rewards/check_gptzero_func/std": 0.4712514281272888, "rewards/check_winston_local_func/mean": 0.6175585985183716, "rewards/check_winston_local_func/std": 0.3558521866798401, "rewards/sentence_count_match_reward_logic/mean": 0.9964285492897034, "rewards/sentence_count_match_reward_logic/std": 0.026726122945547104, "step": 533 }, { "clip_ratio": 0.005551089067012072, "epoch": 0.1868439468159552, "grad_norm": 0.3348199324641618, "kl": 0.6171875, "learning_rate": 9.318181818181819e-06, "loss": 0.0001, "step": 534 }, { "clip_ratio": 0.0181160569190979, "epoch": 0.18719384184744578, "grad_norm": 0.2693218215218903, "kl": 0.625, "learning_rate": 9.335664335664337e-06, "loss": -0.0014, "step": 535 }, { "clip_ratio": 0.03085288777947426, "epoch": 0.18754373687893633, "grad_norm": 0.29414920043693743, "kl": 0.64453125, "learning_rate": 9.353146853146854e-06, "loss": -0.002, "step": 536 }, { "clip_ratio": 0.0021990772802382708, "completions/clipped_ratio": 0.0, "completions/max_length": 227.0, "completions/max_terminated_length": 227.0, "completions/mean_length": 153.0178680419922, "completions/mean_terminated_length": 153.0178680419922, "completions/min_length": 86.0, "completions/min_terminated_length": 86.0, "epoch": 0.18789363191042688, "grad_norm": 0.44609001493563566, "kl": 0.60546875, "learning_rate": 9.370629370629372e-06, "loss": 0.0013, "num_tokens": 2724554.0, "reward": 2.225473165512085, "reward_std": 0.23413650691509247, "rewards/check_gptzero_func/mean": 0.5892857313156128, "rewards/check_gptzero_func/std": 0.49641573429107666, "rewards/check_winston_local_func/mean": 0.6361874341964722, "rewards/check_winston_local_func/std": 0.2751188576221466, "rewards/sentence_count_match_reward_logic/mean": 1.0, "rewards/sentence_count_match_reward_logic/std": 0.0, "step": 537 }, { "clip_ratio": 0.013675306923687458, "epoch": 0.18824352694191743, "grad_norm": 0.4066667851973514, "kl": 0.61328125, "learning_rate": 9.38811188811189e-06, "loss": -0.0002, "step": 538 }, { "clip_ratio": 0.020812558010220528, "epoch": 0.18859342197340798, "grad_norm": 0.2136696783480782, "kl": 0.62109375, "learning_rate": 9.405594405594406e-06, "loss": -0.0013, "step": 539 }, { "clip_ratio": 0.027904940769076347, "epoch": 0.18894331700489853, "grad_norm": 0.32022552051852177, "kl": 0.64453125, "learning_rate": 9.423076923076923e-06, "loss": -0.0017, "step": 540 }, { "clip_ratio": 0.002972039859741926, "completions/clipped_ratio": 0.1785714285714286, "completions/max_length": 256.0, "completions/max_terminated_length": 254.0, "completions/mean_length": 171.8928680419922, "completions/mean_terminated_length": 153.60870361328125, "completions/min_length": 109.0, "completions/min_terminated_length": 109.0, "epoch": 0.18929321203638907, "grad_norm": 0.3221693914271472, "kl": 0.494140625, "learning_rate": 9.44055944055944e-06, "loss": 0.003, "num_tokens": 2745788.0, "reward": 1.9862843751907349, "reward_std": 0.25192350149154663, "rewards/check_gptzero_func/mean": 0.5, "rewards/check_gptzero_func/std": 0.5045249462127686, "rewards/check_winston_local_func/mean": 0.4862841069698334, "rewards/check_winston_local_func/std": 0.3304797410964966, "rewards/sentence_count_match_reward_logic/mean": 1.0, "rewards/sentence_count_match_reward_logic/std": 0.0, "step": 541 }, { "clip_ratio": 0.004940464161336422, "epoch": 0.18964310706787962, "grad_norm": 0.29340415218697896, "kl": 0.50390625, "learning_rate": 9.458041958041958e-06, "loss": 0.0021, "step": 542 }, { "clip_ratio": 0.017884986475110054, "epoch": 0.1899930020993702, "grad_norm": 0.24643780372978785, "kl": 0.51953125, "learning_rate": 9.475524475524476e-06, "loss": 0.0009, "step": 543 }, { "clip_ratio": 0.029218818992376328, "epoch": 0.19034289713086075, "grad_norm": 0.23469159265804063, "kl": 0.5234375, "learning_rate": 9.493006993006994e-06, "loss": -0.0, "step": 544 }, { "clip_ratio": 0.0033024463336914778, "completions/clipped_ratio": 0.0, "completions/max_length": 239.0, "completions/max_terminated_length": 239.0, "completions/mean_length": 165.3928680419922, "completions/mean_terminated_length": 165.3928680419922, "completions/min_length": 78.0, "completions/min_terminated_length": 78.0, "epoch": 0.1906927921623513, "grad_norm": 0.39863676905182666, "kl": 0.58984375, "learning_rate": 9.510489510489511e-06, "loss": 0.0018, "num_tokens": 2765954.0, "reward": 2.2676448822021484, "reward_std": 0.2270658314228058, "rewards/check_gptzero_func/mean": 0.4642857015132904, "rewards/check_gptzero_func/std": 0.5032362341880798, "rewards/check_winston_local_func/mean": 0.8033590316772461, "rewards/check_winston_local_func/std": 0.2981581389904022, "rewards/sentence_count_match_reward_logic/mean": 1.0, "rewards/sentence_count_match_reward_logic/std": 0.0, "step": 545 }, { "clip_ratio": 0.008547174744307995, "epoch": 0.19104268719384185, "grad_norm": 0.3536297678233503, "kl": 0.59765625, "learning_rate": 9.527972027972029e-06, "loss": 0.0001, "step": 546 }, { "clip_ratio": 0.022986769676208496, "epoch": 0.1913925822253324, "grad_norm": 0.24697232520469456, "kl": 0.59765625, "learning_rate": 9.545454545454547e-06, "loss": -0.0015, "step": 547 }, { "clip_ratio": 0.03913202881813049, "epoch": 0.19174247725682295, "grad_norm": 0.23467494785790602, "kl": 0.6015625, "learning_rate": 9.562937062937063e-06, "loss": -0.0029, "step": 548 }, { "clip_ratio": 0.0032430198043584824, "completions/clipped_ratio": 0.0, "completions/max_length": 242.0, "completions/max_terminated_length": 242.0, "completions/mean_length": 171.07144165039062, "completions/mean_terminated_length": 171.07144165039062, "completions/min_length": 119.0, "completions/min_terminated_length": 119.0, "epoch": 0.1920923722883135, "grad_norm": 0.3658949054596778, "kl": 0.5546875, "learning_rate": 9.58041958041958e-06, "loss": 0.0034, "num_tokens": 2786598.0, "reward": 1.874984860420227, "reward_std": 0.2050197869539261, "rewards/check_gptzero_func/mean": 0.25, "rewards/check_gptzero_func/std": 0.43693143129348755, "rewards/check_winston_local_func/mean": 0.6249846816062927, "rewards/check_winston_local_func/std": 0.32008519768714905, "rewards/sentence_count_match_reward_logic/mean": 1.0, "rewards/sentence_count_match_reward_logic/std": 0.0, "step": 549 }, { "clip_ratio": 0.006330687552690506, "epoch": 0.19244226731980407, "grad_norm": 0.33058590509198515, "kl": 0.5546875, "learning_rate": 9.597902097902098e-06, "loss": 0.0016, "step": 550 }, { "clip_ratio": 0.0186875332146883, "epoch": 0.19279216235129462, "grad_norm": 0.22370852055279458, "kl": 0.55859375, "learning_rate": 9.615384615384616e-06, "loss": 0.0001, "step": 551 }, { "clip_ratio": 0.030639251694083214, "epoch": 0.19314205738278517, "grad_norm": 0.16912805163173983, "kl": 0.57421875, "learning_rate": 9.632867132867133e-06, "loss": -0.0005, "step": 552 }, { "clip_ratio": 0.001749050454236567, "completions/clipped_ratio": 0.2857142857142857, "completions/max_length": 256.0, "completions/max_terminated_length": 234.0, "completions/mean_length": 207.82144165039062, "completions/mean_terminated_length": 188.5500030517578, "completions/min_length": 154.0, "completions/min_terminated_length": 154.0, "epoch": 0.19349195241427572, "grad_norm": 0.4105461233668748, "kl": 0.46484375, "learning_rate": 9.650349650349651e-06, "loss": 0.0017, "num_tokens": 2811908.0, "reward": 2.2013697624206543, "reward_std": 0.170657679438591, "rewards/check_gptzero_func/mean": 0.5178571343421936, "rewards/check_gptzero_func/std": 0.5042031407356262, "rewards/check_winston_local_func/mean": 0.7180362343788147, "rewards/check_winston_local_func/std": 0.19982948899269104, "rewards/sentence_count_match_reward_logic/mean": 0.9654761552810669, "rewards/sentence_count_match_reward_logic/std": 0.06259193271398544, "step": 553 }, { "clip_ratio": 0.005959454458206892, "epoch": 0.19384184744576627, "grad_norm": 0.307879719920678, "kl": 0.47265625, "learning_rate": 9.667832167832169e-06, "loss": 0.0001, "step": 554 }, { "clip_ratio": 0.016122745350003242, "epoch": 0.19419174247725682, "grad_norm": 0.329486749308644, "kl": 0.482421875, "learning_rate": 9.685314685314686e-06, "loss": -0.0012, "step": 555 }, { "clip_ratio": 0.025875693187117577, "epoch": 0.19454163750874737, "grad_norm": 0.22994723547783094, "kl": 0.49609375, "learning_rate": 9.702797202797204e-06, "loss": -0.0025, "step": 556 }, { "clip_ratio": 0.0016035677399486303, "completions/clipped_ratio": 0.0535714285714286, "completions/max_length": 256.0, "completions/max_terminated_length": 256.0, "completions/mean_length": 182.37501525878906, "completions/mean_terminated_length": 178.20755004882812, "completions/min_length": 94.0, "completions/min_terminated_length": 94.0, "epoch": 0.19489153254023792, "grad_norm": 0.374669814603083, "kl": 0.5703125, "learning_rate": 9.72027972027972e-06, "loss": 0.0011, "num_tokens": 2834025.0, "reward": 1.9508066177368164, "reward_std": 0.20227386057376862, "rewards/check_gptzero_func/mean": 0.3035714328289032, "rewards/check_gptzero_func/std": 0.4639608860015869, "rewards/check_winston_local_func/mean": 0.6680682897567749, "rewards/check_winston_local_func/std": 0.31016644835472107, "rewards/sentence_count_match_reward_logic/mean": 0.9791666865348816, "rewards/sentence_count_match_reward_logic/std": 0.07662383466959, "step": 557 }, { "clip_ratio": 0.005917107220739126, "epoch": 0.1952414275717285, "grad_norm": 0.3051508083750182, "kl": 0.57421875, "learning_rate": 9.737762237762238e-06, "loss": -0.0006, "step": 558 }, { "clip_ratio": 0.017790958285331726, "epoch": 0.19559132260321904, "grad_norm": 0.23329909991589806, "kl": 0.58984375, "learning_rate": 9.755244755244755e-06, "loss": -0.0017, "step": 559 }, { "clip_ratio": 0.028636660426855087, "epoch": 0.1959412176347096, "grad_norm": 0.2295984385204308, "kl": 0.6015625, "learning_rate": 9.772727272727273e-06, "loss": -0.0025, "step": 560 }, { "clip_ratio": 0.0018821725388988853, "completions/clipped_ratio": 0.2857142857142857, "completions/max_length": 256.0, "completions/max_terminated_length": 227.0, "completions/mean_length": 210.94644165039062, "completions/mean_terminated_length": 192.9250030517578, "completions/min_length": 123.0, "completions/min_terminated_length": 123.0, "epoch": 0.19629111266620014, "grad_norm": 0.384807325232443, "kl": 0.458984375, "learning_rate": 9.79020979020979e-06, "loss": 0.0035, "num_tokens": 2859894.0, "reward": 1.8481383323669434, "reward_std": 0.14198236167430878, "rewards/check_gptzero_func/mean": 0.3928571343421936, "rewards/check_gptzero_func/std": 0.4928053617477417, "rewards/check_winston_local_func/mean": 0.46826809644699097, "rewards/check_winston_local_func/std": 0.2988917827606201, "rewards/sentence_count_match_reward_logic/mean": 0.9870129823684692, "rewards/sentence_count_match_reward_logic/std": 0.03209943696856499, "step": 561 }, { "clip_ratio": 0.005412397440522909, "epoch": 0.1966410076976907, "grad_norm": 0.31142703594702414, "kl": 0.458984375, "learning_rate": 9.807692307692308e-06, "loss": 0.0016, "step": 562 }, { "clip_ratio": 0.01657096855342388, "epoch": 0.19699090272918124, "grad_norm": 0.28908229367745114, "kl": 0.46875, "learning_rate": 9.825174825174826e-06, "loss": 0.0004, "step": 563 }, { "clip_ratio": 0.027738532051444054, "epoch": 0.1973407977606718, "grad_norm": 0.24839189448448634, "kl": 0.474609375, "learning_rate": 9.842657342657344e-06, "loss": -0.0004, "step": 564 }, { "clip_ratio": 0.0018233924638479948, "completions/clipped_ratio": 0.1428571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 240.0, "completions/mean_length": 184.57144165039062, "completions/mean_terminated_length": 172.6666717529297, "completions/min_length": 124.0, "completions/min_terminated_length": 124.0, "epoch": 0.19769069279216236, "grad_norm": 0.3800265189096609, "kl": 0.5234375, "learning_rate": 9.860139860139861e-06, "loss": 0.0006, "num_tokens": 2882494.0, "reward": 1.9404765367507935, "reward_std": 0.24934391677379608, "rewards/check_gptzero_func/mean": 0.3928571343421936, "rewards/check_gptzero_func/std": 0.4928053617477417, "rewards/check_winston_local_func/mean": 0.5634923577308655, "rewards/check_winston_local_func/std": 0.2936957776546478, "rewards/sentence_count_match_reward_logic/mean": 0.9841269850730896, "rewards/sentence_count_match_reward_logic/std": 0.039232656359672546, "step": 565 }, { "clip_ratio": 0.008471180684864521, "epoch": 0.1980405878236529, "grad_norm": 0.33938331766483154, "kl": 0.53515625, "learning_rate": 9.877622377622379e-06, "loss": -0.0013, "step": 566 }, { "clip_ratio": 0.016386518254876137, "epoch": 0.19839048285514346, "grad_norm": 0.2787547607639364, "kl": 0.5390625, "learning_rate": 9.895104895104895e-06, "loss": -0.0028, "step": 567 }, { "clip_ratio": 0.027985284104943275, "epoch": 0.198740377886634, "grad_norm": 0.22754025465901484, "kl": 0.54296875, "learning_rate": 9.912587412587413e-06, "loss": -0.0041, "step": 568 }, { "clip_ratio": 0.002039486076682806, "completions/clipped_ratio": 0.0, "completions/max_length": 194.0, "completions/max_terminated_length": 194.0, "completions/mean_length": 160.23214721679688, "completions/mean_terminated_length": 160.23214721679688, "completions/min_length": 122.0, "completions/min_terminated_length": 122.0, "epoch": 0.19909027291812456, "grad_norm": 0.4115167625714715, "kl": 0.58203125, "learning_rate": 9.93006993006993e-06, "loss": 0.004, "num_tokens": 2902155.0, "reward": 1.9295141696929932, "reward_std": 0.32489901781082153, "rewards/check_gptzero_func/mean": 0.3214285671710968, "rewards/check_gptzero_func/std": 0.4712514281272888, "rewards/check_winston_local_func/mean": 0.6080855131149292, "rewards/check_winston_local_func/std": 0.24930788576602936, "rewards/sentence_count_match_reward_logic/mean": 1.0, "rewards/sentence_count_match_reward_logic/std": 0.0, "step": 569 }, { "clip_ratio": 0.008871501311659813, "epoch": 0.1994401679496151, "grad_norm": 0.3261422127719558, "kl": 0.58984375, "learning_rate": 9.94755244755245e-06, "loss": 0.0026, "step": 570 }, { "clip_ratio": 0.02494584582746029, "epoch": 0.19979006298110566, "grad_norm": 0.24584006883667756, "kl": 0.6171875, "learning_rate": 9.965034965034966e-06, "loss": 0.0011, "step": 571 }, { "clip_ratio": 0.03738444671034813, "epoch": 0.2001399580125962, "grad_norm": 0.28631130409734484, "kl": 0.62890625, "learning_rate": 9.982517482517483e-06, "loss": 0.0007, "step": 572 }, { "clip_ratio": 0.0020128432661294937, "completions/clipped_ratio": 0.0, "completions/max_length": 229.0, "completions/max_terminated_length": 229.0, "completions/mean_length": 154.10714721679688, "completions/mean_terminated_length": 154.10714721679688, "completions/min_length": 89.0, "completions/min_terminated_length": 89.0, "epoch": 0.20048985304408679, "grad_norm": 0.4685681892432141, "kl": 0.6796875, "learning_rate": 1e-05, "loss": 0.005, "num_tokens": 2920961.0, "reward": 2.183530569076538, "reward_std": 0.10606995224952698, "rewards/check_gptzero_func/mean": 0.4464285671710968, "rewards/check_gptzero_func/std": 0.5016207695007324, "rewards/check_winston_local_func/mean": 0.7406731247901917, "rewards/check_winston_local_func/std": 0.23692531883716583, "rewards/sentence_count_match_reward_logic/mean": 0.9964285492897034, "rewards/sentence_count_match_reward_logic/std": 0.026726121082901955, "step": 573 }, { "clip_ratio": 0.01333075575530529, "epoch": 0.20083974807557733, "grad_norm": 0.31419951630005893, "kl": 0.6875, "learning_rate": 9.999999067523662e-06, "loss": 0.0036, "step": 574 }, { "clip_ratio": 0.025941630825400352, "epoch": 0.20118964310706788, "grad_norm": 0.43674404953294643, "kl": 0.6953125, "learning_rate": 9.999996270094992e-06, "loss": 0.0026, "step": 575 }, { "clip_ratio": 0.02915494702756405, "epoch": 0.20153953813855843, "grad_norm": 0.1760891366030249, "kl": 0.6953125, "learning_rate": 9.999991607715036e-06, "loss": 0.0015, "step": 576 }, { "clip_ratio": 0.0028188808355480433, "completions/clipped_ratio": 0.1428571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 245.0, "completions/mean_length": 190.12501525878906, "completions/mean_terminated_length": 179.14584350585938, "completions/min_length": 101.0, "completions/min_terminated_length": 101.0, "epoch": 0.20188943317004898, "grad_norm": 0.3801945264552129, "kl": 0.58984375, "learning_rate": 9.99998508038553e-06, "loss": 0.0024, "num_tokens": 2944704.0, "reward": 1.7951732873916626, "reward_std": 0.14366565644741058, "rewards/check_gptzero_func/mean": 0.4107142984867096, "rewards/check_gptzero_func/std": 0.49641573429107666, "rewards/check_winston_local_func/mean": 0.4185498058795929, "rewards/check_winston_local_func/std": 0.28563863039016724, "rewards/sentence_count_match_reward_logic/mean": 0.9659090638160706, "rewards/sentence_count_match_reward_logic/std": 0.08591663092374802, "step": 577 }, { "clip_ratio": 0.01038728840649128, "epoch": 0.20223932820153953, "grad_norm": 0.5493296821499107, "kl": 0.58984375, "learning_rate": 9.999976688108912e-06, "loss": 0.0013, "step": 578 }, { "clip_ratio": 0.016835907474160194, "epoch": 0.20258922323303008, "grad_norm": 0.29602285070645107, "kl": 0.59765625, "learning_rate": 9.999966430888313e-06, "loss": -0.0005, "step": 579 }, { "clip_ratio": 0.02855054847896099, "epoch": 0.20293911826452066, "grad_norm": 0.31334288912228936, "kl": 0.625, "learning_rate": 9.999954308727553e-06, "loss": -0.0014, "step": 580 }, { "clip_ratio": 0.0034609457943588495, "completions/clipped_ratio": 0.1428571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 192.0, "completions/mean_length": 163.10714721679688, "completions/mean_terminated_length": 147.625, "completions/min_length": 94.0, "completions/min_terminated_length": 94.0, "epoch": 0.2032890132960112, "grad_norm": 0.5302280359560508, "kl": 0.60546875, "learning_rate": 9.999940321631158e-06, "loss": 0.003, "num_tokens": 2964830.0, "reward": 2.189173698425293, "reward_std": 0.21461963653564453, "rewards/check_gptzero_func/mean": 0.3928571343421936, "rewards/check_gptzero_func/std": 0.4928053617477417, "rewards/check_winston_local_func/mean": 0.8034591674804688, "rewards/check_winston_local_func/std": 0.19749878346920013, "rewards/sentence_count_match_reward_logic/mean": 0.9928570985794067, "rewards/sentence_count_match_reward_logic/std": 0.05345224216580391, "step": 581 }, { "clip_ratio": 0.014986534602940083, "epoch": 0.20363890832750176, "grad_norm": 0.3234393989816046, "kl": 0.61328125, "learning_rate": 9.999924469604346e-06, "loss": 0.001, "step": 582 }, { "clip_ratio": 0.03097006306052208, "epoch": 0.2039888033589923, "grad_norm": 0.401015851467787, "kl": 0.6171875, "learning_rate": 9.999906752653027e-06, "loss": -0.0008, "step": 583 }, { "clip_ratio": 0.041626960039138794, "epoch": 0.20433869839048285, "grad_norm": 0.27150380824183207, "kl": 0.62109375, "learning_rate": 9.99988717078381e-06, "loss": -0.0024, "step": 584 }, { "clip_ratio": 0.0019793356768786907, "completions/clipped_ratio": 0.0, "completions/max_length": 256.0, "completions/max_terminated_length": 256.0, "completions/mean_length": 175.9107208251953, "completions/mean_terminated_length": 175.9107208251953, "completions/min_length": 90.0, "completions/min_terminated_length": 90.0, "epoch": 0.2046885934219734, "grad_norm": 0.4214758758194339, "kl": 0.5859375, "learning_rate": 9.999865724003998e-06, "loss": 0.0014, "num_tokens": 2986369.0, "reward": 2.3522846698760986, "reward_std": 0.10233016312122345, "rewards/check_gptzero_func/mean": 0.5892857313156128, "rewards/check_gptzero_func/std": 0.49641573429107666, "rewards/check_winston_local_func/mean": 0.7629989385604858, "rewards/check_winston_local_func/std": 0.21591204404830933, "rewards/sentence_count_match_reward_logic/mean": 1.0, "rewards/sentence_count_match_reward_logic/std": 0.0, "step": 585 }, { "clip_ratio": 0.007400404196232557, "epoch": 0.20503848845346395, "grad_norm": 0.3162623792410467, "kl": 0.5703125, "learning_rate": 9.999842412321593e-06, "loss": -0.0006, "step": 586 }, { "clip_ratio": 0.019566740840673447, "epoch": 0.2053883834849545, "grad_norm": 0.23750586080732652, "kl": 0.56640625, "learning_rate": 9.999817235745289e-06, "loss": -0.0018, "step": 587 }, { "clip_ratio": 0.03245891258120537, "epoch": 0.20573827851644508, "grad_norm": 0.20146624765487647, "kl": 0.56640625, "learning_rate": 9.999790194284474e-06, "loss": -0.0026, "step": 588 }, { "clip_ratio": 0.0034594896715134382, "completions/clipped_ratio": 0.0, "completions/max_length": 239.0, "completions/max_terminated_length": 239.0, "completions/mean_length": 182.83929443359375, "completions/mean_terminated_length": 182.83929443359375, "completions/min_length": 144.0, "completions/min_terminated_length": 144.0, "epoch": 0.20608817354793563, "grad_norm": 0.42071622336122194, "kl": 0.5078125, "learning_rate": 9.999761287949237e-06, "loss": 0.005, "num_tokens": 3008504.0, "reward": 2.543346405029297, "reward_std": 0.158544659614563, "rewards/check_gptzero_func/mean": 0.875, "rewards/check_gptzero_func/std": 0.3337118923664093, "rewards/check_winston_local_func/mean": 0.6683462858200073, "rewards/check_winston_local_func/std": 0.24715319275856018, "rewards/sentence_count_match_reward_logic/mean": 1.0, "rewards/sentence_count_match_reward_logic/std": 0.0, "step": 589 }, { "clip_ratio": 0.008678567595779896, "epoch": 0.20643806857942618, "grad_norm": 0.29961360384571567, "kl": 0.515625, "learning_rate": 9.99973051675036e-06, "loss": 0.003, "step": 590 }, { "clip_ratio": 0.024693036451935768, "epoch": 0.20678796361091673, "grad_norm": 0.3455409408899713, "kl": 0.53515625, "learning_rate": 9.999697880699319e-06, "loss": 0.0021, "step": 591 }, { "clip_ratio": 0.03499480336904526, "epoch": 0.20713785864240727, "grad_norm": 0.2664401775126025, "kl": 0.53515625, "learning_rate": 9.999663379808287e-06, "loss": 0.0013, "step": 592 }, { "clip_ratio": 0.0024503853637725115, "completions/clipped_ratio": 0.0, "completions/max_length": 235.0, "completions/max_terminated_length": 235.0, "completions/mean_length": 178.94644165039062, "completions/mean_terminated_length": 178.94644165039062, "completions/min_length": 116.0, "completions/min_terminated_length": 116.0, "epoch": 0.20748775367389782, "grad_norm": 0.4327168761089829, "kl": 0.62109375, "learning_rate": 9.999627014090133e-06, "loss": -0.0035, "num_tokens": 3030133.0, "reward": 1.8079938888549805, "reward_std": 0.1640446037054062, "rewards/check_gptzero_func/mean": 0.2678571343421936, "rewards/check_gptzero_func/std": 0.44685041904449463, "rewards/check_winston_local_func/mean": 0.5624579191207886, "rewards/check_winston_local_func/std": 0.3882260024547577, "rewards/sentence_count_match_reward_logic/mean": 0.9776785969734192, "rewards/sentence_count_match_reward_logic/std": 0.10419371724128723, "step": 593 }, { "clip_ratio": 0.006960414815694094, "epoch": 0.20783764870538837, "grad_norm": 0.38841606928132816, "kl": 0.6171875, "learning_rate": 9.99958878355842e-06, "loss": -0.0058, "step": 594 }, { "clip_ratio": 0.025229057297110558, "epoch": 0.20818754373687895, "grad_norm": 0.3469091434024004, "kl": 0.62109375, "learning_rate": 9.99954868822741e-06, "loss": -0.0081, "step": 595 }, { "clip_ratio": 0.040896620601415634, "epoch": 0.2085374387683695, "grad_norm": 0.2443811655344421, "kl": 0.625, "learning_rate": 9.999506728112057e-06, "loss": -0.0098, "step": 596 }, { "clip_ratio": 0.0031461443286389112, "completions/clipped_ratio": 0.0, "completions/max_length": 239.0, "completions/max_terminated_length": 239.0, "completions/mean_length": 139.80357360839844, "completions/mean_terminated_length": 139.80357360839844, "completions/min_length": 83.0, "completions/min_terminated_length": 83.0, "epoch": 0.20888733379986005, "grad_norm": 0.4514010902754298, "kl": 0.79296875, "learning_rate": 9.99946290322801e-06, "loss": 0.0036, "num_tokens": 3047258.0, "reward": 2.2834596633911133, "reward_std": 0.18222345411777496, "rewards/check_gptzero_func/mean": 0.4285714328289032, "rewards/check_gptzero_func/std": 0.4993502199649811, "rewards/check_winston_local_func/mean": 0.8682810664176941, "rewards/check_winston_local_func/std": 0.174373596906662, "rewards/sentence_count_match_reward_logic/mean": 0.9866071343421936, "rewards/sentence_count_match_reward_logic/std": 0.10022296011447906, "step": 597 }, { "clip_ratio": 0.009858719073235989, "epoch": 0.2092372288313506, "grad_norm": 0.33591413336203635, "kl": 0.8203125, "learning_rate": 9.999417213591619e-06, "loss": 0.0017, "step": 598 }, { "clip_ratio": 0.02691650576889515, "epoch": 0.20958712386284115, "grad_norm": 0.3241480920681562, "kl": 0.83984375, "learning_rate": 9.999369659219922e-06, "loss": 0.0001, "step": 599 }, { "clip_ratio": 0.04032161086797714, "epoch": 0.2099370188943317, "grad_norm": 0.22886575766072306, "kl": 0.83984375, "learning_rate": 9.999320240130658e-06, "loss": -0.001, "step": 600 }, { "clip_ratio": 0.0026814057491719723, "completions/clipped_ratio": 0.0357142857142857, "completions/max_length": 256.0, "completions/max_terminated_length": 253.0, "completions/mean_length": 184.58929443359375, "completions/mean_terminated_length": 181.94444274902344, "completions/min_length": 109.0, "completions/min_terminated_length": 109.0, "epoch": 0.21028691392582224, "grad_norm": 0.464809086321893, "kl": 0.59765625, "learning_rate": 9.999268956342261e-06, "loss": 0.0008, "num_tokens": 3069539.0, "reward": 2.1768274307250977, "reward_std": 0.15103408694267273, "rewards/check_gptzero_func/mean": 0.5535714030265808, "rewards/check_gptzero_func/std": 0.5016207695007324, "rewards/check_winston_local_func/mean": 0.6232558488845825, "rewards/check_winston_local_func/std": 0.26894286274909973, "rewards/sentence_count_match_reward_logic/mean": 1.0, "rewards/sentence_count_match_reward_logic/std": 0.0, "step": 601 }, { "clip_ratio": 0.008367151021957397, "epoch": 0.2106368089573128, "grad_norm": 0.35413830042437056, "kl": 0.6015625, "learning_rate": 9.999215807873857e-06, "loss": -0.0009, "step": 602 }, { "clip_ratio": 0.02084292285144329, "epoch": 0.21098670398880337, "grad_norm": 0.2709694660877687, "kl": 0.60546875, "learning_rate": 9.999160794745271e-06, "loss": -0.0027, "step": 603 }, { "clip_ratio": 0.03538469597697258, "epoch": 0.21133659902029392, "grad_norm": 0.2382050732926755, "kl": 0.6171875, "learning_rate": 9.999103916977022e-06, "loss": -0.0037, "step": 604 }, { "clip_ratio": 0.002271328354254365, "completions/clipped_ratio": 0.0, "completions/max_length": 225.0, "completions/max_terminated_length": 225.0, "completions/mean_length": 170.6428680419922, "completions/mean_terminated_length": 170.6428680419922, "completions/min_length": 76.0, "completions/min_terminated_length": 76.0, "epoch": 0.21168649405178447, "grad_norm": 0.39645931119241823, "kl": 0.61328125, "learning_rate": 9.999045174590324e-06, "loss": -0.0023, "num_tokens": 3090479.0, "reward": 2.135350227355957, "reward_std": 0.2625274360179901, "rewards/check_gptzero_func/mean": 0.5535714030265808, "rewards/check_gptzero_func/std": 0.5016207695007324, "rewards/check_winston_local_func/mean": 0.6246856451034546, "rewards/check_winston_local_func/std": 0.34083864092826843, "rewards/sentence_count_match_reward_logic/mean": 0.9570932388305664, "rewards/sentence_count_match_reward_logic/std": 0.07893603295087814, "step": 605 }, { "clip_ratio": 0.008572181686758995, "epoch": 0.21203638908327502, "grad_norm": 0.3151449983387175, "kl": 0.6171875, "learning_rate": 9.998984567607091e-06, "loss": -0.0039, "step": 606 }, { "clip_ratio": 0.02438836358487606, "epoch": 0.21238628411476557, "grad_norm": 0.28170211658969313, "kl": 0.6328125, "learning_rate": 9.998922096049925e-06, "loss": -0.0052, "step": 607 }, { "clip_ratio": 0.03670211508870125, "epoch": 0.21273617914625612, "grad_norm": 0.19306379513265967, "kl": 0.63671875, "learning_rate": 9.998857759942132e-06, "loss": -0.0062, "step": 608 }, { "clip_ratio": 0.0012553350534290075, "completions/clipped_ratio": 0.0, "completions/max_length": 247.0, "completions/max_terminated_length": 247.0, "completions/mean_length": 181.0178680419922, "completions/mean_terminated_length": 181.0178680419922, "completions/min_length": 86.0, "completions/min_terminated_length": 86.0, "epoch": 0.21308607417774666, "grad_norm": 0.3941479421334096, "kl": 0.60546875, "learning_rate": 9.998791559307702e-06, "loss": 0.0042, "num_tokens": 3112296.0, "reward": 1.8505374193191528, "reward_std": 0.11425817757844925, "rewards/check_gptzero_func/mean": 0.1607142835855484, "rewards/check_gptzero_func/std": 0.3705909848213196, "rewards/check_winston_local_func/mean": 0.6898230910301208, "rewards/check_winston_local_func/std": 0.32950735092163086, "rewards/sentence_count_match_reward_logic/mean": 1.0, "rewards/sentence_count_match_reward_logic/std": 0.0, "step": 609 }, { "clip_ratio": 0.0069536021910607815, "epoch": 0.21343596920923724, "grad_norm": 0.3243431643772678, "kl": 0.60546875, "learning_rate": 9.998723494171331e-06, "loss": 0.0029, "step": 610 }, { "clip_ratio": 0.019810408353805542, "epoch": 0.2137858642407278, "grad_norm": 0.2580149139692923, "kl": 0.60546875, "learning_rate": 9.998653564558408e-06, "loss": 0.0014, "step": 611 }, { "clip_ratio": 0.03233519569039345, "epoch": 0.21413575927221834, "grad_norm": 0.22067347731828016, "kl": 0.6171875, "learning_rate": 9.998581770495012e-06, "loss": 0.0005, "step": 612 }, { "clip_ratio": 0.0026760040782392025, "completions/clipped_ratio": 0.1428571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 184.0, "completions/mean_length": 170.25, "completions/mean_terminated_length": 155.95834350585938, "completions/min_length": 101.0, "completions/min_terminated_length": 101.0, "epoch": 0.2144856543037089, "grad_norm": 0.45661767893140504, "kl": 0.5625, "learning_rate": 9.998508112007925e-06, "loss": -0.0011, "num_tokens": 3133230.0, "reward": 2.020256757736206, "reward_std": 0.2792660593986511, "rewards/check_gptzero_func/mean": 0.5178571343421936, "rewards/check_gptzero_func/std": 0.5042031407356262, "rewards/check_winston_local_func/mean": 0.5023995041847229, "rewards/check_winston_local_func/std": 0.3850404620170593, "rewards/sentence_count_match_reward_logic/mean": 1.0, "rewards/sentence_count_match_reward_logic/std": 0.0, "step": 613 }, { "clip_ratio": 0.00938838068395853, "epoch": 0.21483554933519944, "grad_norm": 0.29496391237158687, "kl": 0.5703125, "learning_rate": 9.99843258912462e-06, "loss": -0.003, "step": 614 }, { "clip_ratio": 0.021352659910917282, "epoch": 0.21518544436669, "grad_norm": 0.29279685768442404, "kl": 0.5859375, "learning_rate": 9.998355201873267e-06, "loss": -0.0041, "step": 615 }, { "clip_ratio": 0.03293222934007645, "epoch": 0.21553533939818054, "grad_norm": 0.22888396536546107, "kl": 0.60546875, "learning_rate": 9.99827595028273e-06, "loss": -0.0046, "step": 616 }, { "clip_ratio": 0.003571388777345419, "completions/clipped_ratio": 0.0, "completions/max_length": 174.0, "completions/max_terminated_length": 174.0, "completions/mean_length": 136.8928680419922, "completions/mean_terminated_length": 136.8928680419922, "completions/min_length": 107.0, "completions/min_terminated_length": 107.0, "epoch": 0.21588523442967109, "grad_norm": 0.48169436310044333, "kl": 0.78125, "learning_rate": 9.998194834382567e-06, "loss": 0.0021, "num_tokens": 3150184.0, "reward": 1.988501787185669, "reward_std": 0.24264030158519745, "rewards/check_gptzero_func/mean": 0.2857142984867096, "rewards/check_gptzero_func/std": 0.4558423161506653, "rewards/check_winston_local_func/mean": 0.7027875185012817, "rewards/check_winston_local_func/std": 0.2970229983329773, "rewards/sentence_count_match_reward_logic/mean": 1.0, "rewards/sentence_count_match_reward_logic/std": 0.0, "step": 617 }, { "clip_ratio": 0.015166756696999073, "epoch": 0.21623512946116166, "grad_norm": 0.2995537544521729, "kl": 0.7890625, "learning_rate": 9.998111854203037e-06, "loss": 0.0006, "step": 618 }, { "clip_ratio": 0.03157136216759682, "epoch": 0.2165850244926522, "grad_norm": 0.2316194549025426, "kl": 0.80078125, "learning_rate": 9.99802700977509e-06, "loss": -0.0004, "step": 619 }, { "clip_ratio": 0.04313179478049278, "epoch": 0.21693491952414276, "grad_norm": 0.21435584780210015, "kl": 0.8046875, "learning_rate": 9.99794030113037e-06, "loss": -0.0009, "step": 620 }, { "clip_ratio": 0.002756283851340413, "completions/clipped_ratio": 0.1428571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 248.0, "completions/mean_length": 187.07144165039062, "completions/mean_terminated_length": 175.58334350585938, "completions/min_length": 125.0, "completions/min_terminated_length": 125.0, "epoch": 0.2172848145556333, "grad_norm": 0.4296513792343426, "kl": 0.66015625, "learning_rate": 9.997851728301219e-06, "loss": 0.0036, "num_tokens": 3172892.0, "reward": 2.112705707550049, "reward_std": 0.27391138672828674, "rewards/check_gptzero_func/mean": 0.3571428656578064, "rewards/check_gptzero_func/std": 0.48349377512931824, "rewards/check_winston_local_func/mean": 0.7555626034736633, "rewards/check_winston_local_func/std": 0.2774483263492584, "rewards/sentence_count_match_reward_logic/mean": 1.0, "rewards/sentence_count_match_reward_logic/std": 0.0, "step": 621 }, { "clip_ratio": 0.008048386313021183, "epoch": 0.21763470958712386, "grad_norm": 0.3757924344581251, "kl": 0.66015625, "learning_rate": 9.997761291320676e-06, "loss": 0.0019, "step": 622 }, { "clip_ratio": 0.019486909732222557, "epoch": 0.2179846046186144, "grad_norm": 0.23894271611467846, "kl": 0.6640625, "learning_rate": 9.997668990222473e-06, "loss": 0.0003, "step": 623 }, { "clip_ratio": 0.03265978768467903, "epoch": 0.21833449965010496, "grad_norm": 0.22000890287651892, "kl": 0.66796875, "learning_rate": 9.997574825041034e-06, "loss": -0.0003, "step": 624 }, { "clip_ratio": 0.0023628915660083294, "completions/clipped_ratio": 0.2857142857142857, "completions/max_length": 256.0, "completions/max_terminated_length": 229.0, "completions/mean_length": 191.7857208251953, "completions/mean_terminated_length": 166.10000610351562, "completions/min_length": 100.0, "completions/min_terminated_length": 100.0, "epoch": 0.21868439468159553, "grad_norm": 0.47086182717200054, "kl": 0.72265625, "learning_rate": 9.997478795811486e-06, "loss": 0.0031, "num_tokens": 3196312.0, "reward": 2.318049907684326, "reward_std": 0.2676825225353241, "rewards/check_gptzero_func/mean": 0.5357142686843872, "rewards/check_gptzero_func/std": 0.5032362341880798, "rewards/check_winston_local_func/mean": 0.796946108341217, "rewards/check_winston_local_func/std": 0.26937127113342285, "rewards/sentence_count_match_reward_logic/mean": 0.9853896498680115, "rewards/sentence_count_match_reward_logic/std": 0.04166621342301369, "step": 625 }, { "clip_ratio": 0.010334283113479614, "epoch": 0.21903428971308608, "grad_norm": 0.35331881778540347, "kl": 0.72265625, "learning_rate": 9.997380902569644e-06, "loss": 0.0011, "step": 626 }, { "clip_ratio": 0.02448984608054161, "epoch": 0.21938418474457663, "grad_norm": 0.3504987875593637, "kl": 0.73046875, "learning_rate": 9.997281145352022e-06, "loss": -0.0007, "step": 627 }, { "clip_ratio": 0.03588655963540077, "epoch": 0.21973407977606718, "grad_norm": 0.23969711942363608, "kl": 0.7421875, "learning_rate": 9.99717952419583e-06, "loss": -0.0018, "step": 628 }, { "clip_ratio": 0.0021772957406938076, "completions/clipped_ratio": 0.2857142857142857, "completions/max_length": 256.0, "completions/max_terminated_length": 193.0, "completions/mean_length": 187.23214721679688, "completions/mean_terminated_length": 159.72500610351562, "completions/min_length": 113.0, "completions/min_terminated_length": 113.0, "epoch": 0.22008397480755773, "grad_norm": 0.43532601419879297, "kl": 0.55078125, "learning_rate": 9.99707603913897e-06, "loss": 0.0026, "num_tokens": 3219853.0, "reward": 2.2206993103027344, "reward_std": 0.2309809923171997, "rewards/check_gptzero_func/mean": 0.5, "rewards/check_gptzero_func/std": 0.5045249462127686, "rewards/check_winston_local_func/mean": 0.7884076237678528, "rewards/check_winston_local_func/std": 0.26802411675453186, "rewards/sentence_count_match_reward_logic/mean": 0.9322916865348816, "rewards/sentence_count_match_reward_logic/std": 0.12090840935707092, "step": 629 }, { "clip_ratio": 0.00760418688878417, "epoch": 0.22043386983904828, "grad_norm": 0.3051859841281716, "kl": 0.5546875, "learning_rate": 9.996970690220043e-06, "loss": 0.0011, "step": 630 }, { "clip_ratio": 0.02059207297861576, "epoch": 0.22078376487053883, "grad_norm": 0.283682533086931, "kl": 0.55859375, "learning_rate": 9.99686347747834e-06, "loss": -0.0001, "step": 631 }, { "clip_ratio": 0.029217256233096123, "epoch": 0.22113365990202938, "grad_norm": 0.1834375397659748, "kl": 0.55859375, "learning_rate": 9.996754400953852e-06, "loss": -0.0009, "step": 632 }, { "clip_ratio": 0.0018388424068689346, "completions/clipped_ratio": 0.2857142857142857, "completions/max_length": 256.0, "completions/max_terminated_length": 165.0, "completions/mean_length": 163.0, "completions/mean_terminated_length": 125.80000305175781, "completions/min_length": 97.0, "completions/min_terminated_length": 97.0, "epoch": 0.22148355493351995, "grad_norm": 0.39739142812814066, "kl": 0.75390625, "learning_rate": 9.996643460687264e-06, "loss": 0.0036, "num_tokens": 3239685.0, "reward": 2.4456162452697754, "reward_std": 0.22956664860248566, "rewards/check_gptzero_func/mean": 0.6071428656578064, "rewards/check_gptzero_func/std": 0.4928053617477417, "rewards/check_winston_local_func/mean": 0.8474017381668091, "rewards/check_winston_local_func/std": 0.22422274947166443, "rewards/sentence_count_match_reward_logic/mean": 0.9910714030265808, "rewards/sentence_count_match_reward_logic/std": 0.04681408405303955, "step": 633 }, { "clip_ratio": 0.008222242817282677, "epoch": 0.2218334499650105, "grad_norm": 0.30241116180870947, "kl": 0.76171875, "learning_rate": 9.996530656719954e-06, "loss": 0.0021, "step": 634 }, { "clip_ratio": 0.02232546918094158, "epoch": 0.22218334499650105, "grad_norm": 0.2439588572774872, "kl": 0.7734375, "learning_rate": 9.996415989093999e-06, "loss": 0.0009, "step": 635 }, { "clip_ratio": 0.034689806401729584, "epoch": 0.2225332400279916, "grad_norm": 0.2641407655050198, "kl": 0.78125, "learning_rate": 9.996299457852167e-06, "loss": 0.0003, "step": 636 }, { "clip_ratio": 0.0020035835914313793, "completions/clipped_ratio": 0.1964285714285714, "completions/max_length": 256.0, "completions/max_terminated_length": 256.0, "completions/mean_length": 169.7678680419922, "completions/mean_terminated_length": 148.6888885498047, "completions/min_length": 59.0, "completions/min_terminated_length": 59.0, "epoch": 0.22288313505948215, "grad_norm": 0.44138843505766756, "kl": 0.90234375, "learning_rate": 9.996181063037924e-06, "loss": 0.0062, "num_tokens": 3260336.0, "reward": 2.187747001647949, "reward_std": 0.20350363850593567, "rewards/check_gptzero_func/mean": 0.3928571343421936, "rewards/check_gptzero_func/std": 0.4928053617477417, "rewards/check_winston_local_func/mean": 0.8127468228340149, "rewards/check_winston_local_func/std": 0.20689767599105835, "rewards/sentence_count_match_reward_logic/mean": 0.9821428656578064, "rewards/sentence_count_match_reward_logic/std": 0.04413674399256706, "step": 637 }, { "clip_ratio": 0.011545426212251186, "epoch": 0.2232330300909727, "grad_norm": 0.39081789217433555, "kl": 0.9140625, "learning_rate": 9.996060804695431e-06, "loss": 0.0043, "step": 638 }, { "clip_ratio": 0.02618834748864174, "epoch": 0.22358292512246325, "grad_norm": 0.27970640658671125, "kl": 0.9375, "learning_rate": 9.995938682869542e-06, "loss": 0.0023, "step": 639 }, { "clip_ratio": 0.04049890115857124, "epoch": 0.22393282015395383, "grad_norm": 0.2682622367473569, "kl": 0.94140625, "learning_rate": 9.995814697605808e-06, "loss": 0.0017, "step": 640 }, { "clip_ratio": 0.002491435268893838, "completions/clipped_ratio": 0.0, "completions/max_length": 236.0, "completions/max_terminated_length": 236.0, "completions/mean_length": 172.5178680419922, "completions/mean_terminated_length": 172.5178680419922, "completions/min_length": 96.0, "completions/min_terminated_length": 96.0, "epoch": 0.22428271518544438, "grad_norm": 0.4596768565101577, "kl": 0.76171875, "learning_rate": 9.995688848950473e-06, "loss": 0.0043, "num_tokens": 3280949.0, "reward": 2.1275525093078613, "reward_std": 0.29593467712402344, "rewards/check_gptzero_func/mean": 0.4642857015132904, "rewards/check_gptzero_func/std": 0.5032362341880798, "rewards/check_winston_local_func/mean": 0.6632664799690247, "rewards/check_winston_local_func/std": 0.3270185589790344, "rewards/sentence_count_match_reward_logic/mean": 1.0, "rewards/sentence_count_match_reward_logic/std": 0.0, "step": 641 }, { "clip_ratio": 0.012305466458201408, "epoch": 0.22463261021693492, "grad_norm": 0.3514643911245793, "kl": 0.76171875, "learning_rate": 9.995561136950479e-06, "loss": 0.0021, "step": 642 }, { "clip_ratio": 0.031479448080062866, "epoch": 0.22498250524842547, "grad_norm": 0.2760167237950136, "kl": 0.76953125, "learning_rate": 9.99543156165346e-06, "loss": 0.0005, "step": 643 }, { "clip_ratio": 0.04356088116765022, "epoch": 0.22533240027991602, "grad_norm": 0.2316346833507427, "kl": 0.7734375, "learning_rate": 9.995300123107747e-06, "loss": -0.0002, "step": 644 }, { "clip_ratio": 0.002448699902743101, "completions/clipped_ratio": 0.1428571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 198.0, "completions/mean_length": 149.875, "completions/mean_terminated_length": 132.1875, "completions/min_length": 100.0, "completions/min_terminated_length": 100.0, "epoch": 0.22568229531140657, "grad_norm": 0.4716525665789514, "kl": 0.78125, "learning_rate": 9.995166821362368e-06, "loss": 0.0048, "num_tokens": 3299478.0, "reward": 2.1852235794067383, "reward_std": 0.28842389583587646, "rewards/check_gptzero_func/mean": 0.4107142984867096, "rewards/check_gptzero_func/std": 0.49641573429107666, "rewards/check_winston_local_func/mean": 0.8370091319084167, "rewards/check_winston_local_func/std": 0.2091631293296814, "rewards/sentence_count_match_reward_logic/mean": 0.9375, "rewards/sentence_count_match_reward_logic/std": 0.16685594618320465, "step": 645 }, { "clip_ratio": 0.014128478243947029, "epoch": 0.22603219034289712, "grad_norm": 0.3576280165585853, "kl": 0.77734375, "learning_rate": 9.995031656467038e-06, "loss": 0.0032, "step": 646 }, { "clip_ratio": 0.027511117979884148, "epoch": 0.22638208537438767, "grad_norm": 0.3336062666722892, "kl": 0.7890625, "learning_rate": 9.994894628472175e-06, "loss": 0.0021, "step": 647 }, { "clip_ratio": 0.037764351814985275, "epoch": 0.22673198040587825, "grad_norm": 0.20515618987783363, "kl": 0.79296875, "learning_rate": 9.99475573742889e-06, "loss": 0.0013, "step": 648 }, { "clip_ratio": 0.0019987495616078377, "completions/clipped_ratio": 0.0, "completions/max_length": 234.0, "completions/max_terminated_length": 234.0, "completions/mean_length": 191.23214721679688, "completions/mean_terminated_length": 191.23214721679688, "completions/min_length": 122.0, "completions/min_terminated_length": 122.0, "epoch": 0.2270818754373688, "grad_norm": 0.4158185171266911, "kl": 0.6640625, "learning_rate": 9.994614983388986e-06, "loss": 0.0035, "num_tokens": 3322475.0, "reward": 1.9489095211029053, "reward_std": 0.28637683391571045, "rewards/check_gptzero_func/mean": 0.2321428507566452, "rewards/check_gptzero_func/std": 0.4260205328464508, "rewards/check_winston_local_func/mean": 0.7167667746543884, "rewards/check_winston_local_func/std": 0.2767655849456787, "rewards/sentence_count_match_reward_logic/mean": 1.0, "rewards/sentence_count_match_reward_logic/std": 0.0, "step": 649 }, { "clip_ratio": 0.006417104043066502, "epoch": 0.22743177046885935, "grad_norm": 0.3862848073536015, "kl": 0.66796875, "learning_rate": 9.994472366404964e-06, "loss": 0.0014, "step": 650 }, { "clip_ratio": 0.01972854509949684, "epoch": 0.2277816655003499, "grad_norm": 0.23929355848389386, "kl": 0.66796875, "learning_rate": 9.99432788653002e-06, "loss": -0.0002, "step": 651 }, { "clip_ratio": 0.03498641774058342, "epoch": 0.22813156053184044, "grad_norm": 0.2710078754296729, "kl": 0.67578125, "learning_rate": 9.99418154381804e-06, "loss": -0.0009, "step": 652 }, { "clip_ratio": 0.002450495958328247, "completions/clipped_ratio": 0.0, "completions/max_length": 251.0, "completions/max_terminated_length": 251.0, "completions/mean_length": 147.60714721679688, "completions/mean_terminated_length": 147.60714721679688, "completions/min_length": 97.0, "completions/min_terminated_length": 97.0, "epoch": 0.228481455563331, "grad_norm": 0.4125672685834131, "kl": 0.7109375, "learning_rate": 9.994033338323612e-06, "loss": 0.0059, "num_tokens": 3340629.0, "reward": 2.405573606491089, "reward_std": 0.20926527678966522, "rewards/check_gptzero_func/mean": 0.625, "rewards/check_gptzero_func/std": 0.48850420117378235, "rewards/check_winston_local_func/mean": 0.7805735468864441, "rewards/check_winston_local_func/std": 0.30331894755363464, "rewards/sentence_count_match_reward_logic/mean": 1.0, "rewards/sentence_count_match_reward_logic/std": 0.0, "step": 653 }, { "clip_ratio": 0.00833478756248951, "epoch": 0.22883135059482154, "grad_norm": 0.3331792774654101, "kl": 0.71875, "learning_rate": 9.993883270102016e-06, "loss": 0.0043, "step": 654 }, { "clip_ratio": 0.023286839947104454, "epoch": 0.22918124562631212, "grad_norm": 0.2824626102364888, "kl": 0.72265625, "learning_rate": 9.993731339209223e-06, "loss": 0.0031, "step": 655 }, { "clip_ratio": 0.03368522599339485, "epoch": 0.22953114065780267, "grad_norm": 0.1942410363159062, "kl": 0.72265625, "learning_rate": 9.993577545701902e-06, "loss": 0.0022, "step": 656 }, { "clip_ratio": 0.00302191823720932, "completions/clipped_ratio": 0.0, "completions/max_length": 247.0, "completions/max_terminated_length": 247.0, "completions/mean_length": 179.83929443359375, "completions/mean_terminated_length": 179.83929443359375, "completions/min_length": 94.0, "completions/min_terminated_length": 94.0, "epoch": 0.22988103568929322, "grad_norm": 0.48180398073235803, "kl": 0.7578125, "learning_rate": 9.993421889637418e-06, "loss": 0.006, "num_tokens": 3362236.0, "reward": 2.3819539546966553, "reward_std": 0.14268112182617188, "rewards/check_gptzero_func/mean": 0.6785714030265808, "rewards/check_gptzero_func/std": 0.4712514281272888, "rewards/check_winston_local_func/mean": 0.7033823728561401, "rewards/check_winston_local_func/std": 0.27687767148017883, "rewards/sentence_count_match_reward_logic/mean": 1.0, "rewards/sentence_count_match_reward_logic/std": 0.0, "step": 657 }, { "clip_ratio": 0.009342414326965809, "epoch": 0.23023093072078377, "grad_norm": 0.3910136275906483, "kl": 0.76171875, "learning_rate": 9.993264371073828e-06, "loss": 0.004, "step": 658 }, { "clip_ratio": 0.021683702245354652, "epoch": 0.23058082575227432, "grad_norm": 0.311756928310921, "kl": 0.76953125, "learning_rate": 9.993104990069887e-06, "loss": 0.0026, "step": 659 }, { "clip_ratio": 0.03263520449399948, "epoch": 0.23093072078376486, "grad_norm": 0.30086180092092046, "kl": 0.77734375, "learning_rate": 9.99294374668504e-06, "loss": 0.0016, "step": 660 }, { "clip_ratio": 0.0014777973992750049, "completions/clipped_ratio": 0.0, "completions/max_length": 223.0, "completions/max_terminated_length": 223.0, "completions/mean_length": 174.25001525878906, "completions/mean_terminated_length": 174.25001525878906, "completions/min_length": 115.0, "completions/min_terminated_length": 115.0, "epoch": 0.2312806158152554, "grad_norm": 0.4641167386589493, "kl": 0.75390625, "learning_rate": 9.99278064097943e-06, "loss": 0.0039, "num_tokens": 3383130.0, "reward": 2.4648900032043457, "reward_std": 0.15857629477977753, "rewards/check_gptzero_func/mean": 0.7142857313156128, "rewards/check_gptzero_func/std": 0.4558422863483429, "rewards/check_winston_local_func/mean": 0.7506040334701538, "rewards/check_winston_local_func/std": 0.2974800169467926, "rewards/sentence_count_match_reward_logic/mean": 1.0, "rewards/sentence_count_match_reward_logic/std": 0.0, "step": 661 }, { "clip_ratio": 0.008855576626956463, "epoch": 0.23163051084674596, "grad_norm": 0.3841666403410189, "kl": 0.75390625, "learning_rate": 9.992615673013895e-06, "loss": 0.0017, "step": 662 }, { "clip_ratio": 0.026043113321065903, "epoch": 0.23198040587823654, "grad_norm": 0.30295397497179394, "kl": 0.7578125, "learning_rate": 9.992448842849967e-06, "loss": -0.0002, "step": 663 }, { "clip_ratio": 0.04126749187707901, "epoch": 0.2323303009097271, "grad_norm": 0.23619619980627488, "kl": 0.76953125, "learning_rate": 9.992280150549869e-06, "loss": -0.0012, "step": 664 }, { "clip_ratio": 0.0018083828035742044, "completions/clipped_ratio": 0.2857142857142857, "completions/max_length": 256.0, "completions/max_terminated_length": 197.0, "completions/mean_length": 185.42857360839844, "completions/mean_terminated_length": 157.1999969482422, "completions/min_length": 110.0, "completions/min_terminated_length": 110.0, "epoch": 0.23268019594121764, "grad_norm": 0.49712846458110405, "kl": 0.6796875, "learning_rate": 9.992109596176525e-06, "loss": 0.003, "num_tokens": 3406074.0, "reward": 2.2612857818603516, "reward_std": 0.24236086010932922, "rewards/check_gptzero_func/mean": 0.5892857313156128, "rewards/check_gptzero_func/std": 0.49641573429107666, "rewards/check_winston_local_func/mean": 0.6764640808105469, "rewards/check_winston_local_func/std": 0.20414741337299347, "rewards/sentence_count_match_reward_logic/mean": 0.9955357313156128, "rewards/sentence_count_match_reward_logic/std": 0.023407042026519775, "step": 665 }, { "clip_ratio": 0.01082862913608551, "epoch": 0.2330300909727082, "grad_norm": 0.35101993963236294, "kl": 0.6875, "learning_rate": 9.991937179793547e-06, "loss": 0.001, "step": 666 }, { "clip_ratio": 0.029049919918179512, "epoch": 0.23337998600419874, "grad_norm": 0.32991029256850124, "kl": 0.6953125, "learning_rate": 9.991762901465249e-06, "loss": -0.0004, "step": 667 }, { "clip_ratio": 0.039859239012002945, "epoch": 0.23372988103568929, "grad_norm": 0.29086068138914817, "kl": 0.70703125, "learning_rate": 9.99158676125663e-06, "loss": -0.0014, "step": 668 }, { "clip_ratio": 0.004335995297878981, "completions/clipped_ratio": 0.1428571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 181.0, "completions/mean_length": 140.0357208251953, "completions/mean_terminated_length": 120.70833587646484, "completions/min_length": 70.0, "completions/min_terminated_length": 70.0, "epoch": 0.23407977606717983, "grad_norm": 0.44976487698301426, "kl": 0.94140625, "learning_rate": 9.991408759233394e-06, "loss": 0.0044, "num_tokens": 3423684.0, "reward": 1.9511029720306396, "reward_std": 0.20480915904045105, "rewards/check_gptzero_func/mean": 0.2678571343421936, "rewards/check_gptzero_func/std": 0.44685041904449463, "rewards/check_winston_local_func/mean": 0.7240620851516724, "rewards/check_winston_local_func/std": 0.3241875171661377, "rewards/sentence_count_match_reward_logic/mean": 0.9591836929321289, "rewards/sentence_count_match_reward_logic/std": 0.10088397562503815, "step": 669 }, { "clip_ratio": 0.01495009008795023, "epoch": 0.2344296710986704, "grad_norm": 0.33915749877694296, "kl": 0.93359375, "learning_rate": 9.991228895461929e-06, "loss": 0.0028, "step": 670 }, { "clip_ratio": 0.03251033276319504, "epoch": 0.23477956613016096, "grad_norm": 0.27989063160154115, "kl": 0.9375, "learning_rate": 9.991047170009325e-06, "loss": 0.0016, "step": 671 }, { "clip_ratio": 0.04417676851153374, "epoch": 0.2351294611616515, "grad_norm": 0.21865493917061743, "kl": 0.9453125, "learning_rate": 9.990863582943363e-06, "loss": 0.001, "step": 672 }, { "clip_ratio": 0.003388113807886839, "completions/clipped_ratio": 0.0, "completions/max_length": 224.0, "completions/max_terminated_length": 224.0, "completions/mean_length": 172.4107208251953, "completions/mean_terminated_length": 172.4107208251953, "completions/min_length": 101.0, "completions/min_terminated_length": 101.0, "epoch": 0.23547935619314206, "grad_norm": 0.46938682439672585, "kl": 0.7421875, "learning_rate": 9.990678134332521e-06, "loss": 0.0005, "num_tokens": 3444491.0, "reward": 2.0483813285827637, "reward_std": 0.1012936383485794, "rewards/check_gptzero_func/mean": 0.1607142835855484, "rewards/check_gptzero_func/std": 0.3705909848213196, "rewards/check_winston_local_func/mean": 0.8876668810844421, "rewards/check_winston_local_func/std": 0.16483810544013977, "rewards/sentence_count_match_reward_logic/mean": 1.0, "rewards/sentence_count_match_reward_logic/std": 0.0, "step": 673 }, { "clip_ratio": 0.00860748253762722, "epoch": 0.2358292512246326, "grad_norm": 0.33374448964158077, "kl": 0.75, "learning_rate": 9.990490824245968e-06, "loss": -0.0008, "step": 674 }, { "clip_ratio": 0.022069284692406654, "epoch": 0.23617914625612316, "grad_norm": 0.27136812326159093, "kl": 0.76953125, "learning_rate": 9.990301652753568e-06, "loss": -0.0024, "step": 675 }, { "clip_ratio": 0.03384119272232056, "epoch": 0.2365290412876137, "grad_norm": 0.22657497311481392, "kl": 0.75390625, "learning_rate": 9.990110619925884e-06, "loss": -0.0031, "step": 676 }, { "clip_ratio": 0.004566964693367481, "completions/clipped_ratio": 0.0, "completions/max_length": 234.0, "completions/max_terminated_length": 234.0, "completions/mean_length": 183.98214721679688, "completions/mean_terminated_length": 183.98214721679688, "completions/min_length": 127.0, "completions/min_terminated_length": 127.0, "epoch": 0.23687893631910426, "grad_norm": 0.5251407164923836, "kl": 0.69140625, "learning_rate": 9.989917725834166e-06, "loss": 0.0044, "num_tokens": 3466506.0, "reward": 2.140127182006836, "reward_std": 0.1487554907798767, "rewards/check_gptzero_func/mean": 0.4642857015132904, "rewards/check_gptzero_func/std": 0.5032362341880798, "rewards/check_winston_local_func/mean": 0.6780734658241272, "rewards/check_winston_local_func/std": 0.2997058928012848, "rewards/sentence_count_match_reward_logic/mean": 0.9977678656578064, "rewards/sentence_count_match_reward_logic/std": 0.016703827306628227, "step": 677 }, { "clip_ratio": 0.010818206705152988, "epoch": 0.23722883135059483, "grad_norm": 0.40382226623650297, "kl": 0.69921875, "learning_rate": 9.989722970550363e-06, "loss": 0.0022, "step": 678 }, { "clip_ratio": 0.030726999044418335, "epoch": 0.23757872638208538, "grad_norm": 0.2841327388285347, "kl": 0.71875, "learning_rate": 9.989526354147116e-06, "loss": 0.0003, "step": 679 }, { "clip_ratio": 0.046416375786066055, "epoch": 0.23792862141357593, "grad_norm": 0.26194342603397247, "kl": 0.7265625, "learning_rate": 9.98932787669776e-06, "loss": -0.0007, "step": 680 }, { "clip_ratio": 0.0023747452069073915, "completions/clipped_ratio": 0.2678571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 254.0, "completions/mean_length": 199.1428680419922, "completions/mean_terminated_length": 178.34146118164062, "completions/min_length": 108.0, "completions/min_terminated_length": 108.0, "epoch": 0.23827851644506648, "grad_norm": 0.4583027873196669, "kl": 0.62890625, "learning_rate": 9.989127538276329e-06, "loss": 0.0045, "num_tokens": 3490546.0, "reward": 2.0309391021728516, "reward_std": 0.1291850209236145, "rewards/check_gptzero_func/mean": 0.3928571343421936, "rewards/check_gptzero_func/std": 0.4928053617477417, "rewards/check_winston_local_func/mean": 0.6539549231529236, "rewards/check_winston_local_func/std": 0.251346230506897, "rewards/sentence_count_match_reward_logic/mean": 0.9841269850730896, "rewards/sentence_count_match_reward_logic/std": 0.039232656359672546, "step": 681 }, { "clip_ratio": 0.006431466434150934, "epoch": 0.23862841147655703, "grad_norm": 0.3668049240913585, "kl": 0.62890625, "learning_rate": 9.988925338957544e-06, "loss": 0.0033, "step": 682 }, { "clip_ratio": 0.017797382548451424, "epoch": 0.23897830650804758, "grad_norm": 0.2733449189071174, "kl": 0.62890625, "learning_rate": 9.988721278816826e-06, "loss": 0.0015, "step": 683 }, { "clip_ratio": 0.03220158442854881, "epoch": 0.23932820153953813, "grad_norm": 0.23755512933774292, "kl": 0.63671875, "learning_rate": 9.988515357930284e-06, "loss": 0.0008, "step": 684 }, { "clip_ratio": 0.0020391144789755344, "completions/clipped_ratio": 0.1428571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 243.0, "completions/mean_length": 186.75001525878906, "completions/mean_terminated_length": 175.20834350585938, "completions/min_length": 107.0, "completions/min_terminated_length": 107.0, "epoch": 0.2396780965710287, "grad_norm": 0.4670918838398367, "kl": 0.77734375, "learning_rate": 9.988307576374727e-06, "loss": -0.0008, "num_tokens": 3513636.0, "reward": 2.2834346294403076, "reward_std": 0.1509338617324829, "rewards/check_gptzero_func/mean": 0.5, "rewards/check_gptzero_func/std": 0.5045249462127686, "rewards/check_winston_local_func/mean": 0.819148600101471, "rewards/check_winston_local_func/std": 0.24821728467941284, "rewards/sentence_count_match_reward_logic/mean": 0.9642857313156128, "rewards/sentence_count_match_reward_logic/std": 0.08954106271266937, "step": 685 }, { "clip_ratio": 0.008796771988272667, "epoch": 0.24002799160251925, "grad_norm": 0.3597720293445847, "kl": 0.78515625, "learning_rate": 9.988097934227656e-06, "loss": -0.0029, "step": 686 }, { "clip_ratio": 0.022153569385409355, "epoch": 0.2403778866340098, "grad_norm": 0.2655819473247039, "kl": 0.79296875, "learning_rate": 9.987886431567264e-06, "loss": -0.004, "step": 687 }, { "clip_ratio": 0.032517921179533005, "epoch": 0.24072778166550035, "grad_norm": 0.2367440780580049, "kl": 0.8046875, "learning_rate": 9.98767306847244e-06, "loss": -0.0048, "step": 688 }, { "clip_ratio": 0.0035277751740068197, "completions/clipped_ratio": 0.0, "completions/max_length": 181.0, "completions/max_terminated_length": 181.0, "completions/mean_length": 128.21429443359375, "completions/mean_terminated_length": 128.21429443359375, "completions/min_length": 68.0, "completions/min_terminated_length": 68.0, "epoch": 0.2410776766969909, "grad_norm": 0.49443585844110644, "kl": 1.2265625, "learning_rate": 9.987457845022767e-06, "loss": 0.0068, "num_tokens": 3529528.0, "reward": 2.5538852214813232, "reward_std": 0.14030277729034424, "rewards/check_gptzero_func/mean": 0.5714285969734192, "rewards/check_gptzero_func/std": 0.4993502199649811, "rewards/check_winston_local_func/mean": 0.9824565052986145, "rewards/check_winston_local_func/std": 0.014201563782989979, "rewards/sentence_count_match_reward_logic/mean": 1.0, "rewards/sentence_count_match_reward_logic/std": 0.0, "step": 689 }, { "clip_ratio": 0.017958790063858032, "epoch": 0.24142757172848145, "grad_norm": 0.3340998409863947, "kl": 1.2265625, "learning_rate": 9.98724076129852e-06, "loss": 0.0048, "step": 690 }, { "clip_ratio": 0.03603988513350487, "epoch": 0.241777466759972, "grad_norm": 0.2344550127553255, "kl": 1.234375, "learning_rate": 9.98702181738067e-06, "loss": 0.0038, "step": 691 }, { "clip_ratio": 0.04851587861776352, "epoch": 0.24212736179146255, "grad_norm": 0.1997553408949644, "kl": 1.2421875, "learning_rate": 9.986801013350883e-06, "loss": 0.0035, "step": 692 }, { "clip_ratio": 0.005415024701505899, "completions/clipped_ratio": 0.0, "completions/max_length": 247.0, "completions/max_terminated_length": 247.0, "completions/mean_length": 145.5, "completions/mean_terminated_length": 145.5, "completions/min_length": 94.0, "completions/min_terminated_length": 94.0, "epoch": 0.24247725682295312, "grad_norm": 0.4827072176144563, "kl": 0.96875, "learning_rate": 9.986578349291514e-06, "loss": 0.0071, "num_tokens": 3547252.0, "reward": 2.7333779335021973, "reward_std": 0.1260833591222763, "rewards/check_gptzero_func/mean": 0.7678571343421936, "rewards/check_gptzero_func/std": 0.4260205328464508, "rewards/check_winston_local_func/mean": 0.9744489789009094, "rewards/check_winston_local_func/std": 0.02029097080230713, "rewards/sentence_count_match_reward_logic/mean": 0.9910714030265808, "rewards/sentence_count_match_reward_logic/std": 0.06681530922651291, "step": 693 }, { "clip_ratio": 0.013280555605888367, "epoch": 0.24282715185444367, "grad_norm": 0.3568191386049385, "kl": 0.9765625, "learning_rate": 9.986353825285615e-06, "loss": 0.0056, "step": 694 }, { "clip_ratio": 0.02892332337796688, "epoch": 0.24317704688593422, "grad_norm": 0.2900448643652273, "kl": 0.9921875, "learning_rate": 9.98612744141693e-06, "loss": 0.0048, "step": 695 }, { "clip_ratio": 0.04019845277070999, "epoch": 0.24352694191742477, "grad_norm": 0.28074653654295023, "kl": 1.0078125, "learning_rate": 9.985899197769902e-06, "loss": 0.0046, "step": 696 }, { "clip_ratio": 0.002369124908000231, "completions/clipped_ratio": 0.0, "completions/max_length": 243.0, "completions/max_terminated_length": 243.0, "completions/mean_length": 162.0, "completions/mean_terminated_length": 162.0, "completions/min_length": 108.0, "completions/min_terminated_length": 108.0, "epoch": 0.24387683694891532, "grad_norm": 0.5042491520038607, "kl": 0.87109375, "learning_rate": 9.985669094429662e-06, "loss": 0.0042, "num_tokens": 3566948.0, "reward": 2.4528651237487793, "reward_std": 0.07609730213880539, "rewards/check_gptzero_func/mean": 0.5714285969734192, "rewards/check_gptzero_func/std": 0.4993502199649811, "rewards/check_winston_local_func/mean": 0.8814366459846497, "rewards/check_winston_local_func/std": 0.11873003095388412, "rewards/sentence_count_match_reward_logic/mean": 1.0, "rewards/sentence_count_match_reward_logic/std": 0.0, "step": 697 }, { "clip_ratio": 0.010691465809941292, "epoch": 0.24422673198040587, "grad_norm": 0.4606329591185797, "kl": 0.87109375, "learning_rate": 9.985437131482035e-06, "loss": 0.0025, "step": 698 }, { "clip_ratio": 0.02367514744400978, "epoch": 0.24457662701189642, "grad_norm": 0.27688188028402827, "kl": 0.87890625, "learning_rate": 9.98520330901354e-06, "loss": 0.0008, "step": 699 }, { "clip_ratio": 0.03675125911831856, "epoch": 0.244926522043387, "grad_norm": 0.23664727873938513, "kl": 0.8828125, "learning_rate": 9.984967627111395e-06, "loss": 0.0004, "step": 700 }, { "clip_ratio": 0.002918368438258767, "completions/clipped_ratio": 0.0, "completions/max_length": 249.0, "completions/max_terminated_length": 249.0, "completions/mean_length": 187.57144165039062, "completions/mean_terminated_length": 187.57144165039062, "completions/min_length": 152.0, "completions/min_terminated_length": 152.0, "epoch": 0.24527641707487755, "grad_norm": 0.5352144093908778, "kl": 0.78125, "learning_rate": 9.984730085863504e-06, "loss": 0.0018, "num_tokens": 3589340.0, "reward": 2.242097854614258, "reward_std": 0.1488455981016159, "rewards/check_gptzero_func/mean": 0.5357142686843872, "rewards/check_gptzero_func/std": 0.5032362937927246, "rewards/check_winston_local_func/mean": 0.727216899394989, "rewards/check_winston_local_func/std": 0.3058179020881653, "rewards/sentence_count_match_reward_logic/mean": 0.9791666269302368, "rewards/sentence_count_match_reward_logic/std": 0.07825431227684021, "step": 701 }, { "clip_ratio": 0.010457238182425499, "epoch": 0.2456263121063681, "grad_norm": 0.3794619346946274, "kl": 0.7890625, "learning_rate": 9.984490685358468e-06, "loss": -0.0005, "step": 702 }, { "clip_ratio": 0.025612840428948402, "epoch": 0.24597620713785864, "grad_norm": 0.3255868263604941, "kl": 0.80078125, "learning_rate": 9.984249425685582e-06, "loss": -0.0023, "step": 703 }, { "clip_ratio": 0.03947295621037483, "epoch": 0.2463261021693492, "grad_norm": 0.25629783057346117, "kl": 0.80859375, "learning_rate": 9.984006306934832e-06, "loss": -0.0035, "step": 704 }, { "clip_ratio": 0.0030612964183092117, "completions/clipped_ratio": 0.0, "completions/max_length": 217.0, "completions/max_terminated_length": 217.0, "completions/mean_length": 155.25, "completions/mean_terminated_length": 155.25, "completions/min_length": 100.0, "completions/min_terminated_length": 100.0, "epoch": 0.24667599720083974, "grad_norm": 0.46659433447724996, "kl": 0.8125, "learning_rate": 9.9837613291969e-06, "loss": 0.0016, "num_tokens": 3608418.0, "reward": 2.0482494831085205, "reward_std": 0.10775505006313324, "rewards/check_gptzero_func/mean": 0.3214285671710968, "rewards/check_gptzero_func/std": 0.4712514281272888, "rewards/check_winston_local_func/mean": 0.7268209457397461, "rewards/check_winston_local_func/std": 0.347108393907547, "rewards/sentence_count_match_reward_logic/mean": 1.0, "rewards/sentence_count_match_reward_logic/std": 0.0, "step": 705 }, { "clip_ratio": 0.013674231246113777, "epoch": 0.2470258922323303, "grad_norm": 0.31796640200053194, "kl": 0.81640625, "learning_rate": 9.98351449256316e-06, "loss": 0.0002, "step": 706 }, { "clip_ratio": 0.02610214240849018, "epoch": 0.24737578726382084, "grad_norm": 0.26853325229966124, "kl": 0.82421875, "learning_rate": 9.98326579712568e-06, "loss": -0.001, "step": 707 }, { "clip_ratio": 0.03673864156007767, "epoch": 0.24772568229531142, "grad_norm": 0.2382792771869112, "kl": 0.83984375, "learning_rate": 9.983015242977224e-06, "loss": -0.0014, "step": 708 }, { "clip_ratio": 0.0021030993666499853, "completions/clipped_ratio": 0.017857142857142905, "completions/max_length": 256.0, "completions/max_terminated_length": 256.0, "completions/mean_length": 181.0178680419922, "completions/mean_terminated_length": 179.654541015625, "completions/min_length": 109.0, "completions/min_terminated_length": 109.0, "epoch": 0.24807557732680197, "grad_norm": 0.5036296792515066, "kl": 0.7265625, "learning_rate": 9.982762830211239e-06, "loss": 0.0034, "num_tokens": 3630211.0, "reward": 2.3316216468811035, "reward_std": 0.15093272924423218, "rewards/check_gptzero_func/mean": 0.4642857015132904, "rewards/check_gptzero_func/std": 0.5032362341880798, "rewards/check_winston_local_func/mean": 0.8784965872764587, "rewards/check_winston_local_func/std": 0.2252359688282013, "rewards/sentence_count_match_reward_logic/mean": 0.9888392686843872, "rewards/sentence_count_match_reward_logic/std": 0.03596704453229904, "step": 709 }, { "clip_ratio": 0.01049026194959879, "epoch": 0.24842547235829252, "grad_norm": 0.42194877106818535, "kl": 0.72265625, "learning_rate": 9.98250855892188e-06, "loss": 0.0014, "step": 710 }, { "clip_ratio": 0.026015175506472588, "epoch": 0.24877536738978306, "grad_norm": 0.3937176503307433, "kl": 0.7265625, "learning_rate": 9.982252429203983e-06, "loss": 0.0004, "step": 711 }, { "clip_ratio": 0.03642505407333374, "epoch": 0.2491252624212736, "grad_norm": 0.24993704945437548, "kl": 0.73046875, "learning_rate": 9.981994441153084e-06, "loss": -0.0008, "step": 712 }, { "clip_ratio": 0.004890369717031717, "completions/clipped_ratio": 0.0, "completions/max_length": 183.0, "completions/max_terminated_length": 183.0, "completions/mean_length": 136.0178680419922, "completions/mean_terminated_length": 136.0178680419922, "completions/min_length": 100.0, "completions/min_terminated_length": 100.0, "epoch": 0.24947515745276416, "grad_norm": 0.5390126802192955, "kl": 1.03125, "learning_rate": 9.98173459486541e-06, "loss": 0.0091, "num_tokens": 3647036.0, "reward": 2.440314531326294, "reward_std": 0.25325995683670044, "rewards/check_gptzero_func/mean": 0.5714285969734192, "rewards/check_gptzero_func/std": 0.4993502199649811, "rewards/check_winston_local_func/mean": 0.9250084161758423, "rewards/check_winston_local_func/std": 0.09556113183498383, "rewards/sentence_count_match_reward_logic/mean": 0.9438775777816772, "rewards/sentence_count_match_reward_logic/std": 0.09408704191446304, "step": 713 }, { "clip_ratio": 0.019484685733914375, "epoch": 0.2498250524842547, "grad_norm": 0.5553610979362682, "kl": 1.03125, "learning_rate": 9.981472890437881e-06, "loss": 0.0078, "step": 714 }, { "clip_ratio": 0.029681911692023277, "epoch": 0.25017494751574526, "grad_norm": 0.3323638568901079, "kl": 1.0390625, "learning_rate": 9.981209327968111e-06, "loss": 0.0056, "step": 715 }, { "clip_ratio": 0.04266848787665367, "epoch": 0.25052484254723584, "grad_norm": 0.3446086261556098, "kl": 1.0546875, "learning_rate": 9.980943907554404e-06, "loss": 0.005, "step": 716 }, { "clip_ratio": 0.0037358186673372984, "completions/clipped_ratio": 0.0, "completions/max_length": 235.0, "completions/max_terminated_length": 235.0, "completions/mean_length": 163.92857360839844, "completions/mean_terminated_length": 163.92857360839844, "completions/min_length": 110.0, "completions/min_terminated_length": 110.0, "epoch": 0.25087473757872636, "grad_norm": 0.6488865447917765, "kl": 0.9609375, "learning_rate": 9.980676629295763e-06, "loss": 0.0118, "num_tokens": 3666880.0, "reward": 2.3992507457733154, "reward_std": 0.17336322367191315, "rewards/check_gptzero_func/mean": 0.5535714030265808, "rewards/check_gptzero_func/std": 0.5016207695007324, "rewards/check_winston_local_func/mean": 0.8533324003219604, "rewards/check_winston_local_func/std": 0.1658250242471695, "rewards/sentence_count_match_reward_logic/mean": 0.9923468828201294, "rewards/sentence_count_match_reward_logic/std": 0.042375147342681885, "step": 717 }, { "clip_ratio": 0.014471771195530891, "epoch": 0.25122463261021694, "grad_norm": 46.537840846293335, "kl": 5.1875, "learning_rate": 9.980407493291876e-06, "loss": 0.0399, "step": 718 }, { "clip_ratio": 0.025525255128741264, "epoch": 0.2515745276417075, "grad_norm": 0.42465750004958186, "kl": 0.96875, "learning_rate": 9.98013649964313e-06, "loss": 0.0093, "step": 719 }, { "clip_ratio": 0.0347730927169323, "epoch": 0.25192442267319803, "grad_norm": 34.13622690438684, "kl": 4.03125, "learning_rate": 9.979863648450606e-06, "loss": 0.0304, "step": 720 }, { "clip_ratio": 0.0015917530981823802, "completions/clipped_ratio": 0.1428571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 206.0, "completions/mean_length": 158.7678680419922, "completions/mean_terminated_length": 142.5625, "completions/min_length": 55.0, "completions/min_terminated_length": 55.0, "epoch": 0.2522743177046886, "grad_norm": 0.47344811610616594, "kl": 0.8828125, "learning_rate": 9.979588939816071e-06, "loss": 0.0053, "num_tokens": 3686523.0, "reward": 2.5105793476104736, "reward_std": 0.33277636766433716, "rewards/check_gptzero_func/mean": 0.6607142686843872, "rewards/check_gptzero_func/std": 0.477751761674881, "rewards/check_winston_local_func/mean": 0.8498648405075073, "rewards/check_winston_local_func/std": 0.23575522005558014, "rewards/sentence_count_match_reward_logic/mean": 1.0, "rewards/sentence_count_match_reward_logic/std": 0.0, "step": 721 }, { "clip_ratio": 0.00781210558488965, "epoch": 0.25262421273617913, "grad_norm": 0.3279548818617064, "kl": 0.89453125, "learning_rate": 9.97931237384199e-06, "loss": 0.0036, "step": 722 }, { "clip_ratio": 0.02457144856452942, "epoch": 0.2529741077676697, "grad_norm": 0.2620796948540116, "kl": 0.8984375, "learning_rate": 9.97903395063152e-06, "loss": 0.0023, "step": 723 }, { "clip_ratio": 0.037495214492082596, "epoch": 0.25332400279916023, "grad_norm": 0.19582916572769818, "kl": 0.90234375, "learning_rate": 9.978753670288508e-06, "loss": 0.0016, "step": 724 }, { "clip_ratio": 0.0019538509659469128, "completions/clipped_ratio": 0.2857142857142857, "completions/max_length": 256.0, "completions/max_terminated_length": 166.0, "completions/mean_length": 170.55357360839844, "completions/mean_terminated_length": 136.375, "completions/min_length": 62.0, "completions/min_terminated_length": 62.0, "epoch": 0.2536738978306508, "grad_norm": 0.46079407053474924, "kl": 0.8828125, "learning_rate": 9.9784715329175e-06, "loss": 0.016, "num_tokens": 3708842.0, "reward": 2.1441400051116943, "reward_std": 0.26984819769859314, "rewards/check_gptzero_func/mean": 0.4107142984867096, "rewards/check_gptzero_func/std": 0.49641573429107666, "rewards/check_winston_local_func/mean": 0.82866370677948, "rewards/check_winston_local_func/std": 0.16839711368083954, "rewards/sentence_count_match_reward_logic/mean": 0.9047618508338928, "rewards/sentence_count_match_reward_logic/std": 0.11916011571884155, "step": 725 }, { "clip_ratio": 0.006476555485278368, "epoch": 0.2540237928621414, "grad_norm": 0.36260777302472913, "kl": 0.8828125, "learning_rate": 9.978187538623726e-06, "loss": 0.0142, "step": 726 }, { "clip_ratio": 0.02267545275390148, "epoch": 0.2543736878936319, "grad_norm": 0.2690453446086791, "kl": 0.88671875, "learning_rate": 9.977901687513117e-06, "loss": 0.0122, "step": 727 }, { "clip_ratio": 0.03528894484043121, "epoch": 0.2547235829251225, "grad_norm": 0.2273491337640385, "kl": 0.88671875, "learning_rate": 9.977613979692291e-06, "loss": 0.0112, "step": 728 }, { "clip_ratio": 0.0021249186247587204, "completions/clipped_ratio": 0.0, "completions/max_length": 214.0, "completions/max_terminated_length": 214.0, "completions/mean_length": 159.55357360839844, "completions/mean_terminated_length": 159.55357360839844, "completions/min_length": 126.0, "completions/min_terminated_length": 126.0, "epoch": 0.255073477956613, "grad_norm": 0.4595437162565651, "kl": 0.97265625, "learning_rate": 9.97732441526856e-06, "loss": 0.0035, "num_tokens": 3728329.0, "reward": 2.096428394317627, "reward_std": 0.20379523932933807, "rewards/check_gptzero_func/mean": 0.2678571343421936, "rewards/check_gptzero_func/std": 0.44685041904449463, "rewards/check_winston_local_func/mean": 0.8315473794937134, "rewards/check_winston_local_func/std": 0.2210366576910019, "rewards/sentence_count_match_reward_logic/mean": 0.9970237612724304, "rewards/sentence_count_match_reward_logic/std": 0.02227177284657955, "step": 729 }, { "clip_ratio": 0.011858997866511345, "epoch": 0.2554233729881036, "grad_norm": 0.36350469963934573, "kl": 0.98046875, "learning_rate": 9.97703299434993e-06, "loss": 0.0017, "step": 730 }, { "clip_ratio": 0.026113217696547508, "epoch": 0.2557732680195941, "grad_norm": 0.402435122564269, "kl": 0.9921875, "learning_rate": 9.976739717045097e-06, "loss": 0.0003, "step": 731 }, { "clip_ratio": 0.03401445969939232, "epoch": 0.2561231630510847, "grad_norm": 0.2297811079811253, "kl": 0.98828125, "learning_rate": 9.976444583463449e-06, "loss": -0.0008, "step": 732 }, { "clip_ratio": 0.002516402630135417, "completions/clipped_ratio": 0.0, "completions/max_length": 201.0, "completions/max_terminated_length": 201.0, "completions/mean_length": 162.60714721679688, "completions/mean_terminated_length": 162.60714721679688, "completions/min_length": 98.0, "completions/min_terminated_length": 98.0, "epoch": 0.2564730580825752, "grad_norm": 0.48140949845098047, "kl": 0.8359375, "learning_rate": 9.976147593715074e-06, "loss": 0.0022, "num_tokens": 3748131.0, "reward": 2.3943867683410645, "reward_std": 0.2082502692937851, "rewards/check_gptzero_func/mean": 0.5535714030265808, "rewards/check_gptzero_func/std": 0.5016207695007324, "rewards/check_winston_local_func/mean": 0.8408151268959045, "rewards/check_winston_local_func/std": 0.24375399947166443, "rewards/sentence_count_match_reward_logic/mean": 1.0, "rewards/sentence_count_match_reward_logic/std": 0.0, "step": 733 }, { "clip_ratio": 0.00772507768124342, "epoch": 0.2568229531140658, "grad_norm": 0.3677301579943928, "kl": 0.8359375, "learning_rate": 9.975848747910741e-06, "loss": 0.0003, "step": 734 }, { "clip_ratio": 0.023478327319025993, "epoch": 0.25717284814555635, "grad_norm": 0.28344071498760387, "kl": 0.83984375, "learning_rate": 9.975548046161918e-06, "loss": -0.0012, "step": 735 }, { "clip_ratio": 0.041291750967502594, "epoch": 0.2575227431770469, "grad_norm": 0.31692625833956023, "kl": 0.84765625, "learning_rate": 9.975245488580763e-06, "loss": -0.0023, "step": 736 }, { "clip_ratio": 0.0027113966643810272, "completions/clipped_ratio": 0.0, "completions/max_length": 210.0, "completions/max_terminated_length": 210.0, "completions/mean_length": 149.80357360839844, "completions/mean_terminated_length": 149.80357360839844, "completions/min_length": 93.0, "completions/min_terminated_length": 93.0, "epoch": 0.25787263820853745, "grad_norm": 0.4895445670310975, "kl": 1.0390625, "learning_rate": 9.974941075280128e-06, "loss": 0.0022, "num_tokens": 3766224.0, "reward": 2.392362356185913, "reward_std": 0.3096998333930969, "rewards/check_gptzero_func/mean": 0.5, "rewards/check_gptzero_func/std": 0.5045249462127686, "rewards/check_winston_local_func/mean": 0.9146836400032043, "rewards/check_winston_local_func/std": 0.11374460905790329, "rewards/sentence_count_match_reward_logic/mean": 0.9776785969734192, "rewards/sentence_count_match_reward_logic/std": 0.07193408906459808, "step": 737 }, { "clip_ratio": 0.008546770550310612, "epoch": 0.258222533240028, "grad_norm": 0.3750957799837979, "kl": 1.046875, "learning_rate": 9.97463480637356e-06, "loss": -0.0, "step": 738 }, { "clip_ratio": 0.024206319823861122, "epoch": 0.25857242827151855, "grad_norm": 0.2708986040883726, "kl": 1.0546875, "learning_rate": 9.974326681975287e-06, "loss": -0.0018, "step": 739 }, { "clip_ratio": 0.040864575654268265, "epoch": 0.25892232330300907, "grad_norm": 0.2457930289434738, "kl": 1.0703125, "learning_rate": 9.974016702200244e-06, "loss": -0.0028, "step": 740 }, { "clip_ratio": 0.001867760787717998, "completions/clipped_ratio": 0.0, "completions/max_length": 248.0, "completions/max_terminated_length": 248.0, "completions/mean_length": 170.7857208251953, "completions/mean_terminated_length": 170.7857208251953, "completions/min_length": 88.0, "completions/min_terminated_length": 88.0, "epoch": 0.25927221833449965, "grad_norm": 0.519719435994371, "kl": 0.8671875, "learning_rate": 9.973704867164044e-06, "loss": 0.0052, "num_tokens": 3786772.0, "reward": 2.4281985759735107, "reward_std": 0.27692127227783203, "rewards/check_gptzero_func/mean": 0.5357142686843872, "rewards/check_gptzero_func/std": 0.5032362937927246, "rewards/check_winston_local_func/mean": 0.8960556983947754, "rewards/check_winston_local_func/std": 0.1275738924741745, "rewards/sentence_count_match_reward_logic/mean": 0.9964285492897034, "rewards/sentence_count_match_reward_logic/std": 0.026726121082901955, "step": 741 }, { "clip_ratio": 0.009042931720614433, "epoch": 0.2596221133659902, "grad_norm": 0.399957671243347, "kl": 0.875, "learning_rate": 9.973391176983003e-06, "loss": 0.0028, "step": 742 }, { "clip_ratio": 0.024890633299946785, "epoch": 0.25997200839748075, "grad_norm": 0.30129129488710166, "kl": 0.8828125, "learning_rate": 9.973075631774123e-06, "loss": 0.0007, "step": 743 }, { "clip_ratio": 0.041101571172475815, "epoch": 0.2603219034289713, "grad_norm": 0.2624983568253012, "kl": 0.8984375, "learning_rate": 9.972758231655097e-06, "loss": -0.0006, "step": 744 }, { "clip_ratio": 0.0021404761355370283, "completions/clipped_ratio": 0.0357142857142857, "completions/max_length": 256.0, "completions/max_terminated_length": 252.0, "completions/mean_length": 176.10714721679688, "completions/mean_terminated_length": 173.1481475830078, "completions/min_length": 112.0, "completions/min_terminated_length": 112.0, "epoch": 0.26067179846046185, "grad_norm": 0.5906456578327066, "kl": 0.96484375, "learning_rate": 9.972438976744317e-06, "loss": 0.0069, "num_tokens": 3808010.0, "reward": 2.578068733215332, "reward_std": 0.1875874549150467, "rewards/check_gptzero_func/mean": 0.6428571343421936, "rewards/check_gptzero_func/std": 0.48349377512931824, "rewards/check_winston_local_func/mean": 0.9403136372566223, "rewards/check_winston_local_func/std": 0.08694165199995041, "rewards/sentence_count_match_reward_logic/mean": 0.9948979616165161, "rewards/sentence_count_match_reward_logic/std": 0.026750901713967323, "step": 745 }, { "clip_ratio": 0.012134714052081108, "epoch": 0.2610216934919524, "grad_norm": 0.41878607852569116, "kl": 0.96875, "learning_rate": 9.972117867160858e-06, "loss": 0.0041, "step": 746 }, { "clip_ratio": 0.029219618067145348, "epoch": 0.26137158852344294, "grad_norm": 0.34723998103792786, "kl": 0.9765625, "learning_rate": 9.971794903024493e-06, "loss": 0.0021, "step": 747 }, { "clip_ratio": 0.04394904151558876, "epoch": 0.2617214835549335, "grad_norm": 0.2747890779312843, "kl": 0.98828125, "learning_rate": 9.971470084455684e-06, "loss": 0.0009, "step": 748 }, { "clip_ratio": 0.0026724764611572027, "completions/clipped_ratio": 0.0, "completions/max_length": 202.0, "completions/max_terminated_length": 202.0, "completions/mean_length": 139.33929443359375, "completions/mean_terminated_length": 139.33929443359375, "completions/min_length": 91.0, "completions/min_terminated_length": 91.0, "epoch": 0.2620713785864241, "grad_norm": 0.5847990131492271, "kl": 1.1484375, "learning_rate": 9.971143411575585e-06, "loss": 0.005, "num_tokens": 3825309.0, "reward": 2.5943121910095215, "reward_std": 0.25837963819503784, "rewards/check_gptzero_func/mean": 0.7142857313156128, "rewards/check_gptzero_func/std": 0.4558422863483429, "rewards/check_winston_local_func/mean": 0.8934192657470703, "rewards/check_winston_local_func/std": 0.1317412406206131, "rewards/sentence_count_match_reward_logic/mean": 0.9866071343421936, "rewards/sentence_count_match_reward_logic/std": 0.056801944971084595, "step": 749 }, { "clip_ratio": 0.013772539794445038, "epoch": 0.2624212736179146, "grad_norm": 0.3669754300871632, "kl": 1.1640625, "learning_rate": 9.970814884506043e-06, "loss": 0.0029, "step": 750 }, { "clip_ratio": 0.03436356782913208, "epoch": 0.2627711686494052, "grad_norm": 0.37199831052116633, "kl": 1.171875, "learning_rate": 9.970484503369595e-06, "loss": 0.0016, "step": 751 }, { "clip_ratio": 0.046261806041002274, "epoch": 0.2631210636808957, "grad_norm": 0.3408720031535595, "kl": 1.1953125, "learning_rate": 9.970152268289468e-06, "loss": 0.0011, "step": 752 }, { "clip_ratio": 0.002223373856395483, "completions/clipped_ratio": 0.1428571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 220.0, "completions/mean_length": 183.32144165039062, "completions/mean_terminated_length": 171.20834350585938, "completions/min_length": 131.0, "completions/min_terminated_length": 131.0, "epoch": 0.2634709587123863, "grad_norm": 0.5670907496086074, "kl": 0.8828125, "learning_rate": 9.969818179389586e-06, "loss": 0.0048, "num_tokens": 3847615.0, "reward": 2.3329079151153564, "reward_std": 0.17812253534793854, "rewards/check_gptzero_func/mean": 0.4107142984867096, "rewards/check_gptzero_func/std": 0.49641573429107666, "rewards/check_winston_local_func/mean": 0.9543362259864807, "rewards/check_winston_local_func/std": 0.08150999248027802, "rewards/sentence_count_match_reward_logic/mean": 0.9678571820259094, "rewards/sentence_count_match_reward_logic/std": 0.08114440739154816, "step": 753 }, { "clip_ratio": 0.011816210113465786, "epoch": 0.2638208537438768, "grad_norm": 0.4763250663306015, "kl": 0.8828125, "learning_rate": 9.969482236794558e-06, "loss": 0.0023, "step": 754 }, { "clip_ratio": 0.029621455818414688, "epoch": 0.2641707487753674, "grad_norm": 0.3103506124635232, "kl": 0.89453125, "learning_rate": 9.969144440629688e-06, "loss": 0.0, "step": 755 }, { "clip_ratio": 0.043503183871507645, "epoch": 0.26452064380685797, "grad_norm": 0.2545212933518372, "kl": 0.90625, "learning_rate": 9.968804791020972e-06, "loss": -0.001, "step": 756 }, { "clip_ratio": 0.001769211725331843, "completions/clipped_ratio": 0.1428571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 255.0, "completions/mean_length": 173.5178680419922, "completions/mean_terminated_length": 159.77084350585938, "completions/min_length": 87.0, "completions/min_terminated_length": 87.0, "epoch": 0.2648705388383485, "grad_norm": 0.45930368865126986, "kl": 1.0625, "learning_rate": 9.968463288095096e-06, "loss": 0.0061, "num_tokens": 3868804.0, "reward": 2.5513663291931152, "reward_std": 0.1908244788646698, "rewards/check_gptzero_func/mean": 0.6607142686843872, "rewards/check_gptzero_func/std": 0.477751761674881, "rewards/check_winston_local_func/mean": 0.890652060508728, "rewards/check_winston_local_func/std": 0.16517268121242523, "rewards/sentence_count_match_reward_logic/mean": 1.0, "rewards/sentence_count_match_reward_logic/std": 0.0, "step": 757 }, { "clip_ratio": 0.009254776872694492, "epoch": 0.26522043386983907, "grad_norm": 0.3631657172535633, "kl": 1.0625, "learning_rate": 9.968119931979436e-06, "loss": 0.0043, "step": 758 }, { "clip_ratio": 0.02615348808467388, "epoch": 0.2655703289013296, "grad_norm": 0.2670886079863764, "kl": 1.0625, "learning_rate": 9.967774722802062e-06, "loss": 0.0028, "step": 759 }, { "clip_ratio": 0.03938836231827736, "epoch": 0.26592022393282017, "grad_norm": 0.23031640964558486, "kl": 1.0625, "learning_rate": 9.967427660691732e-06, "loss": 0.002, "step": 760 }, { "clip_ratio": 0.0035637568216770887, "completions/clipped_ratio": 0.1071428571428571, "completions/max_length": 256.0, "completions/max_terminated_length": 256.0, "completions/mean_length": 166.92857360839844, "completions/mean_terminated_length": 156.239990234375, "completions/min_length": 109.0, "completions/min_terminated_length": 109.0, "epoch": 0.2662701189643107, "grad_norm": 0.5054158370943923, "kl": 0.9375, "learning_rate": 9.9670787457779e-06, "loss": 0.0037, "num_tokens": 3889104.0, "reward": 2.5062077045440674, "reward_std": 0.2912936806678772, "rewards/check_gptzero_func/mean": 0.625, "rewards/check_gptzero_func/std": 0.48850420117378235, "rewards/check_winston_local_func/mean": 0.9332908987998962, "rewards/check_winston_local_func/std": 0.11419615894556046, "rewards/sentence_count_match_reward_logic/mean": 0.9479166865348816, "rewards/sentence_count_match_reward_logic/std": 0.120565265417099, "step": 761 }, { "clip_ratio": 0.00995944906026125, "epoch": 0.26662001399580126, "grad_norm": 0.4216465836152267, "kl": 0.9375, "learning_rate": 9.966727978190705e-06, "loss": 0.0015, "step": 762 }, { "clip_ratio": 0.02692689187824726, "epoch": 0.2669699090272918, "grad_norm": 0.30492152768574815, "kl": 0.94140625, "learning_rate": 9.966375358060981e-06, "loss": -0.0002, "step": 763 }, { "clip_ratio": 0.04177353158593178, "epoch": 0.26731980405878236, "grad_norm": 0.2608835466839861, "kl": 0.94921875, "learning_rate": 9.966020885520251e-06, "loss": -0.0013, "step": 764 }, { "clip_ratio": 0.0038259183056652546, "completions/clipped_ratio": 0.1428571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 225.0, "completions/mean_length": 173.37501525878906, "completions/mean_terminated_length": 159.6041717529297, "completions/min_length": 87.0, "completions/min_terminated_length": 87.0, "epoch": 0.26766969909027294, "grad_norm": 0.5408435309848149, "kl": 0.98046875, "learning_rate": 9.965664560700734e-06, "loss": -0.0015, "num_tokens": 3910021.0, "reward": 2.3749561309814453, "reward_std": 0.09277269244194031, "rewards/check_gptzero_func/mean": 0.5714285969734192, "rewards/check_gptzero_func/std": 0.4993502199649811, "rewards/check_winston_local_func/mean": 0.8198966383934021, "rewards/check_winston_local_func/std": 0.2772858142852783, "rewards/sentence_count_match_reward_logic/mean": 0.9836309552192688, "rewards/sentence_count_match_reward_logic/std": 0.04812033474445343, "step": 765 }, { "clip_ratio": 0.015187127515673637, "epoch": 0.26801959412176346, "grad_norm": 0.49778044920266173, "kl": 0.98046875, "learning_rate": 9.96530638373533e-06, "loss": -0.0037, "step": 766 }, { "clip_ratio": 0.03287479653954506, "epoch": 0.26836948915325404, "grad_norm": 0.35700411239623026, "kl": 0.99609375, "learning_rate": 9.964946354757638e-06, "loss": -0.0059, "step": 767 }, { "clip_ratio": 0.04862682893872261, "epoch": 0.26871938418474456, "grad_norm": 0.34242681193936325, "kl": 1.0, "learning_rate": 9.964584473901946e-06, "loss": -0.0068, "step": 768 }, { "clip_ratio": 0.0035739310551434755, "completions/clipped_ratio": 0.0, "completions/max_length": 243.0, "completions/max_terminated_length": 243.0, "completions/mean_length": 198.07144165039062, "completions/mean_terminated_length": 198.07144165039062, "completions/min_length": 116.0, "completions/min_terminated_length": 116.0, "epoch": 0.26906927921623514, "grad_norm": 0.5385752680578532, "kl": 0.82421875, "learning_rate": 9.964220741303232e-06, "loss": -0.0055, "num_tokens": 3933673.0, "reward": 2.5385284423828125, "reward_std": 0.11876777559518814, "rewards/check_gptzero_func/mean": 0.7321428656578064, "rewards/check_gptzero_func/std": 0.44685041904449463, "rewards/check_winston_local_func/mean": 0.8432903289794922, "rewards/check_winston_local_func/std": 0.23880302906036377, "rewards/sentence_count_match_reward_logic/mean": 0.9630952477455139, "rewards/sentence_count_match_reward_logic/std": 0.08800333738327026, "step": 769 }, { "clip_ratio": 0.013018137775361538, "epoch": 0.26941917424772566, "grad_norm": 0.40554866319951777, "kl": 0.8359375, "learning_rate": 9.963855157097166e-06, "loss": -0.0074, "step": 770 }, { "clip_ratio": 0.02625296637415886, "epoch": 0.26976906927921623, "grad_norm": 0.48065533265398697, "kl": 0.8359375, "learning_rate": 9.963487721420104e-06, "loss": -0.0092, "step": 771 }, { "clip_ratio": 0.033403430134058, "epoch": 0.2701189643107068, "grad_norm": 0.2450554168392271, "kl": 0.83203125, "learning_rate": 9.9631184344091e-06, "loss": -0.0106, "step": 772 }, { "clip_ratio": 0.004458786454051733, "completions/clipped_ratio": 0.0, "completions/max_length": 180.0, "completions/max_terminated_length": 180.0, "completions/mean_length": 156.17857360839844, "completions/mean_terminated_length": 156.17857360839844, "completions/min_length": 131.0, "completions/min_terminated_length": 131.0, "epoch": 0.27046885934219733, "grad_norm": 0.564358935426947, "kl": 1.0546875, "learning_rate": 9.962747296201891e-06, "loss": 0.0066, "num_tokens": 3952579.0, "reward": 2.3245296478271484, "reward_std": 0.2164779156446457, "rewards/check_gptzero_func/mean": 0.4285714328289032, "rewards/check_gptzero_func/std": 0.4993502199649811, "rewards/check_winston_local_func/mean": 0.8959583640098572, "rewards/check_winston_local_func/std": 0.1775386482477188, "rewards/sentence_count_match_reward_logic/mean": 1.0, "rewards/sentence_count_match_reward_logic/std": 0.0, "step": 773 }, { "clip_ratio": 0.013220853172242641, "epoch": 0.2708187543736879, "grad_norm": 0.4333810038727544, "kl": 1.046875, "learning_rate": 9.96237430693691e-06, "loss": 0.0045, "step": 774 }, { "clip_ratio": 0.027929456904530525, "epoch": 0.27116864940517843, "grad_norm": 0.3122999658161747, "kl": 1.0546875, "learning_rate": 9.961999466753278e-06, "loss": 0.0027, "step": 775 }, { "clip_ratio": 0.04488958790898323, "epoch": 0.271518544436669, "grad_norm": 0.30114896326550183, "kl": 1.0703125, "learning_rate": 9.961622775790808e-06, "loss": 0.0017, "step": 776 }, { "clip_ratio": 0.00269711809232831, "completions/clipped_ratio": 0.0, "completions/max_length": 210.0, "completions/max_terminated_length": 210.0, "completions/mean_length": 150.2857208251953, "completions/mean_terminated_length": 150.2857208251953, "completions/min_length": 90.0, "completions/min_terminated_length": 90.0, "epoch": 0.27186843946815953, "grad_norm": 0.502982471760158, "kl": 1.203125, "learning_rate": 9.961244234190001e-06, "loss": 0.0064, "num_tokens": 3971075.0, "reward": 2.681481122970581, "reward_std": 0.20891883969306946, "rewards/check_gptzero_func/mean": 0.6964285969734192, "rewards/check_gptzero_func/std": 0.4639609158039093, "rewards/check_winston_local_func/mean": 0.9850524067878723, "rewards/check_winston_local_func/std": 0.007645362988114357, "rewards/sentence_count_match_reward_logic/mean": 1.0, "rewards/sentence_count_match_reward_logic/std": 0.0, "step": 777 }, { "clip_ratio": 0.015072030946612358, "epoch": 0.2722183344996501, "grad_norm": 0.3715301680956427, "kl": 1.1953125, "learning_rate": 9.960863842092048e-06, "loss": 0.0042, "step": 778 }, { "clip_ratio": 0.0330452062189579, "epoch": 0.2725682295311407, "grad_norm": 0.34290339198042097, "kl": 1.1953125, "learning_rate": 9.960481599638835e-06, "loss": 0.0031, "step": 779 }, { "clip_ratio": 0.044839952141046524, "epoch": 0.2729181245626312, "grad_norm": 0.29756022219848977, "kl": 1.203125, "learning_rate": 9.960097506972934e-06, "loss": 0.0025, "step": 780 }, { "clip_ratio": 0.002978422911837697, "completions/clipped_ratio": 0.1428571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 203.0, "completions/mean_length": 178.10714721679688, "completions/mean_terminated_length": 165.125, "completions/min_length": 111.0, "completions/min_terminated_length": 111.0, "epoch": 0.2732680195941218, "grad_norm": 0.5027905737022857, "kl": 0.98046875, "learning_rate": 9.959711564237603e-06, "loss": 0.0063, "num_tokens": 3992697.0, "reward": 2.683919668197632, "reward_std": 0.16019971668720245, "rewards/check_gptzero_func/mean": 0.75, "rewards/check_gptzero_func/std": 0.43693143129348755, "rewards/check_winston_local_func/mean": 0.9339196085929871, "rewards/check_winston_local_func/std": 0.1293231099843979, "rewards/sentence_count_match_reward_logic/mean": 1.0, "rewards/sentence_count_match_reward_logic/std": 0.0, "step": 781 }, { "clip_ratio": 0.009762002155184746, "epoch": 0.2736179146256123, "grad_norm": 0.41996001491466006, "kl": 0.98046875, "learning_rate": 9.959323771576804e-06, "loss": 0.0049, "step": 782 }, { "clip_ratio": 0.02180704101920128, "epoch": 0.2739678096571029, "grad_norm": 0.27560517236523163, "kl": 0.9921875, "learning_rate": 9.958934129135172e-06, "loss": 0.0036, "step": 783 }, { "clip_ratio": 0.035928744822740555, "epoch": 0.2743177046885934, "grad_norm": 0.28682776806242016, "kl": 0.99609375, "learning_rate": 9.958542637058045e-06, "loss": 0.0029, "step": 784 }, { "clip_ratio": 0.002802504925057292, "completions/clipped_ratio": 0.2857142857142857, "completions/max_length": 256.0, "completions/max_terminated_length": 232.0, "completions/mean_length": 196.3928680419922, "completions/mean_terminated_length": 172.5500030517578, "completions/min_length": 92.0, "completions/min_terminated_length": 92.0, "epoch": 0.274667599720084, "grad_norm": 0.5183401307276787, "kl": 0.7890625, "learning_rate": 9.958149295491441e-06, "loss": -0.0023, "num_tokens": 4016647.0, "reward": 2.589463233947754, "reward_std": 0.21138635277748108, "rewards/check_gptzero_func/mean": 0.7678571343421936, "rewards/check_gptzero_func/std": 0.4260205328464508, "rewards/check_winston_local_func/mean": 0.8459824323654175, "rewards/check_winston_local_func/std": 0.2112969160079956, "rewards/sentence_count_match_reward_logic/mean": 0.9756235480308533, "rewards/sentence_count_match_reward_logic/std": 0.04727768525481224, "step": 785 }, { "clip_ratio": 0.00966702401638031, "epoch": 0.27501749475157455, "grad_norm": 0.4069901365785277, "kl": 0.7890625, "learning_rate": 9.957754104582076e-06, "loss": -0.0046, "step": 786 }, { "clip_ratio": 0.023108581081032753, "epoch": 0.2753673897830651, "grad_norm": 0.3022486460717183, "kl": 0.7890625, "learning_rate": 9.957357064477354e-06, "loss": -0.0059, "step": 787 }, { "clip_ratio": 0.034962933510541916, "epoch": 0.27571728481455565, "grad_norm": 0.25515033223384803, "kl": 0.80078125, "learning_rate": 9.956958175325362e-06, "loss": -0.0072, "step": 788 }, { "clip_ratio": 0.00529387267306447, "completions/clipped_ratio": 0.2678571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 253.0, "completions/mean_length": 165.6607208251953, "completions/mean_terminated_length": 132.60975646972656, "completions/min_length": 81.0, "completions/min_terminated_length": 81.0, "epoch": 0.2760671798460462, "grad_norm": 0.5804574515600924, "kl": 1.0546875, "learning_rate": 9.956557437274887e-06, "loss": 0.0069, "num_tokens": 4036924.0, "reward": 2.6274611949920654, "reward_std": 0.054875560104846954, "rewards/check_gptzero_func/mean": 0.7142857313156128, "rewards/check_gptzero_func/std": 0.4558422863483429, "rewards/check_winston_local_func/mean": 0.9479967951774597, "rewards/check_winston_local_func/std": 0.06435063481330872, "rewards/sentence_count_match_reward_logic/mean": 0.9651785492897034, "rewards/sentence_count_match_reward_logic/std": 0.07193408906459808, "step": 789 }, { "clip_ratio": 0.015100949443876743, "epoch": 0.27641707487753675, "grad_norm": 0.37964796338352, "kl": 1.0625, "learning_rate": 9.9561548504754e-06, "loss": 0.005, "step": 790 }, { "clip_ratio": 0.03155461326241493, "epoch": 0.27676696990902727, "grad_norm": 0.3147588577928675, "kl": 1.0703125, "learning_rate": 9.955750415077056e-06, "loss": 0.0035, "step": 791 }, { "clip_ratio": 0.042045433074235916, "epoch": 0.27711686494051785, "grad_norm": 0.3081437423972393, "kl": 1.078125, "learning_rate": 9.955344131230714e-06, "loss": 0.003, "step": 792 }, { "clip_ratio": 0.0037939248140901327, "completions/clipped_ratio": 0.0, "completions/max_length": 234.0, "completions/max_terminated_length": 234.0, "completions/mean_length": 160.6428680419922, "completions/mean_terminated_length": 160.6428680419922, "completions/min_length": 72.0, "completions/min_terminated_length": 72.0, "epoch": 0.27746675997200837, "grad_norm": 0.6864553672828277, "kl": 1.078125, "learning_rate": 9.954935999087908e-06, "loss": 0.0088, "num_tokens": 4056256.0, "reward": 2.6864712238311768, "reward_std": 0.15500682592391968, "rewards/check_gptzero_func/mean": 0.8214285969734192, "rewards/check_gptzero_func/std": 0.3864591419696808, "rewards/check_winston_local_func/mean": 0.8854504823684692, "rewards/check_winston_local_func/std": 0.13805678486824036, "rewards/sentence_count_match_reward_logic/mean": 0.9795918464660645, "rewards/sentence_count_match_reward_logic/std": 0.050441987812519073, "step": 793 }, { "clip_ratio": 0.018957534804940224, "epoch": 0.27781665500349895, "grad_norm": 0.49432365802150796, "kl": 1.078125, "learning_rate": 9.954526018800872e-06, "loss": 0.0064, "step": 794 }, { "clip_ratio": 0.0363081693649292, "epoch": 0.2781665500349895, "grad_norm": 0.3781170149493823, "kl": 1.0859375, "learning_rate": 9.954114190522521e-06, "loss": 0.0052, "step": 795 }, { "clip_ratio": 0.042792029678821564, "epoch": 0.27851644506648005, "grad_norm": 0.26124847654306504, "kl": 1.1015625, "learning_rate": 9.953700514406464e-06, "loss": 0.0047, "step": 796 }, { "clip_ratio": 0.004139158874750137, "completions/clipped_ratio": 0.2857142857142857, "completions/max_length": 256.0, "completions/max_terminated_length": 209.0, "completions/mean_length": 186.8928680419922, "completions/mean_terminated_length": 159.25, "completions/min_length": 69.0, "completions/min_terminated_length": 69.0, "epoch": 0.2788663400979706, "grad_norm": 0.575971432763401, "kl": 0.87109375, "learning_rate": 9.953284990607e-06, "loss": 0.0075, "num_tokens": 4080218.0, "reward": 2.485288381576538, "reward_std": 0.2206238955259323, "rewards/check_gptzero_func/mean": 0.7142857313156128, "rewards/check_gptzero_func/std": 0.4558422863483429, "rewards/check_winston_local_func/mean": 0.8218955993652344, "rewards/check_winston_local_func/std": 0.16995634138584137, "rewards/sentence_count_match_reward_logic/mean": 0.9491071701049805, "rewards/sentence_count_match_reward_logic/std": 0.11889301985502243, "step": 797 }, { "clip_ratio": 0.01046223845332861, "epoch": 0.27921623512946114, "grad_norm": 0.37872337174774773, "kl": 0.875, "learning_rate": 9.952867619279112e-06, "loss": 0.0062, "step": 798 }, { "clip_ratio": 0.02060561254620552, "epoch": 0.2795661301609517, "grad_norm": 0.35869788713186, "kl": 0.87890625, "learning_rate": 9.95244840057848e-06, "loss": 0.0047, "step": 799 }, { "clip_ratio": 0.032131683081388474, "epoch": 0.27991602519244224, "grad_norm": 0.3133014671550046, "kl": 0.87890625, "learning_rate": 9.952027334661465e-06, "loss": 0.0041, "step": 800 }, { "clip_ratio": 0.003061501309275627, "completions/clipped_ratio": 0.0, "completions/max_length": 211.0, "completions/max_terminated_length": 211.0, "completions/mean_length": 164.71429443359375, "completions/mean_terminated_length": 164.71429443359375, "completions/min_length": 107.0, "completions/min_terminated_length": 107.0, "epoch": 0.2802659202239328, "grad_norm": 0.524912367093809, "kl": 0.97265625, "learning_rate": 9.951604421685121e-06, "loss": 0.0059, "num_tokens": 4100010.0, "reward": 2.4479613304138184, "reward_std": 0.30964088439941406, "rewards/check_gptzero_func/mean": 0.5357142686843872, "rewards/check_gptzero_func/std": 0.5032362341880798, "rewards/check_winston_local_func/mean": 0.9181994199752808, "rewards/check_winston_local_func/std": 0.09025468677282333, "rewards/sentence_count_match_reward_logic/mean": 0.9940475821495056, "rewards/sentence_count_match_reward_logic/std": 0.031209392473101616, "step": 801 }, { "clip_ratio": 0.010351012460887432, "epoch": 0.2806158152554234, "grad_norm": 0.42038528764092925, "kl": 0.9765625, "learning_rate": 9.951179661807194e-06, "loss": 0.0036, "step": 802 }, { "clip_ratio": 0.027073433622717857, "epoch": 0.2809657102869139, "grad_norm": 0.38310052710920733, "kl": 0.98046875, "learning_rate": 9.95075305518611e-06, "loss": 0.0019, "step": 803 }, { "clip_ratio": 0.04155990853905678, "epoch": 0.2813156053184045, "grad_norm": 0.2488605173041685, "kl": 0.98828125, "learning_rate": 9.950324601980994e-06, "loss": 0.0007, "step": 804 }, { "clip_ratio": 0.003913749475032091, "completions/clipped_ratio": 0.017857142857142905, "completions/max_length": 256.0, "completions/max_terminated_length": 256.0, "completions/mean_length": 187.85714721679688, "completions/mean_terminated_length": 186.61817932128906, "completions/min_length": 91.0, "completions/min_terminated_length": 91.0, "epoch": 0.281665500349895, "grad_norm": 0.5793917710855647, "kl": 0.92578125, "learning_rate": 9.949894302351653e-06, "loss": 0.0355, "num_tokens": 4122482.0, "reward": 2.457976818084717, "reward_std": 0.17606514692306519, "rewards/check_gptzero_func/mean": 0.5714285969734192, "rewards/check_gptzero_func/std": 0.4993502199649811, "rewards/check_winston_local_func/mean": 0.9066587686538696, "rewards/check_winston_local_func/std": 0.14035941660404205, "rewards/sentence_count_match_reward_logic/mean": 0.979889452457428, "rewards/sentence_count_match_reward_logic/std": 0.07762039452791214, "step": 805 }, { "clip_ratio": 0.011805259622633457, "epoch": 0.2820153953813856, "grad_norm": 0.4464053246441084, "kl": 0.9375, "learning_rate": 9.949462156458584e-06, "loss": 0.0328, "step": 806 }, { "clip_ratio": 0.025310201570391655, "epoch": 0.2823652904128761, "grad_norm": 0.3796369603046942, "kl": 0.9453125, "learning_rate": 9.949028164462976e-06, "loss": 0.0307, "step": 807 }, { "clip_ratio": 0.03944176435470581, "epoch": 0.2827151854443667, "grad_norm": 0.31264440879360206, "kl": 0.94921875, "learning_rate": 9.9485923265267e-06, "loss": 0.0291, "step": 808 }, { "clip_ratio": 0.004294027574360371, "completions/clipped_ratio": 0.0, "completions/max_length": 198.0, "completions/max_terminated_length": 198.0, "completions/mean_length": 169.46429443359375, "completions/mean_terminated_length": 169.46429443359375, "completions/min_length": 141.0, "completions/min_terminated_length": 141.0, "epoch": 0.28306508047585727, "grad_norm": 0.6220262253757826, "kl": 1.0, "learning_rate": 9.948154642812321e-06, "loss": 0.0035, "num_tokens": 4143164.0, "reward": 2.642123222351074, "reward_std": 0.40322381258010864, "rewards/check_gptzero_func/mean": 0.75, "rewards/check_gptzero_func/std": 0.43693143129348755, "rewards/check_winston_local_func/mean": 0.9229265451431274, "rewards/check_winston_local_func/std": 0.14670218527317047, "rewards/sentence_count_match_reward_logic/mean": 0.9691964387893677, "rewards/sentence_count_match_reward_logic/std": 0.15030205249786377, "step": 809 }, { "clip_ratio": 0.014438241720199585, "epoch": 0.2834149755073478, "grad_norm": 0.42956500704599265, "kl": 1.0078125, "learning_rate": 9.947715113483091e-06, "loss": 0.0007, "step": 810 }, { "clip_ratio": 0.03250772878527641, "epoch": 0.28376487053883837, "grad_norm": 0.3483145399065421, "kl": 1.0078125, "learning_rate": 9.947273738702952e-06, "loss": -0.0012, "step": 811 }, { "clip_ratio": 0.04685623198747635, "epoch": 0.2841147655703289, "grad_norm": 0.30070001839884436, "kl": 1.015625, "learning_rate": 9.946830518636532e-06, "loss": -0.0022, "step": 812 }, { "clip_ratio": 0.004628131631761789, "completions/clipped_ratio": 0.2857142857142857, "completions/max_length": 256.0, "completions/max_terminated_length": 213.0, "completions/mean_length": 179.75001525878906, "completions/mean_terminated_length": 149.25, "completions/min_length": 101.0, "completions/min_terminated_length": 101.0, "epoch": 0.28446466060181946, "grad_norm": 0.5829884913068483, "kl": 0.96484375, "learning_rate": 9.946385453449145e-06, "loss": 0.0063, "num_tokens": 4165606.0, "reward": 2.4536755084991455, "reward_std": 0.12824247777462006, "rewards/check_gptzero_func/mean": 0.625, "rewards/check_gptzero_func/std": 0.48850420117378235, "rewards/check_winston_local_func/mean": 0.8510674238204956, "rewards/check_winston_local_func/std": 0.15796740353107452, "rewards/sentence_count_match_reward_logic/mean": 0.9776077270507812, "rewards/sentence_count_match_reward_logic/std": 0.05179184675216675, "step": 813 }, { "clip_ratio": 0.012547748163342476, "epoch": 0.28481455563331, "grad_norm": 0.39689687727661105, "kl": 0.96484375, "learning_rate": 9.9459385433068e-06, "loss": 0.0042, "step": 814 }, { "clip_ratio": 0.027801649644970894, "epoch": 0.28516445066480056, "grad_norm": 0.2444245781566773, "kl": 0.97265625, "learning_rate": 9.945489788376188e-06, "loss": 0.0028, "step": 815 }, { "clip_ratio": 0.03810962662100792, "epoch": 0.28551434569629114, "grad_norm": 0.2374745822711848, "kl": 0.98828125, "learning_rate": 9.94503918882469e-06, "loss": 0.0024, "step": 816 }, { "clip_ratio": 0.004093131981790066, "completions/clipped_ratio": 0.0, "completions/max_length": 235.0, "completions/max_terminated_length": 235.0, "completions/mean_length": 181.2678680419922, "completions/mean_terminated_length": 181.2678680419922, "completions/min_length": 125.0, "completions/min_terminated_length": 125.0, "epoch": 0.28586424072778166, "grad_norm": 0.5709091578740522, "kl": 0.97265625, "learning_rate": 9.944586744820377e-06, "loss": 0.0104, "num_tokens": 4187501.0, "reward": 2.508549690246582, "reward_std": 0.11523368209600449, "rewards/check_gptzero_func/mean": 0.6607142686843872, "rewards/check_gptzero_func/std": 0.477751761674881, "rewards/check_winston_local_func/mean": 0.8709008097648621, "rewards/check_winston_local_func/std": 0.2468201220035553, "rewards/sentence_count_match_reward_logic/mean": 0.9769344925880432, "rewards/sentence_count_match_reward_logic/std": 0.05020138993859291, "step": 817 }, { "clip_ratio": 0.013506715185940266, "epoch": 0.28621413575927224, "grad_norm": 0.5218715773944923, "kl": 0.9765625, "learning_rate": 9.944132456532005e-06, "loss": 0.0079, "step": 818 }, { "clip_ratio": 0.02834169752895832, "epoch": 0.28656403079076276, "grad_norm": 0.3777646178577803, "kl": 0.9765625, "learning_rate": 9.94367632412902e-06, "loss": 0.006, "step": 819 }, { "clip_ratio": 0.03947857394814491, "epoch": 0.28691392582225334, "grad_norm": 0.31816096296059654, "kl": 0.98046875, "learning_rate": 9.943218347781553e-06, "loss": 0.005, "step": 820 }, { "clip_ratio": 0.0026376887690275908, "completions/clipped_ratio": 0.2857142857142857, "completions/max_length": 256.0, "completions/max_terminated_length": 256.0, "completions/mean_length": 211.96429443359375, "completions/mean_terminated_length": 194.35000610351562, "completions/min_length": 139.0, "completions/min_terminated_length": 139.0, "epoch": 0.28726382085374386, "grad_norm": 0.5589073426391544, "kl": 0.84375, "learning_rate": 9.942758527660429e-06, "loss": 0.0036, "num_tokens": 4213483.0, "reward": 2.1221437454223633, "reward_std": 0.07436716556549072, "rewards/check_gptzero_func/mean": 0.3035714328289032, "rewards/check_gptzero_func/std": 0.4639608860015869, "rewards/check_winston_local_func/mean": 0.8598986864089966, "rewards/check_winston_local_func/std": 0.2663514018058777, "rewards/sentence_count_match_reward_logic/mean": 0.9586734771728516, "rewards/sentence_count_match_reward_logic/std": 0.07703592628240585, "step": 821 }, { "clip_ratio": 0.009844417683780193, "epoch": 0.28761371588523443, "grad_norm": 0.45479368713384577, "kl": 0.84765625, "learning_rate": 9.942296863937154e-06, "loss": 0.0017, "step": 822 }, { "clip_ratio": 0.0208458062261343, "epoch": 0.287963610916725, "grad_norm": 0.32811400723835854, "kl": 0.8515625, "learning_rate": 9.941833356783924e-06, "loss": -0.0002, "step": 823 }, { "clip_ratio": 0.037309642881155014, "epoch": 0.28831350594821553, "grad_norm": 0.3035366372310371, "kl": 0.85546875, "learning_rate": 9.941368006373622e-06, "loss": -0.0019, "step": 824 }, { "clip_ratio": 0.004103482700884342, "completions/clipped_ratio": 0.1428571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 220.0, "completions/mean_length": 159.4107208251953, "completions/mean_terminated_length": 143.3125, "completions/min_length": 78.0, "completions/min_terminated_length": 78.0, "epoch": 0.2886634009797061, "grad_norm": 0.4508996685162622, "kl": 0.97265625, "learning_rate": 9.940900812879822e-06, "loss": 0.0052, "num_tokens": 4233794.0, "reward": 2.6380531787872314, "reward_std": 0.13684748113155365, "rewards/check_gptzero_func/mean": 0.7857142686843872, "rewards/check_gptzero_func/std": 0.4140393137931824, "rewards/check_winston_local_func/mean": 0.8761484026908875, "rewards/check_winston_local_func/std": 0.25401151180267334, "rewards/sentence_count_match_reward_logic/mean": 0.976190447807312, "rewards/sentence_count_match_reward_logic/std": 0.058848995715379715, "step": 825 }, { "clip_ratio": 0.00927019864320755, "epoch": 0.28901329601119663, "grad_norm": 0.3814591645538538, "kl": 0.9765625, "learning_rate": 9.94043177647678e-06, "loss": 0.0039, "step": 826 }, { "clip_ratio": 0.024457460269331932, "epoch": 0.2893631910426872, "grad_norm": 0.30322268577496914, "kl": 0.9765625, "learning_rate": 9.939960897339444e-06, "loss": 0.0025, "step": 827 }, { "clip_ratio": 0.03635338693857193, "epoch": 0.28971308607417773, "grad_norm": 0.23387079800278962, "kl": 0.98828125, "learning_rate": 9.939488175643446e-06, "loss": 0.0015, "step": 828 }, { "clip_ratio": 0.0037647380959242582, "completions/clipped_ratio": 0.2678571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 255.0, "completions/mean_length": 178.1607208251953, "completions/mean_terminated_length": 149.68292236328125, "completions/min_length": 67.0, "completions/min_terminated_length": 67.0, "epoch": 0.2900629811056683, "grad_norm": 0.6841196046457668, "kl": 1.140625, "learning_rate": 9.93901361156511e-06, "loss": 0.0102, "num_tokens": 4255387.0, "reward": 2.673393964767456, "reward_std": 0.2502482533454895, "rewards/check_gptzero_func/mean": 0.7321428656578064, "rewards/check_gptzero_func/std": 0.44685041904449463, "rewards/check_winston_local_func/mean": 0.9564700722694397, "rewards/check_winston_local_func/std": 0.06554171442985535, "rewards/sentence_count_match_reward_logic/mean": 0.9847808480262756, "rewards/sentence_count_match_reward_logic/std": 0.0353604294359684, "step": 829 }, { "clip_ratio": 0.014306803233921528, "epoch": 0.2904128761371588, "grad_norm": 0.4499924633693885, "kl": 1.1484375, "learning_rate": 9.938537205281438e-06, "loss": 0.0077, "step": 830 }, { "clip_ratio": 0.03535647317767143, "epoch": 0.2907627711686494, "grad_norm": 0.45729802906707967, "kl": 1.1484375, "learning_rate": 9.938058956970132e-06, "loss": 0.0057, "step": 831 }, { "clip_ratio": 0.04841914400458336, "epoch": 0.29111266620014, "grad_norm": 0.3387699922473884, "kl": 1.1640625, "learning_rate": 9.937578866809567e-06, "loss": 0.0043, "step": 832 }, { "clip_ratio": 0.0033647087402641773, "completions/clipped_ratio": 0.1428571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 245.0, "completions/mean_length": 187.0357208251953, "completions/mean_terminated_length": 175.5416717529297, "completions/min_length": 91.0, "completions/min_terminated_length": 91.0, "epoch": 0.2914625612316305, "grad_norm": 0.5161488224276803, "kl": 0.81640625, "learning_rate": 9.937096934978819e-06, "loss": 0.0072, "num_tokens": 4278141.0, "reward": 2.732639789581299, "reward_std": 0.1229407861828804, "rewards/check_gptzero_func/mean": 0.9464285969734192, "rewards/check_gptzero_func/std": 0.22720777988433838, "rewards/check_winston_local_func/mean": 0.7862110137939453, "rewards/check_winston_local_func/std": 0.25605037808418274, "rewards/sentence_count_match_reward_logic/mean": 1.0, "rewards/sentence_count_match_reward_logic/std": 0.0, "step": 833 }, { "clip_ratio": 0.010401026345789433, "epoch": 0.2918124562631211, "grad_norm": 0.4626058342422498, "kl": 0.84765625, "learning_rate": 9.936613161657639e-06, "loss": 0.0053, "step": 834 }, { "clip_ratio": 0.0249341893941164, "epoch": 0.2921623512946116, "grad_norm": 0.3802104078197863, "kl": 0.84375, "learning_rate": 9.936127547026471e-06, "loss": 0.0037, "step": 835 }, { "clip_ratio": 0.037995900958776474, "epoch": 0.2925122463261022, "grad_norm": 0.24420378284772312, "kl": 0.83984375, "learning_rate": 9.935640091266444e-06, "loss": 0.0028, "step": 836 }, { "clip_ratio": 0.003705305280163884, "completions/clipped_ratio": 0.0, "completions/max_length": 219.0, "completions/max_terminated_length": 219.0, "completions/mean_length": 133.7857208251953, "completions/mean_terminated_length": 133.7857208251953, "completions/min_length": 68.0, "completions/min_terminated_length": 68.0, "epoch": 0.2928621413575927, "grad_norm": 0.5994733142513563, "kl": 1.3125, "learning_rate": 9.935150794559379e-06, "loss": 0.0043, "num_tokens": 4294913.0, "reward": 2.1593539714813232, "reward_std": 0.3257354497909546, "rewards/check_gptzero_func/mean": 0.3571428656578064, "rewards/check_gptzero_func/std": 0.48349377512931824, "rewards/check_winston_local_func/mean": 0.8319727778434753, "rewards/check_winston_local_func/std": 0.18410922586917877, "rewards/sentence_count_match_reward_logic/mean": 0.9702380895614624, "rewards/sentence_count_match_reward_logic/std": 0.08472858369350433, "step": 837 }, { "clip_ratio": 0.013119549490511417, "epoch": 0.2932120363890833, "grad_norm": 0.40561911452029986, "kl": 1.3125, "learning_rate": 9.934659657087773e-06, "loss": 0.0023, "step": 838 }, { "clip_ratio": 0.030351975932717323, "epoch": 0.29356193142057385, "grad_norm": 0.27577246874697875, "kl": 1.328125, "learning_rate": 9.93416667903482e-06, "loss": 0.0006, "step": 839 }, { "clip_ratio": 0.04636790230870247, "epoch": 0.2939118264520644, "grad_norm": 0.28849008464722137, "kl": 1.359375, "learning_rate": 9.933671860584394e-06, "loss": -0.0, "step": 840 }, { "clip_ratio": 0.004380852449685335, "completions/clipped_ratio": 0.0, "completions/max_length": 190.0, "completions/max_terminated_length": 190.0, "completions/mean_length": 135.83929443359375, "completions/mean_terminated_length": 135.83929443359375, "completions/min_length": 92.0, "completions/min_terminated_length": 92.0, "epoch": 0.29426172148355495, "grad_norm": 0.5288519924862488, "kl": 1.109375, "learning_rate": 9.933175201921057e-06, "loss": 0.0043, "num_tokens": 4311928.0, "reward": 2.1285274028778076, "reward_std": 0.2776705324649811, "rewards/check_gptzero_func/mean": 0.2678571343421936, "rewards/check_gptzero_func/std": 0.446850448846817, "rewards/check_winston_local_func/mean": 0.8919199109077454, "rewards/check_winston_local_func/std": 0.14939084649085999, "rewards/sentence_count_match_reward_logic/mean": 0.96875, "rewards/sentence_count_match_reward_logic/std": 0.08342797309160233, "step": 841 }, { "clip_ratio": 0.013447691686451435, "epoch": 0.29461161651504547, "grad_norm": 0.3880527413952929, "kl": 1.1171875, "learning_rate": 9.93267670323006e-06, "loss": 0.0026, "step": 842 }, { "clip_ratio": 0.02973771281540394, "epoch": 0.29496151154653605, "grad_norm": 0.29670099886219176, "kl": 1.1328125, "learning_rate": 9.932176364697337e-06, "loss": 0.0013, "step": 843 }, { "clip_ratio": 0.04466579481959343, "epoch": 0.29531140657802657, "grad_norm": 0.2653534434976589, "kl": 1.1328125, "learning_rate": 9.93167418650951e-06, "loss": 0.0003, "step": 844 }, { "clip_ratio": 0.0039049063343554735, "completions/clipped_ratio": 0.1428571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 238.0, "completions/mean_length": 179.98214721679688, "completions/mean_terminated_length": 167.3125, "completions/min_length": 103.0, "completions/min_terminated_length": 103.0, "epoch": 0.29566130160951715, "grad_norm": 0.6444232695334207, "kl": 0.9375, "learning_rate": 9.931170168853886e-06, "loss": 0.0084, "num_tokens": 4333711.0, "reward": 2.314997673034668, "reward_std": 0.236615851521492, "rewards/check_gptzero_func/mean": 0.5, "rewards/check_gptzero_func/std": 0.5045249462127686, "rewards/check_winston_local_func/mean": 0.814997673034668, "rewards/check_winston_local_func/std": 0.25384774804115295, "rewards/sentence_count_match_reward_logic/mean": 1.0, "rewards/sentence_count_match_reward_logic/std": 0.0, "step": 845 }, { "clip_ratio": 0.010286367498338223, "epoch": 0.2960111966410077, "grad_norm": 0.3761494029763389, "kl": 0.94140625, "learning_rate": 9.93066431191846e-06, "loss": 0.0066, "step": 846 }, { "clip_ratio": 0.021231399849057198, "epoch": 0.29636109167249824, "grad_norm": 0.29784032132668675, "kl": 0.94921875, "learning_rate": 9.93015661589191e-06, "loss": 0.0053, "step": 847 }, { "clip_ratio": 0.02913116104900837, "epoch": 0.2967109867039888, "grad_norm": 0.24682604081570936, "kl": 0.96484375, "learning_rate": 9.929647080963604e-06, "loss": 0.0046, "step": 848 }, { "clip_ratio": 0.004334324039518833, "completions/clipped_ratio": 0.0, "completions/max_length": 254.0, "completions/max_terminated_length": 254.0, "completions/mean_length": 165.60714721679688, "completions/mean_terminated_length": 165.60714721679688, "completions/min_length": 108.0, "completions/min_terminated_length": 108.0, "epoch": 0.29706088173547934, "grad_norm": 0.5551025034635725, "kl": 0.91796875, "learning_rate": 9.929135707323592e-06, "loss": -0.001, "num_tokens": 4354097.0, "reward": 2.6581907272338867, "reward_std": 0.16786524653434753, "rewards/check_gptzero_func/mean": 0.8035714030265808, "rewards/check_gptzero_func/std": 0.40089187026023865, "rewards/check_winston_local_func/mean": 0.8831905722618103, "rewards/check_winston_local_func/std": 0.16036024689674377, "rewards/sentence_count_match_reward_logic/mean": 0.9714285731315613, "rewards/sentence_count_match_reward_logic/std": 0.07061877846717834, "step": 849 }, { "clip_ratio": 0.014311215840280056, "epoch": 0.2974107767669699, "grad_norm": 0.4393407318282156, "kl": 0.9296875, "learning_rate": 9.928622495162613e-06, "loss": -0.003, "step": 850 }, { "clip_ratio": 0.03156105801463127, "epoch": 0.29776067179846044, "grad_norm": 0.31455228074511415, "kl": 0.9296875, "learning_rate": 9.92810744467209e-06, "loss": -0.005, "step": 851 }, { "clip_ratio": 0.04452741891145706, "epoch": 0.298110566829951, "grad_norm": 0.2889178956129908, "kl": 0.9296875, "learning_rate": 9.927590556044132e-06, "loss": -0.0058, "step": 852 }, { "clip_ratio": 0.0036896602250635624, "completions/clipped_ratio": 0.0, "completions/max_length": 239.0, "completions/max_terminated_length": 239.0, "completions/mean_length": 158.5357208251953, "completions/mean_terminated_length": 158.5357208251953, "completions/min_length": 106.0, "completions/min_terminated_length": 106.0, "epoch": 0.2984604618614416, "grad_norm": 0.5791883494352584, "kl": 1.109375, "learning_rate": 9.927071829471531e-06, "loss": 0.0077, "num_tokens": 4373175.0, "reward": 2.4950387477874756, "reward_std": 0.33616915345191956, "rewards/check_gptzero_func/mean": 0.5892857313156128, "rewards/check_gptzero_func/std": 0.49641573429107666, "rewards/check_winston_local_func/mean": 0.91964191198349, "rewards/check_winston_local_func/std": 0.11771486699581146, "rewards/sentence_count_match_reward_logic/mean": 0.9861111044883728, "rewards/sentence_count_match_reward_logic/std": 0.10393492877483368, "step": 853 }, { "clip_ratio": 0.013752221129834652, "epoch": 0.2988103568929321, "grad_norm": 0.42634353198839253, "kl": 1.1171875, "learning_rate": 9.926551265147769e-06, "loss": 0.0055, "step": 854 }, { "clip_ratio": 0.028267085552215576, "epoch": 0.2991602519244227, "grad_norm": 0.30395671627714826, "kl": 1.140625, "learning_rate": 9.926028863267012e-06, "loss": 0.004, "step": 855 }, { "clip_ratio": 0.04157895967364311, "epoch": 0.2995101469559132, "grad_norm": 0.24142669339068887, "kl": 1.1328125, "learning_rate": 9.925504624024113e-06, "loss": 0.0032, "step": 856 }, { "clip_ratio": 0.0029610220808535814, "completions/clipped_ratio": 0.017857142857142905, "completions/max_length": 256.0, "completions/max_terminated_length": 213.0, "completions/mean_length": 167.82144165039062, "completions/mean_terminated_length": 166.21817016601562, "completions/min_length": 117.0, "completions/min_terminated_length": 117.0, "epoch": 0.2998600419874038, "grad_norm": 0.6769458801780116, "kl": 1.1015625, "learning_rate": 9.924978547614604e-06, "loss": 0.0126, "num_tokens": 4393285.0, "reward": 2.5055253505706787, "reward_std": 0.1803193986415863, "rewards/check_gptzero_func/mean": 0.5892857313156128, "rewards/check_gptzero_func/std": 0.49641573429107666, "rewards/check_winston_local_func/mean": 0.9202080965042114, "rewards/check_winston_local_func/std": 0.10633788257837296, "rewards/sentence_count_match_reward_logic/mean": 0.9960317611694336, "rewards/sentence_count_match_reward_logic/std": 0.029695691540837288, "step": 857 }, { "clip_ratio": 0.015184189192950726, "epoch": 0.3002099370188943, "grad_norm": 0.4792061507130442, "kl": 1.1015625, "learning_rate": 9.924450634234708e-06, "loss": 0.0094, "step": 858 }, { "clip_ratio": 0.0335192047059536, "epoch": 0.3005598320503849, "grad_norm": 0.33072023382513027, "kl": 1.1015625, "learning_rate": 9.923920884081333e-06, "loss": 0.0074, "step": 859 }, { "clip_ratio": 0.048628270626068115, "epoch": 0.3009097270818754, "grad_norm": 0.2786486235545396, "kl": 1.1171875, "learning_rate": 9.92338929735207e-06, "loss": 0.0067, "step": 860 }, { "clip_ratio": 0.0037826907355338335, "completions/clipped_ratio": 0.1428571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 240.0, "completions/mean_length": 189.32144165039062, "completions/mean_terminated_length": 178.20834350585938, "completions/min_length": 115.0, "completions/min_terminated_length": 115.0, "epoch": 0.301259622113366, "grad_norm": 0.6049589106426485, "kl": 0.98046875, "learning_rate": 9.922855874245197e-06, "loss": 0.0089, "num_tokens": 4416335.0, "reward": 2.7679991722106934, "reward_std": 0.16780278086662292, "rewards/check_gptzero_func/mean": 0.8214285969734192, "rewards/check_gptzero_func/std": 0.3864591419696808, "rewards/check_winston_local_func/mean": 0.9788834452629089, "rewards/check_winston_local_func/std": 0.04436923936009407, "rewards/sentence_count_match_reward_logic/mean": 0.9676870703697205, "rewards/sentence_count_match_reward_logic/std": 0.05399492010474205, "step": 861 }, { "clip_ratio": 0.011782762594521046, "epoch": 0.30160951714485656, "grad_norm": 0.45516482070077086, "kl": 0.98046875, "learning_rate": 9.922320614959672e-06, "loss": 0.0063, "step": 862 }, { "clip_ratio": 0.030533161014318466, "epoch": 0.3019594121763471, "grad_norm": 0.3334748800734821, "kl": 0.9921875, "learning_rate": 9.921783519695144e-06, "loss": 0.0048, "step": 863 }, { "clip_ratio": 0.042516011744737625, "epoch": 0.30230930720783766, "grad_norm": 0.29282202807953933, "kl": 0.98828125, "learning_rate": 9.921244588651946e-06, "loss": 0.0041, "step": 864 }, { "clip_ratio": 0.0025338064879179, "completions/clipped_ratio": 0.0, "completions/max_length": 207.0, "completions/max_terminated_length": 207.0, "completions/mean_length": 134.1428680419922, "completions/mean_terminated_length": 134.1428680419922, "completions/min_length": 75.0, "completions/min_terminated_length": 75.0, "epoch": 0.3026592022393282, "grad_norm": 0.5729931720931228, "kl": 1.25, "learning_rate": 9.920703822031094e-06, "loss": 0.0045, "num_tokens": 4432927.0, "reward": 2.3737456798553467, "reward_std": 0.21462096273899078, "rewards/check_gptzero_func/mean": 0.4285714328289032, "rewards/check_gptzero_func/std": 0.4993502199649811, "rewards/check_winston_local_func/mean": 0.9655823111534119, "rewards/check_winston_local_func/std": 0.05104685574769974, "rewards/sentence_count_match_reward_logic/mean": 0.9795918464660645, "rewards/sentence_count_match_reward_logic/std": 0.050441987812519073, "step": 865 }, { "clip_ratio": 0.019387394189834595, "epoch": 0.30300909727081876, "grad_norm": 0.42964236096555075, "kl": 1.2578125, "learning_rate": 9.920161220034286e-06, "loss": 0.0025, "step": 866 }, { "clip_ratio": 0.033170174807310104, "epoch": 0.3033589923023093, "grad_norm": 0.4113267873400971, "kl": 1.265625, "learning_rate": 9.919616782863911e-06, "loss": 0.0014, "step": 867 }, { "clip_ratio": 0.048200223594903946, "epoch": 0.30370888733379986, "grad_norm": 0.24831199417069277, "kl": 1.265625, "learning_rate": 9.919070510723035e-06, "loss": 0.0002, "step": 868 }, { "clip_ratio": 0.0050671459175646305, "completions/clipped_ratio": 0.0, "completions/max_length": 232.0, "completions/max_terminated_length": 232.0, "completions/mean_length": 179.92857360839844, "completions/mean_terminated_length": 179.92857360839844, "completions/min_length": 133.0, "completions/min_terminated_length": 133.0, "epoch": 0.30405878236529044, "grad_norm": 0.592118506977986, "kl": 0.93359375, "learning_rate": 9.918522403815414e-06, "loss": 0.0078, "num_tokens": 4454763.0, "reward": 2.449553966522217, "reward_std": 0.12930035591125488, "rewards/check_gptzero_func/mean": 0.5357142686843872, "rewards/check_gptzero_func/std": 0.5032362937927246, "rewards/check_winston_local_func/mean": 0.9138396382331848, "rewards/check_winston_local_func/std": 0.15999449789524078, "rewards/sentence_count_match_reward_logic/mean": 1.0, "rewards/sentence_count_match_reward_logic/std": 0.0, "step": 869 }, { "clip_ratio": 0.014838553965091705, "epoch": 0.30440867739678096, "grad_norm": 0.47370606522629954, "kl": 0.93359375, "learning_rate": 9.917972462345488e-06, "loss": 0.0055, "step": 870 }, { "clip_ratio": 0.028375450521707535, "epoch": 0.30475857242827153, "grad_norm": 0.3447276022030995, "kl": 0.9453125, "learning_rate": 9.917420686518378e-06, "loss": 0.0039, "step": 871 }, { "clip_ratio": 0.04177219048142433, "epoch": 0.30510846745976206, "grad_norm": 0.35290939809991906, "kl": 0.94921875, "learning_rate": 9.916867076539894e-06, "loss": 0.003, "step": 872 }, { "clip_ratio": 0.0044726296328008175, "completions/clipped_ratio": 0.0, "completions/max_length": 201.0, "completions/max_terminated_length": 201.0, "completions/mean_length": 146.0357208251953, "completions/mean_terminated_length": 146.0357208251953, "completions/min_length": 72.0, "completions/min_terminated_length": 72.0, "epoch": 0.30545836249125263, "grad_norm": 0.5423707181364139, "kl": 1.1484375, "learning_rate": 9.916311632616525e-06, "loss": 0.004, "num_tokens": 4472797.0, "reward": 2.9428603649139404, "reward_std": 0.11016541719436646, "rewards/check_gptzero_func/mean": 0.9642857313156128, "rewards/check_gptzero_func/std": 0.187256321310997, "rewards/check_winston_local_func/mean": 0.9785744547843933, "rewards/check_winston_local_func/std": 0.03545641899108887, "rewards/sentence_count_match_reward_logic/mean": 1.0, "rewards/sentence_count_match_reward_logic/std": 0.0, "step": 873 }, { "clip_ratio": 0.013362230733036995, "epoch": 0.30580825752274315, "grad_norm": 0.3950351598694136, "kl": 1.140625, "learning_rate": 9.915754354955445e-06, "loss": 0.002, "step": 874 }, { "clip_ratio": 0.028235094621777534, "epoch": 0.30615815255423373, "grad_norm": 0.2864385603749428, "kl": 1.140625, "learning_rate": 9.915195243764514e-06, "loss": 0.0007, "step": 875 }, { "clip_ratio": 0.04141067713499069, "epoch": 0.3065080475857243, "grad_norm": 0.21095661451321507, "kl": 1.140625, "learning_rate": 9.91463429925228e-06, "loss": 0.0, "step": 876 }, { "clip_ratio": 0.004016478545963764, "completions/clipped_ratio": 0.1428571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 253.0, "completions/mean_length": 189.75001525878906, "completions/mean_terminated_length": 178.70834350585938, "completions/min_length": 107.0, "completions/min_terminated_length": 107.0, "epoch": 0.30685794261721483, "grad_norm": 0.6511176643417569, "kl": 1.0234375, "learning_rate": 9.914071521627964e-06, "loss": 0.0003, "num_tokens": 4495711.0, "reward": 2.4810409545898438, "reward_std": 0.3310069441795349, "rewards/check_gptzero_func/mean": 0.5714285969734192, "rewards/check_gptzero_func/std": 0.4993502199649811, "rewards/check_winston_local_func/mean": 0.9317423701286316, "rewards/check_winston_local_func/std": 0.07955629378557205, "rewards/sentence_count_match_reward_logic/mean": 0.9778698682785034, "rewards/sentence_count_match_reward_logic/std": 0.0989401638507843, "step": 877 }, { "clip_ratio": 0.018018564209342003, "epoch": 0.3072078376487054, "grad_norm": 0.47661510472796814, "kl": 1.046875, "learning_rate": 9.91350691110148e-06, "loss": -0.0022, "step": 878 }, { "clip_ratio": 0.032664597034454346, "epoch": 0.3075577326801959, "grad_norm": 0.4853508175375639, "kl": 1.0625, "learning_rate": 9.912940467883421e-06, "loss": -0.0038, "step": 879 }, { "clip_ratio": 0.04038387909531593, "epoch": 0.3079076277116865, "grad_norm": 0.30804223975118405, "kl": 1.0546875, "learning_rate": 9.912372192185064e-06, "loss": -0.0051, "step": 880 }, { "clip_ratio": 0.005289111286401749, "completions/clipped_ratio": 0.0892857142857143, "completions/max_length": 256.0, "completions/max_terminated_length": 254.0, "completions/mean_length": 149.82144165039062, "completions/mean_terminated_length": 139.41177368164062, "completions/min_length": 74.0, "completions/min_terminated_length": 74.0, "epoch": 0.308257522743177, "grad_norm": 0.5215684860547146, "kl": 1.1796875, "learning_rate": 9.911802084218374e-06, "loss": 0.0054, "num_tokens": 4513893.0, "reward": 2.4118592739105225, "reward_std": 0.286175400018692, "rewards/check_gptzero_func/mean": 0.4821428656578064, "rewards/check_gptzero_func/std": 0.5042031407356262, "rewards/check_winston_local_func/mean": 0.9542700052261353, "rewards/check_winston_local_func/std": 0.0585140585899353, "rewards/sentence_count_match_reward_logic/mean": 0.9754464030265808, "rewards/sentence_count_match_reward_logic/std": 0.12481723725795746, "step": 881 }, { "clip_ratio": 0.014631097204983234, "epoch": 0.3086074177746676, "grad_norm": 0.399283188334375, "kl": 1.1796875, "learning_rate": 9.911230144195992e-06, "loss": 0.0036, "step": 882 }, { "clip_ratio": 0.027413949370384216, "epoch": 0.3089573128061582, "grad_norm": 0.2940556384879786, "kl": 1.1875, "learning_rate": 9.910656372331246e-06, "loss": 0.0024, "step": 883 }, { "clip_ratio": 0.038305241614580154, "epoch": 0.3093072078376487, "grad_norm": 0.2593263279712417, "kl": 1.203125, "learning_rate": 9.910080768838153e-06, "loss": 0.0016, "step": 884 }, { "clip_ratio": 0.004520189482718706, "completions/clipped_ratio": 0.0, "completions/max_length": 255.0, "completions/max_terminated_length": 255.0, "completions/mean_length": 171.33929443359375, "completions/mean_terminated_length": 171.33929443359375, "completions/min_length": 105.0, "completions/min_terminated_length": 105.0, "epoch": 0.3096571028691393, "grad_norm": 0.6118666749056894, "kl": 1.140625, "learning_rate": 9.909503333931402e-06, "loss": 0.0055, "num_tokens": 4534736.0, "reward": 2.443995952606201, "reward_std": 0.12068773806095123, "rewards/check_gptzero_func/mean": 0.5357142686843872, "rewards/check_gptzero_func/std": 0.5032362937927246, "rewards/check_winston_local_func/mean": 0.9320909380912781, "rewards/check_winston_local_func/std": 0.11957254260778427, "rewards/sentence_count_match_reward_logic/mean": 0.976190447807312, "rewards/sentence_count_match_reward_logic/std": 0.058848991990089417, "step": 885 }, { "clip_ratio": 0.012470818124711514, "epoch": 0.3100069979006298, "grad_norm": 0.43839950579976017, "kl": 1.140625, "learning_rate": 9.908924067826373e-06, "loss": 0.0037, "step": 886 }, { "clip_ratio": 0.02689972333610058, "epoch": 0.3103568929321204, "grad_norm": 0.31779697757477954, "kl": 1.1484375, "learning_rate": 9.908342970739127e-06, "loss": 0.0019, "step": 887 }, { "clip_ratio": 0.03816540911793709, "epoch": 0.3107067879636109, "grad_norm": 0.261775139427113, "kl": 1.1484375, "learning_rate": 9.907760042886406e-06, "loss": 0.0014, "step": 888 }, { "clip_ratio": 0.004502012860029936, "completions/clipped_ratio": 0.1428571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 195.0, "completions/mean_length": 179.6428680419922, "completions/mean_terminated_length": 166.9166717529297, "completions/min_length": 135.0, "completions/min_terminated_length": 135.0, "epoch": 0.3110566829951015, "grad_norm": 0.6239768058179738, "kl": 0.98828125, "learning_rate": 9.90717528448564e-06, "loss": 0.0029, "num_tokens": 4556108.0, "reward": 2.3956077098846436, "reward_std": 0.17256702482700348, "rewards/check_gptzero_func/mean": 0.5, "rewards/check_gptzero_func/std": 0.5045249462127686, "rewards/check_winston_local_func/mean": 0.9049826860427856, "rewards/check_winston_local_func/std": 0.23429879546165466, "rewards/sentence_count_match_reward_logic/mean": 0.9906249642372131, "rewards/sentence_count_match_reward_logic/std": 0.0406096987426281, "step": 889 }, { "clip_ratio": 0.010051841847598553, "epoch": 0.311406578026592, "grad_norm": 0.44122605035250245, "kl": 0.9921875, "learning_rate": 9.906588695754933e-06, "loss": 0.0005, "step": 890 }, { "clip_ratio": 0.02566792629659176, "epoch": 0.3117564730580826, "grad_norm": 0.33947980765224706, "kl": 1.0, "learning_rate": 9.906000276913081e-06, "loss": -0.0013, "step": 891 }, { "clip_ratio": 0.03743993490934372, "epoch": 0.31210636808957315, "grad_norm": 0.2793366211292734, "kl": 0.99609375, "learning_rate": 9.905410028179559e-06, "loss": -0.0022, "step": 892 }, { "clip_ratio": 0.0025709362234920263, "completions/clipped_ratio": 0.375, "completions/max_length": 256.0, "completions/max_terminated_length": 253.0, "completions/mean_length": 196.50001525878906, "completions/mean_terminated_length": 160.8000030517578, "completions/min_length": 125.0, "completions/min_terminated_length": 125.0, "epoch": 0.31245626312106367, "grad_norm": 0.4861327373384439, "kl": 0.8359375, "learning_rate": 9.904817949774524e-06, "loss": 0.005, "num_tokens": 4580056.0, "reward": 2.76625394821167, "reward_std": 0.06338987499475479, "rewards/check_gptzero_func/mean": 0.8392857313156128, "rewards/check_gptzero_func/std": 0.3705909848213196, "rewards/check_winston_local_func/mean": 0.9547461271286011, "rewards/check_winston_local_func/std": 0.07849010080099106, "rewards/sentence_count_match_reward_logic/mean": 0.9722222089767456, "rewards/sentence_count_match_reward_logic/std": 0.0510825552046299, "step": 893 }, { "clip_ratio": 0.007430867291986942, "epoch": 0.31280615815255425, "grad_norm": 0.3987227359287993, "kl": 0.8359375, "learning_rate": 9.904224041918813e-06, "loss": 0.0034, "step": 894 }, { "clip_ratio": 0.019713977351784706, "epoch": 0.31315605318404477, "grad_norm": 0.35428702453751754, "kl": 0.8359375, "learning_rate": 9.90362830483395e-06, "loss": 0.0021, "step": 895 }, { "clip_ratio": 0.03183629736304283, "epoch": 0.31350594821553535, "grad_norm": 0.2701804794756079, "kl": 0.83984375, "learning_rate": 9.903030738742138e-06, "loss": 0.0005, "step": 896 }, { "clip_ratio": 0.004811465740203857, "completions/clipped_ratio": 0.0, "completions/max_length": 230.0, "completions/max_terminated_length": 230.0, "completions/mean_length": 145.67857360839844, "completions/mean_terminated_length": 145.67857360839844, "completions/min_length": 101.0, "completions/min_terminated_length": 101.0, "epoch": 0.31385584324702587, "grad_norm": 0.5273802749721579, "kl": 1.1640625, "learning_rate": 9.902431343866266e-06, "loss": 0.0058, "num_tokens": 4597990.0, "reward": 2.6732280254364014, "reward_std": 0.12476924806833267, "rewards/check_gptzero_func/mean": 0.7142857313156128, "rewards/check_gptzero_func/std": 0.4558422863483429, "rewards/check_winston_local_func/mean": 0.9589421153068542, "rewards/check_winston_local_func/std": 0.0700051560997963, "rewards/sentence_count_match_reward_logic/mean": 1.0, "rewards/sentence_count_match_reward_logic/std": 0.0, "step": 897 }, { "clip_ratio": 0.00888424925506115, "epoch": 0.31420573827851644, "grad_norm": 0.3885267560123547, "kl": 1.171875, "learning_rate": 9.901830120429899e-06, "loss": 0.0041, "step": 898 }, { "clip_ratio": 0.02621215395629406, "epoch": 0.314555633310007, "grad_norm": 0.31476625287064425, "kl": 1.1796875, "learning_rate": 9.901227068657292e-06, "loss": 0.0027, "step": 899 }, { "clip_ratio": 0.042655568569898605, "epoch": 0.31490552834149754, "grad_norm": 0.2498374882546369, "kl": 1.1875, "learning_rate": 9.900622188773374e-06, "loss": 0.0016, "step": 900 }, { "clip_ratio": 0.003608257044106722, "completions/clipped_ratio": 0.1071428571428571, "completions/max_length": 256.0, "completions/max_terminated_length": 256.0, "completions/mean_length": 168.69644165039062, "completions/mean_terminated_length": 158.22000122070312, "completions/min_length": 86.0, "completions/min_terminated_length": 86.0, "epoch": 0.3152554233729881, "grad_norm": 0.5400725494879518, "kl": 0.9375, "learning_rate": 9.900015481003762e-06, "loss": 0.0062, "num_tokens": 4618445.0, "reward": 2.8980343341827393, "reward_std": 0.09856580197811127, "rewards/check_gptzero_func/mean": 0.9642857313156128, "rewards/check_gptzero_func/std": 0.187256321310997, "rewards/check_winston_local_func/mean": 0.9496214985847473, "rewards/check_winston_local_func/std": 0.061366163194179535, "rewards/sentence_count_match_reward_logic/mean": 0.9841269850730896, "rewards/sentence_count_match_reward_logic/std": 0.039232656359672546, "step": 901 }, { "clip_ratio": 0.012529409490525723, "epoch": 0.31560531840447864, "grad_norm": 0.441735157791935, "kl": 0.9375, "learning_rate": 9.89940694557475e-06, "loss": 0.0043, "step": 902 }, { "clip_ratio": 0.027224896475672722, "epoch": 0.3159552134359692, "grad_norm": 0.32307996685207035, "kl": 0.94140625, "learning_rate": 9.898796582713317e-06, "loss": 0.0027, "step": 903 }, { "clip_ratio": 0.04118110612034798, "epoch": 0.31630510846745974, "grad_norm": 0.22233693261750206, "kl": 0.94140625, "learning_rate": 9.898184392647123e-06, "loss": 0.002, "step": 904 }, { "clip_ratio": 0.0020105827134102583, "completions/clipped_ratio": 0.1428571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 250.0, "completions/mean_length": 202.35714721679688, "completions/mean_terminated_length": 193.4166717529297, "completions/min_length": 116.0, "completions/min_terminated_length": 116.0, "epoch": 0.3166550034989503, "grad_norm": 0.5971587191817159, "kl": 0.82421875, "learning_rate": 9.897570375604508e-06, "loss": 0.0013, "num_tokens": 4642665.0, "reward": 2.52656626701355, "reward_std": 0.32463538646698, "rewards/check_gptzero_func/mean": 0.5892857313156128, "rewards/check_gptzero_func/std": 0.49641573429107666, "rewards/check_winston_local_func/mean": 0.9372802972793579, "rewards/check_winston_local_func/std": 0.09635893255472183, "rewards/sentence_count_match_reward_logic/mean": 1.0, "rewards/sentence_count_match_reward_logic/std": 0.0, "step": 905 }, { "clip_ratio": 0.009831378236413002, "epoch": 0.3170048985304409, "grad_norm": 0.3960059310967986, "kl": 0.8359375, "learning_rate": 9.896954531814496e-06, "loss": -0.0008, "step": 906 }, { "clip_ratio": 0.021724319085478783, "epoch": 0.3173547935619314, "grad_norm": 0.3390426781457867, "kl": 0.84765625, "learning_rate": 9.89633686150679e-06, "loss": -0.0023, "step": 907 }, { "clip_ratio": 0.036219216883182526, "epoch": 0.317704688593422, "grad_norm": 0.31125639296860574, "kl": 0.84765625, "learning_rate": 9.895717364911774e-06, "loss": -0.0039, "step": 908 }, { "clip_ratio": 0.0021278399508446455, "completions/clipped_ratio": 0.4285714285714286, "completions/max_length": 256.0, "completions/max_terminated_length": 233.0, "completions/mean_length": 216.37501525878906, "completions/mean_terminated_length": 186.65625, "completions/min_length": 112.0, "completions/min_terminated_length": 112.0, "epoch": 0.3180545836249125, "grad_norm": 0.49139534442331106, "kl": 0.67578125, "learning_rate": 9.895096042260517e-06, "loss": 0.0064, "num_tokens": 4669966.0, "reward": 2.4895973205566406, "reward_std": 0.30473220348358154, "rewards/check_gptzero_func/mean": 0.6964285969734192, "rewards/check_gptzero_func/std": 0.4639609456062317, "rewards/check_winston_local_func/mean": 0.8804702758789062, "rewards/check_winston_local_func/std": 0.14940808713436127, "rewards/sentence_count_match_reward_logic/mean": 0.9126983880996704, "rewards/sentence_count_match_reward_logic/std": 0.14168186485767365, "step": 909 }, { "clip_ratio": 0.008068223483860493, "epoch": 0.3184044786564031, "grad_norm": 0.42284562874511594, "kl": 0.67578125, "learning_rate": 9.894472893784766e-06, "loss": 0.0052, "step": 910 }, { "clip_ratio": 0.018465235829353333, "epoch": 0.3187543736878936, "grad_norm": 0.3613742625822235, "kl": 0.67578125, "learning_rate": 9.893847919716948e-06, "loss": 0.0034, "step": 911 }, { "clip_ratio": 0.028868865221738815, "epoch": 0.3191042687193842, "grad_norm": 0.25629469769299806, "kl": 0.68359375, "learning_rate": 9.893221120290172e-06, "loss": 0.0022, "step": 912 }, { "clip_ratio": 0.0045161438174545765, "completions/clipped_ratio": 0.0, "completions/max_length": 254.0, "completions/max_terminated_length": 254.0, "completions/mean_length": 154.7857208251953, "completions/mean_terminated_length": 154.7857208251953, "completions/min_length": 105.0, "completions/min_terminated_length": 105.0, "epoch": 0.31945416375087476, "grad_norm": 0.6174739938755652, "kl": 1.3125, "learning_rate": 9.892592495738229e-06, "loss": 0.0042, "num_tokens": 4688738.0, "reward": 2.549290657043457, "reward_std": 0.30794212222099304, "rewards/check_gptzero_func/mean": 0.6428571343421936, "rewards/check_gptzero_func/std": 0.48349377512931824, "rewards/check_winston_local_func/mean": 0.9189051389694214, "rewards/check_winston_local_func/std": 0.17142120003700256, "rewards/sentence_count_match_reward_logic/mean": 0.987528383731842, "rewards/sentence_count_match_reward_logic/std": 0.03653794527053833, "step": 913 }, { "clip_ratio": 0.017508573830127716, "epoch": 0.3198040587823653, "grad_norm": 0.5572072422187392, "kl": 1.3203125, "learning_rate": 9.891962046295593e-06, "loss": 0.0021, "step": 914 }, { "clip_ratio": 0.037663426250219345, "epoch": 0.32015395381385586, "grad_norm": 0.4609216788869545, "kl": 1.328125, "learning_rate": 9.891329772197409e-06, "loss": -0.0001, "step": 915 }, { "clip_ratio": 0.052014462649822235, "epoch": 0.3205038488453464, "grad_norm": 0.3496089344339127, "kl": 1.328125, "learning_rate": 9.890695673679516e-06, "loss": -0.0016, "step": 916 }, { "clip_ratio": 0.003127399366348982, "completions/clipped_ratio": 0.0, "completions/max_length": 239.0, "completions/max_terminated_length": 239.0, "completions/mean_length": 184.3928680419922, "completions/mean_terminated_length": 184.3928680419922, "completions/min_length": 142.0, "completions/min_terminated_length": 142.0, "epoch": 0.32085374387683696, "grad_norm": 0.5336111553383825, "kl": 1.015625, "learning_rate": 9.890059750978425e-06, "loss": 0.011, "num_tokens": 4710816.0, "reward": 2.1725876331329346, "reward_std": 0.15002501010894775, "rewards/check_gptzero_func/mean": 0.2857142984867096, "rewards/check_gptzero_func/std": 0.4558423161506653, "rewards/check_winston_local_func/mean": 0.898777961730957, "rewards/check_winston_local_func/std": 0.14738346636295319, "rewards/sentence_count_match_reward_logic/mean": 0.9880952835083008, "rewards/sentence_count_match_reward_logic/std": 0.062418777495622635, "step": 917 }, { "clip_ratio": 0.009024065919220448, "epoch": 0.3212036389083275, "grad_norm": 0.4707582069807348, "kl": 1.015625, "learning_rate": 9.889422004331326e-06, "loss": 0.0087, "step": 918 }, { "clip_ratio": 0.02500525489449501, "epoch": 0.32155353393981806, "grad_norm": 0.3385248160015866, "kl": 1.015625, "learning_rate": 9.888782433976094e-06, "loss": 0.0067, "step": 919 }, { "clip_ratio": 0.03834707662463188, "epoch": 0.3219034289713086, "grad_norm": 0.28491495278564094, "kl": 1.0234375, "learning_rate": 9.888141040151284e-06, "loss": 0.0057, "step": 920 }, { "clip_ratio": 0.0029402098152786493, "completions/clipped_ratio": 0.125, "completions/max_length": 256.0, "completions/max_terminated_length": 255.0, "completions/mean_length": 178.94644165039062, "completions/mean_terminated_length": 167.9387664794922, "completions/min_length": 105.0, "completions/min_terminated_length": 105.0, "epoch": 0.32225332400279916, "grad_norm": 0.5121210861844383, "kl": 0.96484375, "learning_rate": 9.88749782309613e-06, "loss": 0.006, "num_tokens": 4732397.0, "reward": 2.2911338806152344, "reward_std": 0.2172955870628357, "rewards/check_gptzero_func/mean": 0.4107142984867096, "rewards/check_gptzero_func/std": 0.49641573429107666, "rewards/check_winston_local_func/mean": 0.8804197311401367, "rewards/check_winston_local_func/std": 0.16890762746334076, "rewards/sentence_count_match_reward_logic/mean": 1.0, "rewards/sentence_count_match_reward_logic/std": 0.0, "step": 921 }, { "clip_ratio": 0.007345401681959629, "epoch": 0.32260321903428973, "grad_norm": 0.42970946801483145, "kl": 0.96484375, "learning_rate": 9.886852783050544e-06, "loss": 0.0042, "step": 922 }, { "clip_ratio": 0.020369473844766617, "epoch": 0.32295311406578026, "grad_norm": 0.36148206388736526, "kl": 0.96484375, "learning_rate": 9.886205920255123e-06, "loss": 0.0024, "step": 923 }, { "clip_ratio": 0.033789362758398056, "epoch": 0.32330300909727083, "grad_norm": 0.28247365495163373, "kl": 0.96484375, "learning_rate": 9.885557234951137e-06, "loss": 0.0014, "step": 924 }, { "clip_ratio": 0.002983111422508955, "completions/clipped_ratio": 0.0, "completions/max_length": 249.0, "completions/max_terminated_length": 249.0, "completions/mean_length": 142.58929443359375, "completions/mean_terminated_length": 142.58929443359375, "completions/min_length": 70.0, "completions/min_terminated_length": 70.0, "epoch": 0.32365290412876135, "grad_norm": 0.4753423818047027, "kl": 1.4609375, "learning_rate": 9.88490672738054e-06, "loss": 0.007, "num_tokens": 4749998.0, "reward": 2.91489315032959, "reward_std": 0.16744986176490784, "rewards/check_gptzero_func/mean": 0.9464285969734192, "rewards/check_gptzero_func/std": 0.22720777988433838, "rewards/check_winston_local_func/mean": 0.9684643149375916, "rewards/check_winston_local_func/std": 0.044621508568525314, "rewards/sentence_count_match_reward_logic/mean": 1.0, "rewards/sentence_count_match_reward_logic/std": 0.0, "step": 925 }, { "clip_ratio": 0.009925998747348785, "epoch": 0.32400279916025193, "grad_norm": 0.444968855555156, "kl": 1.4609375, "learning_rate": 9.884254397785966e-06, "loss": 0.0056, "step": 926 }, { "clip_ratio": 0.025306234136223793, "epoch": 0.32435269419174245, "grad_norm": 0.33323711680367996, "kl": 1.4609375, "learning_rate": 9.883600246410729e-06, "loss": 0.0039, "step": 927 }, { "clip_ratio": 0.04186262935400009, "epoch": 0.32470258922323303, "grad_norm": 0.265371425027051, "kl": 1.4765625, "learning_rate": 9.882944273498821e-06, "loss": 0.0032, "step": 928 }, { "clip_ratio": 0.003554000286385417, "completions/clipped_ratio": 0.1428571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 235.0, "completions/mean_length": 175.4107208251953, "completions/mean_terminated_length": 161.9791717529297, "completions/min_length": 72.0, "completions/min_terminated_length": 72.0, "epoch": 0.3250524842547236, "grad_norm": 0.539847400087025, "kl": 1.1484375, "learning_rate": 9.882286479294911e-06, "loss": 0.0093, "num_tokens": 4771349.0, "reward": 2.7445452213287354, "reward_std": 0.15875372290611267, "rewards/check_gptzero_func/mean": 0.8035714030265808, "rewards/check_gptzero_func/std": 0.40089187026023865, "rewards/check_winston_local_func/mean": 0.9490092396736145, "rewards/check_winston_local_func/std": 0.09948525577783585, "rewards/sentence_count_match_reward_logic/mean": 0.9919642806053162, "rewards/sentence_count_match_reward_logic/std": 0.04240152984857559, "step": 929 }, { "clip_ratio": 0.009274025447666645, "epoch": 0.3254023792862141, "grad_norm": 0.40307582708418016, "kl": 1.1640625, "learning_rate": 9.881626864044354e-06, "loss": 0.0069, "step": 930 }, { "clip_ratio": 0.025451352819800377, "epoch": 0.3257522743177047, "grad_norm": 0.357279572512142, "kl": 1.1640625, "learning_rate": 9.880965427993177e-06, "loss": 0.0055, "step": 931 }, { "clip_ratio": 0.03588319569826126, "epoch": 0.3261021693491952, "grad_norm": 0.28983751767390603, "kl": 1.1640625, "learning_rate": 9.88030217138809e-06, "loss": 0.0044, "step": 932 }, { "clip_ratio": 0.00287638115696609, "completions/clipped_ratio": 0.0, "completions/max_length": 207.0, "completions/max_terminated_length": 207.0, "completions/mean_length": 158.98214721679688, "completions/mean_terminated_length": 158.98214721679688, "completions/min_length": 110.0, "completions/min_terminated_length": 110.0, "epoch": 0.3264520643806858, "grad_norm": 0.6013454156790338, "kl": 0.97265625, "learning_rate": 9.879637094476482e-06, "loss": 0.0009, "num_tokens": 4790948.0, "reward": 2.5956521034240723, "reward_std": 0.2955929934978485, "rewards/check_gptzero_func/mean": 0.6964285969734192, "rewards/check_gptzero_func/std": 0.4639609158039093, "rewards/check_winston_local_func/mean": 0.9021996259689331, "rewards/check_winston_local_func/std": 0.18688060343265533, "rewards/sentence_count_match_reward_logic/mean": 0.9970237612724304, "rewards/sentence_count_match_reward_logic/std": 0.02227177284657955, "step": 933 }, { "clip_ratio": 0.00730094313621521, "epoch": 0.3268019594121763, "grad_norm": 0.4240435150467496, "kl": 0.9765625, "learning_rate": 9.87897019750642e-06, "loss": -0.0012, "step": 934 }, { "clip_ratio": 0.025101443752646446, "epoch": 0.3271518544436669, "grad_norm": 0.301908991994695, "kl": 0.99609375, "learning_rate": 9.878301480726654e-06, "loss": -0.0032, "step": 935 }, { "clip_ratio": 0.04186474531888962, "epoch": 0.3275017494751575, "grad_norm": 0.27744131214794415, "kl": 1.015625, "learning_rate": 9.877630944386603e-06, "loss": -0.0039, "step": 936 }, { "clip_ratio": 0.0040021841414272785, "completions/clipped_ratio": 0.0, "completions/max_length": 226.0, "completions/max_terminated_length": 226.0, "completions/mean_length": 168.94644165039062, "completions/mean_terminated_length": 168.94644165039062, "completions/min_length": 36.0, "completions/min_terminated_length": 36.0, "epoch": 0.327851644506648, "grad_norm": 0.5317661432247327, "kl": 1.140625, "learning_rate": 9.876958588736371e-06, "loss": 0.0226, "num_tokens": 4811097.0, "reward": 2.4684581756591797, "reward_std": 0.21393515169620514, "rewards/check_gptzero_func/mean": 0.5714285969734192, "rewards/check_gptzero_func/std": 0.4993502199649811, "rewards/check_winston_local_func/mean": 0.9855216145515442, "rewards/check_winston_local_func/std": 0.03114210069179535, "rewards/sentence_count_match_reward_logic/mean": 0.9115079641342163, "rewards/sentence_count_match_reward_logic/std": 0.21466076374053955, "step": 937 }, { "clip_ratio": 0.011517497710883617, "epoch": 0.3282015395381386, "grad_norm": 0.42610670989203275, "kl": 1.140625, "learning_rate": 9.876284414026747e-06, "loss": 0.0207, "step": 938 }, { "clip_ratio": 0.02687731385231018, "epoch": 0.3285514345696291, "grad_norm": 0.32266304146909175, "kl": 1.140625, "learning_rate": 9.875608420509186e-06, "loss": 0.0181, "step": 939 }, { "clip_ratio": 0.041266508400440216, "epoch": 0.3289013296011197, "grad_norm": 0.2833762309935492, "kl": 1.140625, "learning_rate": 9.874930608435826e-06, "loss": 0.0171, "step": 940 }, { "clip_ratio": 0.0017402692465111613, "completions/clipped_ratio": 0.1428571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 229.0, "completions/mean_length": 183.2857208251953, "completions/mean_terminated_length": 171.1666717529297, "completions/min_length": 136.0, "completions/min_terminated_length": 136.0, "epoch": 0.3292512246326102, "grad_norm": 0.46498364460422426, "kl": 0.75, "learning_rate": 9.874250978059489e-06, "loss": -0.0022, "num_tokens": 4834297.0, "reward": 2.70208477973938, "reward_std": 0.20433631539344788, "rewards/check_gptzero_func/mean": 0.7857142686843872, "rewards/check_gptzero_func/std": 0.4140393137931824, "rewards/check_winston_local_func/mean": 0.9614596962928772, "rewards/check_winston_local_func/std": 0.06497784703969955, "rewards/sentence_count_match_reward_logic/mean": 0.9549106955528259, "rewards/sentence_count_match_reward_logic/std": 0.07468412816524506, "step": 941 }, { "clip_ratio": 0.0058443075977265835, "epoch": 0.3296011196641008, "grad_norm": 0.40217608345079275, "kl": 0.74609375, "learning_rate": 9.873569529633671e-06, "loss": -0.0036, "step": 942 }, { "clip_ratio": 0.016617249697446823, "epoch": 0.32995101469559135, "grad_norm": 0.2607828208508696, "kl": 0.74609375, "learning_rate": 9.87288626341254e-06, "loss": -0.0051, "step": 943 }, { "clip_ratio": 0.027554651722311974, "epoch": 0.33030090972708187, "grad_norm": 0.2119181955723247, "kl": 0.74609375, "learning_rate": 9.872201179650954e-06, "loss": -0.0059, "step": 944 }, { "clip_ratio": 0.003319530515000224, "completions/clipped_ratio": 0.0, "completions/max_length": 225.0, "completions/max_terminated_length": 225.0, "completions/mean_length": 154.21429443359375, "completions/mean_terminated_length": 154.21429443359375, "completions/min_length": 76.0, "completions/min_terminated_length": 76.0, "epoch": 0.33065080475857245, "grad_norm": 0.5569681340020514, "kl": 1.1484375, "learning_rate": 9.871514278604439e-06, "loss": 0.0024, "num_tokens": 4853077.0, "reward": 2.679734468460083, "reward_std": 0.21398071944713593, "rewards/check_gptzero_func/mean": 0.7678571343421936, "rewards/check_gptzero_func/std": 0.4260205328464508, "rewards/check_winston_local_func/mean": 0.9148533940315247, "rewards/check_winston_local_func/std": 0.1460483968257904, "rewards/sentence_count_match_reward_logic/mean": 0.9970237612724304, "rewards/sentence_count_match_reward_logic/std": 0.02227177284657955, "step": 945 }, { "clip_ratio": 0.010953111574053764, "epoch": 0.33100069979006297, "grad_norm": 0.3997312968523163, "kl": 1.1484375, "learning_rate": 9.870825560529205e-06, "loss": 0.0005, "step": 946 }, { "clip_ratio": 0.031348418444395065, "epoch": 0.33135059482155355, "grad_norm": 0.3476746151572426, "kl": 1.171875, "learning_rate": 9.870135025682135e-06, "loss": -0.001, "step": 947 }, { "clip_ratio": 0.045457176864147186, "epoch": 0.33170048985304407, "grad_norm": 0.28405596481487266, "kl": 1.1796875, "learning_rate": 9.869442674320792e-06, "loss": -0.0019, "step": 948 }, { "clip_ratio": 0.0034632179886102676, "completions/clipped_ratio": 0.2678571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 254.0, "completions/mean_length": 173.48214721679688, "completions/mean_terminated_length": 143.2926788330078, "completions/min_length": 105.0, "completions/min_terminated_length": 105.0, "epoch": 0.33205038488453464, "grad_norm": 0.5270875067596552, "kl": 1.171875, "learning_rate": 9.86874850670342e-06, "loss": 0.003, "num_tokens": 4874016.0, "reward": 2.601285696029663, "reward_std": 0.09635166078805923, "rewards/check_gptzero_func/mean": 0.6428571343421936, "rewards/check_gptzero_func/std": 0.48349374532699585, "rewards/check_winston_local_func/mean": 0.9727140069007874, "rewards/check_winston_local_func/std": 0.039449237287044525, "rewards/sentence_count_match_reward_logic/mean": 0.985714316368103, "rewards/sentence_count_match_reward_logic/std": 0.035309404134750366, "step": 949 }, { "clip_ratio": 0.010447409935295582, "epoch": 0.33240027991602517, "grad_norm": 0.4581453041270769, "kl": 1.1640625, "learning_rate": 9.868052523088934e-06, "loss": 0.0015, "step": 950 }, { "clip_ratio": 0.027458591386675835, "epoch": 0.33275017494751574, "grad_norm": 0.3145823304167746, "kl": 1.1796875, "learning_rate": 9.867354723736928e-06, "loss": -0.0005, "step": 951 }, { "clip_ratio": 0.04143347963690758, "epoch": 0.3331000699790063, "grad_norm": 0.3088342360075299, "kl": 1.1796875, "learning_rate": 9.86665510890768e-06, "loss": -0.0011, "step": 952 }, { "clip_ratio": 0.0025196911301463842, "completions/clipped_ratio": 0.1428571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 227.0, "completions/mean_length": 157.125, "completions/mean_terminated_length": 140.64584350585938, "completions/min_length": 68.0, "completions/min_terminated_length": 68.0, "epoch": 0.33344996501049684, "grad_norm": 0.5134116391598476, "kl": 1.296875, "learning_rate": 9.865953678862133e-06, "loss": 0.0116, "num_tokens": 4893183.0, "reward": 2.8424389362335205, "reward_std": 0.10693056881427765, "rewards/check_gptzero_func/mean": 0.8571428656578064, "rewards/check_gptzero_func/std": 0.3530939221382141, "rewards/check_winston_local_func/mean": 0.9852962493896484, "rewards/check_winston_local_func/std": 0.014523893594741821, "rewards/sentence_count_match_reward_logic/mean": 1.0, "rewards/sentence_count_match_reward_logic/std": 0.0, "step": 953 }, { "clip_ratio": 0.009032025933265686, "epoch": 0.3337998600419874, "grad_norm": 0.40283903415550687, "kl": 1.2890625, "learning_rate": 9.865250433861917e-06, "loss": 0.0094, "step": 954 }, { "clip_ratio": 0.024953361600637436, "epoch": 0.33414975507347794, "grad_norm": 0.30243009979256297, "kl": 1.3046875, "learning_rate": 9.864545374169337e-06, "loss": 0.0077, "step": 955 }, { "clip_ratio": 0.03883909434080124, "epoch": 0.3344996501049685, "grad_norm": 0.28521627626823215, "kl": 1.3046875, "learning_rate": 9.863838500047372e-06, "loss": 0.0069, "step": 956 }, { "clip_ratio": 0.004094089847058058, "completions/clipped_ratio": 0.0, "completions/max_length": 218.0, "completions/max_terminated_length": 218.0, "completions/mean_length": 166.05357360839844, "completions/mean_terminated_length": 166.05357360839844, "completions/min_length": 93.0, "completions/min_terminated_length": 93.0, "epoch": 0.33484954513645904, "grad_norm": 0.5583565197486182, "kl": 1.1796875, "learning_rate": 9.863129811759678e-06, "loss": 0.0051, "num_tokens": 4912922.0, "reward": 2.801032781600952, "reward_std": 0.06961309164762497, "rewards/check_gptzero_func/mean": 0.8392857313156128, "rewards/check_gptzero_func/std": 0.3705909848213196, "rewards/check_winston_local_func/mean": 0.9647234678268433, "rewards/check_winston_local_func/std": 0.07164967805147171, "rewards/sentence_count_match_reward_logic/mean": 0.9970237612724304, "rewards/sentence_count_match_reward_logic/std": 0.02227177284657955, "step": 957 }, { "clip_ratio": 0.009542415849864483, "epoch": 0.3351994401679496, "grad_norm": 0.405942084631191, "kl": 1.1875, "learning_rate": 9.862419309570592e-06, "loss": 0.0032, "step": 958 }, { "clip_ratio": 0.02387855388224125, "epoch": 0.3355493351994402, "grad_norm": 0.3739428947003411, "kl": 1.2109375, "learning_rate": 9.861706993745122e-06, "loss": 0.0018, "step": 959 }, { "clip_ratio": 0.0389372743666172, "epoch": 0.3358992302309307, "grad_norm": 0.2682839143803159, "kl": 1.1953125, "learning_rate": 9.860992864548957e-06, "loss": 0.0008, "step": 960 }, { "clip_ratio": 0.004240341950207949, "completions/clipped_ratio": 0.0, "completions/max_length": 240.0, "completions/max_terminated_length": 240.0, "completions/mean_length": 162.98214721679688, "completions/mean_terminated_length": 162.98214721679688, "completions/min_length": 98.0, "completions/min_terminated_length": 98.0, "epoch": 0.3362491252624213, "grad_norm": 0.5987774881359184, "kl": 1.328125, "learning_rate": 9.86027692224846e-06, "loss": 0.005, "num_tokens": 4932185.0, "reward": 2.611877202987671, "reward_std": 0.2159661054611206, "rewards/check_gptzero_func/mean": 0.625, "rewards/check_gptzero_func/std": 0.48850420117378235, "rewards/check_winston_local_func/mean": 0.9898534417152405, "rewards/check_winston_local_func/std": 0.00650352519005537, "rewards/sentence_count_match_reward_logic/mean": 0.9970237612724304, "rewards/sentence_count_match_reward_logic/std": 0.02227177284657955, "step": 961 }, { "clip_ratio": 0.010547228157520294, "epoch": 0.3365990202939118, "grad_norm": 0.46636625402856613, "kl": 1.328125, "learning_rate": 9.85955916711067e-06, "loss": 0.0029, "step": 962 }, { "clip_ratio": 0.02870071493089199, "epoch": 0.3369489153254024, "grad_norm": 0.32968062102466034, "kl": 1.328125, "learning_rate": 9.858839599403303e-06, "loss": 0.0012, "step": 963 }, { "clip_ratio": 0.04699863865971565, "epoch": 0.3372988103568929, "grad_norm": 0.30882882702721215, "kl": 1.3359375, "learning_rate": 9.858118219394753e-06, "loss": 0.0003, "step": 964 }, { "clip_ratio": 0.00508005078881979, "completions/clipped_ratio": 0.0, "completions/max_length": 231.0, "completions/max_terminated_length": 231.0, "completions/mean_length": 166.32144165039062, "completions/mean_terminated_length": 166.32144165039062, "completions/min_length": 97.0, "completions/min_terminated_length": 97.0, "epoch": 0.3376487053883835, "grad_norm": 0.5208185273580731, "kl": 1.1640625, "learning_rate": 9.857395027354085e-06, "loss": 0.0105, "num_tokens": 4952139.0, "reward": 2.7641568183898926, "reward_std": 0.20545341074466705, "rewards/check_gptzero_func/mean": 0.7857142686843872, "rewards/check_gptzero_func/std": 0.4140393137931824, "rewards/check_winston_local_func/mean": 0.9829065203666687, "rewards/check_winston_local_func/std": 0.02834891341626644, "rewards/sentence_count_match_reward_logic/mean": 0.9955357313156128, "rewards/sentence_count_match_reward_logic/std": 0.033407654613256454, "step": 965 }, { "clip_ratio": 0.009836743585765362, "epoch": 0.33799860041987406, "grad_norm": 0.40318896636560897, "kl": 1.1640625, "learning_rate": 9.856670023551043e-06, "loss": 0.0087, "step": 966 }, { "clip_ratio": 0.025130821391940117, "epoch": 0.3383484954513646, "grad_norm": 0.2815289391499299, "kl": 1.1640625, "learning_rate": 9.855943208256046e-06, "loss": 0.0072, "step": 967 }, { "clip_ratio": 0.039088647812604904, "epoch": 0.33869839048285516, "grad_norm": 0.24742780915863996, "kl": 1.171875, "learning_rate": 9.855214581740194e-06, "loss": 0.0063, "step": 968 }, { "clip_ratio": 0.0031754777301102877, "completions/clipped_ratio": 0.1428571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 250.0, "completions/mean_length": 199.2678680419922, "completions/mean_terminated_length": 189.8125, "completions/min_length": 122.0, "completions/min_terminated_length": 122.0, "epoch": 0.3390482855143457, "grad_norm": 0.576058358094256, "kl": 0.875, "learning_rate": 9.854484144275254e-06, "loss": 0.0025, "num_tokens": 4976218.0, "reward": 2.642024040222168, "reward_std": 0.33989211916923523, "rewards/check_gptzero_func/mean": 0.7321428656578064, "rewards/check_gptzero_func/std": 0.44685041904449463, "rewards/check_winston_local_func/mean": 0.9336905479431152, "rewards/check_winston_local_func/std": 0.09587498754262924, "rewards/sentence_count_match_reward_logic/mean": 0.976190447807312, "rewards/sentence_count_match_reward_logic/std": 0.058848995715379715, "step": 969 }, { "clip_ratio": 0.009972062893211842, "epoch": 0.33939818054583626, "grad_norm": 0.452559922952224, "kl": 0.87890625, "learning_rate": 9.853751896133672e-06, "loss": 0.0001, "step": 970 }, { "clip_ratio": 0.026148328557610512, "epoch": 0.3397480755773268, "grad_norm": 0.3545650081585569, "kl": 0.88671875, "learning_rate": 9.853017837588569e-06, "loss": -0.0016, "step": 971 }, { "clip_ratio": 0.03873347118496895, "epoch": 0.34009797060881736, "grad_norm": 0.3181697173326473, "kl": 0.89453125, "learning_rate": 9.852281968913743e-06, "loss": -0.0023, "step": 972 }, { "clip_ratio": 0.00408219825476408, "completions/clipped_ratio": 0.0, "completions/max_length": 247.0, "completions/max_terminated_length": 247.0, "completions/mean_length": 170.75001525878906, "completions/mean_terminated_length": 170.75001525878906, "completions/min_length": 62.0, "completions/min_terminated_length": 62.0, "epoch": 0.34044786564030793, "grad_norm": 0.5733683608856266, "kl": 1.265625, "learning_rate": 9.85154429038367e-06, "loss": 0.0089, "num_tokens": 4996748.0, "reward": 2.9383318424224854, "reward_std": 0.124526247382164, "rewards/check_gptzero_func/mean": 0.9642857313156128, "rewards/check_gptzero_func/std": 0.187256321310997, "rewards/check_winston_local_func/mean": 0.9816989302635193, "rewards/check_winston_local_func/std": 0.03688593953847885, "rewards/sentence_count_match_reward_logic/mean": 0.9923468828201294, "rewards/sentence_count_match_reward_logic/std": 0.03245825320482254, "step": 973 }, { "clip_ratio": 0.011287247762084007, "epoch": 0.34079776067179846, "grad_norm": 0.42428497965010153, "kl": 1.265625, "learning_rate": 9.85080480227349e-06, "loss": 0.0069, "step": 974 }, { "clip_ratio": 0.029963050037622452, "epoch": 0.34114765570328903, "grad_norm": 0.3320676453187029, "kl": 1.2734375, "learning_rate": 9.85006350485903e-06, "loss": 0.0044, "step": 975 }, { "clip_ratio": 0.04517865180969238, "epoch": 0.34149755073477955, "grad_norm": 0.25704833758641005, "kl": 1.2734375, "learning_rate": 9.849320398416784e-06, "loss": 0.0032, "step": 976 }, { "clip_ratio": 0.003289851825684309, "completions/clipped_ratio": 0.1428571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 256.0, "completions/mean_length": 213.42857360839844, "completions/mean_terminated_length": 206.33334350585938, "completions/min_length": 121.0, "completions/min_terminated_length": 121.0, "epoch": 0.34184744576627013, "grad_norm": 0.5808694369755386, "kl": 0.9609375, "learning_rate": 9.848575483223925e-06, "loss": 0.0108, "num_tokens": 5022012.0, "reward": 2.764549732208252, "reward_std": 0.2836875915527344, "rewards/check_gptzero_func/mean": 0.8571428656578064, "rewards/check_gptzero_func/std": 0.3530939221382141, "rewards/check_winston_local_func/mean": 0.9391525387763977, "rewards/check_winston_local_func/std": 0.10629332810640335, "rewards/sentence_count_match_reward_logic/mean": 0.9682539701461792, "rewards/sentence_count_match_reward_logic/std": 0.12302634865045547, "step": 977 }, { "clip_ratio": 0.01071621011942625, "epoch": 0.34219734079776065, "grad_norm": 0.44243431023252855, "kl": 0.95703125, "learning_rate": 9.847828759558302e-06, "loss": 0.0086, "step": 978 }, { "clip_ratio": 0.02718263491988182, "epoch": 0.34254723582925123, "grad_norm": 0.3656850210949629, "kl": 0.95703125, "learning_rate": 9.84708022769843e-06, "loss": 0.0065, "step": 979 }, { "clip_ratio": 0.04072707146406174, "epoch": 0.34289713086074175, "grad_norm": 0.33689379194198044, "kl": 0.9609375, "learning_rate": 9.84632988792351e-06, "loss": 0.0048, "step": 980 }, { "clip_ratio": 0.00369679881259799, "completions/clipped_ratio": 0.017857142857142905, "completions/max_length": 256.0, "completions/max_terminated_length": 248.0, "completions/mean_length": 177.2857208251953, "completions/mean_terminated_length": 175.8545379638672, "completions/min_length": 139.0, "completions/min_terminated_length": 139.0, "epoch": 0.3432470258922323, "grad_norm": 0.5405413734373568, "kl": 1.0078125, "learning_rate": 9.845577740513409e-06, "loss": -0.0007, "num_tokens": 5043060.0, "reward": 2.9204752445220947, "reward_std": 0.0681794211268425, "rewards/check_gptzero_func/mean": 0.9821428656578064, "rewards/check_gptzero_func/std": 0.13363061845302582, "rewards/check_winston_local_func/mean": 0.9633322954177856, "rewards/check_winston_local_func/std": 0.04838621988892555, "rewards/sentence_count_match_reward_logic/mean": 0.9749999642372131, "rewards/sentence_count_match_reward_logic/std": 0.06674237549304962, "step": 981 }, { "clip_ratio": 0.009722827933728695, "epoch": 0.3435969209237229, "grad_norm": 0.4621632810229016, "kl": 1.0078125, "learning_rate": 9.844823785748669e-06, "loss": -0.0032, "step": 982 }, { "clip_ratio": 0.021508390083909035, "epoch": 0.3439468159552134, "grad_norm": 0.40316141700787794, "kl": 1.015625, "learning_rate": 9.844068023910512e-06, "loss": -0.0053, "step": 983 }, { "clip_ratio": 0.03767473250627518, "epoch": 0.344296710986704, "grad_norm": 0.2717454392238654, "kl": 1.015625, "learning_rate": 9.843310455280828e-06, "loss": -0.0068, "step": 984 }, { "clip_ratio": 0.004474524408578873, "completions/clipped_ratio": 0.0, "completions/max_length": 223.0, "completions/max_terminated_length": 223.0, "completions/mean_length": 153.4107208251953, "completions/mean_terminated_length": 153.4107208251953, "completions/min_length": 85.0, "completions/min_terminated_length": 85.0, "epoch": 0.3446466060181945, "grad_norm": 0.6306491387041261, "kl": 1.25, "learning_rate": 9.842551080142182e-06, "loss": 0.0107, "num_tokens": 5061427.0, "reward": 2.595808982849121, "reward_std": 0.19130733609199524, "rewards/check_gptzero_func/mean": 0.7321428656578064, "rewards/check_gptzero_func/std": 0.44685041904449463, "rewards/check_winston_local_func/mean": 0.8681301474571228, "rewards/check_winston_local_func/std": 0.29965493083000183, "rewards/sentence_count_match_reward_logic/mean": 0.9955357313156128, "rewards/sentence_count_match_reward_logic/std": 0.033407654613256454, "step": 985 }, { "clip_ratio": 0.013272779062390327, "epoch": 0.3449965010496851, "grad_norm": 0.44562688670767403, "kl": 1.25, "learning_rate": 9.841789898777817e-06, "loss": 0.0087, "step": 986 }, { "clip_ratio": 0.027754992246627808, "epoch": 0.3453463960811756, "grad_norm": 0.3828721359724303, "kl": 1.2578125, "learning_rate": 9.84102691147164e-06, "loss": 0.0067, "step": 987 }, { "clip_ratio": 0.045303285121917725, "epoch": 0.3456962911126662, "grad_norm": 0.3446850115246854, "kl": 1.2890625, "learning_rate": 9.840262118508245e-06, "loss": 0.0051, "step": 988 }, { "clip_ratio": 0.0028941321652382612, "completions/clipped_ratio": 0.1428571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 255.0, "completions/mean_length": 184.10714721679688, "completions/mean_terminated_length": 172.125, "completions/min_length": 105.0, "completions/min_terminated_length": 105.0, "epoch": 0.3460461861441568, "grad_norm": 0.5509280439241683, "kl": 1.1328125, "learning_rate": 9.83949552017289e-06, "loss": 0.0105, "num_tokens": 5083185.0, "reward": 2.7954275608062744, "reward_std": 0.07599183917045593, "rewards/check_gptzero_func/mean": 0.8214285969734192, "rewards/check_gptzero_func/std": 0.3864591419696808, "rewards/check_winston_local_func/mean": 0.9878876805305481, "rewards/check_winston_local_func/std": 0.007914519868791103, "rewards/sentence_count_match_reward_logic/mean": 0.9861111044883728, "rewards/sentence_count_match_reward_logic/std": 0.0370790958404541, "step": 989 }, { "clip_ratio": 0.009185890667140484, "epoch": 0.3463960811756473, "grad_norm": 0.47510919622960224, "kl": 1.1328125, "learning_rate": 9.838727116751508e-06, "loss": 0.0081, "step": 990 }, { "clip_ratio": 0.02554592490196228, "epoch": 0.3467459762071379, "grad_norm": 0.3314259502330326, "kl": 1.1328125, "learning_rate": 9.837956908530706e-06, "loss": 0.0063, "step": 991 }, { "clip_ratio": 0.039617814123630524, "epoch": 0.3470958712386284, "grad_norm": 0.2752692237427353, "kl": 1.140625, "learning_rate": 9.837184895797765e-06, "loss": 0.005, "step": 992 }, { "clip_ratio": 0.003227891633287072, "completions/clipped_ratio": 0.017857142857142905, "completions/max_length": 256.0, "completions/max_terminated_length": 251.0, "completions/mean_length": 176.57144165039062, "completions/mean_terminated_length": 175.1272735595703, "completions/min_length": 120.0, "completions/min_terminated_length": 120.0, "epoch": 0.34744576627011897, "grad_norm": 0.545333706196628, "kl": 1.1640625, "learning_rate": 9.83641107884064e-06, "loss": 0.0071, "num_tokens": 5104041.0, "reward": 2.6747395992279053, "reward_std": 0.13874465227127075, "rewards/check_gptzero_func/mean": 0.7142857313156128, "rewards/check_gptzero_func/std": 0.4558422863483429, "rewards/check_winston_local_func/mean": 0.9868144392967224, "rewards/check_winston_local_func/std": 0.022091619670391083, "rewards/sentence_count_match_reward_logic/mean": 0.9736394882202148, "rewards/sentence_count_match_reward_logic/std": 0.06543049216270447, "step": 993 }, { "clip_ratio": 0.00980244018137455, "epoch": 0.3477956613016095, "grad_norm": 0.4561328357267762, "kl": 1.1640625, "learning_rate": 9.835635457947957e-06, "loss": 0.0052, "step": 994 }, { "clip_ratio": 0.03001573123037815, "epoch": 0.34814555633310007, "grad_norm": 0.32027727835672354, "kl": 1.1484375, "learning_rate": 9.834858033409012e-06, "loss": 0.0034, "step": 995 }, { "clip_ratio": 0.04414162039756775, "epoch": 0.34849545136459065, "grad_norm": 0.283625428474067, "kl": 1.1640625, "learning_rate": 9.83407880551378e-06, "loss": 0.0024, "step": 996 }, { "clip_ratio": 0.0030551834497600794, "completions/clipped_ratio": 0.0, "completions/max_length": 235.0, "completions/max_terminated_length": 235.0, "completions/mean_length": 160.1428680419922, "completions/mean_terminated_length": 160.1428680419922, "completions/min_length": 92.0, "completions/min_terminated_length": 92.0, "epoch": 0.34884534639608117, "grad_norm": 0.5751388869100045, "kl": 1.1875, "learning_rate": 9.833297774552905e-06, "loss": 0.0057, "num_tokens": 5123505.0, "reward": 2.824868679046631, "reward_std": 0.019445210695266724, "rewards/check_gptzero_func/mean": 0.8571428656578064, "rewards/check_gptzero_func/std": 0.3530939221382141, "rewards/check_winston_local_func/mean": 0.9736781120300293, "rewards/check_winston_local_func/std": 0.06339899450540543, "rewards/sentence_count_match_reward_logic/mean": 0.9940475821495056, "rewards/sentence_count_match_reward_logic/std": 0.031209392473101616, "step": 997 }, { "clip_ratio": 0.010175569914281368, "epoch": 0.34919524142757175, "grad_norm": 0.44411662938960633, "kl": 1.1875, "learning_rate": 9.832514940817705e-06, "loss": 0.0039, "step": 998 }, { "clip_ratio": 0.028519557788968086, "epoch": 0.34954513645906227, "grad_norm": 0.4108125880200289, "kl": 1.1875, "learning_rate": 9.83173030460017e-06, "loss": 0.0014, "step": 999 }, { "clip_ratio": 0.044226646423339844, "epoch": 0.34989503149055284, "grad_norm": 0.2831280898429615, "kl": 1.1875, "learning_rate": 9.830943866192957e-06, "loss": -0.0002, "step": 1000 }, { "clip_ratio": 0.004460901487618685, "completions/clipped_ratio": 0.1428571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 251.0, "completions/mean_length": 156.7678680419922, "completions/mean_terminated_length": 140.2291717529297, "completions/min_length": 14.0, "completions/min_terminated_length": 14.0, "epoch": 0.35024492652204336, "grad_norm": 0.6417532580798346, "kl": 1.09375, "learning_rate": 9.830155625889406e-06, "loss": 0.0133, "num_tokens": 5143492.0, "reward": 2.1607234477996826, "reward_std": 0.2761501669883728, "rewards/check_originality_func/mean": 0.4285714328289032, "rewards/check_originality_func/std": 0.4993502199649811, "rewards/perplexity_reward/mean": 0.8140345811843872, "rewards/perplexity_reward/std": 0.09724966436624527, "rewards/sentence_count_match_reward_logic/mean": 0.9181172251701355, "rewards/sentence_count_match_reward_logic/std": 0.1282411366701126, "step": 1001 }, { "clip_ratio": 0.018666720017790794, "epoch": 0.35059482155353394, "grad_norm": 0.5035488836921288, "kl": 1.0859375, "learning_rate": 9.829365583983518e-06, "loss": 0.0103, "step": 1002 }, { "clip_ratio": 0.04058823361992836, "epoch": 0.3509447165850245, "grad_norm": 0.5186987974801003, "kl": 1.0859375, "learning_rate": 9.828573740769978e-06, "loss": 0.0082, "step": 1003 }, { "clip_ratio": 0.05858086794614792, "epoch": 0.35129461161651504, "grad_norm": 0.29975765370856444, "kl": 1.09375, "learning_rate": 9.827780096544129e-06, "loss": 0.0061, "step": 1004 }, { "clip_ratio": 0.0020286475773900747, "completions/clipped_ratio": 0.1428571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 234.0, "completions/mean_length": 114.58929443359375, "completions/mean_terminated_length": 91.02083587646484, "completions/min_length": 25.0, "completions/min_terminated_length": 25.0, "epoch": 0.3516445066480056, "grad_norm": 0.40524093212322326, "kl": 1.015625, "learning_rate": 9.826984651601998e-06, "loss": 0.0037, "num_tokens": 5158645.0, "reward": 2.007178544998169, "reward_std": 0.17999465763568878, "rewards/check_originality_func/mean": 0.1964285671710968, "rewards/check_originality_func/std": 0.40089187026023865, "rewards/perplexity_reward/mean": 0.8351004719734192, "rewards/perplexity_reward/std": 0.05030488222837448, "rewards/sentence_count_match_reward_logic/mean": 0.975649356842041, "rewards/sentence_count_match_reward_logic/std": 0.06367429345846176, "step": 1005 }, { "clip_ratio": 0.011542818509042263, "epoch": 0.35199440167949614, "grad_norm": 0.37323675691731056, "kl": 1.015625, "learning_rate": 9.826187406240275e-06, "loss": 0.0024, "step": 1006 }, { "clip_ratio": 0.029700295999646187, "epoch": 0.3523442967109867, "grad_norm": 0.3197021935382214, "kl": 1.015625, "learning_rate": 9.825388360756326e-06, "loss": 0.0012, "step": 1007 }, { "clip_ratio": 0.0499030277132988, "epoch": 0.35269419174247724, "grad_norm": 0.22685849819872456, "kl": 1.015625, "learning_rate": 9.824587515448188e-06, "loss": 0.0003, "step": 1008 }, { "clip_ratio": 0.0034899371676146984, "completions/clipped_ratio": 0.2857142857142857, "completions/max_length": 256.0, "completions/max_terminated_length": 102.0, "completions/mean_length": 113.16072082519531, "completions/mean_terminated_length": 56.025001525878906, "completions/min_length": 22.0, "completions/min_terminated_length": 22.0, "epoch": 0.3530440867739678, "grad_norm": 0.4199126534686055, "kl": 1.2578125, "learning_rate": 9.823784870614568e-06, "loss": 0.0042, "num_tokens": 5174142.0, "reward": 1.951906681060791, "reward_std": 0.21821077167987823, "rewards/check_originality_func/mean": 0.25, "rewards/check_originality_func/std": 0.43693143129348755, "rewards/perplexity_reward/mean": 0.7905970811843872, "rewards/perplexity_reward/std": 0.04866195470094681, "rewards/sentence_count_match_reward_logic/mean": 0.911309540271759, "rewards/sentence_count_match_reward_logic/std": 0.12002133578062057, "step": 1009 }, { "clip_ratio": 0.009594294242560863, "epoch": 0.35339398180545833, "grad_norm": 0.37018353668935006, "kl": 1.2578125, "learning_rate": 9.822980426554846e-06, "loss": 0.0029, "step": 1010 }, { "clip_ratio": 0.036841876804828644, "epoch": 0.3537438768369489, "grad_norm": 0.2438581416359941, "kl": 1.2734375, "learning_rate": 9.822174183569071e-06, "loss": 0.0011, "step": 1011 }, { "clip_ratio": 0.054157696664333344, "epoch": 0.3540937718684395, "grad_norm": 0.21516796540932737, "kl": 1.2890625, "learning_rate": 9.821366141957964e-06, "loss": 0.0004, "step": 1012 }, { "clip_ratio": 0.003256747033447027, "completions/clipped_ratio": 0.0, "completions/max_length": 231.0, "completions/max_terminated_length": 231.0, "completions/mean_length": 82.91072082519531, "completions/mean_terminated_length": 82.91072082519531, "completions/min_length": 12.0, "completions/min_terminated_length": 12.0, "epoch": 0.35444366689993, "grad_norm": 0.48032981439019434, "kl": 1.140625, "learning_rate": 9.820556302022916e-06, "loss": 0.005, "num_tokens": 5185145.0, "reward": 1.9647042751312256, "reward_std": 0.22500750422477722, "rewards/check_originality_func/mean": 0.2142857164144516, "rewards/check_originality_func/std": 0.41403928399086, "rewards/perplexity_reward/mean": 0.7504185438156128, "rewards/perplexity_reward/std": 0.0855189859867096, "rewards/sentence_count_match_reward_logic/mean": 1.0, "rewards/sentence_count_match_reward_logic/std": 0.0, "step": 1013 }, { "clip_ratio": 0.028348103165626526, "epoch": 0.3547935619314206, "grad_norm": 0.3164843180030405, "kl": 1.1328125, "learning_rate": 9.819744664065993e-06, "loss": 0.0035, "step": 1014 }, { "clip_ratio": 0.05624133348464966, "epoch": 0.3551434569629111, "grad_norm": 0.2155986727453674, "kl": 1.1328125, "learning_rate": 9.818931228389925e-06, "loss": 0.0025, "step": 1015 }, { "clip_ratio": 0.07626326382160187, "epoch": 0.3554933519944017, "grad_norm": 0.2372241144188812, "kl": 1.140625, "learning_rate": 9.818115995298117e-06, "loss": 0.0023, "step": 1016 }, { "clip_ratio": 0.0019384061452001333, "completions/clipped_ratio": 0.1428571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 44.0, "completions/mean_length": 56.142860412597656, "completions/mean_terminated_length": 22.83333396911621, "completions/min_length": 11.0, "completions/min_terminated_length": 11.0, "epoch": 0.3558432470258922, "grad_norm": 0.2995432626106159, "kl": 1.0546875, "learning_rate": 9.817298965094644e-06, "loss": 0.001, "num_tokens": 5193801.0, "reward": 1.7499536275863647, "reward_std": 0.08738617599010468, "rewards/check_originality_func/mean": 0.0535714291036129, "rewards/check_originality_func/std": 0.22720779478549957, "rewards/perplexity_reward/mean": 0.7440010905265808, "rewards/perplexity_reward/std": 0.0796859860420227, "rewards/sentence_count_match_reward_logic/mean": 0.9523810148239136, "rewards/sentence_count_match_reward_logic/std": 0.11769796907901764, "step": 1017 }, { "clip_ratio": 0.021242577582597733, "epoch": 0.3561931420573828, "grad_norm": 0.21228136772487513, "kl": 1.0390625, "learning_rate": 9.816480138084248e-06, "loss": 0.0001, "step": 1018 }, { "clip_ratio": 0.036156732589006424, "epoch": 0.35654303708887336, "grad_norm": 0.14736834486249534, "kl": 1.0390625, "learning_rate": 9.815659514572347e-06, "loss": -0.0002, "step": 1019 }, { "clip_ratio": 0.045630548149347305, "epoch": 0.3568929321203639, "grad_norm": 0.12654503388729338, "kl": 1.0390625, "learning_rate": 9.814837094865021e-06, "loss": -0.0008, "step": 1020 }, { "clip_ratio": 0.0027981034945696592, "completions/clipped_ratio": 0.2678571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 234.0, "completions/mean_length": 156.7678680419922, "completions/mean_terminated_length": 120.46340942382812, "completions/min_length": 17.0, "completions/min_terminated_length": 17.0, "epoch": 0.35724282715185446, "grad_norm": 0.5189930800160789, "kl": 0.94921875, "learning_rate": 9.814012879269031e-06, "loss": 0.0002, "num_tokens": 5213564.0, "reward": 2.085418462753296, "reward_std": 0.2657741606235504, "rewards/check_originality_func/mean": 0.3214285671710968, "rewards/check_originality_func/std": 0.4712514281272888, "rewards/perplexity_reward/mean": 0.8240094780921936, "rewards/perplexity_reward/std": 0.05634373053908348, "rewards/sentence_count_match_reward_logic/mean": 0.939980149269104, "rewards/sentence_count_match_reward_logic/std": 0.14052338898181915, "step": 1021 }, { "clip_ratio": 0.009884454309940338, "epoch": 0.357592722183345, "grad_norm": 0.44553403041407896, "kl": 0.9453125, "learning_rate": 9.813186868091799e-06, "loss": -0.002, "step": 1022 }, { "clip_ratio": 0.04069020599126816, "epoch": 0.35794261721483556, "grad_norm": 0.31196704642747747, "kl": 0.9375, "learning_rate": 9.812359061641417e-06, "loss": -0.004, "step": 1023 }, { "clip_ratio": 0.054274868220090866, "epoch": 0.3582925122463261, "grad_norm": 0.23948510449467644, "kl": 0.93359375, "learning_rate": 9.811529460226651e-06, "loss": -0.0052, "step": 1024 }, { "clip_ratio": 0.004214856773614883, "completions/clipped_ratio": 0.1428571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 115.0, "completions/mean_length": 71.53572082519531, "completions/mean_terminated_length": 40.79166793823242, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "epoch": 0.35864240727781665, "grad_norm": 0.335595858723195, "kl": 0.95703125, "learning_rate": 9.810698064156935e-06, "loss": 0.0029, "num_tokens": 5224058.0, "reward": 1.7250579595565796, "reward_std": 0.15617945790290833, "rewards/check_originality_func/mean": 0.1071428582072258, "rewards/check_originality_func/std": 0.3120938837528229, "rewards/perplexity_reward/mean": 0.7276087999343872, "rewards/perplexity_reward/std": 0.11160840839147568, "rewards/sentence_count_match_reward_logic/mean": 0.8903061151504517, "rewards/sentence_count_match_reward_logic/std": 0.1862451434135437, "step": 1025 }, { "clip_ratio": 0.011776990257203579, "epoch": 0.35899230230930723, "grad_norm": 0.2812815684193644, "kl": 0.95703125, "learning_rate": 9.809864873742372e-06, "loss": 0.0018, "step": 1026 }, { "clip_ratio": 0.03595331683754921, "epoch": 0.35934219734079775, "grad_norm": 0.21275478276135157, "kl": 0.96484375, "learning_rate": 9.809029889293731e-06, "loss": 0.0009, "step": 1027 }, { "clip_ratio": 0.06251238286495209, "epoch": 0.35969209237228833, "grad_norm": 0.18109835975694807, "kl": 0.9765625, "learning_rate": 9.808193111122457e-06, "loss": 0.0001, "step": 1028 }, { "clip_ratio": 0.0033598653972148895, "completions/clipped_ratio": 0.0, "completions/max_length": 253.0, "completions/max_terminated_length": 253.0, "completions/mean_length": 108.58928680419922, "completions/mean_terminated_length": 108.58928680419922, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "epoch": 0.36004198740377885, "grad_norm": 0.4628872559497099, "kl": 0.9765625, "learning_rate": 9.80735453954066e-06, "loss": 0.0153, "num_tokens": 5237563.0, "reward": 2.0426201820373535, "reward_std": 0.159409761428833, "rewards/check_originality_func/mean": 0.3035714328289032, "rewards/check_originality_func/std": 0.4639608860015869, "rewards/perplexity_reward/mean": 0.75244140625, "rewards/perplexity_reward/std": 0.15415240824222565, "rewards/sentence_count_match_reward_logic/mean": 0.9866071343421936, "rewards/sentence_count_match_reward_logic/std": 0.056801944971084595, "step": 1029 }, { "clip_ratio": 0.00991840846836567, "epoch": 0.36039188243526943, "grad_norm": 0.38393562051903, "kl": 0.9765625, "learning_rate": 9.806514174861117e-06, "loss": 0.0134, "step": 1030 }, { "clip_ratio": 0.0355314165353775, "epoch": 0.36074177746675995, "grad_norm": 0.2682952846809197, "kl": 0.97265625, "learning_rate": 9.805672017397276e-06, "loss": 0.0121, "step": 1031 }, { "clip_ratio": 0.047610051929950714, "epoch": 0.3610916724982505, "grad_norm": 0.20443928822427873, "kl": 0.9765625, "learning_rate": 9.804828067463257e-06, "loss": 0.0112, "step": 1032 }, { "clip_ratio": 0.003443425055593252, "completions/clipped_ratio": 0.1428571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 118.0, "completions/mean_length": 89.91072082519531, "completions/mean_terminated_length": 62.22916793823242, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "epoch": 0.3614415675297411, "grad_norm": 0.40523263714353325, "kl": 0.9609375, "learning_rate": 9.803982325373843e-06, "loss": 0.0052, "num_tokens": 5250606.0, "reward": 2.1329522132873535, "reward_std": 0.09062376618385315, "rewards/check_originality_func/mean": 0.3571428656578064, "rewards/check_originality_func/std": 0.48349377512931824, "rewards/perplexity_reward/mean": 0.8215680718421936, "rewards/perplexity_reward/std": 0.06271760165691376, "rewards/sentence_count_match_reward_logic/mean": 0.9542410969734192, "rewards/sentence_count_match_reward_logic/std": 0.11337479203939438, "step": 1033 }, { "clip_ratio": 0.012065940536558628, "epoch": 0.3617914625612316, "grad_norm": 0.2963148532073566, "kl": 0.95703125, "learning_rate": 9.803134791444488e-06, "loss": 0.0035, "step": 1034 }, { "clip_ratio": 0.037733640521764755, "epoch": 0.3621413575927222, "grad_norm": 0.2512095153988625, "kl": 0.95703125, "learning_rate": 9.802285465991315e-06, "loss": 0.0028, "step": 1035 }, { "clip_ratio": 0.04998208209872246, "epoch": 0.3624912526242127, "grad_norm": 0.2594241146098143, "kl": 0.96484375, "learning_rate": 9.801434349331114e-06, "loss": 0.0023, "step": 1036 }, { "clip_ratio": 0.0017355100717395544, "completions/clipped_ratio": 0.3035714285714286, "completions/max_length": 256.0, "completions/max_terminated_length": 242.0, "completions/mean_length": 119.83929443359375, "completions/mean_terminated_length": 60.487178802490234, "completions/min_length": 12.0, "completions/min_terminated_length": 12.0, "epoch": 0.3628411476557033, "grad_norm": 0.4869387123337017, "kl": 0.84375, "learning_rate": 9.800581441781342e-06, "loss": 0.0083, "num_tokens": 5266941.0, "reward": 2.0211572647094727, "reward_std": 0.1754155308008194, "rewards/check_originality_func/mean": 0.2857142984867096, "rewards/check_originality_func/std": 0.4558423161506653, "rewards/perplexity_reward/mean": 0.7903878092765808, "rewards/perplexity_reward/std": 0.0749247819185257, "rewards/sentence_count_match_reward_logic/mean": 0.9450549483299255, "rewards/sentence_count_match_reward_logic/std": 0.13580535352230072, "step": 1037 }, { "clip_ratio": 0.010457620024681091, "epoch": 0.3631910426871938, "grad_norm": 0.33959377395963575, "kl": 0.84765625, "learning_rate": 9.799726743660128e-06, "loss": 0.0069, "step": 1038 }, { "clip_ratio": 0.03201509267091751, "epoch": 0.3635409377186844, "grad_norm": 0.3054577685367088, "kl": 0.85546875, "learning_rate": 9.798870255286262e-06, "loss": 0.0057, "step": 1039 }, { "clip_ratio": 0.05356757715344429, "epoch": 0.363890832750175, "grad_norm": 0.594001689127495, "kl": 0.91015625, "learning_rate": 9.798011976979211e-06, "loss": 0.0054, "step": 1040 }, { "clip_ratio": 0.005111331585794687, "completions/clipped_ratio": 0.0, "completions/max_length": 124.0, "completions/max_terminated_length": 124.0, "completions/mean_length": 62.78571701049805, "completions/mean_terminated_length": 62.78571701049805, "completions/min_length": 19.0, "completions/min_terminated_length": 19.0, "epoch": 0.3642407277816655, "grad_norm": 0.43765182669987734, "kl": 1.109375, "learning_rate": 9.797151909059102e-06, "loss": 0.0052, "num_tokens": 5275641.0, "reward": 1.9512417316436768, "reward_std": 0.14170262217521667, "rewards/check_originality_func/mean": 0.1964285671710968, "rewards/check_originality_func/std": 0.40089187026023865, "rewards/perplexity_reward/mean": 0.79052734375, "rewards/perplexity_reward/std": 0.04677761346101761, "rewards/sentence_count_match_reward_logic/mean": 0.9642857313156128, "rewards/sentence_count_match_reward_logic/std": 0.08827348053455353, "step": 1041 }, { "clip_ratio": 0.014175469987094402, "epoch": 0.3645906228131561, "grad_norm": 0.3543283019863203, "kl": 1.109375, "learning_rate": 9.796290051846735e-06, "loss": 0.0039, "step": 1042 }, { "clip_ratio": 0.04274800047278404, "epoch": 0.3649405178446466, "grad_norm": 0.24525536259971686, "kl": 1.1171875, "learning_rate": 9.795426405663571e-06, "loss": 0.0029, "step": 1043 }, { "clip_ratio": 0.06675127893686295, "epoch": 0.36529041287613717, "grad_norm": 0.17262247055219465, "kl": 1.109375, "learning_rate": 9.79456097083174e-06, "loss": 0.0023, "step": 1044 }, { "clip_ratio": 0.0032248985953629017, "completions/clipped_ratio": 0.2857142857142857, "completions/max_length": 256.0, "completions/max_terminated_length": 113.0, "completions/mean_length": 126.6964340209961, "completions/mean_terminated_length": 74.9749984741211, "completions/min_length": 16.0, "completions/min_terminated_length": 16.0, "epoch": 0.3656403079076277, "grad_norm": 0.540451585913258, "kl": 0.9765625, "learning_rate": 9.79369374767405e-06, "loss": 0.0064, "num_tokens": 5291912.0, "reward": 2.044642925262451, "reward_std": 0.2582760751247406, "rewards/check_originality_func/mean": 0.2678571343421936, "rewards/check_originality_func/std": 0.44685041904449463, "rewards/perplexity_reward/mean": 0.8214285969734192, "rewards/perplexity_reward/std": 0.03438399359583855, "rewards/sentence_count_match_reward_logic/mean": 0.9553571343421936, "rewards/sentence_count_match_reward_logic/std": 0.08698699623346329, "step": 1045 }, { "clip_ratio": 0.01464919000864029, "epoch": 0.36599020293911827, "grad_norm": 0.39732638707425794, "kl": 1.0234375, "learning_rate": 9.792824736513958e-06, "loss": 0.0048, "step": 1046 }, { "clip_ratio": 0.038724061101675034, "epoch": 0.3663400979706088, "grad_norm": 0.31172986268681907, "kl": 1.0703125, "learning_rate": 9.7919539376756e-06, "loss": 0.0031, "step": 1047 }, { "clip_ratio": 0.05118652805685997, "epoch": 0.36668999300209937, "grad_norm": 0.23233997982489404, "kl": 1.046875, "learning_rate": 9.791081351483777e-06, "loss": 0.0023, "step": 1048 }, { "clip_ratio": 0.005877395626157522, "completions/clipped_ratio": 0.3571428571428571, "completions/max_length": 256.0, "completions/max_terminated_length": 256.0, "completions/mean_length": 139.42857360839844, "completions/mean_terminated_length": 74.66666412353516, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "epoch": 0.36703988803358994, "grad_norm": 0.42232404008458824, "kl": 0.90625, "learning_rate": 9.790206978263955e-06, "loss": 0.0051, "num_tokens": 5309792.0, "reward": 1.8314881324768066, "reward_std": 0.1085570827126503, "rewards/check_originality_func/mean": 0.0357142873108387, "rewards/check_originality_func/std": 0.1872563362121582, "rewards/perplexity_reward/mean": 0.8386579155921936, "rewards/perplexity_reward/std": 0.06647898256778717, "rewards/sentence_count_match_reward_logic/mean": 0.9571157693862915, "rewards/sentence_count_match_reward_logic/std": 0.07461525499820709, "step": 1049 }, { "clip_ratio": 0.01320679485797882, "epoch": 0.36738978306508047, "grad_norm": 0.36860592479982707, "kl": 0.90625, "learning_rate": 9.789330818342266e-06, "loss": 0.0038, "step": 1050 }, { "clip_ratio": 0.031966909766197205, "epoch": 0.36773967809657104, "grad_norm": 0.2649889357489392, "kl": 0.90234375, "learning_rate": 9.788452872045508e-06, "loss": 0.0024, "step": 1051 }, { "clip_ratio": 0.04404625669121742, "epoch": 0.36808957312806156, "grad_norm": 0.19543869485205295, "kl": 0.90234375, "learning_rate": 9.78757313970115e-06, "loss": 0.0022, "step": 1052 }, { "clip_ratio": 0.004266379866749048, "completions/clipped_ratio": 0.1428571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 186.0, "completions/mean_length": 86.42857360839844, "completions/mean_terminated_length": 58.16666793823242, "completions/min_length": 16.0, "completions/min_terminated_length": 16.0, "epoch": 0.36843946815955214, "grad_norm": 0.4904090164399192, "kl": 1.125, "learning_rate": 9.786691621637322e-06, "loss": 0.0009, "num_tokens": 5321216.0, "reward": 1.8653600215911865, "reward_std": 0.12812787294387817, "rewards/check_originality_func/mean": 0.0535714291036129, "rewards/check_originality_func/std": 0.22720779478549957, "rewards/perplexity_reward/mean": 0.8153599500656128, "rewards/perplexity_reward/std": 0.07362950593233109, "rewards/sentence_count_match_reward_logic/mean": 0.9964285492897034, "rewards/sentence_count_match_reward_logic/std": 0.026726122945547104, "step": 1053 }, { "clip_ratio": 0.028216643258929253, "epoch": 0.36878936319104266, "grad_norm": 0.35926489004754064, "kl": 1.109375, "learning_rate": 9.785808318182821e-06, "loss": -0.0005, "step": 1054 }, { "clip_ratio": 0.04196515306830406, "epoch": 0.36913925822253324, "grad_norm": 0.2768061030422268, "kl": 1.1171875, "learning_rate": 9.784923229667113e-06, "loss": -0.0015, "step": 1055 }, { "clip_ratio": 0.052228305488824844, "epoch": 0.3694891532540238, "grad_norm": 0.17782939276733775, "kl": 1.140625, "learning_rate": 9.784036356420327e-06, "loss": -0.0021, "step": 1056 }, { "clip_ratio": 0.005273033399134874, "completions/clipped_ratio": 0.2142857142857143, "completions/max_length": 256.0, "completions/max_terminated_length": 256.0, "completions/mean_length": 131.46429443359375, "completions/mean_terminated_length": 97.5, "completions/min_length": 11.0, "completions/min_terminated_length": 11.0, "epoch": 0.36983904828551434, "grad_norm": 0.5978703966326979, "kl": 1.0625, "learning_rate": 9.783147698773257e-06, "loss": 0.0083, "num_tokens": 5338602.0, "reward": 1.7713184356689453, "reward_std": 0.12629970908164978, "rewards/check_originality_func/mean": 0.0714285746216774, "rewards/check_originality_func/std": 0.25987008213996887, "rewards/perplexity_reward/mean": 0.7908760905265808, "rewards/perplexity_reward/std": 0.08663509041070938, "rewards/sentence_count_match_reward_logic/mean": 0.9090136289596558, "rewards/sentence_count_match_reward_logic/std": 0.18007917702198029, "step": 1057 }, { "clip_ratio": 0.024880653247237206, "epoch": 0.3701889433170049, "grad_norm": 0.36685675955544955, "kl": 1.0546875, "learning_rate": 9.782257257057364e-06, "loss": 0.0061, "step": 1058 }, { "clip_ratio": 0.04139021784067154, "epoch": 0.37053883834849544, "grad_norm": 0.3566900568868868, "kl": 1.046875, "learning_rate": 9.781365031604776e-06, "loss": 0.0049, "step": 1059 }, { "clip_ratio": 0.06453008204698563, "epoch": 0.370888733379986, "grad_norm": 0.26159492866971196, "kl": 1.046875, "learning_rate": 9.780471022748284e-06, "loss": 0.004, "step": 1060 }, { "clip_ratio": 0.003699442371726036, "completions/clipped_ratio": 0.2321428571428571, "completions/max_length": 256.0, "completions/max_terminated_length": 254.0, "completions/mean_length": 117.50000762939453, "completions/mean_terminated_length": 75.6279067993164, "completions/min_length": 19.0, "completions/min_terminated_length": 19.0, "epoch": 0.37123862841147653, "grad_norm": 0.49633451040504145, "kl": 0.9765625, "learning_rate": 9.779575230821344e-06, "loss": 0.0058, "num_tokens": 5354118.0, "reward": 2.065138101577759, "reward_std": 0.16086140275001526, "rewards/check_originality_func/mean": 0.2857142984867096, "rewards/check_originality_func/std": 0.4558423161506653, "rewards/perplexity_reward/mean": 0.8102678656578064, "rewards/perplexity_reward/std": 0.0612529031932354, "rewards/sentence_count_match_reward_logic/mean": 0.9691558480262756, "rewards/sentence_count_match_reward_logic/std": 0.07806214690208435, "step": 1061 }, { "clip_ratio": 0.014377872459590435, "epoch": 0.3715885234429671, "grad_norm": 0.356070018927161, "kl": 0.9609375, "learning_rate": 9.77867765615808e-06, "loss": 0.0038, "step": 1062 }, { "clip_ratio": 0.043709199875593185, "epoch": 0.3719384184744577, "grad_norm": 0.235202444876399, "kl": 0.95703125, "learning_rate": 9.777778299093274e-06, "loss": 0.0028, "step": 1063 }, { "clip_ratio": 0.052749957889318466, "epoch": 0.3722883135059482, "grad_norm": 0.2323010761510333, "kl": 0.95703125, "learning_rate": 9.776877159962384e-06, "loss": 0.0022, "step": 1064 }, { "clip_ratio": 0.00691613694652915, "completions/clipped_ratio": 0.0, "completions/max_length": 239.0, "completions/max_terminated_length": 239.0, "completions/mean_length": 143.23214721679688, "completions/mean_terminated_length": 143.23214721679688, "completions/min_length": 32.0, "completions/min_terminated_length": 32.0, "epoch": 0.3726382085374388, "grad_norm": 0.5567025382302608, "kl": 0.86328125, "learning_rate": 9.775974239101522e-06, "loss": 0.0081, "num_tokens": 5371755.0, "reward": 2.228306531906128, "reward_std": 0.21077503263950348, "rewards/check_originality_func/mean": 0.375, "rewards/check_originality_func/std": 0.48850420117378235, "rewards/perplexity_reward/mean": 0.8533063530921936, "rewards/perplexity_reward/std": 0.034522537142038345, "rewards/sentence_count_match_reward_logic/mean": 1.0, "rewards/sentence_count_match_reward_logic/std": 0.0, "step": 1065 }, { "clip_ratio": 0.018492290750145912, "epoch": 0.3729881035689293, "grad_norm": 0.3977213789078399, "kl": 0.8515625, "learning_rate": 9.77506953684747e-06, "loss": 0.006, "step": 1066 }, { "clip_ratio": 0.03323226049542427, "epoch": 0.3733379986004199, "grad_norm": 0.29580121976274154, "kl": 0.8515625, "learning_rate": 9.774163053537675e-06, "loss": 0.0045, "step": 1067 }, { "clip_ratio": 0.04953654482960701, "epoch": 0.3736878936319104, "grad_norm": 0.2322997353844274, "kl": 0.8515625, "learning_rate": 9.773254789510244e-06, "loss": 0.0042, "step": 1068 }, { "clip_ratio": 0.003935136366635561, "completions/clipped_ratio": 0.0, "completions/max_length": 161.0, "completions/max_terminated_length": 161.0, "completions/mean_length": 66.76786041259766, "completions/mean_terminated_length": 66.76786041259766, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "epoch": 0.374037788663401, "grad_norm": 0.39678851621013345, "kl": 1.2109375, "learning_rate": 9.772344745103955e-06, "loss": 0.0048, "num_tokens": 5381070.0, "reward": 1.9702149629592896, "reward_std": 0.2030467689037323, "rewards/check_originality_func/mean": 0.3035714328289032, "rewards/check_originality_func/std": 0.4639609158039093, "rewards/perplexity_reward/mean": 0.7380719780921936, "rewards/perplexity_reward/std": 0.13122044503688812, "rewards/sentence_count_match_reward_logic/mean": 0.9285714030265808, "rewards/sentence_count_match_reward_logic/std": 0.17654696106910706, "step": 1069 }, { "clip_ratio": 0.010176779702305794, "epoch": 0.37438768369489156, "grad_norm": 0.27491920677455794, "kl": 1.2265625, "learning_rate": 9.77143292065824e-06, "loss": 0.0038, "step": 1070 }, { "clip_ratio": 0.042131297290325165, "epoch": 0.3747375787263821, "grad_norm": 0.18568776183722038, "kl": 1.2578125, "learning_rate": 9.770519316513205e-06, "loss": 0.0029, "step": 1071 }, { "clip_ratio": 0.051760103553533554, "epoch": 0.37508747375787266, "grad_norm": 0.19086224750610886, "kl": 1.328125, "learning_rate": 9.769603933009614e-06, "loss": 0.0028, "step": 1072 }, { "clip_ratio": 0.0035763788037002087, "completions/clipped_ratio": 0.1428571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 123.0, "completions/mean_length": 80.46428680419922, "completions/mean_terminated_length": 51.208335876464844, "completions/min_length": 7.0, "completions/min_terminated_length": 7.0, "epoch": 0.3754373687893632, "grad_norm": 0.4085713834264746, "kl": 0.7578125, "learning_rate": 9.7686867704889e-06, "loss": 0.005, "num_tokens": 5391768.0, "reward": 1.9066686630249023, "reward_std": 0.06412173062562943, "rewards/check_originality_func/mean": 0.125, "rewards/check_originality_func/std": 0.3337118923664093, "rewards/perplexity_reward/mean": 0.7816685438156128, "rewards/perplexity_reward/std": 0.07228022068738937, "rewards/sentence_count_match_reward_logic/mean": 1.0, "rewards/sentence_count_match_reward_logic/std": 0.0, "step": 1073 }, { "clip_ratio": 0.014226826839148998, "epoch": 0.37578726382085376, "grad_norm": 0.3245683719359973, "kl": 0.7578125, "learning_rate": 9.767767829293151e-06, "loss": 0.0036, "step": 1074 }, { "clip_ratio": 0.03799664229154587, "epoch": 0.3761371588523443, "grad_norm": 0.2096275667800665, "kl": 0.7578125, "learning_rate": 9.766847109765126e-06, "loss": 0.0025, "step": 1075 }, { "clip_ratio": 0.051180098205804825, "epoch": 0.37648705388383485, "grad_norm": 0.18345270996697455, "kl": 0.7578125, "learning_rate": 9.765924612248245e-06, "loss": 0.0021, "step": 1076 }, { "clip_ratio": 0.006659400183707476, "completions/clipped_ratio": 0.0, "completions/max_length": 188.0, "completions/max_terminated_length": 188.0, "completions/mean_length": 89.92857360839844, "completions/mean_terminated_length": 89.92857360839844, "completions/min_length": 11.0, "completions/min_terminated_length": 11.0, "epoch": 0.3768369489153254, "grad_norm": 0.44634123912884827, "kl": 0.88671875, "learning_rate": 9.76500033708659e-06, "loss": 0.0048, "num_tokens": 5403620.0, "reward": 1.862444281578064, "reward_std": 0.1119406446814537, "rewards/check_originality_func/mean": 0.0357142873108387, "rewards/check_originality_func/std": 0.1872563362121582, "rewards/perplexity_reward/mean": 0.8267298936843872, "rewards/perplexity_reward/std": 0.053768809884786606, "rewards/sentence_count_match_reward_logic/mean": 1.0, "rewards/sentence_count_match_reward_logic/std": 0.0, "step": 1077 }, { "clip_ratio": 0.01971287652850151, "epoch": 0.37718684394681595, "grad_norm": 0.3123294667509169, "kl": 0.88671875, "learning_rate": 9.764074284624904e-06, "loss": 0.0036, "step": 1078 }, { "clip_ratio": 0.039826519787311554, "epoch": 0.37753673897830653, "grad_norm": 0.21875012335646174, "kl": 0.90625, "learning_rate": 9.763146455208602e-06, "loss": 0.0026, "step": 1079 }, { "clip_ratio": 0.057590026408433914, "epoch": 0.37788663400979705, "grad_norm": 0.18248282330121884, "kl": 0.92578125, "learning_rate": 9.76221684918375e-06, "loss": 0.002, "step": 1080 }, { "clip_ratio": 0.00295533356256783, "completions/clipped_ratio": 0.5714285714285714, "completions/max_length": 256.0, "completions/max_terminated_length": 229.0, "completions/mean_length": 209.67857360839844, "completions/mean_terminated_length": 147.9166717529297, "completions/min_length": 104.0, "completions/min_terminated_length": 104.0, "epoch": 0.37823652904128763, "grad_norm": 0.5666808011874951, "kl": 0.72265625, "learning_rate": 9.761285466897086e-06, "loss": 0.0072, "num_tokens": 5431170.0, "reward": 1.914240837097168, "reward_std": 0.2867667078971863, "rewards/check_originality_func/mean": 0.1964285671710968, "rewards/check_originality_func/std": 0.40089187026023865, "rewards/perplexity_reward/mean": 0.8676060438156128, "rewards/perplexity_reward/std": 0.030546760186553, "rewards/sentence_count_match_reward_logic/mean": 0.8502060770988464, "rewards/sentence_count_match_reward_logic/std": 0.16828709840774536, "step": 1081 }, { "clip_ratio": 0.009891982190310955, "epoch": 0.37858642407277815, "grad_norm": 0.4381599462121888, "kl": 0.72265625, "learning_rate": 9.760352308696005e-06, "loss": 0.0051, "step": 1082 }, { "clip_ratio": 0.023493582382798195, "epoch": 0.3789363191042687, "grad_norm": 0.36307715031535137, "kl": 0.72265625, "learning_rate": 9.759417374928566e-06, "loss": 0.0032, "step": 1083 }, { "clip_ratio": 0.03567522019147873, "epoch": 0.37928621413575925, "grad_norm": 0.2731678907296326, "kl": 0.73828125, "learning_rate": 9.758480665943492e-06, "loss": 0.0018, "step": 1084 }, { "clip_ratio": 0.003786014625802636, "completions/clipped_ratio": 0.2857142857142857, "completions/max_length": 256.0, "completions/max_terminated_length": 144.0, "completions/mean_length": 128.08929443359375, "completions/mean_terminated_length": 76.92500305175781, "completions/min_length": 23.0, "completions/min_terminated_length": 23.0, "epoch": 0.3796361091672498, "grad_norm": 0.48880598764472144, "kl": 0.91015625, "learning_rate": 9.757542182090165e-06, "loss": 0.0042, "num_tokens": 5448023.0, "reward": 1.8544671535491943, "reward_std": 0.14890950918197632, "rewards/check_originality_func/mean": 0.125, "rewards/check_originality_func/std": 0.3337118923664093, "rewards/perplexity_reward/mean": 0.83203125, "rewards/perplexity_reward/std": 0.04404325410723686, "rewards/sentence_count_match_reward_logic/mean": 0.8974358439445496, "rewards/sentence_count_match_reward_logic/std": 0.14652347564697266, "step": 1085 }, { "clip_ratio": 0.016874365508556366, "epoch": 0.3799860041987404, "grad_norm": 0.4191734687617227, "kl": 0.90625, "learning_rate": 9.756601923718631e-06, "loss": 0.0024, "step": 1086 }, { "clip_ratio": 0.03536205738782883, "epoch": 0.3803358992302309, "grad_norm": 0.34482005392496456, "kl": 0.90234375, "learning_rate": 9.755659891179598e-06, "loss": 0.0009, "step": 1087 }, { "clip_ratio": 0.051765136420726776, "epoch": 0.3806857942617215, "grad_norm": 0.23118129076862856, "kl": 0.90625, "learning_rate": 9.754716084824434e-06, "loss": -0.0004, "step": 1088 }, { "clip_ratio": 0.003619401017203927, "completions/clipped_ratio": 0.0, "completions/max_length": 185.0, "completions/max_terminated_length": 185.0, "completions/mean_length": 72.66072082519531, "completions/mean_terminated_length": 72.66072082519531, "completions/min_length": 11.0, "completions/min_terminated_length": 11.0, "epoch": 0.381035689293212, "grad_norm": 0.385964644411969, "kl": 1.046875, "learning_rate": 9.753770505005171e-06, "loss": 0.0041, "num_tokens": 5457836.0, "reward": 2.016299247741699, "reward_std": 0.16558092832565308, "rewards/check_originality_func/mean": 0.2678571343421936, "rewards/check_originality_func/std": 0.44685041904449463, "rewards/perplexity_reward/mean": 0.7990373969078064, "rewards/perplexity_reward/std": 0.07053673267364502, "rewards/sentence_count_match_reward_logic/mean": 0.9494047164916992, "rewards/sentence_count_match_reward_logic/std": 0.11857589334249496, "step": 1089 }, { "clip_ratio": 0.027268562465906143, "epoch": 0.3813855843247026, "grad_norm": 0.26064969106136515, "kl": 1.0390625, "learning_rate": 9.7528231520745e-06, "loss": 0.003, "step": 1090 }, { "clip_ratio": 0.04564419016242027, "epoch": 0.3817354793561931, "grad_norm": 0.21951632333854462, "kl": 1.046875, "learning_rate": 9.751874026385777e-06, "loss": 0.0024, "step": 1091 }, { "clip_ratio": 0.06547866016626358, "epoch": 0.3820853743876837, "grad_norm": 0.16473692871422269, "kl": 1.0390625, "learning_rate": 9.750923128293016e-06, "loss": 0.002, "step": 1092 }, { "clip_ratio": 0.004333388525992632, "completions/clipped_ratio": 0.2857142857142857, "completions/max_length": 256.0, "completions/max_terminated_length": 177.0, "completions/mean_length": 130.94644165039062, "completions/mean_terminated_length": 80.92500305175781, "completions/min_length": 11.0, "completions/min_terminated_length": 11.0, "epoch": 0.3824352694191743, "grad_norm": 0.510602518578193, "kl": 1.1484375, "learning_rate": 9.749970458150893e-06, "loss": 0.0085, "num_tokens": 5475105.0, "reward": 1.9634590148925781, "reward_std": 0.16250184178352356, "rewards/check_originality_func/mean": 0.2321428507566452, "rewards/check_originality_func/std": 0.4260205328464508, "rewards/perplexity_reward/mean": 0.8331473469734192, "rewards/perplexity_reward/std": 0.06279102712869644, "rewards/sentence_count_match_reward_logic/mean": 0.898168683052063, "rewards/sentence_count_match_reward_logic/std": 0.16215775907039642, "step": 1093 }, { "clip_ratio": 0.022182656452059746, "epoch": 0.3827851644506648, "grad_norm": 0.3998819470354718, "kl": 1.1484375, "learning_rate": 9.749016016314744e-06, "loss": 0.0065, "step": 1094 }, { "clip_ratio": 0.04511132463812828, "epoch": 0.38313505948215537, "grad_norm": 0.3083102255887304, "kl": 1.1484375, "learning_rate": 9.748059803140564e-06, "loss": 0.0051, "step": 1095 }, { "clip_ratio": 0.06006563827395439, "epoch": 0.3834849545136459, "grad_norm": 0.22782013960914277, "kl": 1.1640625, "learning_rate": 9.747101818985018e-06, "loss": 0.0041, "step": 1096 }, { "clip_ratio": 0.003071150043979287, "completions/clipped_ratio": 0.0, "completions/max_length": 232.0, "completions/max_terminated_length": 232.0, "completions/mean_length": 81.30357360839844, "completions/mean_terminated_length": 81.30357360839844, "completions/min_length": 11.0, "completions/min_terminated_length": 11.0, "epoch": 0.38383484954513647, "grad_norm": 0.4133711433040694, "kl": 1.0078125, "learning_rate": 9.746142064205422e-06, "loss": 0.0045, "num_tokens": 5486034.0, "reward": 2.126185894012451, "reward_std": 0.15576663613319397, "rewards/check_originality_func/mean": 0.3214285671710968, "rewards/check_originality_func/std": 0.4712514281272888, "rewards/perplexity_reward/mean": 0.8047572374343872, "rewards/perplexity_reward/std": 0.0933510959148407, "rewards/sentence_count_match_reward_logic/mean": 1.0, "rewards/sentence_count_match_reward_logic/std": 0.0, "step": 1097 }, { "clip_ratio": 0.016016289591789246, "epoch": 0.384184744576627, "grad_norm": 0.33457883134062333, "kl": 1.0078125, "learning_rate": 9.745180539159754e-06, "loss": 0.0033, "step": 1098 }, { "clip_ratio": 0.054890699684619904, "epoch": 0.38453463960811757, "grad_norm": 0.1762845596189936, "kl": 1.015625, "learning_rate": 9.744217244206655e-06, "loss": 0.0023, "step": 1099 }, { "clip_ratio": 0.073496975004673, "epoch": 0.38488453463960814, "grad_norm": 0.14367540058378866, "kl": 1.0234375, "learning_rate": 9.743252179705425e-06, "loss": 0.0021, "step": 1100 }, { "clip_ratio": 0.005421882029622793, "completions/clipped_ratio": 0.0, "completions/max_length": 205.0, "completions/max_terminated_length": 205.0, "completions/mean_length": 113.05357360839844, "completions/mean_terminated_length": 113.05357360839844, "completions/min_length": 28.0, "completions/min_terminated_length": 28.0, "epoch": 0.38523442967109867, "grad_norm": 0.5115206651400696, "kl": 1.265625, "learning_rate": 9.742285346016024e-06, "loss": 0.0046, "num_tokens": 5500269.0, "reward": 2.0677876472473145, "reward_std": 0.1874489188194275, "rewards/check_originality_func/mean": 0.2678571343421936, "rewards/check_originality_func/std": 0.446850448846817, "rewards/perplexity_reward/mean": 0.8035016655921936, "rewards/perplexity_reward/std": 0.09390056133270264, "rewards/sentence_count_match_reward_logic/mean": 0.9964285492897034, "rewards/sentence_count_match_reward_logic/std": 0.026726121082901955, "step": 1101 }, { "clip_ratio": 0.013064180500805378, "epoch": 0.38558432470258924, "grad_norm": 0.4224921338826791, "kl": 1.2734375, "learning_rate": 9.741316743499071e-06, "loss": 0.0029, "step": 1102 }, { "clip_ratio": 0.03288503363728523, "epoch": 0.38593421973407976, "grad_norm": 0.3049558564594372, "kl": 1.2734375, "learning_rate": 9.740346372515847e-06, "loss": 0.0016, "step": 1103 }, { "clip_ratio": 0.05601086840033531, "epoch": 0.38628411476557034, "grad_norm": 0.24650484363871492, "kl": 1.2890625, "learning_rate": 9.73937423342829e-06, "loss": 0.0007, "step": 1104 }, { "clip_ratio": 0.004042067565023899, "completions/clipped_ratio": 0.0, "completions/max_length": 207.0, "completions/max_terminated_length": 207.0, "completions/mean_length": 69.5, "completions/mean_terminated_length": 69.5, "completions/min_length": 13.0, "completions/min_terminated_length": 13.0, "epoch": 0.38663400979706086, "grad_norm": 0.3471518264312055, "kl": 1.0703125, "learning_rate": 9.738400326599e-06, "loss": 0.0032, "num_tokens": 5509753.0, "reward": 1.9714704751968384, "reward_std": 0.13145847618579865, "rewards/check_originality_func/mean": 0.1964285671710968, "rewards/check_originality_func/std": 0.40089187026023865, "rewards/perplexity_reward/mean": 0.7750418782234192, "rewards/perplexity_reward/std": 0.05810055881738663, "rewards/sentence_count_match_reward_logic/mean": 1.0, "rewards/sentence_count_match_reward_logic/std": 0.0, "step": 1105 }, { "clip_ratio": 0.015474380925297737, "epoch": 0.38698390482855144, "grad_norm": 0.21508379979966127, "kl": 1.078125, "learning_rate": 9.737424652391232e-06, "loss": 0.0024, "step": 1106 }, { "clip_ratio": 0.02662345953285694, "epoch": 0.38733379986004196, "grad_norm": 0.1638711899628895, "kl": 1.109375, "learning_rate": 9.736447211168908e-06, "loss": 0.0021, "step": 1107 }, { "clip_ratio": 0.03565652295947075, "epoch": 0.38768369489153254, "grad_norm": 0.17228545130433656, "kl": 1.1640625, "learning_rate": 9.735468003296599e-06, "loss": 0.002, "step": 1108 }, { "clip_ratio": 0.003737079445272684, "completions/clipped_ratio": 0.0, "completions/max_length": 146.0, "completions/max_terminated_length": 146.0, "completions/mean_length": 70.58928680419922, "completions/mean_terminated_length": 70.58928680419922, "completions/min_length": 34.0, "completions/min_terminated_length": 34.0, "epoch": 0.3880335899230231, "grad_norm": 0.42132010903491557, "kl": 1.1875, "learning_rate": 9.734487029139544e-06, "loss": 0.0051, "num_tokens": 5519450.0, "reward": 2.074986219406128, "reward_std": 0.10819171369075775, "rewards/check_originality_func/mean": 0.2857142984867096, "rewards/check_originality_func/std": 0.4558423161506653, "rewards/perplexity_reward/mean": 0.7892717719078064, "rewards/perplexity_reward/std": 0.04558122158050537, "rewards/sentence_count_match_reward_logic/mean": 1.0, "rewards/sentence_count_match_reward_logic/std": 0.0, "step": 1109 }, { "clip_ratio": 0.018504129722714424, "epoch": 0.38838348495451364, "grad_norm": 0.2733909338344536, "kl": 1.1875, "learning_rate": 9.733504289063636e-06, "loss": 0.0038, "step": 1110 }, { "clip_ratio": 0.03270285576581955, "epoch": 0.3887333799860042, "grad_norm": 0.22338849663778174, "kl": 1.1953125, "learning_rate": 9.732519783435427e-06, "loss": 0.0034, "step": 1111 }, { "clip_ratio": 0.04828167334198952, "epoch": 0.38908327501749473, "grad_norm": 0.17249027111853535, "kl": 1.203125, "learning_rate": 9.731533512622129e-06, "loss": 0.003, "step": 1112 }, { "clip_ratio": 0.0032456961926072836, "completions/clipped_ratio": 0.1428571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 159.0, "completions/mean_length": 129.2857208251953, "completions/mean_terminated_length": 108.16667175292969, "completions/min_length": 34.0, "completions/min_terminated_length": 34.0, "epoch": 0.3894331700489853, "grad_norm": 0.521149705704551, "kl": 1.0546875, "learning_rate": 9.730545476991613e-06, "loss": 0.0078, "num_tokens": 5536778.0, "reward": 2.307847023010254, "reward_std": 0.14570990204811096, "rewards/check_originality_func/mean": 0.5535714030265808, "rewards/check_originality_func/std": 0.5016207695007324, "rewards/perplexity_reward/mean": 0.8204520344734192, "rewards/perplexity_reward/std": 0.03453093022108078, "rewards/sentence_count_match_reward_logic/mean": 0.9338235259056091, "rewards/sentence_count_match_reward_logic/std": 0.16373375058174133, "step": 1113 }, { "clip_ratio": 0.011543912813067436, "epoch": 0.38978306508047583, "grad_norm": 0.38401566310290247, "kl": 1.0546875, "learning_rate": 9.729555676912405e-06, "loss": 0.0055, "step": 1114 }, { "clip_ratio": 0.02915995940566063, "epoch": 0.3901329601119664, "grad_norm": 0.25499255113388763, "kl": 1.0546875, "learning_rate": 9.72856411275369e-06, "loss": 0.0042, "step": 1115 }, { "clip_ratio": 0.04294339939951897, "epoch": 0.390482855143457, "grad_norm": 0.2060769076821949, "kl": 1.0546875, "learning_rate": 9.727570784885316e-06, "loss": 0.0034, "step": 1116 }, { "clip_ratio": 0.0022891394328325987, "completions/clipped_ratio": 0.0, "completions/max_length": 171.0, "completions/max_terminated_length": 171.0, "completions/mean_length": 74.89286041259766, "completions/mean_terminated_length": 74.89286041259766, "completions/min_length": 12.0, "completions/min_terminated_length": 12.0, "epoch": 0.3908327501749475, "grad_norm": 0.41343161606857787, "kl": 1.1015625, "learning_rate": 9.726575693677782e-06, "loss": 0.0068, "num_tokens": 5546900.0, "reward": 1.9565197229385376, "reward_std": 0.17810125648975372, "rewards/check_originality_func/mean": 0.1607142835855484, "rewards/check_originality_func/std": 0.3705909848213196, "rewards/perplexity_reward/mean": 0.8017578125, "rewards/perplexity_reward/std": 0.06469103693962097, "rewards/sentence_count_match_reward_logic/mean": 0.9940475821495056, "rewards/sentence_count_match_reward_logic/std": 0.031209392473101616, "step": 1117 }, { "clip_ratio": 0.019485559314489365, "epoch": 0.3911826452064381, "grad_norm": 0.3145675519168885, "kl": 1.1015625, "learning_rate": 9.725578839502249e-06, "loss": 0.0053, "step": 1118 }, { "clip_ratio": 0.04540565237402916, "epoch": 0.3915325402379286, "grad_norm": 0.20809924667459817, "kl": 1.109375, "learning_rate": 9.724580222730533e-06, "loss": 0.0043, "step": 1119 }, { "clip_ratio": 0.05924573168158531, "epoch": 0.3918824352694192, "grad_norm": 0.17989978427773118, "kl": 1.125, "learning_rate": 9.723579843735108e-06, "loss": 0.0039, "step": 1120 }, { "clip_ratio": 0.0038158383686095476, "completions/clipped_ratio": 0.2857142857142857, "completions/max_length": 256.0, "completions/max_terminated_length": 132.0, "completions/mean_length": 135.3928680419922, "completions/mean_terminated_length": 87.1500015258789, "completions/min_length": 32.0, "completions/min_terminated_length": 32.0, "epoch": 0.3922323303009097, "grad_norm": 0.48652280644752677, "kl": 1.0859375, "learning_rate": 9.722577702889106e-06, "loss": 0.0066, "num_tokens": 5565234.0, "reward": 1.8180395364761353, "reward_std": 0.20104895532131195, "rewards/check_originality_func/mean": 0.1071428582072258, "rewards/check_originality_func/std": 0.3120938837528229, "rewards/perplexity_reward/mean": 0.8429827094078064, "rewards/perplexity_reward/std": 0.02041902206838131, "rewards/sentence_count_match_reward_logic/mean": 0.8679138422012329, "rewards/sentence_count_match_reward_logic/std": 0.1502300500869751, "step": 1121 }, { "clip_ratio": 0.010891572572290897, "epoch": 0.3925822253324003, "grad_norm": 0.4082704253846289, "kl": 1.078125, "learning_rate": 9.72157380056632e-06, "loss": 0.0049, "step": 1122 }, { "clip_ratio": 0.03291507810354233, "epoch": 0.39293212036389086, "grad_norm": 0.26842862598102796, "kl": 1.078125, "learning_rate": 9.72056813714119e-06, "loss": 0.0032, "step": 1123 }, { "clip_ratio": 0.050630491226911545, "epoch": 0.3932820153953814, "grad_norm": 0.23383895349656897, "kl": 1.078125, "learning_rate": 9.719560712988824e-06, "loss": 0.0022, "step": 1124 }, { "clip_ratio": 0.004065016284584999, "completions/clipped_ratio": 0.1428571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 212.0, "completions/mean_length": 94.4464340209961, "completions/mean_terminated_length": 67.52083587646484, "completions/min_length": 15.0, "completions/min_terminated_length": 15.0, "epoch": 0.39363191042687196, "grad_norm": 0.3971297149098367, "kl": 0.90625, "learning_rate": 9.718551528484979e-06, "loss": 0.0028, "num_tokens": 5577643.0, "reward": 1.8241790533065796, "reward_std": 0.0701049268245697, "rewards/check_originality_func/mean": 0.0357142873108387, "rewards/check_originality_func/std": 0.1872563362121582, "rewards/perplexity_reward/mean": 0.8356584906578064, "rewards/perplexity_reward/std": 0.04163981229066849, "rewards/sentence_count_match_reward_logic/mean": 0.9528061151504517, "rewards/sentence_count_match_reward_logic/std": 0.11896023899316788, "step": 1125 }, { "clip_ratio": 0.01800398714840412, "epoch": 0.3939818054583625, "grad_norm": 0.28678364887800667, "kl": 0.90625, "learning_rate": 9.717540584006074e-06, "loss": 0.0016, "step": 1126 }, { "clip_ratio": 0.038025617599487305, "epoch": 0.39433170048985305, "grad_norm": 0.21120119212055755, "kl": 0.9140625, "learning_rate": 9.716527879929176e-06, "loss": 0.0006, "step": 1127 }, { "clip_ratio": 0.05046340450644493, "epoch": 0.3946815955213436, "grad_norm": 0.18177774607772054, "kl": 0.9296875, "learning_rate": 9.71551341663202e-06, "loss": -0.0002, "step": 1128 }, { "clip_ratio": 0.0025888788513839245, "completions/clipped_ratio": 0.2857142857142857, "completions/max_length": 256.0, "completions/max_terminated_length": 141.0, "completions/mean_length": 104.01786041259766, "completions/mean_terminated_length": 43.22500228881836, "completions/min_length": 7.0, "completions/min_terminated_length": 7.0, "epoch": 0.39503149055283415, "grad_norm": 0.40899734053840875, "kl": 0.875, "learning_rate": 9.714497194492988e-06, "loss": 0.003, "num_tokens": 5593580.0, "reward": 1.8720543384552002, "reward_std": 0.1330038160085678, "rewards/check_originality_func/mean": 0.2142857164144516, "rewards/check_originality_func/std": 0.4140393137931824, "rewards/perplexity_reward/mean": 0.7984793782234192, "rewards/perplexity_reward/std": 0.1097087487578392, "rewards/sentence_count_match_reward_logic/mean": 0.8592891693115234, "rewards/sentence_count_match_reward_logic/std": 0.22636474668979645, "step": 1129 }, { "clip_ratio": 0.012285659089684486, "epoch": 0.39538138558432473, "grad_norm": 0.32607989105113727, "kl": 0.87890625, "learning_rate": 9.713479213891122e-06, "loss": 0.0019, "step": 1130 }, { "clip_ratio": 0.04547170177102089, "epoch": 0.39573128061581525, "grad_norm": 0.1990771633510165, "kl": 0.88671875, "learning_rate": 9.712459475206119e-06, "loss": 0.0009, "step": 1131 }, { "clip_ratio": 0.059153515845537186, "epoch": 0.3960811756473058, "grad_norm": 0.17795856794995982, "kl": 0.90234375, "learning_rate": 9.711437978818332e-06, "loss": 0.0006, "step": 1132 }, { "clip_ratio": 0.003726182272657752, "completions/clipped_ratio": 0.1428571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 194.0, "completions/mean_length": 113.05357360839844, "completions/mean_terminated_length": 89.22917175292969, "completions/min_length": 16.0, "completions/min_terminated_length": 16.0, "epoch": 0.39643107067879635, "grad_norm": 0.4812174822944876, "kl": 1.0546875, "learning_rate": 9.710414725108771e-06, "loss": 0.0053, "num_tokens": 5608823.0, "reward": 1.8736371994018555, "reward_std": 0.20823635160923004, "rewards/check_originality_func/mean": 0.1071428582072258, "rewards/check_originality_func/std": 0.3120938837528229, "rewards/perplexity_reward/mean": 0.8355189561843872, "rewards/perplexity_reward/std": 0.05370814725756645, "rewards/sentence_count_match_reward_logic/mean": 0.9309752583503723, "rewards/sentence_count_match_reward_logic/std": 0.12860073149204254, "step": 1133 }, { "clip_ratio": 0.013400653377175331, "epoch": 0.3967809657102869, "grad_norm": 0.37328443864764144, "kl": 1.0546875, "learning_rate": 9.709389714459098e-06, "loss": 0.004, "step": 1134 }, { "clip_ratio": 0.02775341272354126, "epoch": 0.39713086074177745, "grad_norm": 0.29761254218084116, "kl": 1.0546875, "learning_rate": 9.708362947251632e-06, "loss": 0.0029, "step": 1135 }, { "clip_ratio": 0.03481655940413475, "epoch": 0.397480755773268, "grad_norm": 0.23190483936335662, "kl": 1.0625, "learning_rate": 9.707334423869348e-06, "loss": 0.0025, "step": 1136 }, { "clip_ratio": 0.0037280977703630924, "completions/clipped_ratio": 0.0, "completions/max_length": 206.0, "completions/max_terminated_length": 206.0, "completions/mean_length": 105.6964340209961, "completions/mean_terminated_length": 105.6964340209961, "completions/min_length": 40.0, "completions/min_terminated_length": 40.0, "epoch": 0.39783065080475855, "grad_norm": 0.5689575532137705, "kl": 0.96484375, "learning_rate": 9.706304144695877e-06, "loss": 0.0088, "num_tokens": 5622422.0, "reward": 2.2636022567749023, "reward_std": 0.25684037804603577, "rewards/check_originality_func/mean": 0.4107142984867096, "rewards/check_originality_func/std": 0.49641573429107666, "rewards/perplexity_reward/mean": 0.8528878092765808, "rewards/perplexity_reward/std": 0.027239827439188957, "rewards/sentence_count_match_reward_logic/mean": 1.0, "rewards/sentence_count_match_reward_logic/std": 0.0, "step": 1137 }, { "clip_ratio": 0.01592887192964554, "epoch": 0.3981805458362491, "grad_norm": 0.33897433879804995, "kl": 0.96484375, "learning_rate": 9.705272110115503e-06, "loss": 0.0073, "step": 1138 }, { "clip_ratio": 0.03396547585725784, "epoch": 0.3985304408677397, "grad_norm": 0.24731297300536512, "kl": 0.95703125, "learning_rate": 9.704238320513162e-06, "loss": 0.0062, "step": 1139 }, { "clip_ratio": 0.047399021685123444, "epoch": 0.3988803358992302, "grad_norm": 0.19984408748931484, "kl": 0.9609375, "learning_rate": 9.703202776274452e-06, "loss": 0.0057, "step": 1140 }, { "clip_ratio": 0.007208069320768118, "completions/clipped_ratio": 0.0714285714285714, "completions/max_length": 256.0, "completions/max_terminated_length": 251.0, "completions/mean_length": 97.71428680419922, "completions/mean_terminated_length": 85.53846740722656, "completions/min_length": 6.0, "completions/min_terminated_length": 6.0, "epoch": 0.3992302309307208, "grad_norm": 0.5015606094381554, "kl": 0.9453125, "learning_rate": 9.702165477785618e-06, "loss": 0.0066, "num_tokens": 5635102.0, "reward": 1.9673550128936768, "reward_std": 0.12892407178878784, "rewards/check_originality_func/mean": 0.1607142835855484, "rewards/check_originality_func/std": 0.3705909848213196, "rewards/perplexity_reward/mean": 0.806640625, "rewards/perplexity_reward/std": 0.08472151309251785, "rewards/sentence_count_match_reward_logic/mean": 1.0, "rewards/sentence_count_match_reward_logic/std": 0.0, "step": 1141 }, { "clip_ratio": 0.022536545991897583, "epoch": 0.3995801259622113, "grad_norm": 0.3203607515581717, "kl": 0.94140625, "learning_rate": 9.701126425433564e-06, "loss": 0.005, "step": 1142 }, { "clip_ratio": 0.05144750326871872, "epoch": 0.3999300209937019, "grad_norm": 0.21845040495756166, "kl": 0.9609375, "learning_rate": 9.700085619605846e-06, "loss": 0.0041, "step": 1143 }, { "clip_ratio": 0.06214434653520584, "epoch": 0.4002799160251924, "grad_norm": 0.21551388682539713, "kl": 0.98828125, "learning_rate": 9.699043060690675e-06, "loss": 0.0037, "step": 1144 }, { "clip_ratio": 0.003668899880722165, "completions/clipped_ratio": 0.4285714285714286, "completions/max_length": 256.0, "completions/max_terminated_length": 56.0, "completions/mean_length": 130.57144165039062, "completions/mean_terminated_length": 36.5, "completions/min_length": 15.0, "completions/min_terminated_length": 15.0, "epoch": 0.400629811056683, "grad_norm": 0.44591820665266785, "kl": 0.69140625, "learning_rate": 9.697998749076916e-06, "loss": 0.0047, "num_tokens": 5655846.0, "reward": 1.9984723329544067, "reward_std": 0.24175167083740234, "rewards/check_originality_func/mean": 0.3571428656578064, "rewards/check_originality_func/std": 0.48349377512931824, "rewards/perplexity_reward/mean": 0.8277762532234192, "rewards/perplexity_reward/std": 0.04138866811990738, "rewards/sentence_count_match_reward_logic/mean": 0.8135531544685364, "rewards/sentence_count_match_reward_logic/std": 0.22296017408370972, "step": 1145 }, { "clip_ratio": 0.01618923805654049, "epoch": 0.40097970608817357, "grad_norm": 0.33458955271576274, "kl": 0.69140625, "learning_rate": 9.696952685154088e-06, "loss": 0.0029, "step": 1146 }, { "clip_ratio": 0.04206811636686325, "epoch": 0.4013296011196641, "grad_norm": 0.2644561463581566, "kl": 0.6953125, "learning_rate": 9.69590486931236e-06, "loss": 0.0019, "step": 1147 }, { "clip_ratio": 0.06048130244016647, "epoch": 0.40167949615115467, "grad_norm": 0.23495682518921182, "kl": 0.6875, "learning_rate": 9.694855301942563e-06, "loss": 0.0013, "step": 1148 }, { "clip_ratio": 0.0030692717991769314, "completions/clipped_ratio": 0.0, "completions/max_length": 121.0, "completions/max_terminated_length": 121.0, "completions/mean_length": 62.58928680419922, "completions/mean_terminated_length": 62.58928680419922, "completions/min_length": 12.0, "completions/min_terminated_length": 12.0, "epoch": 0.4020293911826452, "grad_norm": 0.316472715413382, "kl": 0.8515625, "learning_rate": 9.69380398343617e-06, "loss": 0.0035, "num_tokens": 5664751.0, "reward": 1.9080357551574707, "reward_std": 0.16065219044685364, "rewards/check_originality_func/mean": 0.1071428582072258, "rewards/check_originality_func/std": 0.3120938837528229, "rewards/perplexity_reward/mean": 0.8080357313156128, "rewards/perplexity_reward/std": 0.04848918691277504, "rewards/sentence_count_match_reward_logic/mean": 0.9928570985794067, "rewards/sentence_count_match_reward_logic/std": 0.03745126724243164, "step": 1149 }, { "clip_ratio": 0.01728784665465355, "epoch": 0.40237928621413577, "grad_norm": 0.23501108901029988, "kl": 0.84765625, "learning_rate": 9.692750914185314e-06, "loss": 0.0029, "step": 1150 }, { "clip_ratio": 0.03389471024274826, "epoch": 0.4027291812456263, "grad_norm": 0.12552101943965577, "kl": 0.83984375, "learning_rate": 9.691696094582782e-06, "loss": 0.0022, "step": 1151 }, { "clip_ratio": 0.042407598346471786, "epoch": 0.40307907627711687, "grad_norm": 0.11642737468474772, "kl": 0.83203125, "learning_rate": 9.69063952502201e-06, "loss": 0.002, "step": 1152 }, { "clip_ratio": 0.00519039761275053, "completions/clipped_ratio": 0.0, "completions/max_length": 197.0, "completions/max_terminated_length": 197.0, "completions/mean_length": 88.9464340209961, "completions/mean_terminated_length": 88.9464340209961, "completions/min_length": 18.0, "completions/min_terminated_length": 18.0, "epoch": 0.40342897130860744, "grad_norm": 0.5118392143477518, "kl": 1.4921875, "learning_rate": 9.68958120589709e-06, "loss": 0.0091, "num_tokens": 5676340.0, "reward": 2.2030553817749023, "reward_std": 0.2160230129957199, "rewards/check_originality_func/mean": 0.3571428656578064, "rewards/check_originality_func/std": 0.48349377512931824, "rewards/perplexity_reward/mean": 0.8459123969078064, "rewards/perplexity_reward/std": 0.034666888415813446, "rewards/sentence_count_match_reward_logic/mean": 1.0, "rewards/sentence_count_match_reward_logic/std": 0.0, "step": 1153 }, { "clip_ratio": 0.020182287320494652, "epoch": 0.40377886634009796, "grad_norm": 0.3454498061516106, "kl": 1.4921875, "learning_rate": 9.688521137602763e-06, "loss": 0.0073, "step": 1154 }, { "clip_ratio": 0.04389003664255142, "epoch": 0.40412876137158854, "grad_norm": 0.32485806455623484, "kl": 1.4765625, "learning_rate": 9.687459320534428e-06, "loss": 0.0064, "step": 1155 }, { "clip_ratio": 0.06691855192184448, "epoch": 0.40447865640307906, "grad_norm": 0.3067704230841087, "kl": 1.5703125, "learning_rate": 9.686395755088128e-06, "loss": 0.0056, "step": 1156 }, { "clip_ratio": 0.0038874729070812464, "completions/clipped_ratio": 0.0, "completions/max_length": 255.0, "completions/max_terminated_length": 255.0, "completions/mean_length": 108.39286041259766, "completions/mean_terminated_length": 108.39286041259766, "completions/min_length": 17.0, "completions/min_terminated_length": 17.0, "epoch": 0.40482855143456964, "grad_norm": 0.4425499702952419, "kl": 0.76953125, "learning_rate": 9.685330441660564e-06, "loss": 0.0062, "num_tokens": 5690666.0, "reward": 2.103236675262451, "reward_std": 0.3452214300632477, "rewards/check_originality_func/mean": 0.3035714328289032, "rewards/check_originality_func/std": 0.4639609158039093, "rewards/perplexity_reward/mean": 0.80859375, "rewards/perplexity_reward/std": 0.09930567443370819, "rewards/sentence_count_match_reward_logic/mean": 0.9910714030265808, "rewards/sentence_count_match_reward_logic/std": 0.037867967039346695, "step": 1157 }, { "clip_ratio": 0.01392984576523304, "epoch": 0.40517844646606016, "grad_norm": 0.3366183210510295, "kl": 0.7734375, "learning_rate": 9.68426338064909e-06, "loss": 0.0046, "step": 1158 }, { "clip_ratio": 0.04286269471049309, "epoch": 0.40552834149755074, "grad_norm": 0.2567086290344313, "kl": 0.78125, "learning_rate": 9.68319457245171e-06, "loss": 0.0034, "step": 1159 }, { "clip_ratio": 0.05557159334421158, "epoch": 0.4058782365290413, "grad_norm": 0.23757382961459933, "kl": 0.77734375, "learning_rate": 9.682124017467075e-06, "loss": 0.0031, "step": 1160 }, { "clip_ratio": 0.004878993611782789, "completions/clipped_ratio": 0.0, "completions/max_length": 157.0, "completions/max_terminated_length": 157.0, "completions/mean_length": 68.625, "completions/mean_terminated_length": 68.625, "completions/min_length": 17.0, "completions/min_terminated_length": 17.0, "epoch": 0.40622813156053184, "grad_norm": 0.37877443375228576, "kl": 0.64453125, "learning_rate": 9.681051716094497e-06, "loss": 0.0019, "num_tokens": 5700453.0, "reward": 2.037388563156128, "reward_std": 0.22169284522533417, "rewards/check_originality_func/mean": 0.2321428507566452, "rewards/check_originality_func/std": 0.4260205328464508, "rewards/perplexity_reward/mean": 0.8052455186843872, "rewards/perplexity_reward/std": 0.06600786000490189, "rewards/sentence_count_match_reward_logic/mean": 1.0, "rewards/sentence_count_match_reward_logic/std": 0.0, "step": 1161 }, { "clip_ratio": 0.013846869580447674, "epoch": 0.4065780265920224, "grad_norm": 0.2745484472799669, "kl": 0.64453125, "learning_rate": 9.67997766873393e-06, "loss": 0.0009, "step": 1162 }, { "clip_ratio": 0.041829049587249756, "epoch": 0.40692792162351293, "grad_norm": 0.1809481828712593, "kl": 0.65234375, "learning_rate": 9.678901875785986e-06, "loss": 0.0001, "step": 1163 }, { "clip_ratio": 0.060119085013866425, "epoch": 0.4072778166550035, "grad_norm": 0.1654881612923025, "kl": 0.66015625, "learning_rate": 9.677824337651927e-06, "loss": -0.0003, "step": 1164 }, { "clip_ratio": 0.0017642888706177473, "completions/clipped_ratio": 0.1428571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 160.0, "completions/mean_length": 83.17857360839844, "completions/mean_terminated_length": 54.375, "completions/min_length": 12.0, "completions/min_terminated_length": 12.0, "epoch": 0.40762771168649403, "grad_norm": 0.3858492191198908, "kl": 0.9453125, "learning_rate": 9.676745054733661e-06, "loss": 0.0055, "num_tokens": 5712887.0, "reward": 1.963822841644287, "reward_std": 0.20624203979969025, "rewards/check_originality_func/mean": 0.2321428507566452, "rewards/check_originality_func/std": 0.4260205328464508, "rewards/perplexity_reward/mean": 0.75927734375, "rewards/perplexity_reward/std": 0.09363321959972382, "rewards/sentence_count_match_reward_logic/mean": 0.9724025726318359, "rewards/sentence_count_match_reward_logic/std": 0.06916838884353638, "step": 1165 }, { "clip_ratio": 0.009802820160984993, "epoch": 0.4079776067179846, "grad_norm": 0.27503500507995027, "kl": 0.9453125, "learning_rate": 9.675664027433753e-06, "loss": 0.0045, "step": 1166 }, { "clip_ratio": 0.020527968183159828, "epoch": 0.40832750174947513, "grad_norm": 0.22740108374926282, "kl": 0.96484375, "learning_rate": 9.674581256155415e-06, "loss": 0.0038, "step": 1167 }, { "clip_ratio": 0.026253491640090942, "epoch": 0.4086773967809657, "grad_norm": 0.15956248265512532, "kl": 0.9921875, "learning_rate": 9.673496741302509e-06, "loss": 0.0031, "step": 1168 }, { "clip_ratio": 0.002558978972956538, "completions/clipped_ratio": 0.1428571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 132.0, "completions/mean_length": 106.96428680419922, "completions/mean_terminated_length": 82.125, "completions/min_length": 22.0, "completions/min_terminated_length": 22.0, "epoch": 0.4090272918124563, "grad_norm": 0.39726564574662204, "kl": 0.59765625, "learning_rate": 9.67241048327955e-06, "loss": 0.0055, "num_tokens": 5727429.0, "reward": 1.8947173357009888, "reward_std": 0.11666020005941391, "rewards/check_originality_func/mean": 0.1428571492433548, "rewards/check_originality_func/std": 0.3530939221382141, "rewards/perplexity_reward/mean": 0.7890625, "rewards/perplexity_reward/std": 0.09043316543102264, "rewards/sentence_count_match_reward_logic/mean": 0.9627975821495056, "rewards/sentence_count_match_reward_logic/std": 0.08055586367845535, "step": 1169 }, { "clip_ratio": 0.00920100323855877, "epoch": 0.4093771868439468, "grad_norm": 0.2910626649144081, "kl": 0.59765625, "learning_rate": 9.671322482491704e-06, "loss": 0.0043, "step": 1170 }, { "clip_ratio": 0.019903123378753662, "epoch": 0.4097270818754374, "grad_norm": 0.2046174648199445, "kl": 0.59765625, "learning_rate": 9.67023273934478e-06, "loss": 0.0032, "step": 1171 }, { "clip_ratio": 0.028154293075203896, "epoch": 0.4100769769069279, "grad_norm": 0.18358644815977862, "kl": 0.6015625, "learning_rate": 9.669141254245248e-06, "loss": 0.0028, "step": 1172 }, { "clip_ratio": 0.0009203843073919415, "completions/clipped_ratio": 0.1428571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 177.0, "completions/mean_length": 115.5714340209961, "completions/mean_terminated_length": 92.16667175292969, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "epoch": 0.4104268719384185, "grad_norm": 0.39308012885399773, "kl": 0.9609375, "learning_rate": 9.668048027600217e-06, "loss": 0.0042, "num_tokens": 5742413.0, "reward": 2.004603862762451, "reward_std": 0.10620773583650589, "rewards/check_originality_func/mean": 0.1785714328289032, "rewards/check_originality_func/std": 0.3864591419696808, "rewards/perplexity_reward/mean": 0.8260323405265808, "rewards/perplexity_reward/std": 0.07959192991256714, "rewards/sentence_count_match_reward_logic/mean": 1.0, "rewards/sentence_count_match_reward_logic/std": 0.0, "step": 1173 }, { "clip_ratio": 0.005426581483334303, "epoch": 0.410776766969909, "grad_norm": 0.3284606546951862, "kl": 0.9609375, "learning_rate": 9.666953059817452e-06, "loss": 0.0032, "step": 1174 }, { "clip_ratio": 0.018996089696884155, "epoch": 0.4111266620013996, "grad_norm": 0.2368359284832425, "kl": 0.96484375, "learning_rate": 9.665856351305365e-06, "loss": 0.0023, "step": 1175 }, { "clip_ratio": 0.03328803554177284, "epoch": 0.41147655703289016, "grad_norm": 0.17114514332407998, "kl": 0.97265625, "learning_rate": 9.664757902473019e-06, "loss": 0.0017, "step": 1176 }, { "clip_ratio": 0.00459891464561224, "completions/clipped_ratio": 0.0, "completions/max_length": 189.0, "completions/max_terminated_length": 189.0, "completions/mean_length": 108.26786041259766, "completions/mean_terminated_length": 108.26786041259766, "completions/min_length": 28.0, "completions/min_terminated_length": 28.0, "epoch": 0.4118264520643807, "grad_norm": 0.5238359816997366, "kl": 1.125, "learning_rate": 9.663657713730123e-06, "loss": 0.0018, "num_tokens": 5756548.0, "reward": 2.114522933959961, "reward_std": 0.11584653705358505, "rewards/check_originality_func/mean": 0.2857142984867096, "rewards/check_originality_func/std": 0.4558423161506653, "rewards/perplexity_reward/mean": 0.8323799967765808, "rewards/perplexity_reward/std": 0.029366882517933846, "rewards/sentence_count_match_reward_logic/mean": 0.9964285492897034, "rewards/sentence_count_match_reward_logic/std": 0.026726121082901955, "step": 1177 }, { "clip_ratio": 0.013471275568008423, "epoch": 0.41217634709587125, "grad_norm": 0.3406432601909774, "kl": 1.1328125, "learning_rate": 9.66255578548704e-06, "loss": -0.0001, "step": 1178 }, { "clip_ratio": 0.029440900310873985, "epoch": 0.4125262421273618, "grad_norm": 0.2409411012847638, "kl": 1.1328125, "learning_rate": 9.661452118154777e-06, "loss": -0.0014, "step": 1179 }, { "clip_ratio": 0.04484536498785019, "epoch": 0.41287613715885235, "grad_norm": 0.19588291418385165, "kl": 1.1484375, "learning_rate": 9.66034671214499e-06, "loss": -0.002, "step": 1180 }, { "clip_ratio": 0.0030483794398605824, "completions/clipped_ratio": 0.0, "completions/max_length": 238.0, "completions/max_terminated_length": 238.0, "completions/mean_length": 79.8214340209961, "completions/mean_terminated_length": 79.8214340209961, "completions/min_length": 12.0, "completions/min_terminated_length": 12.0, "epoch": 0.4132260321903429, "grad_norm": 0.4301288842485523, "kl": 1.0234375, "learning_rate": 9.659239567869989e-06, "loss": 0.0088, "num_tokens": 5767338.0, "reward": 2.1048877239227295, "reward_std": 0.09962750226259232, "rewards/check_originality_func/mean": 0.375, "rewards/check_originality_func/std": 0.48850420117378235, "rewards/perplexity_reward/mean": 0.7715541124343872, "rewards/perplexity_reward/std": 0.10180802643299103, "rewards/sentence_count_match_reward_logic/mean": 0.9583333730697632, "rewards/sentence_count_match_reward_logic/std": 0.09265989810228348, "step": 1181 }, { "clip_ratio": 0.01202878076583147, "epoch": 0.41357592722183345, "grad_norm": 0.3469174530462174, "kl": 1.0390625, "learning_rate": 9.658130685742724e-06, "loss": 0.0073, "step": 1182 }, { "clip_ratio": 0.03433989733457565, "epoch": 0.413925822253324, "grad_norm": 0.21269254231356352, "kl": 1.0390625, "learning_rate": 9.6570200661768e-06, "loss": 0.0059, "step": 1183 }, { "clip_ratio": 0.04702560976147652, "epoch": 0.41427571728481455, "grad_norm": 0.1853369801821771, "kl": 1.046875, "learning_rate": 9.655907709586465e-06, "loss": 0.0056, "step": 1184 }, { "clip_ratio": 0.003293489571660757, "completions/clipped_ratio": 0.0, "completions/max_length": 235.0, "completions/max_terminated_length": 235.0, "completions/mean_length": 86.12500762939453, "completions/mean_terminated_length": 86.12500762939453, "completions/min_length": 7.0, "completions/min_terminated_length": 7.0, "epoch": 0.4146256123163051, "grad_norm": 0.4298104828351276, "kl": 1.3359375, "learning_rate": 9.654793616386621e-06, "loss": 0.0064, "num_tokens": 5778689.0, "reward": 2.1144533157348633, "reward_std": 0.292834997177124, "rewards/check_originality_func/mean": 0.3214285671710968, "rewards/check_originality_func/std": 0.4712514281272888, "rewards/perplexity_reward/mean": 0.798828125, "rewards/perplexity_reward/std": 0.09209597110748291, "rewards/sentence_count_match_reward_logic/mean": 0.9941964149475098, "rewards/sentence_count_match_reward_logic/std": 0.03125811368227005, "step": 1185 }, { "clip_ratio": 0.014103950001299381, "epoch": 0.41497550734779565, "grad_norm": 0.3325645863563008, "kl": 1.328125, "learning_rate": 9.653677786992815e-06, "loss": 0.0053, "step": 1186 }, { "clip_ratio": 0.045470066368579865, "epoch": 0.4153254023792862, "grad_norm": 0.27897096488242173, "kl": 1.3203125, "learning_rate": 9.652560221821234e-06, "loss": 0.0043, "step": 1187 }, { "clip_ratio": 0.058563947677612305, "epoch": 0.41567529741077675, "grad_norm": 0.19175029811937458, "kl": 1.3203125, "learning_rate": 9.651440921288727e-06, "loss": 0.0039, "step": 1188 }, { "clip_ratio": 0.004268891643732786, "completions/clipped_ratio": 0.0, "completions/max_length": 194.0, "completions/max_terminated_length": 194.0, "completions/mean_length": 141.57144165039062, "completions/mean_terminated_length": 141.57144165039062, "completions/min_length": 78.0, "completions/min_terminated_length": 78.0, "epoch": 0.4160251924422673, "grad_norm": 0.4690297602915386, "kl": 1.265625, "learning_rate": 9.650319885812777e-06, "loss": 0.0098, "num_tokens": 5796473.0, "reward": 2.3946011066436768, "reward_std": 0.2762870490550995, "rewards/check_originality_func/mean": 0.5357142686843872, "rewards/check_originality_func/std": 0.5032362937927246, "rewards/perplexity_reward/mean": 0.85888671875, "rewards/perplexity_reward/std": 0.022310281172394753, "rewards/sentence_count_match_reward_logic/mean": 1.0, "rewards/sentence_count_match_reward_logic/std": 0.0, "step": 1189 }, { "clip_ratio": 0.007849589921534061, "epoch": 0.4163750874737579, "grad_norm": 0.39329104022945366, "kl": 1.265625, "learning_rate": 9.649197115811524e-06, "loss": 0.008, "step": 1190 }, { "clip_ratio": 0.020492922514677048, "epoch": 0.4167249825052484, "grad_norm": 0.2951100980959848, "kl": 1.2578125, "learning_rate": 9.648072611703749e-06, "loss": 0.0062, "step": 1191 }, { "clip_ratio": 0.037400033324956894, "epoch": 0.417074877536739, "grad_norm": 0.2202911772730187, "kl": 1.265625, "learning_rate": 9.64694637390888e-06, "loss": 0.0052, "step": 1192 }, { "clip_ratio": 0.004345451481640339, "completions/clipped_ratio": 0.2857142857142857, "completions/max_length": 256.0, "completions/max_terminated_length": 245.0, "completions/mean_length": 148.08929443359375, "completions/mean_terminated_length": 104.92500305175781, "completions/min_length": 11.0, "completions/min_terminated_length": 11.0, "epoch": 0.4174247725682295, "grad_norm": 0.3879126800043469, "kl": 0.75, "learning_rate": 9.645818402846992e-06, "loss": 0.004, "num_tokens": 5817206.0, "reward": 2.0796749591827393, "reward_std": 0.2797943651676178, "rewards/check_originality_func/mean": 0.3214285671710968, "rewards/check_originality_func/std": 0.4712514281272888, "rewards/perplexity_reward/mean": 0.8444475531578064, "rewards/perplexity_reward/std": 0.07116787880659103, "rewards/sentence_count_match_reward_logic/mean": 0.9137987494468689, "rewards/sentence_count_match_reward_logic/std": 0.17863109707832336, "step": 1193 }, { "clip_ratio": 0.0072298431769013405, "epoch": 0.4177746675997201, "grad_norm": 0.3243347359022924, "kl": 0.75, "learning_rate": 9.644688698938811e-06, "loss": 0.0027, "step": 1194 }, { "clip_ratio": 0.021286819130182266, "epoch": 0.4181245626312106, "grad_norm": 0.2506705238955765, "kl": 0.75390625, "learning_rate": 9.643557262605704e-06, "loss": 0.0012, "step": 1195 }, { "clip_ratio": 0.035406868904829025, "epoch": 0.4184744576627012, "grad_norm": 0.22152063439264075, "kl": 0.75390625, "learning_rate": 9.642424094269685e-06, "loss": 0.0003, "step": 1196 }, { "clip_ratio": 0.0025005133356899023, "completions/clipped_ratio": 0.0, "completions/max_length": 124.0, "completions/max_terminated_length": 124.0, "completions/mean_length": 51.21428680419922, "completions/mean_terminated_length": 51.21428680419922, "completions/min_length": 6.0, "completions/min_terminated_length": 6.0, "epoch": 0.4188243526941917, "grad_norm": 0.261482434430749, "kl": 0.6640625, "learning_rate": 9.641289194353418e-06, "loss": 0.0036, "num_tokens": 5824810.0, "reward": 1.945452094078064, "reward_std": 0.08038129657506943, "rewards/check_originality_func/mean": 0.2321428507566452, "rewards/check_originality_func/std": 0.4260205328464508, "rewards/perplexity_reward/mean": 0.7133091688156128, "rewards/perplexity_reward/std": 0.22477087378501892, "rewards/sentence_count_match_reward_logic/mean": 1.0, "rewards/sentence_count_match_reward_logic/std": 0.0, "step": 1197 }, { "clip_ratio": 0.005855499301105738, "epoch": 0.4191742477256823, "grad_norm": 0.21752517042562802, "kl": 0.6640625, "learning_rate": 9.640152563280207e-06, "loss": 0.0031, "step": 1198 }, { "clip_ratio": 0.014196371659636497, "epoch": 0.41952414275717287, "grad_norm": 0.13835601499772235, "kl": 0.6640625, "learning_rate": 9.639014201474009e-06, "loss": 0.0026, "step": 1199 }, { "clip_ratio": 0.02918110229074955, "epoch": 0.4198740377886634, "grad_norm": 0.1069305176038205, "kl": 0.6640625, "learning_rate": 9.637874109359416e-06, "loss": 0.0023, "step": 1200 }, { "clip_ratio": 0.0025282870046794415, "completions/clipped_ratio": 0.0, "completions/max_length": 180.0, "completions/max_terminated_length": 180.0, "completions/mean_length": 64.96428680419922, "completions/mean_terminated_length": 64.96428680419922, "completions/min_length": 15.0, "completions/min_terminated_length": 15.0, "epoch": 0.42022393282015397, "grad_norm": 0.40781347223438186, "kl": 0.93359375, "learning_rate": 9.636732287361675e-06, "loss": 0.0045, "num_tokens": 5834088.0, "reward": 2.0033483505249023, "reward_std": 0.08092652261257172, "rewards/check_originality_func/mean": 0.25, "rewards/check_originality_func/std": 0.43693143129348755, "rewards/perplexity_reward/mean": 0.7533482313156128, "rewards/perplexity_reward/std": 0.09229285269975662, "rewards/sentence_count_match_reward_logic/mean": 1.0, "rewards/sentence_count_match_reward_logic/std": 0.0, "step": 1201 }, { "clip_ratio": 0.021594999358057976, "epoch": 0.4205738278516445, "grad_norm": 0.2833213065268794, "kl": 0.9375, "learning_rate": 9.635588735906675e-06, "loss": 0.003, "step": 1202 }, { "clip_ratio": 0.04662244766950607, "epoch": 0.42092372288313507, "grad_norm": 0.18333881706751978, "kl": 0.9375, "learning_rate": 9.63444345542095e-06, "loss": 0.0019, "step": 1203 }, { "clip_ratio": 0.06131825968623161, "epoch": 0.4212736179146256, "grad_norm": 0.15372139300112414, "kl": 0.9375, "learning_rate": 9.633296446331678e-06, "loss": 0.0017, "step": 1204 }, { "clip_ratio": 0.009244365617632866, "completions/clipped_ratio": 0.0, "completions/max_length": 161.0, "completions/max_terminated_length": 161.0, "completions/mean_length": 89.10714721679688, "completions/mean_terminated_length": 89.10714721679688, "completions/min_length": 20.0, "completions/min_terminated_length": 20.0, "epoch": 0.42162351294611616, "grad_norm": 0.5232429025233126, "kl": 1.4296875, "learning_rate": 9.632147709066682e-06, "loss": -0.0137, "num_tokens": 5846334.0, "reward": 2.1288645267486572, "reward_std": 0.2131444811820984, "rewards/check_originality_func/mean": 0.3392857015132904, "rewards/check_originality_func/std": 0.477751761674881, "rewards/perplexity_reward/mean": 0.8538643717765808, "rewards/perplexity_reward/std": 0.07399516552686691, "rewards/sentence_count_match_reward_logic/mean": 0.9357143044471741, "rewards/sentence_count_match_reward_logic/std": 0.17521785199642181, "step": 1205 }, { "clip_ratio": 0.01993703842163086, "epoch": 0.42197340797760674, "grad_norm": 0.3618713156366196, "kl": 1.4296875, "learning_rate": 9.630997244054429e-06, "loss": -0.0153, "step": 1206 }, { "clip_ratio": 0.046209417283535004, "epoch": 0.42232330300909726, "grad_norm": 0.22550982801161928, "kl": 1.4296875, "learning_rate": 9.629845051724037e-06, "loss": -0.0164, "step": 1207 }, { "clip_ratio": 0.0707961916923523, "epoch": 0.42267319804058784, "grad_norm": 0.18932355338495263, "kl": 1.4296875, "learning_rate": 9.628691132505257e-06, "loss": -0.0171, "step": 1208 }, { "clip_ratio": 0.003289873478934169, "completions/clipped_ratio": 0.1428571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 195.0, "completions/mean_length": 90.51786041259766, "completions/mean_terminated_length": 62.9375, "completions/min_length": 13.0, "completions/min_terminated_length": 13.0, "epoch": 0.42302309307207836, "grad_norm": 0.47957493472906254, "kl": 0.69921875, "learning_rate": 9.627535486828491e-06, "loss": 0.0078, "num_tokens": 5859323.0, "reward": 2.0267624855041504, "reward_std": 0.2879658341407776, "rewards/check_originality_func/mean": 0.3035714328289032, "rewards/check_originality_func/std": 0.4639608860015869, "rewards/perplexity_reward/mean": 0.8273577094078064, "rewards/perplexity_reward/std": 0.1055789589881897, "rewards/sentence_count_match_reward_logic/mean": 0.8958333134651184, "rewards/sentence_count_match_reward_logic/std": 0.15793009102344513, "step": 1209 }, { "clip_ratio": 0.0148972999304533, "epoch": 0.42337298810356894, "grad_norm": 0.3292509002389228, "kl": 0.70703125, "learning_rate": 9.626378115124788e-06, "loss": 0.0061, "step": 1210 }, { "clip_ratio": 0.03398551419377327, "epoch": 0.42372288313505946, "grad_norm": 0.26298007598366124, "kl": 0.71875, "learning_rate": 9.625219017825833e-06, "loss": 0.0049, "step": 1211 }, { "clip_ratio": 0.04590662941336632, "epoch": 0.42407277816655004, "grad_norm": 0.19653266184007115, "kl": 0.7265625, "learning_rate": 9.624058195363958e-06, "loss": 0.0042, "step": 1212 }, { "clip_ratio": 0.0011424823896959424, "completions/clipped_ratio": 0.0, "completions/max_length": 200.0, "completions/max_terminated_length": 200.0, "completions/mean_length": 76.80357360839844, "completions/mean_terminated_length": 76.80357360839844, "completions/min_length": 11.0, "completions/min_terminated_length": 11.0, "epoch": 0.4244226731980406, "grad_norm": 0.3037466247418452, "kl": 0.9765625, "learning_rate": 9.622895648172141e-06, "loss": 0.0052, "num_tokens": 5870016.0, "reward": 1.8468492031097412, "reward_std": 0.15849658846855164, "rewards/check_originality_func/mean": 0.0892857164144516, "rewards/check_originality_func/std": 0.28773635625839233, "rewards/perplexity_reward/mean": 0.7601143717765808, "rewards/perplexity_reward/std": 0.10562718659639359, "rewards/sentence_count_match_reward_logic/mean": 0.9974489808082581, "rewards/sentence_count_match_reward_logic/std": 0.019090088084340096, "step": 1213 }, { "clip_ratio": 0.009966720826923847, "epoch": 0.42477256822953113, "grad_norm": 0.22807411369712954, "kl": 0.9765625, "learning_rate": 9.621731376683998e-06, "loss": 0.0041, "step": 1214 }, { "clip_ratio": 0.029521064832806587, "epoch": 0.4251224632610217, "grad_norm": 0.16821777002955807, "kl": 0.97265625, "learning_rate": 9.620565381333796e-06, "loss": 0.0034, "step": 1215 }, { "clip_ratio": 0.046096321195364, "epoch": 0.42547235829251223, "grad_norm": 0.16452903904524438, "kl": 0.96875, "learning_rate": 9.619397662556434e-06, "loss": 0.0032, "step": 1216 }, { "clip_ratio": 0.0014021920505911112, "completions/clipped_ratio": 0.1428571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 126.0, "completions/mean_length": 92.96428680419922, "completions/mean_terminated_length": 65.79167175292969, "completions/min_length": 14.0, "completions/min_terminated_length": 14.0, "epoch": 0.4258222533240028, "grad_norm": 0.3918846193705195, "kl": 1.2890625, "learning_rate": 9.618228220787466e-06, "loss": 0.0053, "num_tokens": 5882342.0, "reward": 2.212890625, "reward_std": 0.19113793969154358, "rewards/check_originality_func/mean": 0.4285714328289032, "rewards/check_originality_func/std": 0.4993502199649811, "rewards/perplexity_reward/mean": 0.8021763563156128, "rewards/perplexity_reward/std": 0.06751222163438797, "rewards/sentence_count_match_reward_logic/mean": 0.9821428656578064, "rewards/sentence_count_match_reward_logic/std": 0.06496752798557281, "step": 1217 }, { "clip_ratio": 0.007215688470751047, "epoch": 0.42617214835549333, "grad_norm": 0.3169233850945356, "kl": 1.2890625, "learning_rate": 9.61705705646308e-06, "loss": 0.0043, "step": 1218 }, { "clip_ratio": 0.02858486771583557, "epoch": 0.4265220433869839, "grad_norm": 0.207503592598403, "kl": 1.2890625, "learning_rate": 9.615884170020107e-06, "loss": 0.0033, "step": 1219 }, { "clip_ratio": 0.03927164524793625, "epoch": 0.4268719384184745, "grad_norm": 0.1722965208064582, "kl": 1.2890625, "learning_rate": 9.614709561896027e-06, "loss": 0.0028, "step": 1220 }, { "clip_ratio": 0.0020757378078997135, "completions/clipped_ratio": 0.0, "completions/max_length": 182.0, "completions/max_terminated_length": 182.0, "completions/mean_length": 72.875, "completions/mean_terminated_length": 72.875, "completions/min_length": 17.0, "completions/min_terminated_length": 17.0, "epoch": 0.427221833449965, "grad_norm": 0.43151913244263573, "kl": 0.81640625, "learning_rate": 9.613533232528956e-06, "loss": 0.0055, "num_tokens": 5892351.0, "reward": 1.980120062828064, "reward_std": 0.08783666789531708, "rewards/check_originality_func/mean": 0.2142857164144516, "rewards/check_originality_func/std": 0.4140393137931824, "rewards/perplexity_reward/mean": 0.7658342719078064, "rewards/perplexity_reward/std": 0.07180942595005035, "rewards/sentence_count_match_reward_logic/mean": 1.0, "rewards/sentence_count_match_reward_logic/std": 0.0, "step": 1221 }, { "clip_ratio": 0.012082981877028942, "epoch": 0.4275717284814556, "grad_norm": 0.2846461621153801, "kl": 0.8125, "learning_rate": 9.612355182357654e-06, "loss": 0.0041, "step": 1222 }, { "clip_ratio": 0.03228525444865227, "epoch": 0.4279216235129461, "grad_norm": 0.1967239157388468, "kl": 0.81640625, "learning_rate": 9.611175411821519e-06, "loss": 0.0034, "step": 1223 }, { "clip_ratio": 0.051976025104522705, "epoch": 0.4282715185444367, "grad_norm": 0.15923936240523195, "kl": 0.8203125, "learning_rate": 9.609993921360599e-06, "loss": 0.0029, "step": 1224 }, { "clip_ratio": 0.0030229564290493727, "completions/clipped_ratio": 0.2857142857142857, "completions/max_length": 256.0, "completions/max_terminated_length": 184.0, "completions/mean_length": 135.875, "completions/mean_terminated_length": 87.82500457763672, "completions/min_length": 19.0, "completions/min_terminated_length": 19.0, "epoch": 0.4286214135759272, "grad_norm": 0.4007915573044581, "kl": 0.77734375, "learning_rate": 9.608810711415577e-06, "loss": 0.0062, "num_tokens": 5912416.0, "reward": 2.00508189201355, "reward_std": 0.16108568012714386, "rewards/check_originality_func/mean": 0.2857142984867096, "rewards/check_originality_func/std": 0.4558423161506653, "rewards/perplexity_reward/mean": 0.8291015625, "rewards/perplexity_reward/std": 0.06164295971393585, "rewards/sentence_count_match_reward_logic/mean": 0.8902660608291626, "rewards/sentence_count_match_reward_logic/std": 0.18080627918243408, "step": 1225 }, { "clip_ratio": 0.014190579764544964, "epoch": 0.4289713086074178, "grad_norm": 0.31477121554353577, "kl": 0.77734375, "learning_rate": 9.607625782427779e-06, "loss": 0.0051, "step": 1226 }, { "clip_ratio": 0.023689625784754753, "epoch": 0.42932120363890836, "grad_norm": 0.23942766309163668, "kl": 0.78515625, "learning_rate": 9.606439134839172e-06, "loss": 0.004, "step": 1227 }, { "clip_ratio": 0.041715074330568314, "epoch": 0.4296710986703989, "grad_norm": 0.18529699781946202, "kl": 0.80078125, "learning_rate": 9.605250769092364e-06, "loss": 0.0034, "step": 1228 }, { "clip_ratio": 0.002544895512983203, "completions/clipped_ratio": 0.2857142857142857, "completions/max_length": 256.0, "completions/max_terminated_length": 127.0, "completions/mean_length": 127.26786041259766, "completions/mean_terminated_length": 75.7750015258789, "completions/min_length": 20.0, "completions/min_terminated_length": 20.0, "epoch": 0.43002099370188945, "grad_norm": 0.44276724845024595, "kl": 0.9140625, "learning_rate": 9.604060685630608e-06, "loss": 0.0057, "num_tokens": 5930591.0, "reward": 1.9379496574401855, "reward_std": 0.14822331070899963, "rewards/check_originality_func/mean": 0.2321428507566452, "rewards/check_originality_func/std": 0.4260205328464508, "rewards/perplexity_reward/mean": 0.7931082844734192, "rewards/perplexity_reward/std": 0.05705101788043976, "rewards/sentence_count_match_reward_logic/mean": 0.9126983880996704, "rewards/sentence_count_match_reward_logic/std": 0.17435374855995178, "step": 1229 }, { "clip_ratio": 0.014405058696866035, "epoch": 0.43037088873338, "grad_norm": 0.3390508825806928, "kl": 0.9140625, "learning_rate": 9.602868884897787e-06, "loss": 0.0039, "step": 1230 }, { "clip_ratio": 0.033845677971839905, "epoch": 0.43072078376487055, "grad_norm": 0.23338835088699642, "kl": 0.91015625, "learning_rate": 9.601675367338436e-06, "loss": 0.0028, "step": 1231 }, { "clip_ratio": 0.04458533599972725, "epoch": 0.4310706787963611, "grad_norm": 0.16392689612274586, "kl": 0.91015625, "learning_rate": 9.600480133397726e-06, "loss": 0.0022, "step": 1232 }, { "clip_ratio": 0.0025742079596966505, "completions/clipped_ratio": 0.25, "completions/max_length": 256.0, "completions/max_terminated_length": 256.0, "completions/mean_length": 132.4107208251953, "completions/mean_terminated_length": 91.21428680419922, "completions/min_length": 10.0, "completions/min_terminated_length": 10.0, "epoch": 0.43142057382785165, "grad_norm": 0.49645695301967213, "kl": 0.8671875, "learning_rate": 9.599283183521467e-06, "loss": 0.0033, "num_tokens": 5948694.0, "reward": 2.2628345489501953, "reward_std": 0.28757810592651367, "rewards/check_originality_func/mean": 0.5178571343421936, "rewards/check_originality_func/std": 0.5042031407356262, "rewards/perplexity_reward/mean": 0.8103376030921936, "rewards/perplexity_reward/std": 0.045978158712387085, "rewards/sentence_count_match_reward_logic/mean": 0.9346398115158081, "rewards/sentence_count_match_reward_logic/std": 0.1255393624305725, "step": 1233 }, { "clip_ratio": 0.01112673245370388, "epoch": 0.43177046885934217, "grad_norm": 0.45466190301963993, "kl": 0.90234375, "learning_rate": 9.598084518156107e-06, "loss": 0.0022, "step": 1234 }, { "clip_ratio": 0.03090047836303711, "epoch": 0.43212036389083275, "grad_norm": 0.26682543214643695, "kl": 0.8828125, "learning_rate": 9.596884137748744e-06, "loss": 0.0007, "step": 1235 }, { "clip_ratio": 0.04830467328429222, "epoch": 0.4324702589223233, "grad_norm": 0.17801932662678793, "kl": 0.875, "learning_rate": 9.5956820427471e-06, "loss": -0.0003, "step": 1236 }, { "clip_ratio": 0.001748182694427669, "completions/clipped_ratio": 0.2857142857142857, "completions/max_length": 256.0, "completions/max_terminated_length": 31.0, "completions/mean_length": 88.23214721679688, "completions/mean_terminated_length": 21.125, "completions/min_length": 14.0, "completions/min_terminated_length": 14.0, "epoch": 0.43282015395381385, "grad_norm": 0.3525373023744907, "kl": 0.68359375, "learning_rate": 9.594478233599551e-06, "loss": 0.004, "num_tokens": 5961787.0, "reward": 1.8814282417297363, "reward_std": 0.14212946593761444, "rewards/check_originality_func/mean": 0.1785714328289032, "rewards/check_originality_func/std": 0.3864591419696808, "rewards/perplexity_reward/mean": 0.7805524468421936, "rewards/perplexity_reward/std": 0.06189383938908577, "rewards/sentence_count_match_reward_logic/mean": 0.9223042726516724, "rewards/sentence_count_match_reward_logic/std": 0.1349078267812729, "step": 1237 }, { "clip_ratio": 0.006663133390247822, "epoch": 0.4331700489853044, "grad_norm": 0.2991526804724335, "kl": 0.68359375, "learning_rate": 9.593272710755104e-06, "loss": 0.0034, "step": 1238 }, { "clip_ratio": 0.019043829292058945, "epoch": 0.43351994401679494, "grad_norm": 0.23733974817183134, "kl": 0.6796875, "learning_rate": 9.59206547466341e-06, "loss": 0.002, "step": 1239 }, { "clip_ratio": 0.033133797347545624, "epoch": 0.4338698390482855, "grad_norm": 0.1752178150466255, "kl": 0.68359375, "learning_rate": 9.590856525774753e-06, "loss": 0.0015, "step": 1240 }, { "clip_ratio": 0.004240456968545914, "completions/clipped_ratio": 0.1428571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 145.0, "completions/mean_length": 100.05357360839844, "completions/mean_terminated_length": 74.0625, "completions/min_length": 26.0, "completions/min_terminated_length": 26.0, "epoch": 0.43421973407977604, "grad_norm": 0.410996608924371, "kl": 0.78125, "learning_rate": 9.589645864540061e-06, "loss": 0.0022, "num_tokens": 5976182.0, "reward": 1.9026682376861572, "reward_std": 0.2321634590625763, "rewards/check_originality_func/mean": 0.1607142835855484, "rewards/check_originality_func/std": 0.3705909848213196, "rewards/perplexity_reward/mean": 0.8187081217765808, "rewards/perplexity_reward/std": 0.05367248132824898, "rewards/sentence_count_match_reward_logic/mean": 0.9232456088066101, "rewards/sentence_count_match_reward_logic/std": 0.16766050457954407, "step": 1241 }, { "clip_ratio": 0.014522829093039036, "epoch": 0.4345696291112666, "grad_norm": 0.3084906145669655, "kl": 0.78515625, "learning_rate": 9.5884334914109e-06, "loss": 0.0011, "step": 1242 }, { "clip_ratio": 0.03599288687109947, "epoch": 0.4349195241427572, "grad_norm": 0.2264087102690863, "kl": 0.7890625, "learning_rate": 9.587219406839475e-06, "loss": -0.0, "step": 1243 }, { "clip_ratio": 0.05328714847564697, "epoch": 0.4352694191742477, "grad_norm": 0.16647748425210987, "kl": 0.7890625, "learning_rate": 9.586003611278625e-06, "loss": -0.0006, "step": 1244 }, { "clip_ratio": 0.003165266942232847, "completions/clipped_ratio": 0.0, "completions/max_length": 240.0, "completions/max_terminated_length": 240.0, "completions/mean_length": 87.25000762939453, "completions/mean_terminated_length": 87.25000762939453, "completions/min_length": 7.0, "completions/min_terminated_length": 7.0, "epoch": 0.4356193142057383, "grad_norm": 0.368786325303019, "kl": 1.1875, "learning_rate": 9.584786105181831e-06, "loss": 0.0029, "num_tokens": 5988028.0, "reward": 1.9282925128936768, "reward_std": 0.11038944125175476, "rewards/check_originality_func/mean": 0.1428571492433548, "rewards/check_originality_func/std": 0.3530939221382141, "rewards/perplexity_reward/mean": 0.8010602593421936, "rewards/perplexity_reward/std": 0.07930870354175568, "rewards/sentence_count_match_reward_logic/mean": 0.984375, "rewards/sentence_count_match_reward_logic/std": 0.04171398654580116, "step": 1245 }, { "clip_ratio": 0.014285484328866005, "epoch": 0.4359692092372288, "grad_norm": 0.2987299965780507, "kl": 1.1875, "learning_rate": 9.583566889003212e-06, "loss": 0.0017, "step": 1246 }, { "clip_ratio": 0.0323936827480793, "epoch": 0.4363191042687194, "grad_norm": 0.20755936134932257, "kl": 1.1796875, "learning_rate": 9.582345963197525e-06, "loss": 0.0006, "step": 1247 }, { "clip_ratio": 0.05551514774560928, "epoch": 0.4366689993002099, "grad_norm": 0.1659193470720179, "kl": 1.1796875, "learning_rate": 9.581123328220161e-06, "loss": 0.0003, "step": 1248 }, { "clip_ratio": 0.003726161550730467, "completions/clipped_ratio": 0.0, "completions/max_length": 216.0, "completions/max_terminated_length": 216.0, "completions/mean_length": 112.92857360839844, "completions/mean_terminated_length": 112.92857360839844, "completions/min_length": 20.0, "completions/min_terminated_length": 20.0, "epoch": 0.4370188943317005, "grad_norm": 0.4582346081528588, "kl": 1.0703125, "learning_rate": 9.579898984527154e-06, "loss": 0.0122, "num_tokens": 6002504.0, "reward": 2.2205357551574707, "reward_std": 0.17828527092933655, "rewards/check_originality_func/mean": 0.375, "rewards/check_originality_func/std": 0.48850420117378235, "rewards/perplexity_reward/mean": 0.8705357313156128, "rewards/perplexity_reward/std": 0.035974759608507156, "rewards/sentence_count_match_reward_logic/mean": 0.9749999642372131, "rewards/sentence_count_match_reward_logic/std": 0.06674237549304962, "step": 1249 }, { "clip_ratio": 0.013544348068535328, "epoch": 0.43736878936319107, "grad_norm": 0.3542265561154528, "kl": 1.0703125, "learning_rate": 9.578672932575173e-06, "loss": 0.0106, "step": 1250 }, { "clip_ratio": 0.029031861573457718, "epoch": 0.4377186843946816, "grad_norm": 0.2541526592163876, "kl": 1.0703125, "learning_rate": 9.57744517282152e-06, "loss": 0.0091, "step": 1251 }, { "clip_ratio": 0.043623752892017365, "epoch": 0.43806857942617217, "grad_norm": 0.230279822272752, "kl": 1.0703125, "learning_rate": 9.57621570572414e-06, "loss": 0.0085, "step": 1252 }, { "clip_ratio": 0.002863466739654541, "completions/clipped_ratio": 0.0, "completions/max_length": 210.0, "completions/max_terminated_length": 210.0, "completions/mean_length": 110.17857360839844, "completions/mean_terminated_length": 110.17857360839844, "completions/min_length": 13.0, "completions/min_terminated_length": 13.0, "epoch": 0.4384184744576627, "grad_norm": 0.37588758723034305, "kl": 0.875, "learning_rate": 9.574984531741613e-06, "loss": 0.0035, "num_tokens": 6016786.0, "reward": 2.0963356494903564, "reward_std": 0.14418287575244904, "rewards/check_originality_func/mean": 0.3571428656578064, "rewards/check_originality_func/std": 0.48349377512931824, "rewards/perplexity_reward/mean": 0.8049665093421936, "rewards/perplexity_reward/std": 0.07676319032907486, "rewards/sentence_count_match_reward_logic/mean": 0.9342261552810669, "rewards/sentence_count_match_reward_logic/std": 0.08460661768913269, "step": 1253 }, { "clip_ratio": 0.007479255553334951, "epoch": 0.43876836948915326, "grad_norm": 0.29870415488964797, "kl": 0.8828125, "learning_rate": 9.573751651333156e-06, "loss": 0.0023, "step": 1254 }, { "clip_ratio": 0.017788350582122803, "epoch": 0.4391182645206438, "grad_norm": 0.2104996865570027, "kl": 0.91796875, "learning_rate": 9.57251706495862e-06, "loss": 0.0016, "step": 1255 }, { "clip_ratio": 0.029163358733057976, "epoch": 0.43946815955213436, "grad_norm": 0.19301949393280324, "kl": 0.96484375, "learning_rate": 9.571280773078495e-06, "loss": 0.0011, "step": 1256 }, { "clip_ratio": 0.0028782931622117758, "completions/clipped_ratio": 0.1428571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 174.0, "completions/mean_length": 113.92857360839844, "completions/mean_terminated_length": 90.25, "completions/min_length": 25.0, "completions/min_terminated_length": 25.0, "epoch": 0.43981805458362494, "grad_norm": 0.49469916068970193, "kl": 1.203125, "learning_rate": 9.570042776153904e-06, "loss": 0.0032, "num_tokens": 6033454.0, "reward": 2.0988388061523438, "reward_std": 0.21336546540260315, "rewards/check_originality_func/mean": 0.3928571343421936, "rewards/check_originality_func/std": 0.4928053617477417, "rewards/perplexity_reward/mean": 0.822265625, "rewards/perplexity_reward/std": 0.07256875932216644, "rewards/sentence_count_match_reward_logic/mean": 0.8837160468101501, "rewards/sentence_count_match_reward_logic/std": 0.2000565528869629, "step": 1257 }, { "clip_ratio": 0.012243368662893772, "epoch": 0.44016794961511546, "grad_norm": 0.7161980090380069, "kl": 1.171875, "learning_rate": 9.568803074646614e-06, "loss": 0.0023, "step": 1258 }, { "clip_ratio": 0.024242758750915527, "epoch": 0.44051784464660604, "grad_norm": 1.44656300900154, "kl": 1.4296875, "learning_rate": 9.567561669019014e-06, "loss": 0.0024, "step": 1259 }, { "clip_ratio": 0.041616182774305344, "epoch": 0.44086773967809656, "grad_norm": 0.49219705243412104, "kl": 1.1640625, "learning_rate": 9.566318559734142e-06, "loss": -0.0, "step": 1260 }, { "clip_ratio": 0.003017785493284464, "completions/clipped_ratio": 0.0, "completions/max_length": 199.0, "completions/max_terminated_length": 199.0, "completions/mean_length": 117.00000762939453, "completions/mean_terminated_length": 117.00000762939453, "completions/min_length": 25.0, "completions/min_terminated_length": 25.0, "epoch": 0.44121763470958714, "grad_norm": 0.45154052823893087, "kl": 1.203125, "learning_rate": 9.565073747255665e-06, "loss": 0.0089, "num_tokens": 6048358.0, "reward": 2.044145345687866, "reward_std": 0.021014945581555367, "rewards/check_originality_func/mean": 0.2857142984867096, "rewards/check_originality_func/std": 0.4558423161506653, "rewards/perplexity_reward/mean": 0.8322405219078064, "rewards/perplexity_reward/std": 0.07624667137861252, "rewards/sentence_count_match_reward_logic/mean": 0.9261904954910278, "rewards/sentence_count_match_reward_logic/std": 0.09314142167568207, "step": 1261 }, { "clip_ratio": 0.010497880168259144, "epoch": 0.44156752974107766, "grad_norm": 0.3488879446630639, "kl": 1.203125, "learning_rate": 9.563827232047885e-06, "loss": 0.0075, "step": 1262 }, { "clip_ratio": 0.025769850239157677, "epoch": 0.44191742477256823, "grad_norm": 0.26433022445776516, "kl": 1.203125, "learning_rate": 9.562579014575741e-06, "loss": 0.0066, "step": 1263 }, { "clip_ratio": 0.03894853591918945, "epoch": 0.44226731980405876, "grad_norm": 0.20773431356161123, "kl": 1.2109375, "learning_rate": 9.561329095304805e-06, "loss": 0.0059, "step": 1264 }, { "clip_ratio": 0.0025546865072101355, "completions/clipped_ratio": 0.1428571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 159.0, "completions/mean_length": 100.76786041259766, "completions/mean_terminated_length": 74.89583587646484, "completions/min_length": 22.0, "completions/min_terminated_length": 22.0, "epoch": 0.44261721483554933, "grad_norm": 0.46409753078106103, "kl": 0.69921875, "learning_rate": 9.56007747470129e-06, "loss": 0.0042, "num_tokens": 6062113.0, "reward": 2.061418294906616, "reward_std": 0.20672699809074402, "rewards/check_originality_func/mean": 0.3035714328289032, "rewards/check_originality_func/std": 0.4639608860015869, "rewards/perplexity_reward/mean": 0.7977818250656128, "rewards/perplexity_reward/std": 0.0868430808186531, "rewards/sentence_count_match_reward_logic/mean": 0.9600648880004883, "rewards/sentence_count_match_reward_logic/std": 0.107392817735672, "step": 1265 }, { "clip_ratio": 0.011832990683615208, "epoch": 0.4429671098670399, "grad_norm": 0.35330195296357964, "kl": 0.70703125, "learning_rate": 9.558824153232033e-06, "loss": 0.0024, "step": 1266 }, { "clip_ratio": 0.028167499229311943, "epoch": 0.44331700489853043, "grad_norm": 0.26374758789817015, "kl": 0.71484375, "learning_rate": 9.557569131364512e-06, "loss": 0.0012, "step": 1267 }, { "clip_ratio": 0.042435210198163986, "epoch": 0.443666899930021, "grad_norm": 0.22348699658981555, "kl": 0.72265625, "learning_rate": 9.556312409566842e-06, "loss": 0.0003, "step": 1268 }, { "clip_ratio": 0.005049162544310093, "completions/clipped_ratio": 0.0892857142857143, "completions/max_length": 256.0, "completions/max_terminated_length": 255.0, "completions/mean_length": 114.71429443359375, "completions/mean_terminated_length": 100.86274719238281, "completions/min_length": 19.0, "completions/min_terminated_length": 19.0, "epoch": 0.44401679496151153, "grad_norm": 0.46262662271955746, "kl": 1.171875, "learning_rate": 9.555053988307764e-06, "loss": 0.0231, "num_tokens": 6077393.0, "reward": 2.1334946155548096, "reward_std": 0.12000206112861633, "rewards/check_originality_func/mean": 0.3392857015132904, "rewards/check_originality_func/std": 0.477751761674881, "rewards/perplexity_reward/mean": 0.8219866156578064, "rewards/perplexity_reward/std": 0.05625225603580475, "rewards/sentence_count_match_reward_logic/mean": 0.9722222089767456, "rewards/sentence_count_match_reward_logic/std": 0.06444096565246582, "step": 1269 }, { "clip_ratio": 0.010003849864006042, "epoch": 0.4443666899930021, "grad_norm": 0.37307328038595233, "kl": 1.171875, "learning_rate": 9.55379386805666e-06, "loss": 0.0217, "step": 1270 }, { "clip_ratio": 0.026938391849398613, "epoch": 0.4447165850244926, "grad_norm": 0.2995720740978132, "kl": 1.1796875, "learning_rate": 9.552532049283541e-06, "loss": 0.0198, "step": 1271 }, { "clip_ratio": 0.04161752015352249, "epoch": 0.4450664800559832, "grad_norm": 0.22564888717147683, "kl": 1.1640625, "learning_rate": 9.551268532459055e-06, "loss": 0.0189, "step": 1272 }, { "clip_ratio": 0.002591668860986829, "completions/clipped_ratio": 0.2857142857142857, "completions/max_length": 256.0, "completions/max_terminated_length": 213.0, "completions/mean_length": 133.98214721679688, "completions/mean_terminated_length": 85.17500305175781, "completions/min_length": 17.0, "completions/min_terminated_length": 17.0, "epoch": 0.4454163750874738, "grad_norm": 0.43909046236077254, "kl": 0.78515625, "learning_rate": 9.550003318054482e-06, "loss": 0.0058, "num_tokens": 6097048.0, "reward": 2.058697462081909, "reward_std": 0.08636587113142014, "rewards/check_originality_func/mean": 0.3392857015132904, "rewards/check_originality_func/std": 0.477751761674881, "rewards/perplexity_reward/mean": 0.8395647406578064, "rewards/perplexity_reward/std": 0.027800140902400017, "rewards/sentence_count_match_reward_logic/mean": 0.87984699010849, "rewards/sentence_count_match_reward_logic/std": 0.1668829470872879, "step": 1273 }, { "clip_ratio": 0.005426590796560049, "epoch": 0.4457662701189643, "grad_norm": 0.3886628060359815, "kl": 0.7890625, "learning_rate": 9.548736406541732e-06, "loss": 0.0039, "step": 1274 }, { "clip_ratio": 0.017338993027806282, "epoch": 0.4461161651504549, "grad_norm": 0.2549025295977963, "kl": 0.7890625, "learning_rate": 9.547467798393355e-06, "loss": 0.0021, "step": 1275 }, { "clip_ratio": 0.027860021218657494, "epoch": 0.4464660601819454, "grad_norm": 0.23332298819804245, "kl": 0.79296875, "learning_rate": 9.546197494082528e-06, "loss": 0.0012, "step": 1276 }, { "clip_ratio": 0.0037345443852245808, "completions/clipped_ratio": 0.1428571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 239.0, "completions/mean_length": 147.82144165039062, "completions/mean_terminated_length": 129.7916717529297, "completions/min_length": 61.0, "completions/min_terminated_length": 61.0, "epoch": 0.446815955213436, "grad_norm": 0.5584612440177016, "kl": 1.1640625, "learning_rate": 9.544925494083062e-06, "loss": 0.0066, "num_tokens": 6116326.0, "reward": 2.437469005584717, "reward_std": 0.21441830694675446, "rewards/check_originality_func/mean": 0.6607142686843872, "rewards/check_originality_func/std": 0.477751761674881, "rewards/perplexity_reward/mean": 0.8327985405921936, "rewards/perplexity_reward/std": 0.02067556418478489, "rewards/sentence_count_match_reward_logic/mean": 0.9439560174942017, "rewards/sentence_count_match_reward_logic/std": 0.11986825615167618, "step": 1277 }, { "clip_ratio": 0.012873275205492973, "epoch": 0.4471658502449265, "grad_norm": 0.46380836087190785, "kl": 1.1640625, "learning_rate": 9.5436517988694e-06, "loss": 0.0042, "step": 1278 }, { "clip_ratio": 0.027592970058321953, "epoch": 0.4475157452764171, "grad_norm": 0.3545618820051247, "kl": 1.1640625, "learning_rate": 9.54237640891662e-06, "loss": 0.002, "step": 1279 }, { "clip_ratio": 0.04457196593284607, "epoch": 0.44786564030790765, "grad_norm": 0.25819870042854726, "kl": 1.171875, "learning_rate": 9.541099324700432e-06, "loss": 0.0007, "step": 1280 }, { "clip_ratio": 0.005596727132797241, "completions/clipped_ratio": 0.017857142857142905, "completions/max_length": 256.0, "completions/max_terminated_length": 233.0, "completions/mean_length": 124.60714721679688, "completions/mean_terminated_length": 122.21817779541016, "completions/min_length": 14.0, "completions/min_terminated_length": 14.0, "epoch": 0.4482155353393982, "grad_norm": 0.5533314394823895, "kl": 1.1015625, "learning_rate": 9.539820546697175e-06, "loss": 0.002, "num_tokens": 6132480.0, "reward": 2.1887259483337402, "reward_std": 0.19407513737678528, "rewards/check_originality_func/mean": 0.3928571343421936, "rewards/check_originality_func/std": 0.4928053617477417, "rewards/perplexity_reward/mean": 0.8468889594078064, "rewards/perplexity_reward/std": 0.07743094116449356, "rewards/sentence_count_match_reward_logic/mean": 0.9489795565605164, "rewards/sentence_count_match_reward_logic/std": 0.1455325484275818, "step": 1281 }, { "clip_ratio": 0.021532749757170677, "epoch": 0.44856543037088875, "grad_norm": 0.46451386685305196, "kl": 1.1015625, "learning_rate": 9.53854007538382e-06, "loss": -0.0005, "step": 1282 }, { "clip_ratio": 0.05161803960800171, "epoch": 0.4489153254023793, "grad_norm": 0.3148012622668804, "kl": 1.109375, "learning_rate": 9.537257911237968e-06, "loss": -0.0028, "step": 1283 }, { "clip_ratio": 0.0661654993891716, "epoch": 0.44926522043386985, "grad_norm": 0.2340773641780161, "kl": 1.109375, "learning_rate": 9.53597405473786e-06, "loss": -0.004, "step": 1284 }, { "clip_ratio": 0.0057115075178444386, "completions/clipped_ratio": 0.0, "completions/max_length": 134.0, "completions/max_terminated_length": 134.0, "completions/mean_length": 61.19643020629883, "completions/mean_terminated_length": 61.19643020629883, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "epoch": 0.44961511546536037, "grad_norm": 0.3806761346580955, "kl": 0.9375, "learning_rate": 9.53468850636236e-06, "loss": 0.0013, "num_tokens": 6141571.0, "reward": 1.9760046005249023, "reward_std": 0.1528196632862091, "rewards/check_originality_func/mean": 0.1428571492433548, "rewards/check_originality_func/std": 0.3530939221382141, "rewards/perplexity_reward/mean": 0.8331473469734192, "rewards/perplexity_reward/std": 0.053189270198345184, "rewards/sentence_count_match_reward_logic/mean": 1.0, "rewards/sentence_count_match_reward_logic/std": 0.0, "step": 1285 }, { "clip_ratio": 0.02537727728486061, "epoch": 0.44996501049685095, "grad_norm": 0.26056107619054064, "kl": 0.9375, "learning_rate": 9.533401266590964e-06, "loss": 0.0002, "step": 1286 }, { "clip_ratio": 0.03767083212733269, "epoch": 0.4503149055283415, "grad_norm": 0.19029562851145354, "kl": 0.9375, "learning_rate": 9.532112335903803e-06, "loss": -0.0007, "step": 1287 }, { "clip_ratio": 0.04932733252644539, "epoch": 0.45066480055983205, "grad_norm": 0.16837315691746582, "kl": 0.9609375, "learning_rate": 9.530821714781632e-06, "loss": -0.0011, "step": 1288 }, { "clip_ratio": 0.0020020226947963238, "completions/clipped_ratio": 0.0, "completions/max_length": 219.0, "completions/max_terminated_length": 219.0, "completions/mean_length": 84.67857360839844, "completions/mean_terminated_length": 84.67857360839844, "completions/min_length": 6.0, "completions/min_terminated_length": 6.0, "epoch": 0.4510146955913226, "grad_norm": 0.34276605168180085, "kl": 0.6484375, "learning_rate": 9.529529403705844e-06, "loss": 0.0012, "num_tokens": 6153193.0, "reward": 1.9035296440124512, "reward_std": 0.17751462757587433, "rewards/check_originality_func/mean": 0.125, "rewards/check_originality_func/std": 0.3337118923664093, "rewards/perplexity_reward/mean": 0.7785295844078064, "rewards/perplexity_reward/std": 0.1079372763633728, "rewards/sentence_count_match_reward_logic/mean": 1.0, "rewards/sentence_count_match_reward_logic/std": 0.0, "step": 1289 }, { "clip_ratio": 0.005649688187986612, "epoch": 0.45136459062281314, "grad_norm": 0.2866091624642747, "kl": 0.6484375, "learning_rate": 9.528235403158457e-06, "loss": 0.0004, "step": 1290 }, { "clip_ratio": 0.020567024126648903, "epoch": 0.4517144856543037, "grad_norm": 0.18115363718267666, "kl": 0.6484375, "learning_rate": 9.526939713622121e-06, "loss": -0.0005, "step": 1291 }, { "clip_ratio": 0.030202524736523628, "epoch": 0.45206438068579424, "grad_norm": 0.16885268123551303, "kl": 0.65234375, "learning_rate": 9.525642335580116e-06, "loss": -0.0011, "step": 1292 }, { "clip_ratio": 0.003607138991355896, "completions/clipped_ratio": 0.4285714285714286, "completions/max_length": 256.0, "completions/max_terminated_length": 103.0, "completions/mean_length": 139.58929443359375, "completions/mean_terminated_length": 52.28125, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "epoch": 0.4524142757172848, "grad_norm": 0.44893077498042894, "kl": 0.87890625, "learning_rate": 9.524343269516354e-06, "loss": 0.0063, "num_tokens": 6173498.0, "reward": 2.125297784805298, "reward_std": 0.2607511281967163, "rewards/check_originality_func/mean": 0.4285714328289032, "rewards/check_originality_func/std": 0.4993502199649811, "rewards/perplexity_reward/mean": 0.8169642686843872, "rewards/perplexity_reward/std": 0.05817048251628876, "rewards/sentence_count_match_reward_logic/mean": 0.8797618746757507, "rewards/sentence_count_match_reward_logic/std": 0.1929754763841629, "step": 1293 }, { "clip_ratio": 0.019344137981534004, "epoch": 0.45276417074877534, "grad_norm": 0.364074300108073, "kl": 0.8828125, "learning_rate": 9.523042515915371e-06, "loss": 0.0045, "step": 1294 }, { "clip_ratio": 0.03875714913010597, "epoch": 0.4531140657802659, "grad_norm": 0.28032167258334023, "kl": 0.91796875, "learning_rate": 9.521740075262338e-06, "loss": 0.0029, "step": 1295 }, { "clip_ratio": 0.05782461166381836, "epoch": 0.4534639608117565, "grad_norm": 0.24027688777127462, "kl": 0.9453125, "learning_rate": 9.520435948043051e-06, "loss": 0.0012, "step": 1296 }, { "clip_ratio": 0.006041354034096003, "completions/clipped_ratio": 0.1428571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 142.0, "completions/mean_length": 96.35714721679688, "completions/mean_terminated_length": 69.75, "completions/min_length": 24.0, "completions/min_terminated_length": 24.0, "epoch": 0.453813855843247, "grad_norm": 0.40181007624023113, "kl": 1.0859375, "learning_rate": 9.519130134743938e-06, "loss": 0.0085, "num_tokens": 6187286.0, "reward": 2.0980050563812256, "reward_std": 0.09066580981016159, "rewards/check_originality_func/mean": 0.3392857015132904, "rewards/check_originality_func/std": 0.477751761674881, "rewards/perplexity_reward/mean": 0.8078264594078064, "rewards/perplexity_reward/std": 0.06428306549787521, "rewards/sentence_count_match_reward_logic/mean": 0.9508928656578064, "rewards/sentence_count_match_reward_logic/std": 0.1218302994966507, "step": 1297 }, { "clip_ratio": 0.015519640408456326, "epoch": 0.4541637508747376, "grad_norm": 0.28387999987497653, "kl": 1.0859375, "learning_rate": 9.517822635852059e-06, "loss": 0.0074, "step": 1298 }, { "clip_ratio": 0.034810177981853485, "epoch": 0.4545136459062281, "grad_norm": 0.18583335510625745, "kl": 1.0859375, "learning_rate": 9.516513451855093e-06, "loss": 0.0063, "step": 1299 }, { "clip_ratio": 0.0454939641058445, "epoch": 0.4548635409377187, "grad_norm": 0.15036563630731847, "kl": 1.0859375, "learning_rate": 9.515202583241355e-06, "loss": 0.006, "step": 1300 }, { "clip_ratio": 0.002568537136539817, "completions/clipped_ratio": 0.0, "completions/max_length": 141.0, "completions/max_terminated_length": 141.0, "completions/mean_length": 76.76786041259766, "completions/mean_terminated_length": 76.76786041259766, "completions/min_length": 12.0, "completions/min_terminated_length": 12.0, "epoch": 0.4552134359692092, "grad_norm": 0.4112867194318592, "kl": 1.1796875, "learning_rate": 9.513890030499786e-06, "loss": 0.0061, "num_tokens": 6197865.0, "reward": 1.9799200296401978, "reward_std": 0.20412799715995789, "rewards/check_originality_func/mean": 0.2321428507566452, "rewards/check_originality_func/std": 0.4260205328464508, "rewards/perplexity_reward/mean": 0.7900390625, "rewards/perplexity_reward/std": 0.09335929900407791, "rewards/sentence_count_match_reward_logic/mean": 0.9577380418777466, "rewards/sentence_count_match_reward_logic/std": 0.08177106082439423, "step": 1301 }, { "clip_ratio": 0.009359468705952168, "epoch": 0.4555633310006998, "grad_norm": 0.25552418809413613, "kl": 1.1796875, "learning_rate": 9.512575794119957e-06, "loss": 0.0047, "step": 1302 }, { "clip_ratio": 0.023315204307436943, "epoch": 0.45591322603219037, "grad_norm": 0.26869616790062056, "kl": 1.2265625, "learning_rate": 9.511259874592067e-06, "loss": 0.0044, "step": 1303 }, { "clip_ratio": 0.0429060198366642, "epoch": 0.4562631210636809, "grad_norm": 0.20810724530788585, "kl": 1.2265625, "learning_rate": 9.509942272406938e-06, "loss": 0.0039, "step": 1304 }, { "clip_ratio": 0.0030550833325833082, "completions/clipped_ratio": 0.0, "completions/max_length": 241.0, "completions/max_terminated_length": 241.0, "completions/mean_length": 155.2678680419922, "completions/mean_terminated_length": 155.2678680419922, "completions/min_length": 28.0, "completions/min_terminated_length": 28.0, "epoch": 0.45661301609517146, "grad_norm": 0.5574541961363407, "kl": 0.96875, "learning_rate": 9.508622988056026e-06, "loss": 0.0066, "num_tokens": 6217256.0, "reward": 2.345242738723755, "reward_std": 0.15656709671020508, "rewards/check_originality_func/mean": 0.5357142686843872, "rewards/check_originality_func/std": 0.5032362937927246, "rewards/perplexity_reward/mean": 0.8514927625656128, "rewards/perplexity_reward/std": 0.05747024714946747, "rewards/sentence_count_match_reward_logic/mean": 0.9580357670783997, "rewards/sentence_count_match_reward_logic/std": 0.07309818267822266, "step": 1305 }, { "clip_ratio": 0.010489254258573055, "epoch": 0.456962911126662, "grad_norm": 0.4641669845836053, "kl": 0.97265625, "learning_rate": 9.50730202203141e-06, "loss": 0.004, "step": 1306 }, { "clip_ratio": 0.03126826882362366, "epoch": 0.45731280615815256, "grad_norm": 0.3148741941284479, "kl": 0.96875, "learning_rate": 9.5059793748258e-06, "loss": 0.0012, "step": 1307 }, { "clip_ratio": 0.04968094453215599, "epoch": 0.4576627011896431, "grad_norm": 0.25817070166921013, "kl": 0.98046875, "learning_rate": 9.504655046932528e-06, "loss": -0.0003, "step": 1308 }, { "clip_ratio": 0.0014011607272550464, "completions/clipped_ratio": 0.1428571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 207.0, "completions/mean_length": 102.42857360839844, "completions/mean_terminated_length": 76.83333587646484, "completions/min_length": 13.0, "completions/min_terminated_length": 13.0, "epoch": 0.45801259622113366, "grad_norm": 0.3164175014058867, "kl": 0.74609375, "learning_rate": 9.503329038845556e-06, "loss": 0.0038, "num_tokens": 6232336.0, "reward": 2.089670181274414, "reward_std": 0.11285890638828278, "rewards/check_originality_func/mean": 0.4285714328289032, "rewards/check_originality_func/std": 0.4993502199649811, "rewards/perplexity_reward/mean": 0.73828125, "rewards/perplexity_reward/std": 0.15212231874465942, "rewards/sentence_count_match_reward_logic/mean": 0.9228175282478333, "rewards/sentence_count_match_reward_logic/std": 0.17586621642112732, "step": 1309 }, { "clip_ratio": 0.008879530243575573, "epoch": 0.45836249125262424, "grad_norm": 0.2451898222930066, "kl": 0.7421875, "learning_rate": 9.502001351059477e-06, "loss": 0.0026, "step": 1310 }, { "clip_ratio": 0.020167699083685875, "epoch": 0.45871238628411476, "grad_norm": 0.1836328080769614, "kl": 0.7421875, "learning_rate": 9.500671984069501e-06, "loss": 0.002, "step": 1311 }, { "clip_ratio": 0.02798602357506752, "epoch": 0.45906228131560534, "grad_norm": 0.15602093407131243, "kl": 0.7421875, "learning_rate": 9.49934093837147e-06, "loss": 0.0012, "step": 1312 }, { "clip_ratio": 0.004468298517167568, "completions/clipped_ratio": 0.1428571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 205.0, "completions/mean_length": 138.80357360839844, "completions/mean_terminated_length": 119.27083587646484, "completions/min_length": 22.0, "completions/min_terminated_length": 22.0, "epoch": 0.45941217634709586, "grad_norm": 0.41053059847269485, "kl": 0.65234375, "learning_rate": 9.498008214461854e-06, "loss": 0.0062, "num_tokens": 6250653.0, "reward": 2.2758169174194336, "reward_std": 0.17903192341327667, "rewards/check_originality_func/mean": 0.4821428656578064, "rewards/check_originality_func/std": 0.5042031407356262, "rewards/perplexity_reward/mean": 0.8690708875656128, "rewards/perplexity_reward/std": 0.050328463315963745, "rewards/sentence_count_match_reward_logic/mean": 0.9246031641960144, "rewards/sentence_count_match_reward_logic/std": 0.09559816122055054, "step": 1313 }, { "clip_ratio": 0.008803869597613811, "epoch": 0.45976207137858643, "grad_norm": 0.33618823727479336, "kl": 0.65234375, "learning_rate": 9.496673812837742e-06, "loss": 0.0046, "step": 1314 }, { "clip_ratio": 0.027456296607851982, "epoch": 0.46011196641007696, "grad_norm": 0.22389166510871547, "kl": 0.6484375, "learning_rate": 9.495337733996858e-06, "loss": 0.0034, "step": 1315 }, { "clip_ratio": 0.04018176719546318, "epoch": 0.46046186144156753, "grad_norm": 0.20555361299674713, "kl": 0.6484375, "learning_rate": 9.493999978437544e-06, "loss": 0.0024, "step": 1316 }, { "clip_ratio": 0.0023120909463614225, "completions/clipped_ratio": 0.0, "completions/max_length": 228.0, "completions/max_terminated_length": 228.0, "completions/mean_length": 107.64286041259766, "completions/mean_terminated_length": 107.64286041259766, "completions/min_length": 11.0, "completions/min_terminated_length": 11.0, "epoch": 0.4608117564730581, "grad_norm": 0.3963587916365904, "kl": 0.8359375, "learning_rate": 9.492660546658771e-06, "loss": 0.0057, "num_tokens": 6264833.0, "reward": 2.1104912757873535, "reward_std": 0.19136138260364532, "rewards/check_originality_func/mean": 0.3571428656578064, "rewards/check_originality_func/std": 0.48349374532699585, "rewards/perplexity_reward/mean": 0.7533482313156128, "rewards/perplexity_reward/std": 0.13770127296447754, "rewards/sentence_count_match_reward_logic/mean": 1.0, "rewards/sentence_count_match_reward_logic/std": 0.0, "step": 1317 }, { "clip_ratio": 0.012419181875884533, "epoch": 0.46116165150454863, "grad_norm": 0.338087361565223, "kl": 0.8359375, "learning_rate": 9.491319439160134e-06, "loss": 0.0043, "step": 1318 }, { "clip_ratio": 0.028975775465369225, "epoch": 0.4615115465360392, "grad_norm": 0.2377729410200103, "kl": 0.83203125, "learning_rate": 9.489976656441855e-06, "loss": 0.003, "step": 1319 }, { "clip_ratio": 0.038430336862802505, "epoch": 0.46186144156752973, "grad_norm": 0.18979537107369754, "kl": 0.83203125, "learning_rate": 9.488632199004777e-06, "loss": 0.0026, "step": 1320 }, { "clip_ratio": 0.0022999767679721117, "completions/clipped_ratio": 0.1428571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 191.0, "completions/mean_length": 118.51786041259766, "completions/mean_terminated_length": 95.60417175292969, "completions/min_length": 24.0, "completions/min_terminated_length": 24.0, "epoch": 0.4622113365990203, "grad_norm": 0.378264223579146, "kl": 0.62890625, "learning_rate": 9.48728606735037e-06, "loss": 0.002, "num_tokens": 6281526.0, "reward": 2.177485466003418, "reward_std": 0.22770610451698303, "rewards/check_originality_func/mean": 0.3928571343421936, "rewards/check_originality_func/std": 0.4928053617477417, "rewards/perplexity_reward/mean": 0.86083984375, "rewards/perplexity_reward/std": 0.0296422578394413, "rewards/sentence_count_match_reward_logic/mean": 0.9237882494926453, "rewards/sentence_count_match_reward_logic/std": 0.13912498950958252, "step": 1321 }, { "clip_ratio": 0.011008714325726032, "epoch": 0.4625612316305108, "grad_norm": 0.32409593038517, "kl": 0.6328125, "learning_rate": 9.48593826198073e-06, "loss": 0.0007, "step": 1322 }, { "clip_ratio": 0.02120712772011757, "epoch": 0.4629111266620014, "grad_norm": 0.2315815881680291, "kl": 0.62890625, "learning_rate": 9.484588783398573e-06, "loss": -0.0004, "step": 1323 }, { "clip_ratio": 0.03462797775864601, "epoch": 0.4632610216934919, "grad_norm": 0.18194928233561253, "kl": 0.6328125, "learning_rate": 9.483237632107245e-06, "loss": -0.0014, "step": 1324 }, { "clip_ratio": 0.0036798540968447924, "completions/clipped_ratio": 0.2857142857142857, "completions/max_length": 256.0, "completions/max_terminated_length": 203.0, "completions/mean_length": 124.73214721679688, "completions/mean_terminated_length": 72.2249984741211, "completions/min_length": 17.0, "completions/min_terminated_length": 17.0, "epoch": 0.4636109167249825, "grad_norm": 0.39389868337112083, "kl": 0.7890625, "learning_rate": 9.481884808610712e-06, "loss": 0.0069, "num_tokens": 6297975.0, "reward": 1.9479310512542725, "reward_std": 0.20874430239200592, "rewards/check_originality_func/mean": 0.1785714328289032, "rewards/check_originality_func/std": 0.3864591419696808, "rewards/perplexity_reward/mean": 0.8231026530265808, "rewards/perplexity_reward/std": 0.07708006352186203, "rewards/sentence_count_match_reward_logic/mean": 0.9462569355964661, "rewards/sentence_count_match_reward_logic/std": 0.07260096818208694, "step": 1325 }, { "clip_ratio": 0.015782814472913742, "epoch": 0.4639608117564731, "grad_norm": 0.326445679347816, "kl": 0.7890625, "learning_rate": 9.480530313413563e-06, "loss": 0.0053, "step": 1326 }, { "clip_ratio": 0.036831311881542206, "epoch": 0.4643107067879636, "grad_norm": 0.20500825958986718, "kl": 0.8046875, "learning_rate": 9.47917414702101e-06, "loss": 0.0042, "step": 1327 }, { "clip_ratio": 0.05084250122308731, "epoch": 0.4646606018194542, "grad_norm": 0.18255239578016913, "kl": 0.8203125, "learning_rate": 9.477816309938896e-06, "loss": 0.0036, "step": 1328 }, { "clip_ratio": 0.0018333130283281207, "completions/clipped_ratio": 0.1428571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 137.0, "completions/mean_length": 83.85714721679688, "completions/mean_terminated_length": 55.16666793823242, "completions/min_length": 15.0, "completions/min_terminated_length": 15.0, "epoch": 0.4650104968509447, "grad_norm": 0.37467448163725453, "kl": 1.0546875, "learning_rate": 9.476456802673677e-06, "loss": 0.0048, "num_tokens": 6310455.0, "reward": 2.0579285621643066, "reward_std": 0.19985972344875336, "rewards/check_originality_func/mean": 0.3035714328289032, "rewards/check_originality_func/std": 0.4639608860015869, "rewards/perplexity_reward/mean": 0.8010602593421936, "rewards/perplexity_reward/std": 0.09485866874456406, "rewards/sentence_count_match_reward_logic/mean": 0.9532966613769531, "rewards/sentence_count_match_reward_logic/std": 0.11613143980503082, "step": 1329 }, { "clip_ratio": 0.013765263371169567, "epoch": 0.4653603918824353, "grad_norm": 0.27035216484983743, "kl": 1.0625, "learning_rate": 9.475095625732437e-06, "loss": 0.0032, "step": 1330 }, { "clip_ratio": 0.03564935177564621, "epoch": 0.4657102869139258, "grad_norm": 0.21153105729086694, "kl": 1.0703125, "learning_rate": 9.473732779622882e-06, "loss": 0.0022, "step": 1331 }, { "clip_ratio": 0.052291031926870346, "epoch": 0.4660601819454164, "grad_norm": 0.14127836087016085, "kl": 1.0703125, "learning_rate": 9.472368264853342e-06, "loss": 0.0018, "step": 1332 }, { "clip_ratio": 0.0034869599621742964, "completions/clipped_ratio": 0.1428571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 89.0, "completions/mean_length": 63.017860412597656, "completions/mean_terminated_length": 30.854167938232422, "completions/min_length": 13.0, "completions/min_terminated_length": 13.0, "epoch": 0.46641007697690695, "grad_norm": 0.25583894682334085, "kl": 0.875, "learning_rate": 9.471002081932767e-06, "loss": 0.0015, "num_tokens": 6320192.0, "reward": 1.8426339626312256, "reward_std": 0.13546420633792877, "rewards/check_originality_func/mean": 0.1071428582072258, "rewards/check_originality_func/std": 0.3120938837528229, "rewards/perplexity_reward/mean": 0.7756696343421936, "rewards/perplexity_reward/std": 0.06782964617013931, "rewards/sentence_count_match_reward_logic/mean": 0.9598214030265808, "rewards/sentence_count_match_reward_logic/std": 0.10049255937337875, "step": 1333 }, { "clip_ratio": 0.01112015824764967, "epoch": 0.4667599720083975, "grad_norm": 0.20052024803223245, "kl": 0.875, "learning_rate": 9.46963423137073e-06, "loss": 0.001, "step": 1334 }, { "clip_ratio": 0.03381561115384102, "epoch": 0.46710986703988805, "grad_norm": 0.13006512842119067, "kl": 0.875, "learning_rate": 9.468264713677427e-06, "loss": 0.0005, "step": 1335 }, { "clip_ratio": 0.04296246916055679, "epoch": 0.46745976207137857, "grad_norm": 0.10084450394486297, "kl": 0.8671875, "learning_rate": 9.466893529363676e-06, "loss": 0.0003, "step": 1336 }, { "clip_ratio": 0.002494320971891284, "completions/clipped_ratio": 0.3035714285714286, "completions/max_length": 256.0, "completions/max_terminated_length": 218.0, "completions/mean_length": 154.4107208251953, "completions/mean_terminated_length": 110.12820434570312, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "epoch": 0.46780965710286915, "grad_norm": 0.43105771064533316, "kl": 0.71484375, "learning_rate": 9.465520678940913e-06, "loss": 0.0171, "num_tokens": 6340919.0, "reward": 2.3352084159851074, "reward_std": 0.12836739420890808, "rewards/check_originality_func/mean": 0.5535714030265808, "rewards/check_originality_func/std": 0.5016207695007324, "rewards/perplexity_reward/mean": 0.8341936469078064, "rewards/perplexity_reward/std": 0.10110506415367126, "rewards/sentence_count_match_reward_logic/mean": 0.9474431276321411, "rewards/sentence_count_match_reward_logic/std": 0.1251138299703598, "step": 1337 }, { "clip_ratio": 0.014182101003825665, "epoch": 0.46815955213435967, "grad_norm": 0.38857869679907225, "kl": 0.71484375, "learning_rate": 9.464146162921201e-06, "loss": 0.0156, "step": 1338 }, { "clip_ratio": 0.03853065147995949, "epoch": 0.46850944716585025, "grad_norm": 0.29526908609009067, "kl": 0.7265625, "learning_rate": 9.462769981817221e-06, "loss": 0.0138, "step": 1339 }, { "clip_ratio": 0.055917780846357346, "epoch": 0.4688593421973408, "grad_norm": 0.21818089613928388, "kl": 0.74609375, "learning_rate": 9.461392136142276e-06, "loss": 0.0124, "step": 1340 }, { "clip_ratio": 0.0020259064622223377, "completions/clipped_ratio": 0.0892857142857143, "completions/max_length": 256.0, "completions/max_terminated_length": 247.0, "completions/mean_length": 126.6964340209961, "completions/mean_terminated_length": 114.01961517333984, "completions/min_length": 13.0, "completions/min_terminated_length": 13.0, "epoch": 0.46920923722883134, "grad_norm": 0.44118337715338524, "kl": 1.359375, "learning_rate": 9.460012626410286e-06, "loss": 0.0076, "num_tokens": 6357518.0, "reward": 2.1736228466033936, "reward_std": 0.28322935104370117, "rewards/check_originality_func/mean": 0.3571428656578064, "rewards/check_originality_func/std": 0.48349377512931824, "rewards/perplexity_reward/mean": 0.8625836968421936, "rewards/perplexity_reward/std": 0.024023303762078285, "rewards/sentence_count_match_reward_logic/mean": 0.9538961052894592, "rewards/sentence_count_match_reward_logic/std": 0.09020981937646866, "step": 1341 }, { "clip_ratio": 0.007230711635202169, "epoch": 0.4695591322603219, "grad_norm": 0.3768601706941308, "kl": 1.359375, "learning_rate": 9.458631453135799e-06, "loss": 0.006, "step": 1342 }, { "clip_ratio": 0.021343957632780075, "epoch": 0.46990902729181244, "grad_norm": 0.2795630055196881, "kl": 1.3671875, "learning_rate": 9.457248616833978e-06, "loss": 0.004, "step": 1343 }, { "clip_ratio": 0.044151220470666885, "epoch": 0.470258922323303, "grad_norm": 0.20834904979672073, "kl": 1.375, "learning_rate": 9.455864118020607e-06, "loss": 0.0025, "step": 1344 }, { "clip_ratio": 0.004872799851000309, "completions/clipped_ratio": 0.1428571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 145.0, "completions/mean_length": 82.0, "completions/mean_terminated_length": 53.0, "completions/min_length": 11.0, "completions/min_terminated_length": 11.0, "epoch": 0.47060881735479354, "grad_norm": 0.35851775684045123, "kl": 0.76953125, "learning_rate": 9.454477957212092e-06, "loss": 0.0036, "num_tokens": 6369918.0, "reward": 2.164839744567871, "reward_std": 0.01355635467916727, "rewards/check_originality_func/mean": 0.4285714328289032, "rewards/check_originality_func/std": 0.4993502199649811, "rewards/perplexity_reward/mean": 0.7847377061843872, "rewards/perplexity_reward/std": 0.08279936015605927, "rewards/sentence_count_match_reward_logic/mean": 0.9515305757522583, "rewards/sentence_count_match_reward_logic/std": 0.12037907540798187, "step": 1345 }, { "clip_ratio": 0.013797236606478691, "epoch": 0.4709587123862841, "grad_norm": 0.29784260686110214, "kl": 0.77734375, "learning_rate": 9.453090134925459e-06, "loss": 0.0026, "step": 1346 }, { "clip_ratio": 0.04360436648130417, "epoch": 0.4713086074177747, "grad_norm": 0.19068869990656032, "kl": 0.77734375, "learning_rate": 9.45170065167835e-06, "loss": 0.0015, "step": 1347 }, { "clip_ratio": 0.060564637184143066, "epoch": 0.4716585024492652, "grad_norm": 0.17817097088836345, "kl": 0.77734375, "learning_rate": 9.450309507989031e-06, "loss": 0.0009, "step": 1348 }, { "clip_ratio": 0.0014505592407658696, "completions/clipped_ratio": 0.1428571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 171.0, "completions/mean_length": 122.1964340209961, "completions/mean_terminated_length": 99.89583587646484, "completions/min_length": 13.0, "completions/min_terminated_length": 13.0, "epoch": 0.4720083974807558, "grad_norm": 0.3785966162252588, "kl": 1.109375, "learning_rate": 9.448916704376384e-06, "loss": 0.0036, "num_tokens": 6386321.0, "reward": 2.3025062084198, "reward_std": 0.13304826617240906, "rewards/check_originality_func/mean": 0.4642857015132904, "rewards/check_originality_func/std": 0.5032362341880798, "rewards/perplexity_reward/mean": 0.83984375, "rewards/perplexity_reward/std": 0.056065235286951065, "rewards/sentence_count_match_reward_logic/mean": 0.9983766674995422, "rewards/sentence_count_match_reward_logic/std": 0.012148234993219376, "step": 1349 }, { "clip_ratio": 0.011677952483296394, "epoch": 0.4723582925122463, "grad_norm": 0.2713552279178417, "kl": 1.109375, "learning_rate": 9.447522241359911e-06, "loss": 0.0026, "step": 1350 }, { "clip_ratio": 0.026105323806405067, "epoch": 0.4727081875437369, "grad_norm": 0.1923928531770189, "kl": 1.109375, "learning_rate": 9.446126119459739e-06, "loss": 0.0014, "step": 1351 }, { "clip_ratio": 0.0413205623626709, "epoch": 0.4730580825752274, "grad_norm": 0.17020138161227982, "kl": 1.125, "learning_rate": 9.444728339196601e-06, "loss": 0.001, "step": 1352 }, { "clip_ratio": 0.0036350570153445005, "completions/clipped_ratio": 0.1071428571428571, "completions/max_length": 256.0, "completions/max_terminated_length": 251.0, "completions/mean_length": 133.35714721679688, "completions/mean_terminated_length": 118.63999938964844, "completions/min_length": 12.0, "completions/min_terminated_length": 12.0, "epoch": 0.473407977606718, "grad_norm": 0.5126067612276869, "kl": 1.0703125, "learning_rate": 9.44332890109186e-06, "loss": 0.0086, "num_tokens": 6404005.0, "reward": 2.1817104816436768, "reward_std": 0.21687668561935425, "rewards/check_originality_func/mean": 0.3928571343421936, "rewards/check_originality_func/std": 0.4928053617477417, "rewards/perplexity_reward/mean": 0.8267996907234192, "rewards/perplexity_reward/std": 0.07143313437700272, "rewards/sentence_count_match_reward_logic/mean": 0.9620535969734192, "rewards/sentence_count_match_reward_logic/std": 0.10391774028539658, "step": 1353 }, { "clip_ratio": 0.012126200832426548, "epoch": 0.4737578726382085, "grad_norm": 0.40350384138168766, "kl": 1.0625, "learning_rate": 9.441927805667493e-06, "loss": 0.0065, "step": 1354 }, { "clip_ratio": 0.037785641849040985, "epoch": 0.4741077676696991, "grad_norm": 0.29264708549071905, "kl": 1.0703125, "learning_rate": 9.440525053446092e-06, "loss": 0.0044, "step": 1355 }, { "clip_ratio": 0.049919288605451584, "epoch": 0.47445766270118966, "grad_norm": 0.22952372117804207, "kl": 1.0703125, "learning_rate": 9.439120644950875e-06, "loss": 0.0034, "step": 1356 }, { "clip_ratio": 0.002409138949587941, "completions/clipped_ratio": 0.1428571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 97.0, "completions/mean_length": 88.37500762939453, "completions/mean_terminated_length": 60.4375, "completions/min_length": 13.0, "completions/min_terminated_length": 13.0, "epoch": 0.4748075577326802, "grad_norm": 0.3298032958862111, "kl": 1.078125, "learning_rate": 9.437714580705671e-06, "loss": 0.0042, "num_tokens": 6417474.0, "reward": 2.1170082092285156, "reward_std": 0.13890738785266876, "rewards/check_originality_func/mean": 0.375, "rewards/check_originality_func/std": 0.48850420117378235, "rewards/perplexity_reward/mean": 0.8025948405265808, "rewards/perplexity_reward/std": 0.038258519023656845, "rewards/sentence_count_match_reward_logic/mean": 0.9394132494926453, "rewards/sentence_count_match_reward_logic/std": 0.1415393054485321, "step": 1357 }, { "clip_ratio": 0.012692444957792759, "epoch": 0.47515745276417076, "grad_norm": 0.25240607542390153, "kl": 1.0703125, "learning_rate": 9.436306861234926e-06, "loss": 0.0034, "step": 1358 }, { "clip_ratio": 0.032967500388622284, "epoch": 0.4755073477956613, "grad_norm": 0.1801181821595916, "kl": 1.078125, "learning_rate": 9.43489748706371e-06, "loss": 0.0025, "step": 1359 }, { "clip_ratio": 0.0511624701321125, "epoch": 0.47585724282715186, "grad_norm": 0.14473427350426965, "kl": 1.078125, "learning_rate": 9.433486458717705e-06, "loss": 0.0019, "step": 1360 }, { "clip_ratio": 0.0021458002738654613, "completions/clipped_ratio": 0.1428571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 116.0, "completions/mean_length": 83.875, "completions/mean_terminated_length": 55.1875, "completions/min_length": 14.0, "completions/min_terminated_length": 14.0, "epoch": 0.4762071378586424, "grad_norm": 0.3517677406943864, "kl": 1.1484375, "learning_rate": 9.43207377672321e-06, "loss": -0.0004, "num_tokens": 6430675.0, "reward": 1.9061721563339233, "reward_std": 0.20024029910564423, "rewards/check_originality_func/mean": 0.2321428507566452, "rewards/check_originality_func/std": 0.4260205328464508, "rewards/perplexity_reward/mean": 0.7917829155921936, "rewards/perplexity_reward/std": 0.08901987224817276, "rewards/sentence_count_match_reward_logic/mean": 0.8822464346885681, "rewards/sentence_count_match_reward_logic/std": 0.19072380661964417, "step": 1361 }, { "clip_ratio": 0.006044987589120865, "epoch": 0.47655703289013296, "grad_norm": 0.2972849401532017, "kl": 1.140625, "learning_rate": 9.430659441607145e-06, "loss": -0.0016, "step": 1362 }, { "clip_ratio": 0.021860165521502495, "epoch": 0.47690692792162354, "grad_norm": 0.2547105701063421, "kl": 1.1484375, "learning_rate": 9.429243453897039e-06, "loss": -0.0029, "step": 1363 }, { "clip_ratio": 0.033561158925294876, "epoch": 0.47725682295311406, "grad_norm": 0.22256115510809793, "kl": 1.1484375, "learning_rate": 9.427825814121047e-06, "loss": -0.0036, "step": 1364 }, { "clip_ratio": 0.0037349748890846968, "completions/clipped_ratio": 0.0, "completions/max_length": 189.0, "completions/max_terminated_length": 189.0, "completions/mean_length": 96.60714721679688, "completions/mean_terminated_length": 96.60714721679688, "completions/min_length": 15.0, "completions/min_terminated_length": 15.0, "epoch": 0.47760671798460463, "grad_norm": 0.3834996494303664, "kl": 0.7109375, "learning_rate": 9.426406522807932e-06, "loss": 0.0048, "num_tokens": 6444021.0, "reward": 1.9492188692092896, "reward_std": 0.015069333836436272, "rewards/check_originality_func/mean": 0.1428571492433548, "rewards/check_originality_func/std": 0.3530939221382141, "rewards/perplexity_reward/mean": 0.8197544813156128, "rewards/perplexity_reward/std": 0.03511620685458183, "rewards/sentence_count_match_reward_logic/mean": 0.9866071343421936, "rewards/sentence_count_match_reward_logic/std": 0.03901173919439316, "step": 1365 }, { "clip_ratio": 0.007244210224598646, "epoch": 0.47795661301609516, "grad_norm": 0.3252009238215533, "kl": 0.7109375, "learning_rate": 9.424985580487075e-06, "loss": 0.0034, "step": 1366 }, { "clip_ratio": 0.025209249928593636, "epoch": 0.47830650804758573, "grad_norm": 0.2176644770362538, "kl": 0.71484375, "learning_rate": 9.423562987688479e-06, "loss": 0.0019, "step": 1367 }, { "clip_ratio": 0.03594032675027847, "epoch": 0.47865640307907625, "grad_norm": 0.16681007462585012, "kl": 0.71484375, "learning_rate": 9.422138744942753e-06, "loss": 0.001, "step": 1368 }, { "clip_ratio": 0.0031936396844685078, "completions/clipped_ratio": 0.0, "completions/max_length": 217.0, "completions/max_terminated_length": 217.0, "completions/mean_length": 101.73214721679688, "completions/mean_terminated_length": 101.73214721679688, "completions/min_length": 19.0, "completions/min_terminated_length": 19.0, "epoch": 0.47900629811056683, "grad_norm": 0.4346706732800255, "kl": 1.3671875, "learning_rate": 9.420712852781129e-06, "loss": 0.0045, "num_tokens": 6457630.0, "reward": 2.264284133911133, "reward_std": 0.119565449655056, "rewards/check_originality_func/mean": 0.4285714328289032, "rewards/check_originality_func/std": 0.4993502199649811, "rewards/perplexity_reward/mean": 0.8456333875656128, "rewards/perplexity_reward/std": 0.029165102168917656, "rewards/sentence_count_match_reward_logic/mean": 0.9900793433189392, "rewards/sentence_count_match_reward_logic/std": 0.043817728757858276, "step": 1369 }, { "clip_ratio": 0.007583016995340586, "epoch": 0.4793561931420574, "grad_norm": 0.3840886033006283, "kl": 1.3671875, "learning_rate": 9.419285311735449e-06, "loss": 0.0028, "step": 1370 }, { "clip_ratio": 0.02099606767296791, "epoch": 0.47970608817354793, "grad_norm": 0.2922361827587624, "kl": 1.3671875, "learning_rate": 9.417856122338174e-06, "loss": 0.0005, "step": 1371 }, { "clip_ratio": 0.03587241843342781, "epoch": 0.4800559832050385, "grad_norm": 0.2221652629179817, "kl": 1.375, "learning_rate": 9.416425285122376e-06, "loss": -0.0012, "step": 1372 }, { "clip_ratio": 0.0042788987047970295, "completions/clipped_ratio": 0.0, "completions/max_length": 197.0, "completions/max_terminated_length": 197.0, "completions/mean_length": 89.03572082519531, "completions/mean_terminated_length": 89.03572082519531, "completions/min_length": 12.0, "completions/min_terminated_length": 12.0, "epoch": 0.480405878236529, "grad_norm": 0.36894506357329593, "kl": 1.046875, "learning_rate": 9.414992800621749e-06, "loss": 0.0027, "num_tokens": 6469816.0, "reward": 1.9490095376968384, "reward_std": 0.0675404742360115, "rewards/check_originality_func/mean": 0.1607142835855484, "rewards/check_originality_func/std": 0.3705909848213196, "rewards/perplexity_reward/mean": 0.7927594780921936, "rewards/perplexity_reward/std": 0.04213366657495499, "rewards/sentence_count_match_reward_logic/mean": 0.9955357313156128, "rewards/sentence_count_match_reward_logic/std": 0.033407654613256454, "step": 1373 }, { "clip_ratio": 0.016078615561127663, "epoch": 0.4807557732680196, "grad_norm": 0.30541686846538835, "kl": 1.0390625, "learning_rate": 9.41355866937059e-06, "loss": 0.0017, "step": 1374 }, { "clip_ratio": 0.03569522500038147, "epoch": 0.4811056682995101, "grad_norm": 0.19962379641521721, "kl": 1.03125, "learning_rate": 9.412122891903819e-06, "loss": 0.0003, "step": 1375 }, { "clip_ratio": 0.04713445156812668, "epoch": 0.4814555633310007, "grad_norm": 0.17931069237348313, "kl": 1.0234375, "learning_rate": 9.410685468756966e-06, "loss": -0.0002, "step": 1376 }, { "clip_ratio": 0.0033197510056197643, "completions/clipped_ratio": 0.4821428571428571, "completions/max_length": 256.0, "completions/max_terminated_length": 256.0, "completions/mean_length": 166.80357360839844, "completions/mean_terminated_length": 83.75862121582031, "completions/min_length": 16.0, "completions/min_terminated_length": 16.0, "epoch": 0.4818054583624913, "grad_norm": 0.4705720849907722, "kl": 0.8203125, "learning_rate": 9.409246400466178e-06, "loss": 0.0054, "num_tokens": 6492165.0, "reward": 2.1334035396575928, "reward_std": 0.2984181344509125, "rewards/check_originality_func/mean": 0.3928571343421936, "rewards/check_originality_func/std": 0.4928053617477417, "rewards/perplexity_reward/mean": 0.8200334906578064, "rewards/perplexity_reward/std": 0.061044201254844666, "rewards/sentence_count_match_reward_logic/mean": 0.920512855052948, "rewards/sentence_count_match_reward_logic/std": 0.12854592502117157, "step": 1377 }, { "clip_ratio": 0.007894184440374374, "epoch": 0.4821553533939818, "grad_norm": 0.4209952116657401, "kl": 0.8203125, "learning_rate": 9.407805687568214e-06, "loss": 0.0035, "step": 1378 }, { "clip_ratio": 0.027220185846090317, "epoch": 0.4825052484254724, "grad_norm": 0.3416234974216898, "kl": 0.82421875, "learning_rate": 9.406363330600445e-06, "loss": 0.001, "step": 1379 }, { "clip_ratio": 0.04110557585954666, "epoch": 0.4828551434569629, "grad_norm": 0.2983569944844382, "kl": 0.83203125, "learning_rate": 9.404919330100858e-06, "loss": -0.0009, "step": 1380 }, { "clip_ratio": 0.0017513353377580643, "completions/clipped_ratio": 0.0, "completions/max_length": 165.0, "completions/max_terminated_length": 165.0, "completions/mean_length": 82.60714721679688, "completions/mean_terminated_length": 82.60714721679688, "completions/min_length": 10.0, "completions/min_terminated_length": 10.0, "epoch": 0.4832050384884535, "grad_norm": 0.40083670322363507, "kl": 0.75390625, "learning_rate": 9.40347368660805e-06, "loss": 0.0115, "num_tokens": 6503519.0, "reward": 2.307408094406128, "reward_std": 0.08564241975545883, "rewards/check_originality_func/mean": 0.5178571343421936, "rewards/check_originality_func/std": 0.5042030811309814, "rewards/perplexity_reward/mean": 0.78955078125, "rewards/perplexity_reward/std": 0.0918397381901741, "rewards/sentence_count_match_reward_logic/mean": 1.0, "rewards/sentence_count_match_reward_logic/std": 0.0, "step": 1381 }, { "clip_ratio": 0.021559713408350945, "epoch": 0.483554933519944, "grad_norm": 0.2685396853800156, "kl": 0.7578125, "learning_rate": 9.402026400661233e-06, "loss": 0.0098, "step": 1382 }, { "clip_ratio": 0.03516506031155586, "epoch": 0.4839048285514346, "grad_norm": 0.22531571450170745, "kl": 0.7734375, "learning_rate": 9.40057747280023e-06, "loss": 0.009, "step": 1383 }, { "clip_ratio": 0.04912763833999634, "epoch": 0.4842547235829251, "grad_norm": 0.19808127619199567, "kl": 0.80078125, "learning_rate": 9.39912690356548e-06, "loss": 0.0082, "step": 1384 }, { "clip_ratio": 0.001788423745892942, "completions/clipped_ratio": 0.1428571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 115.0, "completions/mean_length": 75.01786041259766, "completions/mean_terminated_length": 44.85416793823242, "completions/min_length": 15.0, "completions/min_terminated_length": 15.0, "epoch": 0.48460461861441567, "grad_norm": 0.3157731331620494, "kl": 1.015625, "learning_rate": 9.39767469349803e-06, "loss": 0.0043, "num_tokens": 6514392.0, "reward": 1.8326218128204346, "reward_std": 0.1684548407793045, "rewards/check_originality_func/mean": 0.1785714328289032, "rewards/check_originality_func/std": 0.3864591121673584, "rewards/perplexity_reward/mean": 0.7796456217765808, "rewards/perplexity_reward/std": 0.06716758012771606, "rewards/sentence_count_match_reward_logic/mean": 0.874404788017273, "rewards/sentence_count_match_reward_logic/std": 0.1786584109067917, "step": 1385 }, { "clip_ratio": 0.005994910839945078, "epoch": 0.48495451364590625, "grad_norm": 0.2724410773609594, "kl": 1.015625, "learning_rate": 9.396220843139538e-06, "loss": 0.0032, "step": 1386 }, { "clip_ratio": 0.019223524257540703, "epoch": 0.48530440867739677, "grad_norm": 0.2140663982682392, "kl": 1.015625, "learning_rate": 9.39476535303228e-06, "loss": 0.0021, "step": 1387 }, { "clip_ratio": 0.03158998861908913, "epoch": 0.48565430370888735, "grad_norm": 0.17610182435067656, "kl": 1.0234375, "learning_rate": 9.393308223719139e-06, "loss": 0.0012, "step": 1388 }, { "clip_ratio": 0.0045739649794995785, "completions/clipped_ratio": 0.1428571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 215.0, "completions/mean_length": 152.6607208251953, "completions/mean_terminated_length": 135.4375, "completions/min_length": 19.0, "completions/min_terminated_length": 19.0, "epoch": 0.48600419874037787, "grad_norm": 0.44858465167446293, "kl": 0.66015625, "learning_rate": 9.39184945574361e-06, "loss": 0.0077, "num_tokens": 6534885.0, "reward": 2.145503044128418, "reward_std": 0.2910657823085785, "rewards/check_originality_func/mean": 0.4107142984867096, "rewards/check_originality_func/std": 0.49641573429107666, "rewards/perplexity_reward/mean": 0.8341936469078064, "rewards/perplexity_reward/std": 0.046308841556310654, "rewards/sentence_count_match_reward_logic/mean": 0.9005952477455139, "rewards/sentence_count_match_reward_logic/std": 0.1480652540922165, "step": 1389 }, { "clip_ratio": 0.012303956784307957, "epoch": 0.48635409377186845, "grad_norm": 0.35198773378387177, "kl": 0.66015625, "learning_rate": 9.3903890496498e-06, "loss": 0.006, "step": 1390 }, { "clip_ratio": 0.029030438512563705, "epoch": 0.48670398880335897, "grad_norm": 0.2820174071718824, "kl": 0.66015625, "learning_rate": 9.388927005982427e-06, "loss": 0.0046, "step": 1391 }, { "clip_ratio": 0.04530486837029457, "epoch": 0.48705388383484954, "grad_norm": 0.22129720243402307, "kl": 0.66015625, "learning_rate": 9.387463325286822e-06, "loss": 0.0033, "step": 1392 }, { "clip_ratio": 0.0021249742712825537, "completions/clipped_ratio": 0.1428571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 199.0, "completions/mean_length": 109.17857360839844, "completions/mean_terminated_length": 84.70833587646484, "completions/min_length": 21.0, "completions/min_terminated_length": 21.0, "epoch": 0.4874037788663401, "grad_norm": 0.41623637708145556, "kl": 0.80078125, "learning_rate": 9.385998008108917e-06, "loss": 0.0071, "num_tokens": 6549855.0, "reward": 2.2160141468048096, "reward_std": 0.13444934785366058, "rewards/check_originality_func/mean": 0.4642857015132904, "rewards/check_originality_func/std": 0.5032362341880798, "rewards/perplexity_reward/mean": 0.8152204155921936, "rewards/perplexity_reward/std": 0.07351799309253693, "rewards/sentence_count_match_reward_logic/mean": 0.9365078806877136, "rewards/sentence_count_match_reward_logic/std": 0.10746991634368896, "step": 1393 }, { "clip_ratio": 0.006565341260284185, "epoch": 0.48775367389783064, "grad_norm": 0.3802364395491071, "kl": 0.80078125, "learning_rate": 9.384531054995267e-06, "loss": 0.0054, "step": 1394 }, { "clip_ratio": 0.030537644401192665, "epoch": 0.4881035689293212, "grad_norm": 0.2667786995287575, "kl": 0.80078125, "learning_rate": 9.383062466493029e-06, "loss": 0.0037, "step": 1395 }, { "clip_ratio": 0.04291558265686035, "epoch": 0.48845346396081174, "grad_norm": 0.25019789842320306, "kl": 0.80859375, "learning_rate": 9.381592243149976e-06, "loss": 0.0021, "step": 1396 }, { "clip_ratio": 0.002686348045244813, "completions/clipped_ratio": 0.1071428571428571, "completions/max_length": 256.0, "completions/max_terminated_length": 252.0, "completions/mean_length": 117.16072082519531, "completions/mean_terminated_length": 100.5, "completions/min_length": 12.0, "completions/min_terminated_length": 12.0, "epoch": 0.4888033589923023, "grad_norm": 0.40273005668097245, "kl": 1.3125, "learning_rate": 9.380120385514484e-06, "loss": 0.0172, "num_tokens": 6564848.0, "reward": 2.228571653366089, "reward_std": 0.25507110357284546, "rewards/check_originality_func/mean": 0.3928571343421936, "rewards/check_originality_func/std": 0.4928053617477417, "rewards/perplexity_reward/mean": 0.8482142686843872, "rewards/perplexity_reward/std": 0.055315129458904266, "rewards/sentence_count_match_reward_logic/mean": 0.987500011920929, "rewards/sentence_count_match_reward_logic/std": 0.033371198922395706, "step": 1397 }, { "clip_ratio": 0.01473112590610981, "epoch": 0.48915325402379284, "grad_norm": 0.3235876705130528, "kl": 1.3125, "learning_rate": 9.378646894135544e-06, "loss": 0.0162, "step": 1398 }, { "clip_ratio": 0.023136410862207413, "epoch": 0.4895031490552834, "grad_norm": 0.25483275298130215, "kl": 1.3359375, "learning_rate": 9.377171769562751e-06, "loss": 0.0144, "step": 1399 }, { "clip_ratio": 0.03475634753704071, "epoch": 0.489853044086774, "grad_norm": 0.1993050404513927, "kl": 1.4140625, "learning_rate": 9.375695012346319e-06, "loss": 0.0133, "step": 1400 }, { "clip_ratio": 0.0015360800316557288, "completions/clipped_ratio": 0.0, "completions/max_length": 217.0, "completions/max_terminated_length": 217.0, "completions/mean_length": 79.0714340209961, "completions/mean_terminated_length": 79.0714340209961, "completions/min_length": 10.0, "completions/min_terminated_length": 10.0, "epoch": 0.4902029391182645, "grad_norm": 0.23187751785394292, "kl": 0.60546875, "learning_rate": 9.374216623037057e-06, "loss": 0.0058, "num_tokens": 6575948.0, "reward": 1.9354772567749023, "reward_std": 0.10576310008764267, "rewards/check_originality_func/mean": 0.1785714328289032, "rewards/check_originality_func/std": 0.3864591419696808, "rewards/perplexity_reward/mean": 0.7569056749343872, "rewards/perplexity_reward/std": 0.13721397519111633, "rewards/sentence_count_match_reward_logic/mean": 1.0, "rewards/sentence_count_match_reward_logic/std": 0.0, "step": 1401 }, { "clip_ratio": 0.002378890523687005, "epoch": 0.4905528341497551, "grad_norm": 0.22375865098783812, "kl": 0.60546875, "learning_rate": 9.372736602186396e-06, "loss": 0.0051, "step": 1402 }, { "clip_ratio": 0.009852872230112553, "epoch": 0.4909027291812456, "grad_norm": 0.15986496419808027, "kl": 0.6015625, "learning_rate": 9.371254950346366e-06, "loss": 0.0042, "step": 1403 }, { "clip_ratio": 0.016061317175626755, "epoch": 0.4912526242127362, "grad_norm": 0.14974606373195268, "kl": 0.6015625, "learning_rate": 9.369771668069612e-06, "loss": 0.0038, "step": 1404 }, { "clip_ratio": 0.0010163845727220178, "completions/clipped_ratio": 0.1428571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 184.0, "completions/mean_length": 129.05357360839844, "completions/mean_terminated_length": 107.89583587646484, "completions/min_length": 15.0, "completions/min_terminated_length": 15.0, "epoch": 0.4916025192442267, "grad_norm": 0.37027854523331066, "kl": 0.87109375, "learning_rate": 9.368286755909383e-06, "loss": 0.0024, "num_tokens": 6593831.0, "reward": 2.456822156906128, "reward_std": 0.09276057034730911, "rewards/check_originality_func/mean": 0.6607142686843872, "rewards/check_originality_func/std": 0.477751761674881, "rewards/perplexity_reward/mean": 0.8507952094078064, "rewards/perplexity_reward/std": 0.09168360382318497, "rewards/sentence_count_match_reward_logic/mean": 0.9453125, "rewards/sentence_count_match_reward_logic/std": 0.1192200556397438, "step": 1405 }, { "clip_ratio": 0.006015886086970568, "epoch": 0.4919524142757173, "grad_norm": 0.31522384437941536, "kl": 0.8671875, "learning_rate": 9.366800214419536e-06, "loss": 0.001, "step": 1406 }, { "clip_ratio": 0.01737252064049244, "epoch": 0.49230230930720786, "grad_norm": 0.2552490828798314, "kl": 0.86328125, "learning_rate": 9.36531204415454e-06, "loss": -0.0007, "step": 1407 }, { "clip_ratio": 0.034615349024534225, "epoch": 0.4926522043386984, "grad_norm": 0.2310811140645595, "kl": 0.86328125, "learning_rate": 9.363822245669462e-06, "loss": -0.0021, "step": 1408 }, { "clip_ratio": 0.004694334231317043, "completions/clipped_ratio": 0.4285714285714286, "completions/max_length": 256.0, "completions/max_terminated_length": 136.0, "completions/mean_length": 155.48214721679688, "completions/mean_terminated_length": 80.09375, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "epoch": 0.49300209937018896, "grad_norm": 0.40513163911461275, "kl": 1.09375, "learning_rate": 9.362330819519991e-06, "loss": 0.0048, "num_tokens": 6616186.0, "reward": 2.2187631130218506, "reward_std": 0.2875395715236664, "rewards/check_originality_func/mean": 0.5178571343421936, "rewards/check_originality_func/std": 0.5042031407356262, "rewards/perplexity_reward/mean": 0.8261021375656128, "rewards/perplexity_reward/std": 0.06599762290716171, "rewards/sentence_count_match_reward_logic/mean": 0.8748038411140442, "rewards/sentence_count_match_reward_logic/std": 0.18744277954101562, "step": 1409 }, { "clip_ratio": 0.005403453018516302, "epoch": 0.4933519944016795, "grad_norm": 0.3826686568974154, "kl": 1.1015625, "learning_rate": 9.36083776626241e-06, "loss": 0.0032, "step": 1410 }, { "clip_ratio": 0.026828110218048096, "epoch": 0.49370188943317006, "grad_norm": 0.29867837198641417, "kl": 1.1015625, "learning_rate": 9.359343086453613e-06, "loss": 0.0006, "step": 1411 }, { "clip_ratio": 0.042069707065820694, "epoch": 0.4940517844646606, "grad_norm": 0.2538936738047552, "kl": 1.1015625, "learning_rate": 9.357846780651105e-06, "loss": -0.001, "step": 1412 }, { "clip_ratio": 0.0023069039452821016, "completions/clipped_ratio": 0.0, "completions/max_length": 148.0, "completions/max_terminated_length": 148.0, "completions/mean_length": 74.89286041259766, "completions/mean_terminated_length": 74.89286041259766, "completions/min_length": 13.0, "completions/min_terminated_length": 13.0, "epoch": 0.49440167949615116, "grad_norm": 0.29441859036625107, "kl": 1.3125, "learning_rate": 9.356348849412991e-06, "loss": 0.0058, "num_tokens": 6626852.0, "reward": 2.1089565753936768, "reward_std": 0.009761897847056389, "rewards/check_originality_func/mean": 0.2857142984867096, "rewards/check_originality_func/std": 0.4558423161506653, "rewards/perplexity_reward/mean": 0.8232421875, "rewards/perplexity_reward/std": 0.04006698727607727, "rewards/sentence_count_match_reward_logic/mean": 1.0, "rewards/sentence_count_match_reward_logic/std": 0.0, "step": 1413 }, { "clip_ratio": 0.005429446697235107, "epoch": 0.4947515745276417, "grad_norm": 0.2561990228212461, "kl": 1.3125, "learning_rate": 9.354849293297987e-06, "loss": 0.0049, "step": 1414 }, { "clip_ratio": 0.01779494807124138, "epoch": 0.49510146955913226, "grad_norm": 0.18547076638094542, "kl": 1.3046875, "learning_rate": 9.353348112865413e-06, "loss": 0.004, "step": 1415 }, { "clip_ratio": 0.03501694276928902, "epoch": 0.49545136459062283, "grad_norm": 0.15328027306605857, "kl": 1.3125, "learning_rate": 9.351845308675193e-06, "loss": 0.0034, "step": 1416 }, { "clip_ratio": 0.0032814505975693464, "completions/clipped_ratio": 0.0, "completions/max_length": 249.0, "completions/max_terminated_length": 249.0, "completions/mean_length": 88.30357360839844, "completions/mean_terminated_length": 88.30357360839844, "completions/min_length": 18.0, "completions/min_terminated_length": 18.0, "epoch": 0.49580125962211335, "grad_norm": 0.32890589987376795, "kl": 0.96484375, "learning_rate": 9.350340881287861e-06, "loss": 0.0055, "num_tokens": 6639341.0, "reward": 1.83682119846344, "reward_std": 0.09971024096012115, "rewards/check_originality_func/mean": 0.0535714291036129, "rewards/check_originality_func/std": 0.22720779478549957, "rewards/perplexity_reward/mean": 0.8249163031578064, "rewards/perplexity_reward/std": 0.06457453221082687, "rewards/sentence_count_match_reward_logic/mean": 0.9583333730697632, "rewards/sentence_count_match_reward_logic/std": 0.1112372949719429, "step": 1417 }, { "clip_ratio": 0.007054097950458527, "epoch": 0.49615115465360393, "grad_norm": 0.3120414231187872, "kl": 0.9609375, "learning_rate": 9.348834831264552e-06, "loss": 0.0046, "step": 1418 }, { "clip_ratio": 0.02161918766796589, "epoch": 0.49650104968509445, "grad_norm": 0.22475172476052332, "kl": 0.95703125, "learning_rate": 9.347327159167013e-06, "loss": 0.003, "step": 1419 }, { "clip_ratio": 0.05197850987315178, "epoch": 0.49685094471658503, "grad_norm": 0.17417066803243803, "kl": 0.9609375, "learning_rate": 9.345817865557584e-06, "loss": 0.0022, "step": 1420 }, { "clip_ratio": 0.0003627325058914721, "completions/clipped_ratio": 0.0, "completions/max_length": 168.0, "completions/max_terminated_length": 168.0, "completions/mean_length": 66.89286041259766, "completions/mean_terminated_length": 66.89286041259766, "completions/min_length": 11.0, "completions/min_terminated_length": 11.0, "epoch": 0.49720083974807555, "grad_norm": 0.2990517149225106, "kl": 1.1953125, "learning_rate": 9.344306950999226e-06, "loss": 0.01, "num_tokens": 6649335.0, "reward": 2.069056987762451, "reward_std": 0.06054607033729553, "rewards/check_originality_func/mean": 0.2678571343421936, "rewards/check_originality_func/std": 0.44685041904449463, "rewards/perplexity_reward/mean": 0.8011997938156128, "rewards/perplexity_reward/std": 0.10006733983755112, "rewards/sentence_count_match_reward_logic/mean": 1.0, "rewards/sentence_count_match_reward_logic/std": 0.0, "step": 1421 }, { "clip_ratio": 0.0036870932672172785, "epoch": 0.49755073477956613, "grad_norm": 0.2545929234084511, "kl": 1.203125, "learning_rate": 9.342794416055489e-06, "loss": 0.0093, "step": 1422 }, { "clip_ratio": 0.026417981833219528, "epoch": 0.4979006298110567, "grad_norm": 0.18944449510715738, "kl": 1.2109375, "learning_rate": 9.341280261290537e-06, "loss": 0.0084, "step": 1423 }, { "clip_ratio": 0.04930273815989494, "epoch": 0.4982505248425472, "grad_norm": 0.14896866594869676, "kl": 1.2265625, "learning_rate": 9.339764487269133e-06, "loss": 0.0077, "step": 1424 }, { "clip_ratio": 0.0006886518676765263, "completions/clipped_ratio": 0.0, "completions/max_length": 145.0, "completions/max_terminated_length": 145.0, "completions/mean_length": 63.57143020629883, "completions/mean_terminated_length": 63.57143020629883, "completions/min_length": 14.0, "completions/min_terminated_length": 14.0, "epoch": 0.4986004198740378, "grad_norm": 0.35668163447218487, "kl": 1.203125, "learning_rate": 9.338247094556651e-06, "loss": -0.0025, "num_tokens": 6658831.0, "reward": 1.9282227754592896, "reward_std": 0.08415751904249191, "rewards/check_originality_func/mean": 0.1607142835855484, "rewards/check_originality_func/std": 0.3705909848213196, "rewards/perplexity_reward/mean": 0.7764369249343872, "rewards/perplexity_reward/std": 0.12353992462158203, "rewards/sentence_count_match_reward_logic/mean": 0.9910714030265808, "rewards/sentence_count_match_reward_logic/std": 0.06681530922651291, "step": 1425 }, { "clip_ratio": 0.013875369913876057, "epoch": 0.4989503149055283, "grad_norm": 0.2738184715549074, "kl": 1.1953125, "learning_rate": 9.33672808371906e-06, "loss": -0.0034, "step": 1426 }, { "clip_ratio": 0.021747274324297905, "epoch": 0.4993002099370189, "grad_norm": 0.18586113896431578, "kl": 1.1875, "learning_rate": 9.33520745532294e-06, "loss": -0.0044, "step": 1427 }, { "clip_ratio": 0.03998757153749466, "epoch": 0.4996501049685094, "grad_norm": 0.15070627966275565, "kl": 1.1875, "learning_rate": 9.333685209935465e-06, "loss": -0.005, "step": 1428 }, { "clip_ratio": 0.0020949423778802156, "completions/clipped_ratio": 0.2678571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 231.0, "completions/mean_length": 109.58928680419922, "completions/mean_terminated_length": 56.02438735961914, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "epoch": 0.5, "grad_norm": 0.3312141449587405, "kl": 0.83984375, "learning_rate": 9.332161348124426e-06, "loss": 0.0053, "num_tokens": 6674120.0, "reward": 2.0979034900665283, "reward_std": 0.13653436303138733, "rewards/check_originality_func/mean": 0.375, "rewards/check_originality_func/std": 0.48850420117378235, "rewards/perplexity_reward/mean": 0.7671595811843872, "rewards/perplexity_reward/std": 0.12498153001070023, "rewards/sentence_count_match_reward_logic/mean": 0.9557437300682068, "rewards/sentence_count_match_reward_logic/std": 0.0679202526807785, "step": 1429 }, { "clip_ratio": 0.006258868146687746, "epoch": 0.5003498950314905, "grad_norm": 0.31004978146209683, "kl": 0.83984375, "learning_rate": 9.330635870458205e-06, "loss": 0.0044, "step": 1430 }, { "clip_ratio": 0.022839535027742386, "epoch": 0.5006997900629812, "grad_norm": 0.2421469232352637, "kl": 0.8515625, "learning_rate": 9.329108777505788e-06, "loss": 0.0027, "step": 1431 }, { "clip_ratio": 0.04634132981300354, "epoch": 0.5010496850944717, "grad_norm": 0.19524265659290327, "kl": 0.8671875, "learning_rate": 9.327580069836773e-06, "loss": 0.0014, "step": 1432 }, { "clip_ratio": 0.0013110991567373276, "completions/clipped_ratio": 0.0, "completions/max_length": 217.0, "completions/max_terminated_length": 217.0, "completions/mean_length": 96.28572082519531, "completions/mean_terminated_length": 96.28572082519531, "completions/min_length": 17.0, "completions/min_terminated_length": 17.0, "epoch": 0.5013995801259622, "grad_norm": 0.3071597987185799, "kl": 1.109375, "learning_rate": 9.326049748021348e-06, "loss": 0.0076, "num_tokens": 6687360.0, "reward": 2.167868137359619, "reward_std": 0.10749713331460953, "rewards/check_originality_func/mean": 0.3928571343421936, "rewards/check_originality_func/std": 0.4928053617477417, "rewards/perplexity_reward/mean": 0.7792271375656128, "rewards/perplexity_reward/std": 0.11726321280002594, "rewards/sentence_count_match_reward_logic/mean": 0.9957837462425232, "rewards/sentence_count_match_reward_logic/std": 0.022146278992295265, "step": 1433 }, { "clip_ratio": 0.0056223189458251, "epoch": 0.5017494751574527, "grad_norm": 0.27940655985068025, "kl": 1.1171875, "learning_rate": 9.32451781263031e-06, "loss": 0.0067, "step": 1434 }, { "clip_ratio": 0.024753954261541367, "epoch": 0.5020993701889434, "grad_norm": 0.19611631767027735, "kl": 1.1171875, "learning_rate": 9.322984264235055e-06, "loss": 0.0056, "step": 1435 }, { "clip_ratio": 0.03688439354300499, "epoch": 0.5024492652204339, "grad_norm": 0.15920799809540603, "kl": 1.125, "learning_rate": 9.321449103407587e-06, "loss": 0.0045, "step": 1436 }, { "clip_ratio": 0.0009550862014293671, "completions/clipped_ratio": 0.1428571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 233.0, "completions/mean_length": 115.78572082519531, "completions/mean_terminated_length": 92.41667175292969, "completions/min_length": 11.0, "completions/min_terminated_length": 11.0, "epoch": 0.5027991602519244, "grad_norm": 0.37923449892502664, "kl": 1.25, "learning_rate": 9.319912330720502e-06, "loss": 0.0062, "num_tokens": 6703500.0, "reward": 2.0801479816436768, "reward_std": 0.22973471879959106, "rewards/check_originality_func/mean": 0.3214285671710968, "rewards/check_originality_func/std": 0.4712514281272888, "rewards/perplexity_reward/mean": 0.8033621907234192, "rewards/perplexity_reward/std": 0.08098554611206055, "rewards/sentence_count_match_reward_logic/mean": 0.9553571343421936, "rewards/sentence_count_match_reward_logic/std": 0.1134449914097786, "step": 1437 }, { "clip_ratio": 0.009495668113231659, "epoch": 0.503149055283415, "grad_norm": 0.3369315873680901, "kl": 1.25, "learning_rate": 9.318373946747e-06, "loss": 0.0049, "step": 1438 }, { "clip_ratio": 0.02849905751645565, "epoch": 0.5034989503149055, "grad_norm": 0.2953039052831751, "kl": 1.2578125, "learning_rate": 9.316833952060891e-06, "loss": 0.0029, "step": 1439 }, { "clip_ratio": 0.04730107635259628, "epoch": 0.5038488453463961, "grad_norm": 0.25481997976912374, "kl": 1.265625, "learning_rate": 9.315292347236571e-06, "loss": 0.0013, "step": 1440 }, { "clip_ratio": 0.00029743084451183677, "completions/clipped_ratio": 0.0, "completions/max_length": 138.0, "completions/max_terminated_length": 138.0, "completions/mean_length": 63.392860412597656, "completions/mean_terminated_length": 63.392860412597656, "completions/min_length": 10.0, "completions/min_terminated_length": 10.0, "epoch": 0.5041987403778866, "grad_norm": 0.22390908667475634, "kl": 1.3359375, "learning_rate": 9.313749132849048e-06, "loss": 0.003, "num_tokens": 6713114.0, "reward": 2.050502300262451, "reward_std": 0.19474227726459503, "rewards/check_originality_func/mean": 0.2857142984867096, "rewards/check_originality_func/std": 0.4558423161506653, "rewards/perplexity_reward/mean": 0.7647879719734192, "rewards/perplexity_reward/std": 0.118437260389328, "rewards/sentence_count_match_reward_logic/mean": 1.0, "rewards/sentence_count_match_reward_logic/std": 0.0, "step": 1441 }, { "clip_ratio": 0.003273051232099533, "epoch": 0.5045486354093772, "grad_norm": 0.21148866536616223, "kl": 1.3359375, "learning_rate": 9.312204309473924e-06, "loss": 0.0024, "step": 1442 }, { "clip_ratio": 0.014035103842616081, "epoch": 0.5048985304408677, "grad_norm": 0.16188124972141935, "kl": 1.3359375, "learning_rate": 9.310657877687406e-06, "loss": 0.0017, "step": 1443 }, { "clip_ratio": 0.035331547260284424, "epoch": 0.5052484254723583, "grad_norm": 0.13093710469600284, "kl": 1.3359375, "learning_rate": 9.309109838066297e-06, "loss": 0.0011, "step": 1444 }, { "clip_ratio": 0.0016978875501081347, "completions/clipped_ratio": 0.0, "completions/max_length": 219.0, "completions/max_terminated_length": 219.0, "completions/mean_length": 77.98214721679688, "completions/mean_terminated_length": 77.98214721679688, "completions/min_length": 13.0, "completions/min_terminated_length": 13.0, "epoch": 0.5055983205038489, "grad_norm": 0.30738390022800677, "kl": 1.3984375, "learning_rate": 9.307560191188e-06, "loss": 0.0047, "num_tokens": 6724057.0, "reward": 2.2066237926483154, "reward_std": 0.07450532168149948, "rewards/check_originality_func/mean": 0.4464285671710968, "rewards/check_originality_func/std": 0.5016207695007324, "rewards/perplexity_reward/mean": 0.7693219780921936, "rewards/perplexity_reward/std": 0.12551699578762054, "rewards/sentence_count_match_reward_logic/mean": 0.9908729791641235, "rewards/sentence_count_match_reward_logic/std": 0.03507933393120766, "step": 1445 }, { "clip_ratio": 0.009281610138714314, "epoch": 0.5059482155353394, "grad_norm": 0.2651152739093177, "kl": 1.3984375, "learning_rate": 9.306008937630523e-06, "loss": 0.0038, "step": 1446 }, { "clip_ratio": 0.023853374645113945, "epoch": 0.5062981105668299, "grad_norm": 0.20467409559620925, "kl": 1.3984375, "learning_rate": 9.304456077972463e-06, "loss": 0.0026, "step": 1447 }, { "clip_ratio": 0.03543611243367195, "epoch": 0.5066480055983205, "grad_norm": 0.16004703043211896, "kl": 1.3984375, "learning_rate": 9.302901612793028e-06, "loss": 0.0018, "step": 1448 }, { "clip_ratio": 0.002341242739930749, "completions/clipped_ratio": 0.0, "completions/max_length": 227.0, "completions/max_terminated_length": 227.0, "completions/mean_length": 93.98214721679688, "completions/mean_terminated_length": 93.98214721679688, "completions/min_length": 20.0, "completions/min_terminated_length": 20.0, "epoch": 0.5069979006298111, "grad_norm": 0.3230359106993492, "kl": 0.9921875, "learning_rate": 9.301345542672012e-06, "loss": 0.0033, "num_tokens": 6736952.0, "reward": 2.1883370876312256, "reward_std": 0.19645443558692932, "rewards/check_originality_func/mean": 0.375, "rewards/check_originality_func/std": 0.48850420117378235, "rewards/perplexity_reward/mean": 0.8334263563156128, "rewards/perplexity_reward/std": 0.049575548619031906, "rewards/sentence_count_match_reward_logic/mean": 0.9799107313156128, "rewards/sentence_count_match_reward_logic/std": 0.04632386937737465, "step": 1449 }, { "clip_ratio": 0.006601852364838123, "epoch": 0.5073477956613016, "grad_norm": 0.28292536154851794, "kl": 0.9921875, "learning_rate": 9.299787868189821e-06, "loss": 0.0023, "step": 1450 }, { "clip_ratio": 0.02282727137207985, "epoch": 0.5076976906927921, "grad_norm": 0.23262485652116666, "kl": 0.9921875, "learning_rate": 9.298228589927446e-06, "loss": 0.0012, "step": 1451 }, { "clip_ratio": 0.036402542144060135, "epoch": 0.5080475857242828, "grad_norm": 0.17754052999238396, "kl": 1.0, "learning_rate": 9.29666770846649e-06, "loss": 0.0002, "step": 1452 }, { "clip_ratio": 0.0010136057389900088, "completions/clipped_ratio": 0.0357142857142857, "completions/max_length": 256.0, "completions/max_terminated_length": 256.0, "completions/mean_length": 97.21428680419922, "completions/mean_terminated_length": 91.33333587646484, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "epoch": 0.5083974807557733, "grad_norm": 0.2996036876926513, "kl": 1.1796875, "learning_rate": 9.295105224389144e-06, "loss": 0.0035, "num_tokens": 6750252.0, "reward": 1.9847718477249146, "reward_std": 0.19513684511184692, "rewards/check_originality_func/mean": 0.3035714328289032, "rewards/check_originality_func/std": 0.4639608860015869, "rewards/perplexity_reward/mean": 0.7924107313156128, "rewards/perplexity_reward/std": 0.13007588684558868, "rewards/sentence_count_match_reward_logic/mean": 0.8887896537780762, "rewards/sentence_count_match_reward_logic/std": 0.17169351875782013, "step": 1453 }, { "clip_ratio": 0.005154279060661793, "epoch": 0.5087473757872638, "grad_norm": 0.2719583929720405, "kl": 1.171875, "learning_rate": 9.293541138278199e-06, "loss": 0.0028, "step": 1454 }, { "clip_ratio": 0.030729761347174644, "epoch": 0.5090972708187543, "grad_norm": 0.19470736250492407, "kl": 1.171875, "learning_rate": 9.291975450717043e-06, "loss": 0.0016, "step": 1455 }, { "clip_ratio": 0.043928179889917374, "epoch": 0.509447165850245, "grad_norm": 0.16920290395182983, "kl": 1.171875, "learning_rate": 9.290408162289668e-06, "loss": 0.001, "step": 1456 }, { "clip_ratio": 0.0019998699426651, "completions/clipped_ratio": 0.0, "completions/max_length": 168.0, "completions/max_terminated_length": 168.0, "completions/mean_length": 63.16071701049805, "completions/mean_terminated_length": 63.16071701049805, "completions/min_length": 15.0, "completions/min_terminated_length": 15.0, "epoch": 0.5097970608817355, "grad_norm": 0.30708113056818837, "kl": 1.171875, "learning_rate": 9.288839273580652e-06, "loss": 0.0017, "num_tokens": 6760109.0, "reward": 2.0555081367492676, "reward_std": 0.18673212826251984, "rewards/check_originality_func/mean": 0.2321428507566452, "rewards/check_originality_func/std": 0.4260205328464508, "rewards/perplexity_reward/mean": 0.8378208875656128, "rewards/perplexity_reward/std": 0.054692529141902924, "rewards/sentence_count_match_reward_logic/mean": 0.9855442047119141, "rewards/sentence_count_match_reward_logic/std": 0.0647973120212555, "step": 1457 }, { "clip_ratio": 0.00460544740781188, "epoch": 0.510146955913226, "grad_norm": 0.26288206508443573, "kl": 1.1640625, "learning_rate": 9.28726878517518e-06, "loss": 0.0006, "step": 1458 }, { "clip_ratio": 0.02636905573308468, "epoch": 0.5104968509447166, "grad_norm": 0.20897238909485094, "kl": 1.171875, "learning_rate": 9.285696697659026e-06, "loss": -0.0003, "step": 1459 }, { "clip_ratio": 0.04851185530424118, "epoch": 0.5108467459762072, "grad_norm": 0.16927045211251127, "kl": 1.1640625, "learning_rate": 9.284123011618564e-06, "loss": -0.0013, "step": 1460 }, { "clip_ratio": 0.0033843107521533966, "completions/clipped_ratio": 0.0, "completions/max_length": 139.0, "completions/max_terminated_length": 139.0, "completions/mean_length": 76.375, "completions/mean_terminated_length": 76.375, "completions/min_length": 33.0, "completions/min_terminated_length": 33.0, "epoch": 0.5111966410076977, "grad_norm": 0.3241856355231886, "kl": 1.265625, "learning_rate": 9.282547727640767e-06, "loss": -0.0001, "num_tokens": 6771098.0, "reward": 1.893559455871582, "reward_std": 0.26823803782463074, "rewards/check_originality_func/mean": 0.1428571492433548, "rewards/check_originality_func/std": 0.3530939221382141, "rewards/perplexity_reward/mean": 0.8030831217765808, "rewards/perplexity_reward/std": 0.02964426390826702, "rewards/sentence_count_match_reward_logic/mean": 0.9476190209388733, "rewards/sentence_count_match_reward_logic/std": 0.08992541581392288, "step": 1461 }, { "clip_ratio": 0.006781436502933502, "epoch": 0.5115465360391882, "grad_norm": 0.2802198486162179, "kl": 1.265625, "learning_rate": 9.280970846313199e-06, "loss": -0.001, "step": 1462 }, { "clip_ratio": 0.023223433643579483, "epoch": 0.5118964310706788, "grad_norm": 0.2086787913952038, "kl": 1.2734375, "learning_rate": 9.279392368224022e-06, "loss": -0.0022, "step": 1463 }, { "clip_ratio": 0.03967491164803505, "epoch": 0.5122463261021694, "grad_norm": 0.16370222547734548, "kl": 1.2734375, "learning_rate": 9.277812293961992e-06, "loss": -0.003, "step": 1464 }, { "clip_ratio": 0.0019925020169466734, "completions/clipped_ratio": 0.1428571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 153.0, "completions/mean_length": 102.89286041259766, "completions/mean_terminated_length": 77.375, "completions/min_length": 7.0, "completions/min_terminated_length": 7.0, "epoch": 0.5125962211336599, "grad_norm": 0.33844644430596965, "kl": 1.0390625, "learning_rate": 9.276230624116464e-06, "loss": 0.0022, "num_tokens": 6785740.0, "reward": 2.077770948410034, "reward_std": 0.08636149019002914, "rewards/check_originality_func/mean": 0.3214285671710968, "rewards/check_originality_func/std": 0.4712514579296112, "rewards/perplexity_reward/mean": 0.7845284342765808, "rewards/perplexity_reward/std": 0.13912509381771088, "rewards/sentence_count_match_reward_logic/mean": 0.9718137383460999, "rewards/sentence_count_match_reward_logic/std": 0.06596735119819641, "step": 1465 }, { "clip_ratio": 0.005335938185453415, "epoch": 0.5129461161651504, "grad_norm": 0.31725196636445924, "kl": 1.0390625, "learning_rate": 9.274647359277386e-06, "loss": 0.001, "step": 1466 }, { "clip_ratio": 0.015242750756442547, "epoch": 0.513296011196641, "grad_norm": 0.26077335100010673, "kl": 1.0390625, "learning_rate": 9.273062500035298e-06, "loss": -0.0006, "step": 1467 }, { "clip_ratio": 0.026909658685326576, "epoch": 0.5136459062281316, "grad_norm": 0.18940759325027995, "kl": 1.0390625, "learning_rate": 9.27147604698134e-06, "loss": -0.0019, "step": 1468 }, { "clip_ratio": 0.0008940818370319903, "completions/clipped_ratio": 0.0, "completions/max_length": 233.0, "completions/max_terminated_length": 233.0, "completions/mean_length": 96.5714340209961, "completions/mean_terminated_length": 96.5714340209961, "completions/min_length": 10.0, "completions/min_terminated_length": 10.0, "epoch": 0.5139958012596221, "grad_norm": 0.3160893337588327, "kl": 1.1953125, "learning_rate": 9.269888000707243e-06, "loss": 0.0054, "num_tokens": 6798876.0, "reward": 2.195591688156128, "reward_std": 0.3285331726074219, "rewards/check_originality_func/mean": 0.375, "rewards/check_originality_func/std": 0.48850420117378235, "rewards/perplexity_reward/mean": 0.8205915093421936, "rewards/perplexity_reward/std": 0.06366047263145447, "rewards/sentence_count_match_reward_logic/mean": 1.0, "rewards/sentence_count_match_reward_logic/std": 0.0, "step": 1469 }, { "clip_ratio": 0.007243782747536898, "epoch": 0.5143456962911127, "grad_norm": 0.27182851872924385, "kl": 1.1953125, "learning_rate": 9.268298361805333e-06, "loss": 0.0044, "step": 1470 }, { "clip_ratio": 0.017183544114232063, "epoch": 0.5146955913226032, "grad_norm": 0.20722649942478222, "kl": 1.1953125, "learning_rate": 9.266707130868531e-06, "loss": 0.0031, "step": 1471 }, { "clip_ratio": 0.03198588266968727, "epoch": 0.5150454863540938, "grad_norm": 0.15842422312878576, "kl": 1.203125, "learning_rate": 9.265114308490352e-06, "loss": 0.0022, "step": 1472 }, { "clip_ratio": 0.0057344636879861355, "completions/clipped_ratio": 0.0, "completions/max_length": 145.0, "completions/max_terminated_length": 145.0, "completions/mean_length": 53.16071701049805, "completions/mean_terminated_length": 53.16071701049805, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "epoch": 0.5153953813855843, "grad_norm": 0.2914419631393785, "kl": 1.3671875, "learning_rate": 9.263519895264901e-06, "loss": 0.0094, "num_tokens": 6807317.0, "reward": 2.1165597438812256, "reward_std": 0.08808951824903488, "rewards/check_originality_func/mean": 0.3571428656578064, "rewards/check_originality_func/std": 0.48349377512931824, "rewards/perplexity_reward/mean": 0.7594168782234192, "rewards/perplexity_reward/std": 0.06267189234495163, "rewards/sentence_count_match_reward_logic/mean": 1.0, "rewards/sentence_count_match_reward_logic/std": 0.0, "step": 1473 }, { "clip_ratio": 0.016731848940253258, "epoch": 0.5157452764170749, "grad_norm": 0.2641765424676016, "kl": 1.375, "learning_rate": 9.261923891786881e-06, "loss": 0.0086, "step": 1474 }, { "clip_ratio": 0.034365180879831314, "epoch": 0.5160951714485654, "grad_norm": 0.20722941989663282, "kl": 1.3828125, "learning_rate": 9.260326298651586e-06, "loss": 0.0075, "step": 1475 }, { "clip_ratio": 0.055926866829395294, "epoch": 0.516445066480056, "grad_norm": 0.17337504377736784, "kl": 1.390625, "learning_rate": 9.258727116454901e-06, "loss": 0.0064, "step": 1476 }, { "clip_ratio": 0.002364944200962782, "completions/clipped_ratio": 0.0, "completions/max_length": 126.0, "completions/max_terminated_length": 126.0, "completions/mean_length": 42.642860412597656, "completions/mean_terminated_length": 42.642860412597656, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "epoch": 0.5167949615115466, "grad_norm": 0.2136095867114474, "kl": 1.1953125, "learning_rate": 9.25712634579331e-06, "loss": 0.0034, "num_tokens": 6814441.0, "reward": 2.018833875656128, "reward_std": 0.06443503499031067, "rewards/check_originality_func/mean": 0.2678571343421936, "rewards/check_originality_func/std": 0.44685041904449463, "rewards/perplexity_reward/mean": 0.7509765625, "rewards/perplexity_reward/std": 0.11220034211874008, "rewards/sentence_count_match_reward_logic/mean": 1.0, "rewards/sentence_count_match_reward_logic/std": 0.0, "step": 1477 }, { "clip_ratio": 0.011947044171392918, "epoch": 0.5171448565430371, "grad_norm": 0.17854864977331344, "kl": 1.1953125, "learning_rate": 9.255523987263884e-06, "loss": 0.0029, "step": 1478 }, { "clip_ratio": 0.023852365091443062, "epoch": 0.5174947515745276, "grad_norm": 0.14294467953453946, "kl": 1.1953125, "learning_rate": 9.253920041464283e-06, "loss": 0.0023, "step": 1479 }, { "clip_ratio": 0.03762130066752434, "epoch": 0.5178446466060181, "grad_norm": 0.11477900633787087, "kl": 1.1953125, "learning_rate": 9.25231450899277e-06, "loss": 0.0021, "step": 1480 }, { "clip_ratio": 0.0005236880970187485, "completions/clipped_ratio": 0.0, "completions/max_length": 225.0, "completions/max_terminated_length": 225.0, "completions/mean_length": 91.12500762939453, "completions/mean_terminated_length": 91.12500762939453, "completions/min_length": 10.0, "completions/min_terminated_length": 10.0, "epoch": 0.5181945416375088, "grad_norm": 0.30040433518528215, "kl": 1.25, "learning_rate": 9.250707390448187e-06, "loss": 0.0037, "num_tokens": 6827328.0, "reward": 2.2213621139526367, "reward_std": 0.2002595067024231, "rewards/check_originality_func/mean": 0.4642857015132904, "rewards/check_originality_func/std": 0.5032362341880798, "rewards/perplexity_reward/mean": 0.7774134874343872, "rewards/perplexity_reward/std": 0.07531335949897766, "rewards/sentence_count_match_reward_logic/mean": 0.9796627163887024, "rewards/sentence_count_match_reward_logic/std": 0.050109606236219406, "step": 1481 }, { "clip_ratio": 0.005190645810216665, "epoch": 0.5185444366689993, "grad_norm": 0.2399844360941374, "kl": 1.25, "learning_rate": 9.249098686429983e-06, "loss": 0.0025, "step": 1482 }, { "clip_ratio": 0.022883964702486992, "epoch": 0.5188943317004898, "grad_norm": 0.1882702687238126, "kl": 1.2578125, "learning_rate": 9.24748839753818e-06, "loss": 0.0016, "step": 1483 }, { "clip_ratio": 0.03292112052440643, "epoch": 0.5192442267319805, "grad_norm": 0.16512958486026424, "kl": 1.2734375, "learning_rate": 9.245876524373405e-06, "loss": 0.0007, "step": 1484 }, { "clip_ratio": 0.0011705710785463452, "completions/clipped_ratio": 0.0, "completions/max_length": 219.0, "completions/max_terminated_length": 219.0, "completions/mean_length": 80.60714721679688, "completions/mean_terminated_length": 80.60714721679688, "completions/min_length": 15.0, "completions/min_terminated_length": 15.0, "epoch": 0.519594121763471, "grad_norm": 0.23691734073615667, "kl": 1.3203125, "learning_rate": 9.244263067536872e-06, "loss": 0.0041, "num_tokens": 6838850.0, "reward": 2.030971050262451, "reward_std": 0.12797771394252777, "rewards/check_originality_func/mean": 0.25, "rewards/check_originality_func/std": 0.43693143129348755, "rewards/perplexity_reward/mean": 0.8010602593421936, "rewards/perplexity_reward/std": 0.05547253414988518, "rewards/sentence_count_match_reward_logic/mean": 0.9799107313156128, "rewards/sentence_count_match_reward_logic/std": 0.052096858620643616, "step": 1485 }, { "clip_ratio": 0.0014125456800684333, "epoch": 0.5199440167949615, "grad_norm": 0.23929343887427393, "kl": 1.328125, "learning_rate": 9.242648027630382e-06, "loss": 0.0037, "step": 1486 }, { "clip_ratio": 0.008868126198649406, "epoch": 0.520293911826452, "grad_norm": 0.18574615653669424, "kl": 1.328125, "learning_rate": 9.241031405256334e-06, "loss": 0.0028, "step": 1487 }, { "clip_ratio": 0.019138649106025696, "epoch": 0.5206438068579426, "grad_norm": 0.15549637068540298, "kl": 1.328125, "learning_rate": 9.239413201017709e-06, "loss": 0.0019, "step": 1488 }, { "clip_ratio": 0.0008127206237986684, "completions/clipped_ratio": 0.1428571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 201.0, "completions/mean_length": 103.41072082519531, "completions/mean_terminated_length": 77.97917175292969, "completions/min_length": 15.0, "completions/min_terminated_length": 15.0, "epoch": 0.5209937018894332, "grad_norm": 0.2726878138561515, "kl": 0.91015625, "learning_rate": 9.237793415518083e-06, "loss": 0.0003, "num_tokens": 6854585.0, "reward": 2.0448522567749023, "reward_std": 0.16558392345905304, "rewards/check_originality_func/mean": 0.2857142984867096, "rewards/check_originality_func/std": 0.4558423161506653, "rewards/perplexity_reward/mean": 0.83056640625, "rewards/perplexity_reward/std": 0.09006907790899277, "rewards/sentence_count_match_reward_logic/mean": 0.9285714030265808, "rewards/sentence_count_match_reward_logic/std": 0.15604694187641144, "step": 1489 }, { "clip_ratio": 0.00765222916379571, "epoch": 0.5213435969209237, "grad_norm": 0.23676764841063735, "kl": 0.9140625, "learning_rate": 9.23617204936162e-06, "loss": -0.0004, "step": 1490 }, { "clip_ratio": 0.01860973611474037, "epoch": 0.5216934919524143, "grad_norm": 0.20134422039681094, "kl": 0.91796875, "learning_rate": 9.234549103153077e-06, "loss": -0.0015, "step": 1491 }, { "clip_ratio": 0.029038535431027412, "epoch": 0.5220433869839048, "grad_norm": 0.16844075446595616, "kl": 0.9296875, "learning_rate": 9.232924577497797e-06, "loss": -0.0024, "step": 1492 }, { "clip_ratio": 0.0009611580171622336, "completions/clipped_ratio": 0.1428571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 153.0, "completions/mean_length": 123.51786041259766, "completions/mean_terminated_length": 101.4375, "completions/min_length": 29.0, "completions/min_terminated_length": 29.0, "epoch": 0.5223932820153954, "grad_norm": 0.31157251884153375, "kl": 0.97265625, "learning_rate": 9.23129847300171e-06, "loss": 0.0083, "num_tokens": 6871462.0, "reward": 2.250499725341797, "reward_std": 0.16025510430335999, "rewards/check_originality_func/mean": 0.4642857015132904, "rewards/check_originality_func/std": 0.5032362341880798, "rewards/perplexity_reward/mean": 0.8352399468421936, "rewards/perplexity_reward/std": 0.07810051739215851, "rewards/sentence_count_match_reward_logic/mean": 0.9509740471839905, "rewards/sentence_count_match_reward_logic/std": 0.11535302549600601, "step": 1493 }, { "clip_ratio": 0.0030905939638614655, "epoch": 0.5227431770468859, "grad_norm": 0.2788483235252086, "kl": 0.9765625, "learning_rate": 9.229670790271339e-06, "loss": 0.0073, "step": 1494 }, { "clip_ratio": 0.011663360521197319, "epoch": 0.5230930720783765, "grad_norm": 0.21570239685849601, "kl": 0.9765625, "learning_rate": 9.228041529913794e-06, "loss": 0.0061, "step": 1495 }, { "clip_ratio": 0.02210506796836853, "epoch": 0.523442967109867, "grad_norm": 0.1760393125044522, "kl": 0.98828125, "learning_rate": 9.226410692536773e-06, "loss": 0.0047, "step": 1496 }, { "clip_ratio": 0.002816050546243787, "completions/clipped_ratio": 0.0, "completions/max_length": 134.0, "completions/max_terminated_length": 134.0, "completions/mean_length": 56.767860412597656, "completions/mean_terminated_length": 56.767860412597656, "completions/min_length": 12.0, "completions/min_terminated_length": 12.0, "epoch": 0.5237928621413576, "grad_norm": 0.24768357972168006, "kl": 1.5390625, "learning_rate": 9.224778278748567e-06, "loss": 0.0072, "num_tokens": 6880361.0, "reward": 2.0326452255249023, "reward_std": 0.13507340848445892, "rewards/check_originality_func/mean": 0.2678571343421936, "rewards/check_originality_func/std": 0.44685041904449463, "rewards/perplexity_reward/mean": 0.7647879719734192, "rewards/perplexity_reward/std": 0.09030947834253311, "rewards/sentence_count_match_reward_logic/mean": 1.0, "rewards/sentence_count_match_reward_logic/std": 0.0, "step": 1497 }, { "clip_ratio": 0.006357364822179079, "epoch": 0.5241427571728482, "grad_norm": 0.23552679038812988, "kl": 1.5390625, "learning_rate": 9.223144289158046e-06, "loss": 0.0065, "step": 1498 }, { "clip_ratio": 0.01792307011783123, "epoch": 0.5244926522043387, "grad_norm": 0.19006986706672716, "kl": 1.546875, "learning_rate": 9.221508724374674e-06, "loss": 0.0057, "step": 1499 }, { "clip_ratio": 0.04054740071296692, "epoch": 0.5248425472358292, "grad_norm": 0.15355566668913184, "kl": 1.5546875, "learning_rate": 9.219871585008503e-06, "loss": 0.0049, "step": 1500 }, { "clip_ratio": 0.0010768873617053032, "completions/clipped_ratio": 0.0, "completions/max_length": 113.0, "completions/max_terminated_length": 113.0, "completions/mean_length": 47.750003814697266, "completions/mean_terminated_length": 47.750003814697266, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "epoch": 0.5251924422673198, "grad_norm": 0.2333117842744348, "kl": 1.8125, "learning_rate": 9.218232871670168e-06, "loss": 0.0054, "num_tokens": 6888131.0, "reward": 2.1522274017333984, "reward_std": 0.12222620844841003, "rewards/check_originality_func/mean": 0.375, "rewards/check_originality_func/std": 0.48850420117378235, "rewards/perplexity_reward/mean": 0.7802037000656128, "rewards/perplexity_reward/std": 0.11463966220617294, "rewards/sentence_count_match_reward_logic/mean": 0.9970237612724304, "rewards/sentence_count_match_reward_logic/std": 0.02227177284657955, "step": 1501 }, { "clip_ratio": 0.009232011623680592, "epoch": 0.5255423372988104, "grad_norm": 0.18577559580162148, "kl": 1.8125, "learning_rate": 9.216592584970893e-06, "loss": 0.0049, "step": 1502 }, { "clip_ratio": 0.019949516281485558, "epoch": 0.5258922323303009, "grad_norm": 0.1425090757965434, "kl": 1.8203125, "learning_rate": 9.214950725522495e-06, "loss": 0.0043, "step": 1503 }, { "clip_ratio": 0.03399015590548515, "epoch": 0.5262421273617914, "grad_norm": 0.13192843621233688, "kl": 1.8359375, "learning_rate": 9.213307293937366e-06, "loss": 0.004, "step": 1504 }, { "clip_ratio": 0.0022476986050605774, "completions/clipped_ratio": 0.0, "completions/max_length": 97.0, "completions/max_terminated_length": 97.0, "completions/mean_length": 35.017860412597656, "completions/mean_terminated_length": 35.017860412597656, "completions/min_length": 10.0, "completions/min_terminated_length": 10.0, "epoch": 0.5265920223932821, "grad_norm": 0.2559654521744165, "kl": 1.4140625, "learning_rate": 9.211662290828493e-06, "loss": 0.0063, "num_tokens": 6894348.0, "reward": 1.8655134439468384, "reward_std": 0.09432785958051682, "rewards/check_originality_func/mean": 0.1607142835855484, "rewards/check_originality_func/std": 0.3705909848213196, "rewards/perplexity_reward/mean": 0.7405133843421936, "rewards/perplexity_reward/std": 0.12610892951488495, "rewards/sentence_count_match_reward_logic/mean": 0.9642857313156128, "rewards/sentence_count_match_reward_logic/std": 0.08827348798513412, "step": 1505 }, { "clip_ratio": 0.013701475225389004, "epoch": 0.5269419174247726, "grad_norm": 0.2170069062957424, "kl": 1.421875, "learning_rate": 9.210015716809446e-06, "loss": 0.0057, "step": 1506 }, { "clip_ratio": 0.043041396886110306, "epoch": 0.5272918124562631, "grad_norm": 0.15153713592797474, "kl": 1.4375, "learning_rate": 9.208367572494381e-06, "loss": 0.005, "step": 1507 }, { "clip_ratio": 0.06403742730617523, "epoch": 0.5276417074877536, "grad_norm": 0.12530718175302305, "kl": 1.453125, "learning_rate": 9.206717858498042e-06, "loss": 0.0046, "step": 1508 }, { "clip_ratio": 0.0023098750971257687, "completions/clipped_ratio": 0.0, "completions/max_length": 253.0, "completions/max_terminated_length": 253.0, "completions/mean_length": 78.92857360839844, "completions/mean_terminated_length": 78.92857360839844, "completions/min_length": 13.0, "completions/min_terminated_length": 13.0, "epoch": 0.5279916025192443, "grad_norm": 0.25860472181805133, "kl": 1.484375, "learning_rate": 9.205066575435754e-06, "loss": 0.0058, "num_tokens": 6905776.0, "reward": 2.166713237762451, "reward_std": 0.21561016142368317, "rewards/check_originality_func/mean": 0.3571428656578064, "rewards/check_originality_func/std": 0.48349377512931824, "rewards/perplexity_reward/mean": 0.8095703125, "rewards/perplexity_reward/std": 0.0948949083685875, "rewards/sentence_count_match_reward_logic/mean": 1.0, "rewards/sentence_count_match_reward_logic/std": 0.0, "step": 1509 }, { "clip_ratio": 0.012534283101558685, "epoch": 0.5283414975507348, "grad_norm": 0.232306957935186, "kl": 1.484375, "learning_rate": 9.203413723923433e-06, "loss": 0.0053, "step": 1510 }, { "clip_ratio": 0.02179817296564579, "epoch": 0.5286913925822253, "grad_norm": 0.18927266800044312, "kl": 1.484375, "learning_rate": 9.201759304577576e-06, "loss": 0.0041, "step": 1511 }, { "clip_ratio": 0.03419487550854683, "epoch": 0.5290412876137159, "grad_norm": 0.16628627181419264, "kl": 1.4921875, "learning_rate": 9.200103318015266e-06, "loss": 0.0033, "step": 1512 }, { "clip_ratio": 0.0007029272383078933, "completions/clipped_ratio": 0.2321428571428571, "completions/max_length": 256.0, "completions/max_terminated_length": 255.0, "completions/mean_length": 121.76786041259766, "completions/mean_terminated_length": 81.18604278564453, "completions/min_length": 6.0, "completions/min_terminated_length": 6.0, "epoch": 0.5293911826452065, "grad_norm": 0.34109761623709695, "kl": 1.6875, "learning_rate": 9.198445764854166e-06, "loss": 0.0021, "num_tokens": 6922891.0, "reward": 2.178920269012451, "reward_std": 0.30833113193511963, "rewards/check_originality_func/mean": 0.3928571343421936, "rewards/check_originality_func/std": 0.4928053617477417, "rewards/perplexity_reward/mean": 0.8128487467765808, "rewards/perplexity_reward/std": 0.1468917727470398, "rewards/sentence_count_match_reward_logic/mean": 0.9732142686843872, "rewards/sentence_count_match_reward_logic/std": 0.06946247816085815, "step": 1513 }, { "clip_ratio": 0.004393118899315596, "epoch": 0.529741077676697, "grad_norm": 0.29080289223095385, "kl": 1.6875, "learning_rate": 9.196786645712535e-06, "loss": 0.0009, "step": 1514 }, { "clip_ratio": 0.011492418125271797, "epoch": 0.5300909727081875, "grad_norm": 0.24032373353309072, "kl": 1.6875, "learning_rate": 9.195125961209203e-06, "loss": -0.0007, "step": 1515 }, { "clip_ratio": 0.023237818852066994, "epoch": 0.5304408677396781, "grad_norm": 0.24064587153491565, "kl": 1.6953125, "learning_rate": 9.193463711963593e-06, "loss": -0.0018, "step": 1516 }, { "clip_ratio": 0.0013612568145617843, "completions/clipped_ratio": 0.0, "completions/max_length": 237.0, "completions/max_terminated_length": 237.0, "completions/mean_length": 71.64286041259766, "completions/mean_terminated_length": 71.64286041259766, "completions/min_length": 17.0, "completions/min_terminated_length": 17.0, "epoch": 0.5307907627711687, "grad_norm": 0.2879565690281878, "kl": 1.390625, "learning_rate": 9.191799898595706e-06, "loss": -0.0002, "num_tokens": 6933727.0, "reward": 1.8428263664245605, "reward_std": 0.15329542756080627, "rewards/check_originality_func/mean": 0.1071428582072258, "rewards/check_originality_func/std": 0.3120938837528229, "rewards/perplexity_reward/mean": 0.74853515625, "rewards/perplexity_reward/std": 0.12554603815078735, "rewards/sentence_count_match_reward_logic/mean": 0.9871482849121094, "rewards/sentence_count_match_reward_logic/std": 0.03620026260614395, "step": 1517 }, { "clip_ratio": 0.005972649436444044, "epoch": 0.5311406578026592, "grad_norm": 0.25337572093713445, "kl": 1.390625, "learning_rate": 9.19013452172613e-06, "loss": -0.0011, "step": 1518 }, { "clip_ratio": 0.013868997804820538, "epoch": 0.5314905528341498, "grad_norm": 0.22440781517987485, "kl": 1.390625, "learning_rate": 9.188467581976035e-06, "loss": -0.002, "step": 1519 }, { "clip_ratio": 0.02709914930164814, "epoch": 0.5318404478656403, "grad_norm": 0.179840428885986, "kl": 1.3828125, "learning_rate": 9.18679907996717e-06, "loss": -0.0033, "step": 1520 }, { "clip_ratio": 0.0021989999804645777, "completions/clipped_ratio": 0.0, "completions/max_length": 163.0, "completions/max_terminated_length": 163.0, "completions/mean_length": 74.21428680419922, "completions/mean_terminated_length": 74.21428680419922, "completions/min_length": 15.0, "completions/min_terminated_length": 15.0, "epoch": 0.5321903428971309, "grad_norm": 0.3373474960908129, "kl": 1.7265625, "learning_rate": 9.185129016321877e-06, "loss": 0.0076, "num_tokens": 6944851.0, "reward": 2.16019344329834, "reward_std": 0.23666399717330933, "rewards/check_originality_func/mean": 0.3928571343421936, "rewards/check_originality_func/std": 0.4928053617477417, "rewards/perplexity_reward/mean": 0.8113839030265808, "rewards/perplexity_reward/std": 0.04558909684419632, "rewards/sentence_count_match_reward_logic/mean": 0.9559523463249207, "rewards/sentence_count_match_reward_logic/std": 0.09746424108743668, "step": 1521 }, { "clip_ratio": 0.010377918370068073, "epoch": 0.5325402379286214, "grad_norm": 0.28613688245623, "kl": 1.734375, "learning_rate": 9.183457391663068e-06, "loss": 0.0065, "step": 1522 }, { "clip_ratio": 0.02782023325562477, "epoch": 0.532890132960112, "grad_norm": 0.22142207210808723, "kl": 1.734375, "learning_rate": 9.181784206614247e-06, "loss": 0.0054, "step": 1523 }, { "clip_ratio": 0.04028984531760216, "epoch": 0.5332400279916025, "grad_norm": 0.18938814671087104, "kl": 1.7265625, "learning_rate": 9.180109461799493e-06, "loss": 0.0042, "step": 1524 }, { "clip_ratio": 0.0014689641539007425, "completions/clipped_ratio": 0.0357142857142857, "completions/max_length": 256.0, "completions/max_terminated_length": 253.0, "completions/mean_length": 78.8214340209961, "completions/mean_terminated_length": 72.25926208496094, "completions/min_length": 11.0, "completions/min_terminated_length": 11.0, "epoch": 0.533589923023093, "grad_norm": 0.3009706032019002, "kl": 1.625, "learning_rate": 9.178433157843474e-06, "loss": 0.0061, "num_tokens": 6956521.0, "reward": 2.0283286571502686, "reward_std": 0.1812846064567566, "rewards/check_originality_func/mean": 0.2857142984867096, "rewards/check_originality_func/std": 0.4558423161506653, "rewards/perplexity_reward/mean": 0.7762276530265808, "rewards/perplexity_reward/std": 0.08991540223360062, "rewards/sentence_count_match_reward_logic/mean": 0.9663864970207214, "rewards/sentence_count_match_reward_logic/std": 0.08532242476940155, "step": 1525 }, { "clip_ratio": 0.0125093013048172, "epoch": 0.5339398180545836, "grad_norm": 0.2511356129155856, "kl": 1.6328125, "learning_rate": 9.176755295371433e-06, "loss": 0.0052, "step": 1526 }, { "clip_ratio": 0.03444748371839523, "epoch": 0.5342897130860742, "grad_norm": 0.19045970663876127, "kl": 1.6484375, "learning_rate": 9.175075875009197e-06, "loss": 0.0041, "step": 1527 }, { "clip_ratio": 0.050720155239105225, "epoch": 0.5346396081175647, "grad_norm": 0.1619885297248618, "kl": 1.6953125, "learning_rate": 9.173394897383172e-06, "loss": 0.0034, "step": 1528 }, { "clip_ratio": 0.0017682922771200538, "completions/clipped_ratio": 0.0, "completions/max_length": 188.0, "completions/max_terminated_length": 188.0, "completions/mean_length": 66.625, "completions/mean_terminated_length": 66.625, "completions/min_length": 15.0, "completions/min_terminated_length": 15.0, "epoch": 0.5349895031490552, "grad_norm": 0.2948941223346599, "kl": 1.828125, "learning_rate": 9.171712363120351e-06, "loss": -0.0013, "num_tokens": 6967020.0, "reward": 2.0208866596221924, "reward_std": 0.16561436653137207, "rewards/check_originality_func/mean": 0.2321428507566452, "rewards/check_originality_func/std": 0.4260205328464508, "rewards/perplexity_reward/mean": 0.8270089030265808, "rewards/perplexity_reward/std": 0.10909489542245865, "rewards/sentence_count_match_reward_logic/mean": 0.9617347121238708, "rewards/sentence_count_match_reward_logic/std": 0.09476924687623978, "step": 1529 }, { "clip_ratio": 0.006472429726272821, "epoch": 0.5353393981805459, "grad_norm": 0.2704830928871167, "kl": 1.828125, "learning_rate": 9.1700282728483e-06, "loss": -0.002, "step": 1530 }, { "clip_ratio": 0.019848722964525223, "epoch": 0.5356892932120364, "grad_norm": 0.2026668003160986, "kl": 1.828125, "learning_rate": 9.16834262719517e-06, "loss": -0.0031, "step": 1531 }, { "clip_ratio": 0.029885130003094673, "epoch": 0.5360391882435269, "grad_norm": 0.16835310506275558, "kl": 1.828125, "learning_rate": 9.166655426789692e-06, "loss": -0.0042, "step": 1532 }, { "clip_ratio": 0.0015411832137033343, "completions/clipped_ratio": 0.0, "completions/max_length": 86.0, "completions/max_terminated_length": 86.0, "completions/mean_length": 28.910715103149414, "completions/mean_terminated_length": 28.910715103149414, "completions/min_length": 7.0, "completions/min_terminated_length": 7.0, "epoch": 0.5363890832750174, "grad_norm": 0.2199561436984791, "kl": 2.0, "learning_rate": 9.164966672261171e-06, "loss": 0.0076, "num_tokens": 6972567.0, "reward": 1.8564454317092896, "reward_std": 0.045547965914011, "rewards/check_originality_func/mean": 0.1428571492433548, "rewards/check_originality_func/std": 0.3530939221382141, "rewards/perplexity_reward/mean": 0.7180524468421936, "rewards/perplexity_reward/std": 0.07928159087896347, "rewards/sentence_count_match_reward_logic/mean": 0.9955357313156128, "rewards/sentence_count_match_reward_logic/std": 0.033407654613256454, "step": 1533 }, { "clip_ratio": 0.013357090763747692, "epoch": 0.5367389783065081, "grad_norm": 0.1855607940743597, "kl": 2.015625, "learning_rate": 9.163276364239504e-06, "loss": 0.0071, "step": 1534 }, { "clip_ratio": 0.022613124921917915, "epoch": 0.5370888733379986, "grad_norm": 0.15443441738373925, "kl": 2.0, "learning_rate": 9.161584503355155e-06, "loss": 0.0064, "step": 1535 }, { "clip_ratio": 0.05549781769514084, "epoch": 0.5374387683694891, "grad_norm": 0.11017370569899483, "kl": 2.0, "learning_rate": 9.15989109023917e-06, "loss": 0.0059, "step": 1536 }, { "clip_ratio": 0.0044107926078140736, "completions/clipped_ratio": 0.1428571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 97.0, "completions/mean_length": 70.4464340209961, "completions/mean_terminated_length": 39.520835876464844, "completions/min_length": 13.0, "completions/min_terminated_length": 13.0, "epoch": 0.5377886634009797, "grad_norm": 0.25686221097221923, "kl": 1.9609375, "learning_rate": 9.158196125523182e-06, "loss": 0.0015, "num_tokens": 6983312.0, "reward": 1.9959542751312256, "reward_std": 0.15677860379219055, "rewards/check_originality_func/mean": 0.2678571343421936, "rewards/check_originality_func/std": 0.44685041904449463, "rewards/perplexity_reward/mean": 0.7280970811843872, "rewards/perplexity_reward/std": 0.11823321133852005, "rewards/sentence_count_match_reward_logic/mean": 1.0, "rewards/sentence_count_match_reward_logic/std": 0.0, "step": 1537 }, { "clip_ratio": 0.006306316237896681, "epoch": 0.5381385584324703, "grad_norm": 0.2463389022813649, "kl": 1.9609375, "learning_rate": 9.156499609839394e-06, "loss": 0.0007, "step": 1538 }, { "clip_ratio": 0.03172195702791214, "epoch": 0.5384884534639608, "grad_norm": 0.1925101409024503, "kl": 1.9765625, "learning_rate": 9.154801543820589e-06, "loss": -0.0004, "step": 1539 }, { "clip_ratio": 0.042528051882982254, "epoch": 0.5388383484954513, "grad_norm": 0.16644606221323144, "kl": 1.9765625, "learning_rate": 9.15310192810013e-06, "loss": -0.0012, "step": 1540 }, { "clip_ratio": 0.0010924605885520577, "completions/clipped_ratio": 0.2857142857142857, "completions/max_length": 256.0, "completions/max_terminated_length": 182.0, "completions/mean_length": 134.21429443359375, "completions/mean_terminated_length": 85.5, "completions/min_length": 13.0, "completions/min_terminated_length": 13.0, "epoch": 0.539188243526942, "grad_norm": 0.34095999642599234, "kl": 1.015625, "learning_rate": 9.151400763311958e-06, "loss": 0.0045, "num_tokens": 7003900.0, "reward": 2.093071699142456, "reward_std": 0.1680314838886261, "rewards/check_originality_func/mean": 0.4107142984867096, "rewards/check_originality_func/std": 0.49641573429107666, "rewards/perplexity_reward/mean": 0.8330078125, "rewards/perplexity_reward/std": 0.059195537120103836, "rewards/sentence_count_match_reward_logic/mean": 0.8493494987487793, "rewards/sentence_count_match_reward_logic/std": 0.22269190847873688, "step": 1541 }, { "clip_ratio": 0.005962715018540621, "epoch": 0.5395381385584325, "grad_norm": 0.31389420267240165, "kl": 1.015625, "learning_rate": 9.149698050090594e-06, "loss": 0.0029, "step": 1542 }, { "clip_ratio": 0.02025124430656433, "epoch": 0.539888033589923, "grad_norm": 0.25063197285177996, "kl": 1.015625, "learning_rate": 9.147993789071128e-06, "loss": 0.0016, "step": 1543 }, { "clip_ratio": 0.04207068681716919, "epoch": 0.5402379286214136, "grad_norm": 0.2564702301448178, "kl": 1.015625, "learning_rate": 9.146287980889239e-06, "loss": 0.0004, "step": 1544 }, { "clip_ratio": 0.001976375700905919, "completions/clipped_ratio": 0.017857142857142905, "completions/max_length": 256.0, "completions/max_terminated_length": 149.0, "completions/mean_length": 45.642860412597656, "completions/mean_terminated_length": 41.818180084228516, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "epoch": 0.5405878236529041, "grad_norm": 0.2570145690774294, "kl": 1.8203125, "learning_rate": 9.144580626181176e-06, "loss": 0.0246, "num_tokens": 7011424.0, "reward": 1.9778878688812256, "reward_std": 0.12519027292728424, "rewards/check_originality_func/mean": 0.1607142835855484, "rewards/check_originality_func/std": 0.3705909848213196, "rewards/perplexity_reward/mean": 0.8171735405921936, "rewards/perplexity_reward/std": 0.09969719499349594, "rewards/sentence_count_match_reward_logic/mean": 1.0, "rewards/sentence_count_match_reward_logic/std": 0.0, "step": 1545 }, { "clip_ratio": 0.008951976895332336, "epoch": 0.5409377186843947, "grad_norm": 0.21120395280904966, "kl": 1.8203125, "learning_rate": 9.142871725583763e-06, "loss": 0.024, "step": 1546 }, { "clip_ratio": 0.023675542324781418, "epoch": 0.5412876137158852, "grad_norm": 0.1729244250856967, "kl": 1.828125, "learning_rate": 9.141161279734408e-06, "loss": 0.0231, "step": 1547 }, { "clip_ratio": 0.050027865916490555, "epoch": 0.5416375087473758, "grad_norm": 0.13776214739013154, "kl": 1.8359375, "learning_rate": 9.139449289271087e-06, "loss": 0.0223, "step": 1548 }, { "clip_ratio": 0.0023365344386547804, "completions/clipped_ratio": 0.0, "completions/max_length": 232.0, "completions/max_terminated_length": 232.0, "completions/mean_length": 72.1964340209961, "completions/mean_terminated_length": 72.1964340209961, "completions/min_length": 16.0, "completions/min_terminated_length": 16.0, "epoch": 0.5419874037788663, "grad_norm": 0.3371719347607059, "kl": 1.4140625, "learning_rate": 9.13773575483236e-06, "loss": 0.0078, "num_tokens": 7022211.0, "reward": 1.9162946939468384, "reward_std": 0.022209325805306435, "rewards/check_originality_func/mean": 0.1428571492433548, "rewards/check_originality_func/std": 0.3530939221382141, "rewards/perplexity_reward/mean": 0.7734375, "rewards/perplexity_reward/std": 0.09594090282917023, "rewards/sentence_count_match_reward_logic/mean": 1.0, "rewards/sentence_count_match_reward_logic/std": 0.0, "step": 1549 }, { "clip_ratio": 0.019061146304011345, "epoch": 0.5423372988103569, "grad_norm": 0.26247951795577457, "kl": 1.4140625, "learning_rate": 9.136020677057357e-06, "loss": 0.0068, "step": 1550 }, { "clip_ratio": 0.036818087100982666, "epoch": 0.5426871938418475, "grad_norm": 0.20176761902189014, "kl": 1.4140625, "learning_rate": 9.134304056585786e-06, "loss": 0.0056, "step": 1551 }, { "clip_ratio": 0.07030648738145828, "epoch": 0.543037088873338, "grad_norm": 0.15344932379539789, "kl": 1.4140625, "learning_rate": 9.13258589405793e-06, "loss": 0.0047, "step": 1552 }, { "clip_ratio": 0.0009990009712055326, "completions/clipped_ratio": 0.0, "completions/max_length": 116.0, "completions/max_terminated_length": 116.0, "completions/mean_length": 49.73214340209961, "completions/mean_terminated_length": 49.73214340209961, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "epoch": 0.5433869839048285, "grad_norm": 0.22467808173642614, "kl": 1.8125, "learning_rate": 9.130866190114649e-06, "loss": 0.0042, "num_tokens": 7030564.0, "reward": 2.0733120441436768, "reward_std": 0.20234306156635284, "rewards/check_originality_func/mean": 0.2857142984867096, "rewards/check_originality_func/std": 0.4558423161506653, "rewards/perplexity_reward/mean": 0.78759765625, "rewards/perplexity_reward/std": 0.08135906606912613, "rewards/sentence_count_match_reward_logic/mean": 1.0, "rewards/sentence_count_match_reward_logic/std": 0.0, "step": 1553 }, { "clip_ratio": 0.013379223644733429, "epoch": 0.5437368789363191, "grad_norm": 0.18866984878261353, "kl": 1.8203125, "learning_rate": 9.129144945397373e-06, "loss": 0.0038, "step": 1554 }, { "clip_ratio": 0.03108138032257557, "epoch": 0.5440867739678097, "grad_norm": 0.1571775661151774, "kl": 1.8359375, "learning_rate": 9.127422160548114e-06, "loss": 0.0032, "step": 1555 }, { "clip_ratio": 0.04562517628073692, "epoch": 0.5444366689993002, "grad_norm": 0.14376616365235062, "kl": 1.859375, "learning_rate": 9.12569783620945e-06, "loss": 0.0025, "step": 1556 }, { "clip_ratio": 0.002814797917380929, "completions/clipped_ratio": 0.0, "completions/max_length": 172.0, "completions/max_terminated_length": 172.0, "completions/mean_length": 70.25, "completions/mean_terminated_length": 70.25, "completions/min_length": 7.0, "completions/min_terminated_length": 7.0, "epoch": 0.5447865640307907, "grad_norm": 0.2345465429649092, "kl": 0.96875, "learning_rate": 9.123971973024543e-06, "loss": 0.0085, "num_tokens": 7041506.0, "reward": 2.0384347438812256, "reward_std": 0.2351645976305008, "rewards/check_originality_func/mean": 0.25, "rewards/check_originality_func/std": 0.43693143129348755, "rewards/perplexity_reward/mean": 0.8241490125656128, "rewards/perplexity_reward/std": 0.09098518639802933, "rewards/sentence_count_match_reward_logic/mean": 0.9642857313156128, "rewards/sentence_count_match_reward_logic/std": 0.08827348053455353, "step": 1557 }, { "clip_ratio": 0.01067777443677187, "epoch": 0.5451364590622814, "grad_norm": 0.20522473037462485, "kl": 0.9765625, "learning_rate": 9.12224457163712e-06, "loss": 0.0078, "step": 1558 }, { "clip_ratio": 0.022570082917809486, "epoch": 0.5454863540937719, "grad_norm": 0.18080569885680292, "kl": 0.9765625, "learning_rate": 9.120515632691486e-06, "loss": 0.0071, "step": 1559 }, { "clip_ratio": 0.032569799572229385, "epoch": 0.5458362491252624, "grad_norm": 0.129445309759073, "kl": 0.98828125, "learning_rate": 9.118785156832518e-06, "loss": 0.0063, "step": 1560 }, { "clip_ratio": 0.0009217190672643483, "completions/clipped_ratio": 0.0, "completions/max_length": 184.0, "completions/max_terminated_length": 184.0, "completions/mean_length": 81.33928680419922, "completions/mean_terminated_length": 81.33928680419922, "completions/min_length": 14.0, "completions/min_terminated_length": 14.0, "epoch": 0.5461861441567529, "grad_norm": 0.3562148041872904, "kl": 1.9453125, "learning_rate": 9.11705314470567e-06, "loss": 0.0072, "num_tokens": 7053933.0, "reward": 2.0779855251312256, "reward_std": 0.20363499224185944, "rewards/check_originality_func/mean": 0.25, "rewards/check_originality_func/std": 0.43693143129348755, "rewards/perplexity_reward/mean": 0.8636997938156128, "rewards/perplexity_reward/std": 0.04096691682934761, "rewards/sentence_count_match_reward_logic/mean": 0.9642857313156128, "rewards/sentence_count_match_reward_logic/std": 0.08827348053455353, "step": 1561 }, { "clip_ratio": 0.011857636272907257, "epoch": 0.5465360391882436, "grad_norm": 0.2975308189522665, "kl": 1.9453125, "learning_rate": 9.115319596956962e-06, "loss": 0.0058, "step": 1562 }, { "clip_ratio": 0.042689841240644455, "epoch": 0.5468859342197341, "grad_norm": 0.22062062246879038, "kl": 1.953125, "learning_rate": 9.113584514232994e-06, "loss": 0.0045, "step": 1563 }, { "clip_ratio": 0.058293212205171585, "epoch": 0.5472358292512246, "grad_norm": 0.17913512161063677, "kl": 1.953125, "learning_rate": 9.111847897180935e-06, "loss": 0.0032, "step": 1564 }, { "clip_ratio": 0.0016217200318351388, "completions/clipped_ratio": 0.0, "completions/max_length": 136.0, "completions/max_terminated_length": 136.0, "completions/mean_length": 49.017860412597656, "completions/mean_terminated_length": 49.017860412597656, "completions/min_length": 6.0, "completions/min_terminated_length": 6.0, "epoch": 0.5475857242827152, "grad_norm": 0.2539657483608485, "kl": 1.921875, "learning_rate": 9.110109746448527e-06, "loss": 0.0042, "num_tokens": 7062462.0, "reward": 2.132357120513916, "reward_std": 0.16346925497055054, "rewards/check_originality_func/mean": 0.3035714328289032, "rewards/check_originality_func/std": 0.4639608860015869, "rewards/perplexity_reward/mean": 0.8383091688156128, "rewards/perplexity_reward/std": 0.08413224667310715, "rewards/sentence_count_match_reward_logic/mean": 0.9904761910438538, "rewards/sentence_count_match_reward_logic/std": 0.051527880132198334, "step": 1565 }, { "clip_ratio": 0.003725674469023943, "epoch": 0.5479356193142058, "grad_norm": 0.2364169875736241, "kl": 1.9296875, "learning_rate": 9.108370062684082e-06, "loss": 0.0036, "step": 1566 }, { "clip_ratio": 0.026776092126965523, "epoch": 0.5482855143456963, "grad_norm": 0.19324441636404344, "kl": 1.9453125, "learning_rate": 9.106628846536488e-06, "loss": 0.0028, "step": 1567 }, { "clip_ratio": 0.04314931854605675, "epoch": 0.5486354093771868, "grad_norm": 0.14770614723100942, "kl": 1.984375, "learning_rate": 9.1048860986552e-06, "loss": 0.0021, "step": 1568 }, { "clip_ratio": 0.001581028918735683, "completions/clipped_ratio": 0.0, "completions/max_length": 178.0, "completions/max_terminated_length": 178.0, "completions/mean_length": 49.10714340209961, "completions/mean_terminated_length": 49.10714340209961, "completions/min_length": 14.0, "completions/min_terminated_length": 14.0, "epoch": 0.5489853044086774, "grad_norm": 0.2901518110498929, "kl": 2.046875, "learning_rate": 9.103141819690246e-06, "loss": 0.0143, "num_tokens": 7070468.0, "reward": 1.993638515472412, "reward_std": 0.14118826389312744, "rewards/check_originality_func/mean": 0.2321428507566452, "rewards/check_originality_func/std": 0.4260205328464508, "rewards/perplexity_reward/mean": 0.8342633843421936, "rewards/perplexity_reward/std": 0.05208848789334297, "rewards/sentence_count_match_reward_logic/mean": 0.9272321462631226, "rewards/sentence_count_match_reward_logic/std": 0.16735079884529114, "step": 1569 }, { "clip_ratio": 0.010440348647534847, "epoch": 0.549335199440168, "grad_norm": 0.25023843730577655, "kl": 2.046875, "learning_rate": 9.101396010292228e-06, "loss": 0.0133, "step": 1570 }, { "clip_ratio": 0.04383311793208122, "epoch": 0.5496850944716585, "grad_norm": 0.18426149677926706, "kl": 2.078125, "learning_rate": 9.099648671112315e-06, "loss": 0.0123, "step": 1571 }, { "clip_ratio": 0.0695062205195427, "epoch": 0.5500349895031491, "grad_norm": 0.15822592668683577, "kl": 2.09375, "learning_rate": 9.097899802802247e-06, "loss": 0.0115, "step": 1572 }, { "clip_ratio": 0.0026680175215005875, "completions/clipped_ratio": 0.2857142857142857, "completions/max_length": 256.0, "completions/max_terminated_length": 178.0, "completions/mean_length": 126.85714721679688, "completions/mean_terminated_length": 75.20000457763672, "completions/min_length": 15.0, "completions/min_terminated_length": 15.0, "epoch": 0.5503848845346396, "grad_norm": 0.3876863702692961, "kl": 2.078125, "learning_rate": 9.09614940601434e-06, "loss": -0.0047, "num_tokens": 7091652.0, "reward": 2.1260945796966553, "reward_std": 0.38125550746917725, "rewards/check_originality_func/mean": 0.4464285671710968, "rewards/check_originality_func/std": 0.5016207695007324, "rewards/perplexity_reward/mean": 0.82421875, "rewards/perplexity_reward/std": 0.08633045852184296, "rewards/sentence_count_match_reward_logic/mean": 0.8554472923278809, "rewards/sentence_count_match_reward_logic/std": 0.17068199813365936, "step": 1573 }, { "clip_ratio": 0.005133072845637798, "epoch": 0.5507347795661302, "grad_norm": 0.3550525483886813, "kl": 2.078125, "learning_rate": 9.09439748140147e-06, "loss": -0.0067, "step": 1574 }, { "clip_ratio": 0.02796805277466774, "epoch": 0.5510846745976207, "grad_norm": 0.2916132987424848, "kl": 2.078125, "learning_rate": 9.092644029617088e-06, "loss": -0.009, "step": 1575 }, { "clip_ratio": 0.053286273032426834, "epoch": 0.5514345696291113, "grad_norm": 0.25391891415204776, "kl": 2.078125, "learning_rate": 9.090889051315222e-06, "loss": -0.0114, "step": 1576 }, { "clip_ratio": 0.0022564935497939587, "completions/clipped_ratio": 0.0, "completions/max_length": 232.0, "completions/max_terminated_length": 232.0, "completions/mean_length": 81.78572082519531, "completions/mean_terminated_length": 81.78572082519531, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "epoch": 0.5517844646606018, "grad_norm": 0.349737890325205, "kl": 2.578125, "learning_rate": 9.089132547150453e-06, "loss": 0.0088, "num_tokens": 7104600.0, "reward": 2.3238468170166016, "reward_std": 0.25223463773727417, "rewards/check_originality_func/mean": 0.5535714030265808, "rewards/check_originality_func/std": 0.5016207695007324, "rewards/perplexity_reward/mean": 0.78515625, "rewards/perplexity_reward/std": 0.11967065930366516, "rewards/sentence_count_match_reward_logic/mean": 0.9851190447807312, "rewards/sentence_count_match_reward_logic/std": 0.0637766420841217, "step": 1577 }, { "clip_ratio": 0.015272485092282295, "epoch": 0.5521343596920923, "grad_norm": 0.29516801407646165, "kl": 2.578125, "learning_rate": 9.087374517777947e-06, "loss": 0.0078, "step": 1578 }, { "clip_ratio": 0.044972557574510574, "epoch": 0.552484254723583, "grad_norm": 0.24347544473648752, "kl": 2.546875, "learning_rate": 9.085614963853429e-06, "loss": 0.0062, "step": 1579 }, { "clip_ratio": 0.07244467735290527, "epoch": 0.5528341497550735, "grad_norm": 0.19515693191000003, "kl": 2.546875, "learning_rate": 9.083853886033198e-06, "loss": 0.0048, "step": 1580 }, { "clip_ratio": 0.0036476280074566603, "completions/clipped_ratio": 0.125, "completions/max_length": 256.0, "completions/max_terminated_length": 252.0, "completions/mean_length": 95.53572082519531, "completions/mean_terminated_length": 72.61224365234375, "completions/min_length": 12.0, "completions/min_terminated_length": 12.0, "epoch": 0.553184044786564, "grad_norm": 0.39833663872580777, "kl": 2.359375, "learning_rate": 9.08209128497412e-06, "loss": -0.0, "num_tokens": 7119174.0, "reward": 2.2669897079467773, "reward_std": 0.32362279295921326, "rewards/check_originality_func/mean": 0.4821428656578064, "rewards/check_originality_func/std": 0.5042031407356262, "rewards/perplexity_reward/mean": 0.8475167155265808, "rewards/perplexity_reward/std": 0.04319712147116661, "rewards/sentence_count_match_reward_logic/mean": 0.9373299479484558, "rewards/sentence_count_match_reward_logic/std": 0.09756793081760406, "step": 1581 }, { "clip_ratio": 0.01798766665160656, "epoch": 0.5535339398180545, "grad_norm": 0.35589920559003996, "kl": 2.359375, "learning_rate": 9.080327161333624e-06, "loss": -0.0019, "step": 1582 }, { "clip_ratio": 0.03986953943967819, "epoch": 0.5538838348495452, "grad_norm": 0.2820698789146724, "kl": 2.359375, "learning_rate": 9.078561515769717e-06, "loss": -0.004, "step": 1583 }, { "clip_ratio": 0.06111055240035057, "epoch": 0.5542337298810357, "grad_norm": 0.24219749699451334, "kl": 2.359375, "learning_rate": 9.076794348940965e-06, "loss": -0.0058, "step": 1584 }, { "clip_ratio": 0.0021853556390851736, "completions/clipped_ratio": 0.0, "completions/max_length": 243.0, "completions/max_terminated_length": 243.0, "completions/mean_length": 76.60714721679688, "completions/mean_terminated_length": 76.60714721679688, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "epoch": 0.5545836249125262, "grad_norm": 0.382912987153857, "kl": 2.53125, "learning_rate": 9.075025661506505e-06, "loss": 0.0037, "num_tokens": 7131192.0, "reward": 2.1363542079925537, "reward_std": 0.27761682868003845, "rewards/check_originality_func/mean": 0.3571428656578064, "rewards/check_originality_func/std": 0.48349374532699585, "rewards/perplexity_reward/mean": 0.8035016655921936, "rewards/perplexity_reward/std": 0.10601557046175003, "rewards/sentence_count_match_reward_logic/mean": 0.9757097363471985, "rewards/sentence_count_match_reward_logic/std": 0.05371803045272827, "step": 1585 }, { "clip_ratio": 0.0188764501363039, "epoch": 0.5549335199440167, "grad_norm": 0.2879894153006928, "kl": 2.546875, "learning_rate": 9.07325545412604e-06, "loss": 0.0023, "step": 1586 }, { "clip_ratio": 0.038343966007232666, "epoch": 0.5552834149755074, "grad_norm": 0.23651091349527562, "kl": 2.578125, "learning_rate": 9.071483727459842e-06, "loss": 0.0007, "step": 1587 }, { "clip_ratio": 0.057617492973804474, "epoch": 0.5556333100069979, "grad_norm": 0.18557455482804153, "kl": 2.59375, "learning_rate": 9.069710482168746e-06, "loss": -0.0007, "step": 1588 }, { "clip_ratio": 0.004244310315698385, "completions/clipped_ratio": 0.017857142857142905, "completions/max_length": 256.0, "completions/max_terminated_length": 244.0, "completions/mean_length": 103.35714721679688, "completions/mean_terminated_length": 100.58181762695312, "completions/min_length": 29.0, "completions/min_terminated_length": 29.0, "epoch": 0.5559832050384884, "grad_norm": 0.4298451251625754, "kl": 1.921875, "learning_rate": 9.06793571891416e-06, "loss": 0.0036, "num_tokens": 7146652.0, "reward": 2.3890953063964844, "reward_std": 0.23004592955112457, "rewards/check_originality_func/mean": 0.6428571343421936, "rewards/check_originality_func/std": 0.48349377512931824, "rewards/perplexity_reward/mean": 0.8328682780265808, "rewards/perplexity_reward/std": 0.06281311064958572, "rewards/sentence_count_match_reward_logic/mean": 0.9133700132369995, "rewards/sentence_count_match_reward_logic/std": 0.1341174840927124, "step": 1589 }, { "clip_ratio": 0.0087786465883255, "epoch": 0.556333100069979, "grad_norm": 0.39073245631122744, "kl": 1.9296875, "learning_rate": 9.066159438358049e-06, "loss": 0.0018, "step": 1590 }, { "clip_ratio": 0.03364763781428337, "epoch": 0.5566829951014696, "grad_norm": 0.30411839511771754, "kl": 1.9296875, "learning_rate": 9.064381641162952e-06, "loss": -0.0008, "step": 1591 }, { "clip_ratio": 0.058180101215839386, "epoch": 0.5570328901329601, "grad_norm": 0.2637855495496936, "kl": 1.9375, "learning_rate": 9.06260232799197e-06, "loss": -0.0026, "step": 1592 }, { "clip_ratio": 0.000774432613980025, "completions/clipped_ratio": 0.0, "completions/max_length": 85.0, "completions/max_terminated_length": 85.0, "completions/mean_length": 41.66071701049805, "completions/mean_terminated_length": 41.66071701049805, "completions/min_length": 13.0, "completions/min_terminated_length": 13.0, "epoch": 0.5573827851644506, "grad_norm": 0.33471481804617487, "kl": 2.84375, "learning_rate": 9.060821499508769e-06, "loss": 0.0112, "num_tokens": 7154033.0, "reward": 2.081264019012451, "reward_std": 0.13679139316082, "rewards/check_originality_func/mean": 0.2678571343421936, "rewards/check_originality_func/std": 0.446850448846817, "rewards/perplexity_reward/mean": 0.8134068250656128, "rewards/perplexity_reward/std": 0.049438606947660446, "rewards/sentence_count_match_reward_logic/mean": 1.0, "rewards/sentence_count_match_reward_logic/std": 0.0, "step": 1593 }, { "clip_ratio": 0.007110603153705597, "epoch": 0.5577326801959412, "grad_norm": 0.26359975398755986, "kl": 2.84375, "learning_rate": 9.05903915637758e-06, "loss": 0.0099, "step": 1594 }, { "clip_ratio": 0.03383966535329819, "epoch": 0.5580825752274318, "grad_norm": 0.18311367008557908, "kl": 2.84375, "learning_rate": 9.057255299263204e-06, "loss": 0.009, "step": 1595 }, { "clip_ratio": 0.059263769537210464, "epoch": 0.5584324702589223, "grad_norm": 0.1335414181507139, "kl": 2.859375, "learning_rate": 9.055469928831e-06, "loss": 0.0083, "step": 1596 }, { "clip_ratio": 0.003161221742630005, "completions/clipped_ratio": 0.1428571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 163.0, "completions/mean_length": 85.87500762939453, "completions/mean_terminated_length": 57.520835876464844, "completions/min_length": 11.0, "completions/min_terminated_length": 11.0, "epoch": 0.5587823652904129, "grad_norm": 0.375237427154082, "kl": 2.171875, "learning_rate": 9.053683045746897e-06, "loss": -0.0041, "num_tokens": 7168474.0, "reward": 2.0242741107940674, "reward_std": 0.31381455063819885, "rewards/check_originality_func/mean": 0.3392857015132904, "rewards/check_originality_func/std": 0.477751761674881, "rewards/perplexity_reward/mean": 0.8165457844734192, "rewards/perplexity_reward/std": 0.10186243802309036, "rewards/sentence_count_match_reward_logic/mean": 0.8684425354003906, "rewards/sentence_count_match_reward_logic/std": 0.21850062906742096, "step": 1597 }, { "clip_ratio": 0.01209851261228323, "epoch": 0.5591322603219034, "grad_norm": 0.30491532750190176, "kl": 2.15625, "learning_rate": 9.05189465067738e-06, "loss": -0.0054, "step": 1598 }, { "clip_ratio": 0.025339141488075256, "epoch": 0.559482155353394, "grad_norm": 0.22760730592143721, "kl": 2.15625, "learning_rate": 9.05010474428951e-06, "loss": -0.0074, "step": 1599 }, { "clip_ratio": 0.041012559086084366, "epoch": 0.5598320503848845, "grad_norm": 0.18000601683975556, "kl": 2.171875, "learning_rate": 9.0483133272509e-06, "loss": -0.0085, "step": 1600 }, { "clip_ratio": 0.0016113587189465761, "completions/clipped_ratio": 0.1428571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 169.0, "completions/mean_length": 87.21428680419922, "completions/mean_terminated_length": 59.083335876464844, "completions/min_length": 10.0, "completions/min_terminated_length": 10.0, "epoch": 0.5601819454163751, "grad_norm": 0.3976843311590089, "kl": 1.9453125, "learning_rate": 9.046520400229734e-06, "loss": 0.0122, "num_tokens": 7182158.0, "reward": 2.0363190174102783, "reward_std": 0.09845466166734695, "rewards/check_originality_func/mean": 0.3571428656578064, "rewards/check_originality_func/std": 0.48349377512931824, "rewards/perplexity_reward/mean": 0.7163783311843872, "rewards/perplexity_reward/std": 0.1836635172367096, "rewards/sentence_count_match_reward_logic/mean": 0.9627976417541504, "rewards/sentence_count_match_reward_logic/std": 0.09255034476518631, "step": 1601 }, { "clip_ratio": 0.02042665146291256, "epoch": 0.5605318404478656, "grad_norm": 0.3053980995647396, "kl": 1.953125, "learning_rate": 9.044725963894756e-06, "loss": 0.0107, "step": 1602 }, { "clip_ratio": 0.04522139951586723, "epoch": 0.5608817354793562, "grad_norm": 0.23542198915937734, "kl": 1.9765625, "learning_rate": 9.042930018915274e-06, "loss": 0.0091, "step": 1603 }, { "clip_ratio": 0.06214931979775429, "epoch": 0.5612316305108468, "grad_norm": 0.180429080812519, "kl": 1.9921875, "learning_rate": 9.041132565961159e-06, "loss": 0.0076, "step": 1604 }, { "clip_ratio": 0.0009578161989338696, "completions/clipped_ratio": 0.0, "completions/max_length": 102.0, "completions/max_terminated_length": 102.0, "completions/mean_length": 54.000003814697266, "completions/mean_terminated_length": 54.000003814697266, "completions/min_length": 24.0, "completions/min_terminated_length": 24.0, "epoch": 0.5615815255423373, "grad_norm": 0.3621249261009195, "kl": 2.296875, "learning_rate": 9.039333605702844e-06, "loss": 0.0087, "num_tokens": 7191654.0, "reward": 2.032701015472412, "reward_std": 0.13737539947032928, "rewards/check_originality_func/mean": 0.1964285671710968, "rewards/check_originality_func/std": 0.40089187026023865, "rewards/perplexity_reward/mean": 0.83984375, "rewards/perplexity_reward/std": 0.06630381941795349, "rewards/sentence_count_match_reward_logic/mean": 0.9964285492897034, "rewards/sentence_count_match_reward_logic/std": 0.026726122945547104, "step": 1605 }, { "clip_ratio": 0.016322974115610123, "epoch": 0.5619314205738278, "grad_norm": 0.2589396896435312, "kl": 2.296875, "learning_rate": 9.037533138811325e-06, "loss": 0.0072, "step": 1606 }, { "clip_ratio": 0.03802242875099182, "epoch": 0.5622813156053184, "grad_norm": 0.19381707383524566, "kl": 2.296875, "learning_rate": 9.035731165958156e-06, "loss": 0.0061, "step": 1607 }, { "clip_ratio": 0.06051983684301376, "epoch": 0.562631210636809, "grad_norm": 0.14937499436649848, "kl": 2.328125, "learning_rate": 9.033927687815458e-06, "loss": 0.0051, "step": 1608 }, { "clip_ratio": 0.007758478634059429, "completions/clipped_ratio": 0.1071428571428571, "completions/max_length": 256.0, "completions/max_terminated_length": 251.0, "completions/mean_length": 77.17857360839844, "completions/mean_terminated_length": 55.71999740600586, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "epoch": 0.5629811056682995, "grad_norm": 0.3882862117127372, "kl": 2.953125, "learning_rate": 9.032122705055912e-06, "loss": 0.0027, "num_tokens": 7204776.0, "reward": 2.0252749919891357, "reward_std": 0.17708860337734222, "rewards/check_originality_func/mean": 0.3035714328289032, "rewards/check_originality_func/std": 0.4639608860015869, "rewards/perplexity_reward/mean": 0.7822265625, "rewards/perplexity_reward/std": 0.07235147058963776, "rewards/sentence_count_match_reward_logic/mean": 0.9394770860671997, "rewards/sentence_count_match_reward_logic/std": 0.12327740341424942, "step": 1609 }, { "clip_ratio": 0.020372701808810234, "epoch": 0.56333100069979, "grad_norm": 0.34591534495794657, "kl": 2.953125, "learning_rate": 9.030316218352757e-06, "loss": 0.0013, "step": 1610 }, { "clip_ratio": 0.037839967757463455, "epoch": 0.5636808957312807, "grad_norm": 0.2862487445633234, "kl": 2.953125, "learning_rate": 9.028508228379798e-06, "loss": -0.0004, "step": 1611 }, { "clip_ratio": 0.058520641177892685, "epoch": 0.5640307907627712, "grad_norm": 0.22950046273267277, "kl": 2.953125, "learning_rate": 9.026698735811395e-06, "loss": -0.0029, "step": 1612 }, { "clip_ratio": 0.004278481937944889, "completions/clipped_ratio": 0.1785714285714286, "completions/max_length": 256.0, "completions/max_terminated_length": 256.0, "completions/mean_length": 139.5357208251953, "completions/mean_terminated_length": 114.21739196777344, "completions/min_length": 55.0, "completions/min_terminated_length": 55.0, "epoch": 0.5643806857942617, "grad_norm": 0.480651080617604, "kl": 1.484375, "learning_rate": 9.024887741322475e-06, "loss": 0.0212, "num_tokens": 7227454.0, "reward": 2.469017505645752, "reward_std": 0.30782777070999146, "rewards/check_originality_func/mean": 0.7321428656578064, "rewards/check_originality_func/std": 0.44685041904449463, "rewards/perplexity_reward/mean": 0.8292410969734192, "rewards/perplexity_reward/std": 0.04708590358495712, "rewards/sentence_count_match_reward_logic/mean": 0.9076334238052368, "rewards/sentence_count_match_reward_logic/std": 0.1594698131084442, "step": 1613 }, { "clip_ratio": 0.01129467599093914, "epoch": 0.5647305808257522, "grad_norm": 0.42857930638473135, "kl": 1.4921875, "learning_rate": 9.023075245588521e-06, "loss": 0.0191, "step": 1614 }, { "clip_ratio": 0.030204644426703453, "epoch": 0.5650804758572429, "grad_norm": 0.34847065754903683, "kl": 1.4921875, "learning_rate": 9.021261249285575e-06, "loss": 0.0161, "step": 1615 }, { "clip_ratio": 0.049640681594610214, "epoch": 0.5654303708887334, "grad_norm": 0.2665924781812312, "kl": 1.4921875, "learning_rate": 9.019445753090243e-06, "loss": 0.0133, "step": 1616 }, { "clip_ratio": 0.0023316971492022276, "completions/clipped_ratio": 0.017857142857142905, "completions/max_length": 256.0, "completions/max_terminated_length": 243.0, "completions/mean_length": 119.33929443359375, "completions/mean_terminated_length": 116.85453796386719, "completions/min_length": 14.0, "completions/min_terminated_length": 14.0, "epoch": 0.5657802659202239, "grad_norm": 0.4423323062522358, "kl": 1.7265625, "learning_rate": 9.017628757679685e-06, "loss": 0.0216, "num_tokens": 7246137.0, "reward": 2.4452064037323, "reward_std": 0.1710011065006256, "rewards/check_originality_func/mean": 0.6607142686843872, "rewards/check_originality_func/std": 0.477751761674881, "rewards/perplexity_reward/mean": 0.8487025499343872, "rewards/perplexity_reward/std": 0.06679362058639526, "rewards/sentence_count_match_reward_logic/mean": 0.9357895255088806, "rewards/sentence_count_match_reward_logic/std": 0.08530940860509872, "step": 1617 }, { "clip_ratio": 0.00840250588953495, "epoch": 0.5661301609517145, "grad_norm": 0.3701726578514878, "kl": 1.734375, "learning_rate": 9.015810263731626e-06, "loss": 0.019, "step": 1618 }, { "clip_ratio": 0.02013670839369297, "epoch": 0.566480055983205, "grad_norm": 0.3048792642086698, "kl": 1.734375, "learning_rate": 9.013990271924345e-06, "loss": 0.0168, "step": 1619 }, { "clip_ratio": 0.034188952296972275, "epoch": 0.5668299510146956, "grad_norm": 0.2698238082647886, "kl": 1.734375, "learning_rate": 9.012168782936684e-06, "loss": 0.0147, "step": 1620 }, { "clip_ratio": 0.006895849946886301, "completions/clipped_ratio": 0.0, "completions/max_length": 106.0, "completions/max_terminated_length": 106.0, "completions/mean_length": 34.82143020629883, "completions/mean_terminated_length": 34.82143020629883, "completions/min_length": 10.0, "completions/min_terminated_length": 10.0, "epoch": 0.5671798460461861, "grad_norm": 0.333947086877514, "kl": 3.59375, "learning_rate": 9.010345797448037e-06, "loss": 0.0004, "num_tokens": 7254111.0, "reward": 1.7246513366699219, "reward_std": 0.11407818645238876, "rewards/check_originality_func/mean": 0.0535714291036129, "rewards/check_originality_func/std": 0.22720779478549957, "rewards/perplexity_reward/mean": 0.7630440592765808, "rewards/perplexity_reward/std": 0.10353010147809982, "rewards/sentence_count_match_reward_logic/mean": 0.9080356955528259, "rewards/sentence_count_match_reward_logic/std": 0.13721585273742676, "step": 1621 }, { "clip_ratio": 0.022646600380539894, "epoch": 0.5675297410776767, "grad_norm": 0.21538340212886448, "kl": 3.578125, "learning_rate": 9.008521316138364e-06, "loss": -0.0006, "step": 1622 }, { "clip_ratio": 0.05126434564590454, "epoch": 0.5678796361091673, "grad_norm": 0.16080259300736335, "kl": 3.578125, "learning_rate": 9.006695339688179e-06, "loss": -0.0011, "step": 1623 }, { "clip_ratio": 0.0721311941742897, "epoch": 0.5682295311406578, "grad_norm": 0.14491615096229388, "kl": 3.59375, "learning_rate": 9.00486786877855e-06, "loss": -0.0018, "step": 1624 }, { "clip_ratio": 0.0028666870202869177, "completions/clipped_ratio": 0.0, "completions/max_length": 243.0, "completions/max_terminated_length": 243.0, "completions/mean_length": 110.12500762939453, "completions/mean_terminated_length": 110.12500762939453, "completions/min_length": 27.0, "completions/min_terminated_length": 27.0, "epoch": 0.5685794261721484, "grad_norm": 0.40190453925090003, "kl": 1.8203125, "learning_rate": 9.003038904091113e-06, "loss": -0.0009, "num_tokens": 7270302.0, "reward": 2.345256805419922, "reward_std": 0.42070063948631287, "rewards/check_originality_func/mean": 0.5178571343421936, "rewards/check_originality_func/std": 0.5042031407356262, "rewards/perplexity_reward/mean": 0.861328125, "rewards/perplexity_reward/std": 0.07010702043771744, "rewards/sentence_count_match_reward_logic/mean": 0.9660714268684387, "rewards/sentence_count_match_reward_logic/std": 0.0920015498995781, "step": 1625 }, { "clip_ratio": 0.0066363513469696045, "epoch": 0.5689293212036389, "grad_norm": 0.3550740256964286, "kl": 1.8203125, "learning_rate": 9.00120844630805e-06, "loss": -0.0023, "step": 1626 }, { "clip_ratio": 0.03169134259223938, "epoch": 0.5692792162351294, "grad_norm": 0.26692734604144347, "kl": 1.8203125, "learning_rate": 8.999376496112105e-06, "loss": -0.0046, "step": 1627 }, { "clip_ratio": 0.05357546731829643, "epoch": 0.56962911126662, "grad_norm": 0.23305503462868477, "kl": 1.8203125, "learning_rate": 8.997543054186577e-06, "loss": -0.0064, "step": 1628 }, { "clip_ratio": 0.0027590696699917316, "completions/clipped_ratio": 0.0, "completions/max_length": 133.0, "completions/max_terminated_length": 133.0, "completions/mean_length": 64.375, "completions/mean_terminated_length": 64.375, "completions/min_length": 7.0, "completions/min_terminated_length": 7.0, "epoch": 0.5699790062981106, "grad_norm": 0.30929570768838094, "kl": 1.8515625, "learning_rate": 8.995708121215325e-06, "loss": 0.0019, "num_tokens": 7281011.0, "reward": 2.275158166885376, "reward_std": 0.18951481580734253, "rewards/check_originality_func/mean": 0.4821428656578064, "rewards/check_originality_func/std": 0.5042031407356262, "rewards/perplexity_reward/mean": 0.8078961968421936, "rewards/perplexity_reward/std": 0.10059114545583725, "rewards/sentence_count_match_reward_logic/mean": 0.9851189851760864, "rewards/sentence_count_match_reward_logic/std": 0.047956064343452454, "step": 1629 }, { "clip_ratio": 0.012794939801096916, "epoch": 0.5703289013296011, "grad_norm": 0.25807914724046893, "kl": 1.8515625, "learning_rate": 8.99387169788276e-06, "loss": 0.0006, "step": 1630 }, { "clip_ratio": 0.038290079683065414, "epoch": 0.5706787963610916, "grad_norm": 0.2199476899410731, "kl": 1.8671875, "learning_rate": 8.992033784873854e-06, "loss": -0.0002, "step": 1631 }, { "clip_ratio": 0.05819316580891609, "epoch": 0.5710286913925823, "grad_norm": 0.1873234241489962, "kl": 1.890625, "learning_rate": 8.990194382874126e-06, "loss": -0.0013, "step": 1632 }, { "clip_ratio": 0.003486021887511015, "completions/clipped_ratio": 0.1428571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 244.0, "completions/mean_length": 116.14286041259766, "completions/mean_terminated_length": 92.83333587646484, "completions/min_length": 10.0, "completions/min_terminated_length": 10.0, "epoch": 0.5713785864240728, "grad_norm": 0.41541023522433085, "kl": 1.2890625, "learning_rate": 8.988353492569657e-06, "loss": -0.0092, "num_tokens": 7298971.0, "reward": 2.350703477859497, "reward_std": 0.32098516821861267, "rewards/check_originality_func/mean": 0.5892857313156128, "rewards/check_originality_func/std": 0.49641573429107666, "rewards/perplexity_reward/mean": 0.8346121907234192, "rewards/perplexity_reward/std": 0.053927741944789886, "rewards/sentence_count_match_reward_logic/mean": 0.9268053770065308, "rewards/sentence_count_match_reward_logic/std": 0.11047185212373734, "step": 1633 }, { "clip_ratio": 0.007368516642600298, "epoch": 0.5717284814555633, "grad_norm": 0.3756332412100725, "kl": 1.2890625, "learning_rate": 8.986511114647085e-06, "loss": -0.0108, "step": 1634 }, { "clip_ratio": 0.02598569355905056, "epoch": 0.5720783764870538, "grad_norm": 0.28799218155568956, "kl": 1.2890625, "learning_rate": 8.984667249793594e-06, "loss": -0.0135, "step": 1635 }, { "clip_ratio": 0.04172828793525696, "epoch": 0.5724282715185445, "grad_norm": 0.2439635329280563, "kl": 1.2890625, "learning_rate": 8.982821898696933e-06, "loss": -0.0151, "step": 1636 }, { "clip_ratio": 0.0025704801082611084, "completions/clipped_ratio": 0.0, "completions/max_length": 85.0, "completions/max_terminated_length": 85.0, "completions/mean_length": 28.196430206298828, "completions/mean_terminated_length": 28.196430206298828, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "epoch": 0.572778166550035, "grad_norm": 0.2624887583310617, "kl": 3.578125, "learning_rate": 8.980975062045398e-06, "loss": 0.0078, "num_tokens": 7304894.0, "reward": 1.9761998653411865, "reward_std": 0.0856780856847763, "rewards/check_originality_func/mean": 0.2678571343421936, "rewards/check_originality_func/std": 0.44685041904449463, "rewards/perplexity_reward/mean": 0.7654854655265808, "rewards/perplexity_reward/std": 0.07401086390018463, "rewards/sentence_count_match_reward_logic/mean": 0.9428571462631226, "rewards/sentence_count_match_reward_logic/std": 0.1412375569343567, "step": 1637 }, { "clip_ratio": 0.015993211418390274, "epoch": 0.5731280615815255, "grad_norm": 0.20367044893556224, "kl": 3.578125, "learning_rate": 8.979126740527842e-06, "loss": 0.0073, "step": 1638 }, { "clip_ratio": 0.06019607186317444, "epoch": 0.5734779566130161, "grad_norm": 0.14339988151636057, "kl": 3.59375, "learning_rate": 8.977276934833673e-06, "loss": 0.0065, "step": 1639 }, { "clip_ratio": 0.08993850648403168, "epoch": 0.5738278516445067, "grad_norm": 0.10967484578580902, "kl": 3.59375, "learning_rate": 8.975425645652848e-06, "loss": 0.006, "step": 1640 }, { "clip_ratio": 0.004027164541184902, "completions/clipped_ratio": 0.0535714285714286, "completions/max_length": 256.0, "completions/max_terminated_length": 252.0, "completions/mean_length": 92.87500762939453, "completions/mean_terminated_length": 83.64151000976562, "completions/min_length": 6.0, "completions/min_terminated_length": 6.0, "epoch": 0.5741777466759972, "grad_norm": 0.42345359264854954, "kl": 2.765625, "learning_rate": 8.973572873675882e-06, "loss": -0.0012, "num_tokens": 7319463.0, "reward": 2.2842342853546143, "reward_std": 0.1997881978750229, "rewards/check_originality_func/mean": 0.5, "rewards/check_originality_func/std": 0.5045249462127686, "rewards/perplexity_reward/mean": 0.8111048936843872, "rewards/perplexity_reward/std": 0.08319630473852158, "rewards/sentence_count_match_reward_logic/mean": 0.9731292724609375, "rewards/sentence_count_match_reward_logic/std": 0.06785031408071518, "step": 1641 }, { "clip_ratio": 0.014237583614885807, "epoch": 0.5745276417074877, "grad_norm": 0.3771696320740147, "kl": 2.78125, "learning_rate": 8.971718619593844e-06, "loss": -0.0032, "step": 1642 }, { "clip_ratio": 0.03136584162712097, "epoch": 0.5748775367389783, "grad_norm": 0.31652155584824976, "kl": 2.828125, "learning_rate": 8.969862884098346e-06, "loss": -0.0062, "step": 1643 }, { "clip_ratio": 0.05782448872923851, "epoch": 0.5752274317704689, "grad_norm": 0.2540082618610974, "kl": 2.890625, "learning_rate": 8.968005667881567e-06, "loss": -0.0083, "step": 1644 }, { "clip_ratio": 0.0024082178715616465, "completions/clipped_ratio": 0.0, "completions/max_length": 100.0, "completions/max_terminated_length": 100.0, "completions/mean_length": 54.375003814697266, "completions/mean_terminated_length": 54.375003814697266, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "epoch": 0.5755773268019594, "grad_norm": 0.3896589888031421, "kl": 3.140625, "learning_rate": 8.96614697163623e-06, "loss": 0.0024, "num_tokens": 7329036.0, "reward": 1.8891369104385376, "reward_std": 0.24519164860248566, "rewards/check_originality_func/mean": 0.1607142835855484, "rewards/check_originality_func/std": 0.3705909848213196, "rewards/perplexity_reward/mean": 0.8058035969734192, "rewards/perplexity_reward/std": 0.07310473173856735, "rewards/sentence_count_match_reward_logic/mean": 0.9226189851760864, "rewards/sentence_count_match_reward_logic/std": 0.15879689157009125, "step": 1645 }, { "clip_ratio": 0.024621034041047096, "epoch": 0.57592722183345, "grad_norm": 0.30260288479634756, "kl": 3.140625, "learning_rate": 8.964286796055608e-06, "loss": 0.0009, "step": 1646 }, { "clip_ratio": 0.056836459785699844, "epoch": 0.5762771168649405, "grad_norm": 0.19358466872705177, "kl": 3.15625, "learning_rate": 8.96242514183353e-06, "loss": -0.0003, "step": 1647 }, { "clip_ratio": 0.08140458166599274, "epoch": 0.5766270118964311, "grad_norm": 0.1462364035460655, "kl": 3.234375, "learning_rate": 8.960562009664376e-06, "loss": -0.0011, "step": 1648 }, { "clip_ratio": 0.002828629221767187, "completions/clipped_ratio": 0.017857142857142905, "completions/max_length": 256.0, "completions/max_terminated_length": 254.0, "completions/mean_length": 52.05357360839844, "completions/mean_terminated_length": 48.34545135498047, "completions/min_length": 11.0, "completions/min_terminated_length": 11.0, "epoch": 0.5769769069279216, "grad_norm": 0.44960536833224635, "kl": 2.640625, "learning_rate": 8.958697400243077e-06, "loss": -0.0515, "num_tokens": 7340007.0, "reward": 2.0633602142333984, "reward_std": 0.18990224599838257, "rewards/check_originality_func/mean": 0.3392857015132904, "rewards/check_originality_func/std": 0.477751761674881, "rewards/perplexity_reward/mean": 0.8074079155921936, "rewards/perplexity_reward/std": 0.08935271948575974, "rewards/sentence_count_match_reward_logic/mean": 0.9166666865348816, "rewards/sentence_count_match_reward_logic/std": 0.24187816679477692, "step": 1649 }, { "clip_ratio": 0.02136259339749813, "epoch": 0.5773268019594122, "grad_norm": 0.3495594077356462, "kl": 2.640625, "learning_rate": 8.956831314265116e-06, "loss": -0.0533, "step": 1650 }, { "clip_ratio": 0.05045580863952637, "epoch": 0.5776766969909027, "grad_norm": 0.24477089853961126, "kl": 2.65625, "learning_rate": 8.95496375242652e-06, "loss": -0.0549, "step": 1651 }, { "clip_ratio": 0.07504520565271378, "epoch": 0.5780265920223933, "grad_norm": 0.197838515821037, "kl": 2.71875, "learning_rate": 8.953094715423878e-06, "loss": -0.056, "step": 1652 }, { "clip_ratio": 0.004375683143734932, "completions/clipped_ratio": 0.0, "completions/max_length": 178.0, "completions/max_terminated_length": 178.0, "completions/mean_length": 47.17857360839844, "completions/mean_terminated_length": 47.17857360839844, "completions/min_length": 11.0, "completions/min_terminated_length": 11.0, "epoch": 0.5783764870538838, "grad_norm": 0.32183193261165083, "kl": 2.71875, "learning_rate": 8.95122420395432e-06, "loss": 0.0142, "num_tokens": 7348705.0, "reward": 1.9342114925384521, "reward_std": 0.17396481335163116, "rewards/check_originality_func/mean": 0.1607142835855484, "rewards/check_originality_func/std": 0.3705909848213196, "rewards/perplexity_reward/mean": 0.7785295844078064, "rewards/perplexity_reward/std": 0.12431211769580841, "rewards/sentence_count_match_reward_logic/mean": 0.994967520236969, "rewards/sentence_count_match_reward_logic/std": 0.021367212757468224, "step": 1653 }, { "clip_ratio": 0.01958659291267395, "epoch": 0.5787263820853744, "grad_norm": 0.21384554130830136, "kl": 2.734375, "learning_rate": 8.94935221871553e-06, "loss": 0.0135, "step": 1654 }, { "clip_ratio": 0.03716461732983589, "epoch": 0.5790762771168649, "grad_norm": 0.16488583738181425, "kl": 2.765625, "learning_rate": 8.94747876040574e-06, "loss": 0.0127, "step": 1655 }, { "clip_ratio": 0.06334926187992096, "epoch": 0.5794261721483555, "grad_norm": 0.1361640622564588, "kl": 2.828125, "learning_rate": 8.945603829723732e-06, "loss": 0.0121, "step": 1656 }, { "clip_ratio": 0.0013654585927724838, "completions/clipped_ratio": 0.0, "completions/max_length": 130.0, "completions/max_terminated_length": 130.0, "completions/mean_length": 29.017858505249023, "completions/mean_terminated_length": 29.017858505249023, "completions/min_length": 6.0, "completions/min_terminated_length": 6.0, "epoch": 0.5797760671798461, "grad_norm": 0.43981007635383823, "kl": 4.5625, "learning_rate": 8.94372742736884e-06, "loss": 0.0087, "num_tokens": 7354714.0, "reward": 1.9269671440124512, "reward_std": 0.03190129995346069, "rewards/check_originality_func/mean": 0.1428571492433548, "rewards/check_originality_func/std": 0.3530939221382141, "rewards/perplexity_reward/mean": 0.78857421875, "rewards/perplexity_reward/std": 0.0877174362540245, "rewards/sentence_count_match_reward_logic/mean": 0.9955357313156128, "rewards/sentence_count_match_reward_logic/std": 0.023407040163874626, "step": 1657 }, { "clip_ratio": 0.029143912717700005, "epoch": 0.5801259622113366, "grad_norm": 2.789887906757777, "kl": 10.1875, "learning_rate": 8.94184955404094e-06, "loss": 0.0109, "step": 1658 }, { "clip_ratio": 0.03388317674398422, "epoch": 0.5804758572428271, "grad_norm": 2.7898529864422805, "kl": 4.53125, "learning_rate": 8.939970210440466e-06, "loss": 0.012, "step": 1659 }, { "clip_ratio": 0.0609675832092762, "epoch": 0.5808257522743177, "grad_norm": 0.23664805689272086, "kl": 4.90625, "learning_rate": 8.93808939726839e-06, "loss": 0.0068, "step": 1660 }, { "clip_ratio": 0.007296890486031771, "completions/clipped_ratio": 0.0, "completions/max_length": 149.0, "completions/max_terminated_length": 149.0, "completions/mean_length": 75.9464340209961, "completions/mean_terminated_length": 75.9464340209961, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "epoch": 0.5811756473058083, "grad_norm": 0.4079694971583642, "kl": 2.203125, "learning_rate": 8.936207115226242e-06, "loss": 0.0125, "num_tokens": 7366967.0, "reward": 2.3973774909973145, "reward_std": 0.16334746778011322, "rewards/check_originality_func/mean": 0.5892857313156128, "rewards/check_originality_func/std": 0.49641573429107666, "rewards/perplexity_reward/mean": 0.833984375, "rewards/perplexity_reward/std": 0.047252267599105835, "rewards/sentence_count_match_reward_logic/mean": 0.9741071462631226, "rewards/sentence_count_match_reward_logic/std": 0.07567337900400162, "step": 1661 }, { "clip_ratio": 0.016358425840735435, "epoch": 0.5815255423372988, "grad_norm": 0.3727764320264893, "kl": 2.21875, "learning_rate": 8.934323365016094e-06, "loss": 0.0108, "step": 1662 }, { "clip_ratio": 0.046904198825359344, "epoch": 0.5818754373687893, "grad_norm": 0.2551328599957674, "kl": 2.21875, "learning_rate": 8.932438147340565e-06, "loss": 0.009, "step": 1663 }, { "clip_ratio": 0.07854080200195312, "epoch": 0.58222533240028, "grad_norm": 0.1955778198086469, "kl": 2.203125, "learning_rate": 8.930551462902826e-06, "loss": 0.0078, "step": 1664 }, { "clip_ratio": 0.0036528620403259993, "completions/clipped_ratio": 0.0, "completions/max_length": 127.0, "completions/max_terminated_length": 127.0, "completions/mean_length": 52.58928680419922, "completions/mean_terminated_length": 52.58928680419922, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "epoch": 0.5825752274317705, "grad_norm": 0.3576470778289664, "kl": 2.59375, "learning_rate": 8.928663312406593e-06, "loss": 0.0008, "num_tokens": 7376976.0, "reward": 1.8841139078140259, "reward_std": 0.11202973127365112, "rewards/check_originality_func/mean": 0.2142857164144516, "rewards/check_originality_func/std": 0.4140393137931824, "rewards/perplexity_reward/mean": 0.8035016655921936, "rewards/perplexity_reward/std": 0.10541459918022156, "rewards/sentence_count_match_reward_logic/mean": 0.8663265109062195, "rewards/sentence_count_match_reward_logic/std": 0.27962905168533325, "step": 1665 }, { "clip_ratio": 0.018389808014035225, "epoch": 0.582925122463261, "grad_norm": 0.24538307871467827, "kl": 2.59375, "learning_rate": 8.926773696556125e-06, "loss": -0.0004, "step": 1666 }, { "clip_ratio": 0.045198798179626465, "epoch": 0.5832750174947515, "grad_norm": 0.17982638858790823, "kl": 2.59375, "learning_rate": 8.924882616056231e-06, "loss": -0.0011, "step": 1667 }, { "clip_ratio": 0.05717826634645462, "epoch": 0.5836249125262422, "grad_norm": 0.15703471639424413, "kl": 2.59375, "learning_rate": 8.92299007161227e-06, "loss": -0.0015, "step": 1668 }, { "clip_ratio": 0.0032701645977795124, "completions/clipped_ratio": 0.017857142857142905, "completions/max_length": 256.0, "completions/max_terminated_length": 213.0, "completions/mean_length": 99.42857360839844, "completions/mean_terminated_length": 96.58181762695312, "completions/min_length": 10.0, "completions/min_terminated_length": 10.0, "epoch": 0.5839748075577327, "grad_norm": 0.46573792503334804, "kl": 3.0, "learning_rate": 8.921096063930141e-06, "loss": 0.023, "num_tokens": 7392320.0, "reward": 2.272993564605713, "reward_std": 0.22894228994846344, "rewards/check_originality_func/mean": 0.4821428656578064, "rewards/check_originality_func/std": 0.5042031407356262, "rewards/perplexity_reward/mean": 0.8168247938156128, "rewards/perplexity_reward/std": 0.10197055339813232, "rewards/sentence_count_match_reward_logic/mean": 0.9740260243415833, "rewards/sentence_count_match_reward_logic/std": 0.07479913532733917, "step": 1669 }, { "clip_ratio": 0.012805625796318054, "epoch": 0.5843247025892232, "grad_norm": 0.3637118451095016, "kl": 3.0, "learning_rate": 8.91920059371629e-06, "loss": 0.0209, "step": 1670 }, { "clip_ratio": 0.045033909380435944, "epoch": 0.5846745976207138, "grad_norm": 0.28258299257885566, "kl": 3.015625, "learning_rate": 8.917303661677712e-06, "loss": 0.0184, "step": 1671 }, { "clip_ratio": 0.06495944410562515, "epoch": 0.5850244926522044, "grad_norm": 0.2445232807912298, "kl": 3.03125, "learning_rate": 8.91540526852194e-06, "loss": 0.0173, "step": 1672 }, { "clip_ratio": 0.006211334373801947, "completions/clipped_ratio": 0.1428571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 211.0, "completions/mean_length": 82.375, "completions/mean_terminated_length": 53.4375, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "epoch": 0.5853743876836949, "grad_norm": 0.5088955656180011, "kl": 2.5, "learning_rate": 8.91350541495706e-06, "loss": -0.0048, "num_tokens": 7406893.0, "reward": 1.9179083108901978, "reward_std": 0.11357246339321136, "rewards/check_originality_func/mean": 0.25, "rewards/check_originality_func/std": 0.43693143129348755, "rewards/perplexity_reward/mean": 0.7815987467765808, "rewards/perplexity_reward/std": 0.10305200517177582, "rewards/sentence_count_match_reward_logic/mean": 0.8863095045089722, "rewards/sentence_count_match_reward_logic/std": 0.2029668092727661, "step": 1673 }, { "clip_ratio": 0.03208116069436073, "epoch": 0.5857242827151854, "grad_norm": 0.30728180032911817, "kl": 2.5, "learning_rate": 8.9116041016917e-06, "loss": -0.0066, "step": 1674 }, { "clip_ratio": 0.05584792047739029, "epoch": 0.586074177746676, "grad_norm": 0.250029854388435, "kl": 2.5, "learning_rate": 8.909701329435032e-06, "loss": -0.0085, "step": 1675 }, { "clip_ratio": 0.08846212923526764, "epoch": 0.5864240727781665, "grad_norm": 0.22598549855838024, "kl": 2.5, "learning_rate": 8.907797098896768e-06, "loss": -0.0095, "step": 1676 }, { "clip_ratio": 0.003844761522486806, "completions/clipped_ratio": 0.0892857142857143, "completions/max_length": 256.0, "completions/max_terminated_length": 246.0, "completions/mean_length": 107.98214721679688, "completions/mean_terminated_length": 93.47058868408203, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "epoch": 0.5867739678096571, "grad_norm": 0.48050785199441687, "kl": 3.0, "learning_rate": 8.905891410787174e-06, "loss": -0.006, "num_tokens": 7424676.0, "reward": 2.3948893547058105, "reward_std": 0.08138473331928253, "rewards/check_originality_func/mean": 0.5892857313156128, "rewards/check_originality_func/std": 0.49641573429107666, "rewards/perplexity_reward/mean": 0.8389369249343872, "rewards/perplexity_reward/std": 0.053528666496276855, "rewards/sentence_count_match_reward_logic/mean": 0.9666666984558105, "rewards/sentence_count_match_reward_logic/std": 0.08989330381155014, "step": 1677 }, { "clip_ratio": 0.007347181439399719, "epoch": 0.5871238628411477, "grad_norm": 0.4292060623265885, "kl": 3.0, "learning_rate": 8.903984265817048e-06, "loss": -0.0079, "step": 1678 }, { "clip_ratio": 0.03594581037759781, "epoch": 0.5874737578726382, "grad_norm": 0.34105351800014555, "kl": 3.0, "learning_rate": 8.90207566469774e-06, "loss": -0.0109, "step": 1679 }, { "clip_ratio": 0.06115730479359627, "epoch": 0.5878236529041287, "grad_norm": 0.29047082133057545, "kl": 3.015625, "learning_rate": 8.90016560814114e-06, "loss": -0.0132, "step": 1680 }, { "clip_ratio": 0.004676785785704851, "completions/clipped_ratio": 0.1607142857142857, "completions/max_length": 256.0, "completions/max_terminated_length": 154.0, "completions/mean_length": 118.42857360839844, "completions/mean_terminated_length": 92.0851058959961, "completions/min_length": 13.0, "completions/min_terminated_length": 13.0, "epoch": 0.5881735479356193, "grad_norm": 0.5749285566586654, "kl": 2.640625, "learning_rate": 8.898254096859681e-06, "loss": 0.0254, "num_tokens": 7442420.0, "reward": 2.252621650695801, "reward_std": 0.18281489610671997, "rewards/check_originality_func/mean": 0.4464285671710968, "rewards/check_originality_func/std": 0.5016207695007324, "rewards/perplexity_reward/mean": 0.8374720811843872, "rewards/perplexity_reward/std": 0.09381116926670074, "rewards/sentence_count_match_reward_logic/mean": 0.9687208533287048, "rewards/sentence_count_match_reward_logic/std": 0.07542993128299713, "step": 1681 }, { "clip_ratio": 0.014407132752239704, "epoch": 0.5885234429671099, "grad_norm": 0.42240731598430425, "kl": 2.640625, "learning_rate": 8.89634113156634e-06, "loss": 0.0227, "step": 1682 }, { "clip_ratio": 0.04145793244242668, "epoch": 0.5888733379986004, "grad_norm": 0.324040938788475, "kl": 2.640625, "learning_rate": 8.89442671297463e-06, "loss": 0.0205, "step": 1683 }, { "clip_ratio": 0.0680667832493782, "epoch": 0.5892232330300909, "grad_norm": 0.26467212487895125, "kl": 2.65625, "learning_rate": 8.892510841798616e-06, "loss": 0.0187, "step": 1684 }, { "clip_ratio": 0.002948099048808217, "completions/clipped_ratio": 0.0, "completions/max_length": 148.0, "completions/max_terminated_length": 148.0, "completions/mean_length": 51.19643020629883, "completions/mean_terminated_length": 51.19643020629883, "completions/min_length": 10.0, "completions/min_terminated_length": 10.0, "epoch": 0.5895731280615816, "grad_norm": 0.45047386330379513, "kl": 3.765625, "learning_rate": 8.8905935187529e-06, "loss": 0.0101, "num_tokens": 7451503.0, "reward": 2.2108912467956543, "reward_std": 0.21813979744911194, "rewards/check_originality_func/mean": 0.4464285671710968, "rewards/check_originality_func/std": 0.5016207695007324, "rewards/perplexity_reward/mean": 0.8120814561843872, "rewards/perplexity_reward/std": 0.08052657544612885, "rewards/sentence_count_match_reward_logic/mean": 0.9523810148239136, "rewards/sentence_count_match_reward_logic/std": 0.11769796907901764, "step": 1685 }, { "clip_ratio": 0.01880783773958683, "epoch": 0.5899230230930721, "grad_norm": 0.3250154096284779, "kl": 3.765625, "learning_rate": 8.888674744552619e-06, "loss": 0.0084, "step": 1686 }, { "clip_ratio": 0.05761024355888367, "epoch": 0.5902729181245626, "grad_norm": 0.22513010353455515, "kl": 3.765625, "learning_rate": 8.886754519913465e-06, "loss": 0.0074, "step": 1687 }, { "clip_ratio": 0.08474991470575333, "epoch": 0.5906228131560531, "grad_norm": 0.1919506709290572, "kl": 3.765625, "learning_rate": 8.88483284555166e-06, "loss": 0.0068, "step": 1688 }, { "clip_ratio": 0.004012841731309891, "completions/clipped_ratio": 0.0, "completions/max_length": 100.0, "completions/max_terminated_length": 100.0, "completions/mean_length": 62.91071701049805, "completions/mean_terminated_length": 62.91071701049805, "completions/min_length": 7.0, "completions/min_terminated_length": 7.0, "epoch": 0.5909727081875438, "grad_norm": 0.4048094614236816, "kl": 3.484375, "learning_rate": 8.882909722183973e-06, "loss": 0.0078, "num_tokens": 7462298.0, "reward": 2.3838727474212646, "reward_std": 0.21680279076099396, "rewards/check_originality_func/mean": 0.5892857313156128, "rewards/check_originality_func/std": 0.49641573429107666, "rewards/perplexity_reward/mean": 0.8088727593421936, "rewards/perplexity_reward/std": 0.07844698429107666, "rewards/sentence_count_match_reward_logic/mean": 0.985714316368103, "rewards/sentence_count_match_reward_logic/std": 0.051974017173051834, "step": 1689 }, { "clip_ratio": 0.011484584771096706, "epoch": 0.5913226032190343, "grad_norm": 0.2676865900714854, "kl": 3.484375, "learning_rate": 8.880985150527705e-06, "loss": 0.0066, "step": 1690 }, { "clip_ratio": 0.02958468534052372, "epoch": 0.5916724982505248, "grad_norm": 0.19137448740099386, "kl": 3.5, "learning_rate": 8.879059131300709e-06, "loss": 0.0059, "step": 1691 }, { "clip_ratio": 0.04066962003707886, "epoch": 0.5920223932820154, "grad_norm": 0.19727221627017197, "kl": 3.546875, "learning_rate": 8.87713166522137e-06, "loss": 0.0055, "step": 1692 }, { "clip_ratio": 0.004475111607462168, "completions/clipped_ratio": 0.0, "completions/max_length": 197.0, "completions/max_terminated_length": 197.0, "completions/mean_length": 64.53572082519531, "completions/mean_terminated_length": 64.53572082519531, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "epoch": 0.592372288313506, "grad_norm": 0.5475884756791181, "kl": 3.515625, "learning_rate": 8.875202753008614e-06, "loss": -0.005, "num_tokens": 7473072.0, "reward": 2.1970272064208984, "reward_std": 0.23326599597930908, "rewards/check_originality_func/mean": 0.4107142984867096, "rewards/check_originality_func/std": 0.49641573429107666, "rewards/perplexity_reward/mean": 0.8018973469734192, "rewards/perplexity_reward/std": 0.11433244496583939, "rewards/sentence_count_match_reward_logic/mean": 0.984415590763092, "rewards/sentence_count_match_reward_logic/std": 0.04883970692753792, "step": 1693 }, { "clip_ratio": 0.024747032672166824, "epoch": 0.5927221833449965, "grad_norm": 0.40104062801299517, "kl": 3.515625, "learning_rate": 8.873272395381907e-06, "loss": -0.0079, "step": 1694 }, { "clip_ratio": 0.053570456802845, "epoch": 0.593072078376487, "grad_norm": 0.327912491715425, "kl": 3.515625, "learning_rate": 8.871340593061255e-06, "loss": -0.0097, "step": 1695 }, { "clip_ratio": 0.06988386809825897, "epoch": 0.5934219734079776, "grad_norm": 0.23893938163561518, "kl": 3.515625, "learning_rate": 8.869407346767202e-06, "loss": -0.0113, "step": 1696 }, { "clip_ratio": 0.0052078780718147755, "completions/clipped_ratio": 0.1428571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 147.0, "completions/mean_length": 70.03572082519531, "completions/mean_terminated_length": 39.04166793823242, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "epoch": 0.5937718684394682, "grad_norm": 0.4668199634958981, "kl": 3.90625, "learning_rate": 8.867472657220829e-06, "loss": 0.009, "num_tokens": 7485330.0, "reward": 1.97511625289917, "reward_std": 0.13916873931884766, "rewards/check_originality_func/mean": 0.2857142984867096, "rewards/check_originality_func/std": 0.4558423161506653, "rewards/perplexity_reward/mean": 0.7560686469078064, "rewards/perplexity_reward/std": 0.07519328594207764, "rewards/sentence_count_match_reward_logic/mean": 0.9333333373069763, "rewards/sentence_count_match_reward_logic/std": 0.14702436327934265, "step": 1697 }, { "clip_ratio": 0.018657425418496132, "epoch": 0.5941217634709587, "grad_norm": 0.34359096527649097, "kl": 3.90625, "learning_rate": 8.865536525143758e-06, "loss": 0.0068, "step": 1698 }, { "clip_ratio": 0.040095873177051544, "epoch": 0.5944716585024493, "grad_norm": 0.20034940005508511, "kl": 3.90625, "learning_rate": 8.86359895125815e-06, "loss": 0.0052, "step": 1699 }, { "clip_ratio": 0.07173658162355423, "epoch": 0.5948215535339398, "grad_norm": 0.1751290999080419, "kl": 3.90625, "learning_rate": 8.861659936286698e-06, "loss": 0.0046, "step": 1700 }, { "clip_ratio": 0.0038754690904170275, "completions/clipped_ratio": 0.0, "completions/max_length": 154.0, "completions/max_terminated_length": 154.0, "completions/mean_length": 50.55357360839844, "completions/mean_terminated_length": 50.55357360839844, "completions/min_length": 6.0, "completions/min_terminated_length": 6.0, "epoch": 0.5951714485654304, "grad_norm": 0.40436207686632963, "kl": 3.609375, "learning_rate": 8.859719480952637e-06, "loss": 0.008, "num_tokens": 7494241.0, "reward": 2.067582845687866, "reward_std": 0.04025455564260483, "rewards/check_originality_func/mean": 0.2857142984867096, "rewards/check_originality_func/std": 0.4558423161506653, "rewards/perplexity_reward/mean": 0.7931780219078064, "rewards/perplexity_reward/std": 0.0594208650290966, "rewards/sentence_count_match_reward_logic/mean": 0.9886904954910278, "rewards/sentence_count_match_reward_logic/std": 0.037753477692604065, "step": 1701 }, { "clip_ratio": 0.0185269583016634, "epoch": 0.5955213435969209, "grad_norm": 0.31128489698627937, "kl": 3.640625, "learning_rate": 8.85777758597974e-06, "loss": 0.0066, "step": 1702 }, { "clip_ratio": 0.0348929762840271, "epoch": 0.5958712386284115, "grad_norm": 0.253016420663198, "kl": 3.734375, "learning_rate": 8.855834252092315e-06, "loss": 0.0052, "step": 1703 }, { "clip_ratio": 0.06661706417798996, "epoch": 0.596221133659902, "grad_norm": 0.23423936513959634, "kl": 3.796875, "learning_rate": 8.853889480015208e-06, "loss": 0.0039, "step": 1704 }, { "clip_ratio": 0.003353037405759096, "completions/clipped_ratio": 0.0357142857142857, "completions/max_length": 256.0, "completions/max_terminated_length": 244.0, "completions/mean_length": 75.33928680419922, "completions/mean_terminated_length": 68.64814758300781, "completions/min_length": 6.0, "completions/min_terminated_length": 6.0, "epoch": 0.5965710286913926, "grad_norm": 0.4383139662204817, "kl": 2.203125, "learning_rate": 8.851943270473797e-06, "loss": 0.0065, "num_tokens": 7506380.0, "reward": 2.2457218170166016, "reward_std": 0.3381548523902893, "rewards/check_originality_func/mean": 0.4464285671710968, "rewards/check_originality_func/std": 0.5016207695007324, "rewards/perplexity_reward/mean": 0.8141741156578064, "rewards/perplexity_reward/std": 0.07151690125465393, "rewards/sentence_count_match_reward_logic/mean": 0.9851189851760864, "rewards/sentence_count_match_reward_logic/std": 0.05033119022846222, "step": 1705 }, { "clip_ratio": 0.017210569232702255, "epoch": 0.5969209237228832, "grad_norm": 0.34763304783583304, "kl": 2.203125, "learning_rate": 8.849995624194006e-06, "loss": 0.0045, "step": 1706 }, { "clip_ratio": 0.04431062564253807, "epoch": 0.5972708187543737, "grad_norm": 0.2331320990036096, "kl": 2.21875, "learning_rate": 8.84804654190228e-06, "loss": 0.0028, "step": 1707 }, { "clip_ratio": 0.06992732733488083, "epoch": 0.5976207137858642, "grad_norm": 0.20029573244442686, "kl": 2.21875, "learning_rate": 8.846096024325616e-06, "loss": 0.0015, "step": 1708 }, { "clip_ratio": 0.006225175224244595, "completions/clipped_ratio": 0.0, "completions/max_length": 235.0, "completions/max_terminated_length": 235.0, "completions/mean_length": 80.08928680419922, "completions/mean_terminated_length": 80.08928680419922, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "epoch": 0.5979706088173548, "grad_norm": 0.45934891987320176, "kl": 3.03125, "learning_rate": 8.844144072191537e-06, "loss": 0.0072, "num_tokens": 7519113.0, "reward": 2.141587734222412, "reward_std": 0.20402923226356506, "rewards/check_originality_func/mean": 0.3928571343421936, "rewards/check_originality_func/std": 0.4928053915500641, "rewards/perplexity_reward/mean": 0.8058733344078064, "rewards/perplexity_reward/std": 0.08876707404851913, "rewards/sentence_count_match_reward_logic/mean": 0.9428571462631226, "rewards/sentence_count_match_reward_logic/std": 0.1112600639462471, "step": 1709 }, { "clip_ratio": 0.011603184975683689, "epoch": 0.5983205038488454, "grad_norm": 0.3253055615319764, "kl": 3.03125, "learning_rate": 8.842190686228098e-06, "loss": 0.0049, "step": 1710 }, { "clip_ratio": 0.02985481731593609, "epoch": 0.5986703988803359, "grad_norm": 0.24834985280010738, "kl": 3.03125, "learning_rate": 8.8402358671639e-06, "loss": 0.0035, "step": 1711 }, { "clip_ratio": 0.05873802304267883, "epoch": 0.5990202939118264, "grad_norm": 0.20928902654012205, "kl": 3.03125, "learning_rate": 8.838279615728067e-06, "loss": 0.0026, "step": 1712 }, { "clip_ratio": 0.004002052824944258, "completions/clipped_ratio": 0.0, "completions/max_length": 243.0, "completions/max_terminated_length": 243.0, "completions/mean_length": 105.67857360839844, "completions/mean_terminated_length": 105.67857360839844, "completions/min_length": 10.0, "completions/min_terminated_length": 10.0, "epoch": 0.599370188943317, "grad_norm": 0.8981639111448058, "kl": 2.84375, "learning_rate": 8.836321932650266e-06, "loss": 0.0175, "num_tokens": 7535463.0, "reward": 2.4649577140808105, "reward_std": 0.07754355669021606, "rewards/check_originality_func/mean": 0.6964285969734192, "rewards/check_originality_func/std": 0.4639609158039093, "rewards/perplexity_reward/mean": 0.78759765625, "rewards/perplexity_reward/std": 0.0902198851108551, "rewards/sentence_count_match_reward_logic/mean": 0.9809315800666809, "rewards/sentence_count_match_reward_logic/std": 0.05113613232970238, "step": 1713 }, { "clip_ratio": 0.022291725501418114, "epoch": 0.5997200839748076, "grad_norm": 0.5514559384003078, "kl": 2.84375, "learning_rate": 8.834362818660692e-06, "loss": 0.0117, "step": 1714 }, { "clip_ratio": 0.06944454461336136, "epoch": 0.6000699790062981, "grad_norm": 0.46677159959684494, "kl": 2.859375, "learning_rate": 8.832402274490075e-06, "loss": 0.0085, "step": 1715 }, { "clip_ratio": 0.09807431697845459, "epoch": 0.6004198740377886, "grad_norm": 0.33338940671971007, "kl": 2.859375, "learning_rate": 8.830440300869681e-06, "loss": 0.0061, "step": 1716 }, { "clip_ratio": 0.003703679423779249, "completions/clipped_ratio": 0.0, "completions/max_length": 157.0, "completions/max_terminated_length": 157.0, "completions/mean_length": 77.21428680419922, "completions/mean_terminated_length": 77.21428680419922, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "epoch": 0.6007697690692793, "grad_norm": 0.5351490997784328, "kl": 2.78125, "learning_rate": 8.828476898531308e-06, "loss": 0.0064, "num_tokens": 7547755.0, "reward": 2.078836441040039, "reward_std": 0.37736430764198303, "rewards/check_originality_func/mean": 0.3035714328289032, "rewards/check_originality_func/std": 0.4639609158039093, "rewards/perplexity_reward/mean": 0.7984793782234192, "rewards/perplexity_reward/std": 0.05704037845134735, "rewards/sentence_count_match_reward_logic/mean": 0.9767857193946838, "rewards/sentence_count_match_reward_logic/std": 0.12356974184513092, "step": 1717 }, { "clip_ratio": 0.01998080313205719, "epoch": 0.6011196641007698, "grad_norm": 0.41916056020720877, "kl": 2.78125, "learning_rate": 8.826512068207286e-06, "loss": 0.0042, "step": 1718 }, { "clip_ratio": 0.050352420657873154, "epoch": 0.6014695591322603, "grad_norm": 0.35965523876324795, "kl": 2.78125, "learning_rate": 8.824545810630478e-06, "loss": 0.0018, "step": 1719 }, { "clip_ratio": 0.07533912360668182, "epoch": 0.6018194541637508, "grad_norm": 0.38032769336694333, "kl": 2.78125, "learning_rate": 8.822578126534278e-06, "loss": 0.0001, "step": 1720 }, { "clip_ratio": 0.0034243452828377485, "completions/clipped_ratio": 0.0, "completions/max_length": 232.0, "completions/max_terminated_length": 232.0, "completions/mean_length": 106.89286041259766, "completions/mean_terminated_length": 106.89286041259766, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "epoch": 0.6021693491952415, "grad_norm": 0.6410028607106282, "kl": 2.203125, "learning_rate": 8.820609016652616e-06, "loss": 0.0216, "num_tokens": 7563949.0, "reward": 2.5268802642822266, "reward_std": 0.1104428619146347, "rewards/check_originality_func/mean": 0.6964285969734192, "rewards/check_originality_func/std": 0.4639609456062317, "rewards/perplexity_reward/mean": 0.8739537000656128, "rewards/perplexity_reward/std": 0.08896555006504059, "rewards/sentence_count_match_reward_logic/mean": 0.9564980268478394, "rewards/sentence_count_match_reward_logic/std": 0.09489978849887848, "step": 1721 }, { "clip_ratio": 0.02218545600771904, "epoch": 0.602519244226732, "grad_norm": 0.5200462135849483, "kl": 2.234375, "learning_rate": 8.818638481719952e-06, "loss": 0.0176, "step": 1722 }, { "clip_ratio": 0.05034158006310463, "epoch": 0.6028691392582225, "grad_norm": 0.47458097614283135, "kl": 2.265625, "learning_rate": 8.816666522471274e-06, "loss": 0.0142, "step": 1723 }, { "clip_ratio": 0.08449045568704605, "epoch": 0.6032190342897131, "grad_norm": 0.42467977538284984, "kl": 2.28125, "learning_rate": 8.814693139642105e-06, "loss": 0.0119, "step": 1724 }, { "clip_ratio": 0.007527665235102177, "completions/clipped_ratio": 0.1071428571428571, "completions/max_length": 256.0, "completions/max_terminated_length": 250.0, "completions/mean_length": 94.37500762939453, "completions/mean_terminated_length": 74.97999572753906, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "epoch": 0.6035689293212037, "grad_norm": 1.4682890481478077, "kl": 3.375, "learning_rate": 8.812718333968498e-06, "loss": 0.0032, "num_tokens": 7578826.0, "reward": 2.237539052963257, "reward_std": 0.18826062977313995, "rewards/check_originality_func/mean": 0.5178571343421936, "rewards/check_originality_func/std": 0.5042031407356262, "rewards/perplexity_reward/mean": 0.7956891655921936, "rewards/perplexity_reward/std": 0.09428387135267258, "rewards/sentence_count_match_reward_logic/mean": 0.9239926934242249, "rewards/sentence_count_match_reward_logic/std": 0.13877609372138977, "step": 1725 }, { "clip_ratio": 0.04181591793894768, "epoch": 0.6039188243526942, "grad_norm": 1.3100785020594534, "kl": 3.171875, "learning_rate": 8.810742106187039e-06, "loss": -0.0011, "step": 1726 }, { "clip_ratio": 0.06605441868305206, "epoch": 0.6042687193841847, "grad_norm": 0.7775642067029738, "kl": 3.171875, "learning_rate": 8.808764457034839e-06, "loss": -0.0056, "step": 1727 }, { "clip_ratio": 0.10945888608694077, "epoch": 0.6046186144156753, "grad_norm": 0.5912415085112077, "kl": 3.328125, "learning_rate": 8.806785387249546e-06, "loss": -0.0089, "step": 1728 }, { "clip_ratio": 0.0038852097932249308, "completions/clipped_ratio": 0.0, "completions/max_length": 245.0, "completions/max_terminated_length": 245.0, "completions/mean_length": 93.91072082519531, "completions/mean_terminated_length": 93.91072082519531, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "epoch": 0.6049685094471658, "grad_norm": 0.5820763289185916, "kl": 3.984375, "learning_rate": 8.80480489756933e-06, "loss": 0.0059, "num_tokens": 7593037.0, "reward": 2.271883487701416, "reward_std": 0.12532773613929749, "rewards/check_originality_func/mean": 0.5178571343421936, "rewards/check_originality_func/std": 0.5042031407356262, "rewards/perplexity_reward/mean": 0.7676478624343872, "rewards/perplexity_reward/std": 0.1315932273864746, "rewards/sentence_count_match_reward_logic/mean": 0.9863783717155457, "rewards/sentence_count_match_reward_logic/std": 0.042601458728313446, "step": 1729 }, { "clip_ratio": 0.021824782714247704, "epoch": 0.6053184044786564, "grad_norm": 0.5646903454061094, "kl": 4.03125, "learning_rate": 8.8028229887329e-06, "loss": 0.0041, "step": 1730 }, { "clip_ratio": 0.05014881119132042, "epoch": 0.605668299510147, "grad_norm": 0.36135149449457976, "kl": 3.984375, "learning_rate": 8.800839661479483e-06, "loss": 0.0023, "step": 1731 }, { "clip_ratio": 0.07358580082654953, "epoch": 0.6060181945416375, "grad_norm": 0.262911040799685, "kl": 3.984375, "learning_rate": 8.798854916548847e-06, "loss": 0.0008, "step": 1732 }, { "clip_ratio": 0.0086418017745018, "completions/clipped_ratio": 0.2678571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 230.0, "completions/mean_length": 151.67857360839844, "completions/mean_terminated_length": 113.51219177246094, "completions/min_length": 7.0, "completions/min_terminated_length": 7.0, "epoch": 0.606368089573128, "grad_norm": 1.5832650256375302, "kl": 2.125, "learning_rate": 8.79686875468128e-06, "loss": -0.0107, "num_tokens": 7615563.0, "reward": 2.318070650100708, "reward_std": 0.36050766706466675, "rewards/check_originality_func/mean": 0.5892857313156128, "rewards/check_originality_func/std": 0.49641573429107666, "rewards/perplexity_reward/mean": 0.8451451063156128, "rewards/perplexity_reward/std": 0.0503639355301857, "rewards/sentence_count_match_reward_logic/mean": 0.8836396932601929, "rewards/sentence_count_match_reward_logic/std": 0.15165714919567108, "step": 1733 }, { "clip_ratio": 0.03461849316954613, "epoch": 0.6067179846046186, "grad_norm": 0.8549238822875922, "kl": 2.09375, "learning_rate": 8.794881176617602e-06, "loss": -0.0186, "step": 1734 }, { "clip_ratio": 0.09466976672410965, "epoch": 0.6070678796361092, "grad_norm": 0.8281711444568712, "kl": 2.109375, "learning_rate": 8.792892183099162e-06, "loss": -0.027, "step": 1735 }, { "clip_ratio": 0.1453481763601303, "epoch": 0.6074177746675997, "grad_norm": 0.7179200108973468, "kl": 2.21875, "learning_rate": 8.790901774867834e-06, "loss": -0.0313, "step": 1736 }, { "clip_ratio": 0.005226475652307272, "completions/clipped_ratio": 0.25, "completions/max_length": 256.0, "completions/max_terminated_length": 238.0, "completions/mean_length": 125.21429443359375, "completions/mean_terminated_length": 81.61904907226562, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "epoch": 0.6077676696990902, "grad_norm": 2.7054472740286877, "kl": 3.796875, "learning_rate": 8.788909952666024e-06, "loss": 0.0162, "num_tokens": 7634615.0, "reward": 2.2829606533050537, "reward_std": 0.15658137202262878, "rewards/check_originality_func/mean": 0.5357142686843872, "rewards/check_originality_func/std": 0.5032362937927246, "rewards/perplexity_reward/mean": 0.82080078125, "rewards/perplexity_reward/std": 0.07067481428384781, "rewards/sentence_count_match_reward_logic/mean": 0.9264455437660217, "rewards/sentence_count_match_reward_logic/std": 0.12050669640302658, "step": 1737 }, { "clip_ratio": 0.03734007477760315, "epoch": 0.6081175647305809, "grad_norm": 0.9452919372953931, "kl": 3.921875, "learning_rate": 8.78691671723666e-06, "loss": 0.0091, "step": 1738 }, { "clip_ratio": 0.09284043312072754, "epoch": 0.6084674597620714, "grad_norm": 1.0475137593562898, "kl": 4.375, "learning_rate": 8.784922069323202e-06, "loss": 0.006, "step": 1739 }, { "clip_ratio": 0.1213303804397583, "epoch": 0.6088173547935619, "grad_norm": 1.133683860184026, "kl": 4.4375, "learning_rate": 8.782926009669635e-06, "loss": 0.0038, "step": 1740 }, { "clip_ratio": 0.005994891282171011, "completions/clipped_ratio": 0.1428571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 132.0, "completions/mean_length": 94.41072082519531, "completions/mean_terminated_length": 67.47917175292969, "completions/min_length": 12.0, "completions/min_terminated_length": 12.0, "epoch": 0.6091672498250524, "grad_norm": 1.763804901208944, "kl": 2.53125, "learning_rate": 8.780928539020467e-06, "loss": 0.0142, "num_tokens": 7650094.0, "reward": 2.354069471359253, "reward_std": 0.21868975460529327, "rewards/check_originality_func/mean": 0.6071428656578064, "rewards/check_originality_func/std": 0.4928053617477417, "rewards/perplexity_reward/mean": 0.8097098469734192, "rewards/perplexity_reward/std": 0.04729754105210304, "rewards/sentence_count_match_reward_logic/mean": 0.937216579914093, "rewards/sentence_count_match_reward_logic/std": 0.11426442861557007, "step": 1741 }, { "clip_ratio": 0.06809429824352264, "epoch": 0.6095171448565431, "grad_norm": 0.8356140309538688, "kl": 2.546875, "learning_rate": 8.77892965812074e-06, "loss": 0.0103, "step": 1742 }, { "clip_ratio": 0.09361071139574051, "epoch": 0.6098670398880336, "grad_norm": 0.7726420622532352, "kl": 2.578125, "learning_rate": 8.776929367716013e-06, "loss": 0.0081, "step": 1743 }, { "clip_ratio": 0.10735733062028885, "epoch": 0.6102169349195241, "grad_norm": 0.8841985121098968, "kl": 2.59375, "learning_rate": 8.77492766855238e-06, "loss": 0.0035, "step": 1744 }, { "clip_ratio": 0.00909586064517498, "completions/clipped_ratio": 0.017857142857142905, "completions/max_length": 256.0, "completions/max_terminated_length": 193.0, "completions/mean_length": 63.000003814697266, "completions/mean_terminated_length": 59.49090576171875, "completions/min_length": 6.0, "completions/min_terminated_length": 6.0, "epoch": 0.6105668299510147, "grad_norm": 0.8288167422307733, "kl": 4.09375, "learning_rate": 8.772924561376454e-06, "loss": 0.0303, "num_tokens": 7659958.0, "reward": 2.192305326461792, "reward_std": 0.11910615861415863, "rewards/check_originality_func/mean": 0.4107142984867096, "rewards/check_originality_func/std": 0.49641573429107666, "rewards/perplexity_reward/mean": 0.7999442219734192, "rewards/perplexity_reward/std": 0.11870867013931274, "rewards/sentence_count_match_reward_logic/mean": 0.9816468358039856, "rewards/sentence_count_match_reward_logic/std": 0.0899365022778511, "step": 1745 }, { "clip_ratio": 0.023494375869631767, "epoch": 0.6109167249825053, "grad_norm": 0.7524913868475346, "kl": 4.21875, "learning_rate": 8.770920046935374e-06, "loss": 0.0272, "step": 1746 }, { "clip_ratio": 0.043859031051397324, "epoch": 0.6112666200139958, "grad_norm": 0.4737934979453514, "kl": 4.1875, "learning_rate": 8.768914125976806e-06, "loss": 0.0244, "step": 1747 }, { "clip_ratio": 0.08228971809148788, "epoch": 0.6116165150454863, "grad_norm": 0.3631469720339583, "kl": 4.09375, "learning_rate": 8.76690679924894e-06, "loss": 0.0225, "step": 1748 }, { "clip_ratio": 0.007326432503759861, "completions/clipped_ratio": 0.125, "completions/max_length": 256.0, "completions/max_terminated_length": 248.0, "completions/mean_length": 90.58928680419922, "completions/mean_terminated_length": 66.95918273925781, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "epoch": 0.6119664100769769, "grad_norm": 0.7258263616781971, "kl": 3.171875, "learning_rate": 8.764898067500488e-06, "loss": -0.0074, "num_tokens": 7675247.0, "reward": 2.291123867034912, "reward_std": 0.31652647256851196, "rewards/check_originality_func/mean": 0.5714285969734192, "rewards/check_originality_func/std": 0.4993502199649811, "rewards/perplexity_reward/mean": 0.8163365125656128, "rewards/perplexity_reward/std": 0.09914767742156982, "rewards/sentence_count_match_reward_logic/mean": 0.9033588767051697, "rewards/sentence_count_match_reward_logic/std": 0.18881022930145264, "step": 1749 }, { "clip_ratio": 0.02555449679493904, "epoch": 0.6123163051084675, "grad_norm": 0.49886687646420835, "kl": 3.171875, "learning_rate": 8.76288793148069e-06, "loss": -0.0112, "step": 1750 }, { "clip_ratio": 0.0696243867278099, "epoch": 0.612666200139958, "grad_norm": 0.3497140054006379, "kl": 3.171875, "learning_rate": 8.760876391939308e-06, "loss": -0.0142, "step": 1751 }, { "clip_ratio": 0.1023741289973259, "epoch": 0.6130160951714486, "grad_norm": 0.31613195720004866, "kl": 3.203125, "learning_rate": 8.758863449626624e-06, "loss": -0.0158, "step": 1752 }, { "clip_ratio": 0.0041266269981861115, "completions/clipped_ratio": 0.1428571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 214.0, "completions/mean_length": 90.66072082519531, "completions/mean_terminated_length": 63.10416793823242, "completions/min_length": 6.0, "completions/min_terminated_length": 6.0, "epoch": 0.6133659902029391, "grad_norm": 0.618404342034881, "kl": 3.859375, "learning_rate": 8.756849105293447e-06, "loss": 0.0077, "num_tokens": 7688956.0, "reward": 2.2475199699401855, "reward_std": 0.12779180705547333, "rewards/check_originality_func/mean": 0.4464285671710968, "rewards/check_originality_func/std": 0.5016207695007324, "rewards/perplexity_reward/mean": 0.8236607313156128, "rewards/perplexity_reward/std": 0.10166741162538528, "rewards/sentence_count_match_reward_logic/mean": 0.977430522441864, "rewards/sentence_count_match_reward_logic/std": 0.07342445105314255, "step": 1753 }, { "clip_ratio": 0.015782257542014122, "epoch": 0.6137158852344297, "grad_norm": 0.46032856751942375, "kl": 3.859375, "learning_rate": 8.754833359691115e-06, "loss": 0.0052, "step": 1754 }, { "clip_ratio": 0.06238937005400658, "epoch": 0.6140657802659202, "grad_norm": 0.36065128981733446, "kl": 3.875, "learning_rate": 8.752816213571474e-06, "loss": 0.0025, "step": 1755 }, { "clip_ratio": 0.11290116608142853, "epoch": 0.6144156752974108, "grad_norm": 0.2757472874402253, "kl": 3.921875, "learning_rate": 8.750797667686902e-06, "loss": 0.0003, "step": 1756 }, { "clip_ratio": 0.005416807718575001, "completions/clipped_ratio": 0.125, "completions/max_length": 256.0, "completions/max_terminated_length": 254.0, "completions/mean_length": 112.46429443359375, "completions/mean_terminated_length": 91.95918273925781, "completions/min_length": 10.0, "completions/min_terminated_length": 10.0, "epoch": 0.6147655703289013, "grad_norm": 1.0772323807638968, "kl": 2.78125, "learning_rate": 8.7487777227903e-06, "loss": 0.0099, "num_tokens": 7706406.0, "reward": 2.1525440216064453, "reward_std": 0.2604409456253052, "rewards/check_originality_func/mean": 0.5, "rewards/check_originality_func/std": 0.5045249462127686, "rewards/perplexity_reward/mean": 0.7805524468421936, "rewards/perplexity_reward/std": 0.11058609187602997, "rewards/sentence_count_match_reward_logic/mean": 0.8719915747642517, "rewards/sentence_count_match_reward_logic/std": 0.18551042675971985, "step": 1757 }, { "clip_ratio": 0.03159254044294357, "epoch": 0.6151154653603919, "grad_norm": 0.690240980684232, "kl": 2.796875, "learning_rate": 8.746756379635088e-06, "loss": 0.0032, "step": 1758 }, { "clip_ratio": 0.08461690694093704, "epoch": 0.6154653603918825, "grad_norm": 0.5273215615517454, "kl": 2.84375, "learning_rate": 8.744733638975206e-06, "loss": 0.0002, "step": 1759 }, { "clip_ratio": 0.13711126148700714, "epoch": 0.615815255423373, "grad_norm": 0.5123744347535555, "kl": 2.890625, "learning_rate": 8.742709501565118e-06, "loss": -0.0019, "step": 1760 }, { "clip_ratio": 0.010366156697273254, "completions/clipped_ratio": 0.1428571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 179.0, "completions/mean_length": 99.16072082519531, "completions/mean_terminated_length": 73.02083587646484, "completions/min_length": 6.0, "completions/min_terminated_length": 6.0, "epoch": 0.6161651504548635, "grad_norm": 1.1330574593497742, "kl": 2.921875, "learning_rate": 8.740683968159808e-06, "loss": 0.0136, "num_tokens": 7721783.0, "reward": 2.345520496368408, "reward_std": 0.22780542075634003, "rewards/check_originality_func/mean": 0.5892857313156128, "rewards/check_originality_func/std": 0.49641573429107666, "rewards/perplexity_reward/mean": 0.8170340657234192, "rewards/perplexity_reward/std": 0.087653249502182, "rewards/sentence_count_match_reward_logic/mean": 0.9392006993293762, "rewards/sentence_count_match_reward_logic/std": 0.10781842470169067, "step": 1761 }, { "clip_ratio": 0.04458782449364662, "epoch": 0.616515045486354, "grad_norm": 0.5427833138965922, "kl": 2.953125, "learning_rate": 8.738657039514782e-06, "loss": 0.0084, "step": 1762 }, { "clip_ratio": 0.09041338413953781, "epoch": 0.6168649405178447, "grad_norm": 0.44583422353669694, "kl": 3.0, "learning_rate": 8.736628716386064e-06, "loss": 0.0067, "step": 1763 }, { "clip_ratio": 0.12737199664115906, "epoch": 0.6172148355493352, "grad_norm": 0.7357935865778393, "kl": 3.109375, "learning_rate": 8.7345989995302e-06, "loss": 0.0069, "step": 1764 }, { "clip_ratio": 0.015565791167318821, "completions/clipped_ratio": 0.1964285714285714, "completions/max_length": 256.0, "completions/max_terminated_length": 256.0, "completions/mean_length": 126.71429443359375, "completions/mean_terminated_length": 95.11111450195312, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "epoch": 0.6175647305808257, "grad_norm": 2.554549442864004, "kl": 3.59375, "learning_rate": 8.732567889704253e-06, "loss": -0.0191, "num_tokens": 7741447.0, "reward": 2.2398526668548584, "reward_std": 0.20005087554454803, "rewards/check_originality_func/mean": 0.5178571343421936, "rewards/check_originality_func/std": 0.5042031407356262, "rewards/perplexity_reward/mean": 0.8399134874343872, "rewards/perplexity_reward/std": 0.06963248550891876, "rewards/sentence_count_match_reward_logic/mean": 0.8820819854736328, "rewards/sentence_count_match_reward_logic/std": 0.2234332114458084, "step": 1765 }, { "clip_ratio": 0.049983296543359756, "epoch": 0.6179146256123164, "grad_norm": 0.9345492550412965, "kl": 3.703125, "learning_rate": 8.730535387665812e-06, "loss": -0.0255, "step": 1766 }, { "clip_ratio": 0.10213087499141693, "epoch": 0.6182645206438069, "grad_norm": 1.3159832087076906, "kl": 3.921875, "learning_rate": 8.728501494172976e-06, "loss": -0.0279, "step": 1767 }, { "clip_ratio": 0.13417337834835052, "epoch": 0.6186144156752974, "grad_norm": 0.8003157365341573, "kl": 3.828125, "learning_rate": 8.726466209984372e-06, "loss": -0.0312, "step": 1768 }, { "clip_ratio": 0.007245491724461317, "completions/clipped_ratio": 0.017857142857142905, "completions/max_length": 256.0, "completions/max_terminated_length": 143.0, "completions/mean_length": 54.07143020629883, "completions/mean_terminated_length": 50.39999771118164, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "epoch": 0.6189643107067879, "grad_norm": 2.6309386673581803, "kl": 5.53125, "learning_rate": 8.72442953585914e-06, "loss": 0.0353, "num_tokens": 7750611.0, "reward": 2.0578126907348633, "reward_std": 0.2510308623313904, "rewards/check_originality_func/mean": 0.3571428656578064, "rewards/check_originality_func/std": 0.48349377512931824, "rewards/perplexity_reward/mean": 0.7444196343421936, "rewards/perplexity_reward/std": 0.14879192411899567, "rewards/sentence_count_match_reward_logic/mean": 0.956250011920929, "rewards/sentence_count_match_reward_logic/std": 0.16179430484771729, "step": 1769 }, { "clip_ratio": 0.07250972837209702, "epoch": 0.6193142057382786, "grad_norm": 1.2914704423970258, "kl": 5.53125, "learning_rate": 8.72239147255694e-06, "loss": 0.0292, "step": 1770 }, { "clip_ratio": 0.1298639327287674, "epoch": 0.6196641007697691, "grad_norm": 4.435993915686819, "kl": 6.1875, "learning_rate": 8.720352020837951e-06, "loss": 0.0275, "step": 1771 }, { "clip_ratio": 0.19612431526184082, "epoch": 0.6200139958012596, "grad_norm": 0.46240502453694526, "kl": 5.75, "learning_rate": 8.71831118146287e-06, "loss": 0.0231, "step": 1772 }, { "clip_ratio": 0.0189441479742527, "completions/clipped_ratio": 0.1607142857142857, "completions/max_length": 256.0, "completions/max_terminated_length": 248.0, "completions/mean_length": 125.28572082519531, "completions/mean_terminated_length": 100.25531768798828, "completions/min_length": 12.0, "completions/min_terminated_length": 12.0, "epoch": 0.6203638908327501, "grad_norm": 2.9960265604516136, "kl": 2.53125, "learning_rate": 8.716268955192908e-06, "loss": 0.0433, "num_tokens": 7770035.0, "reward": 2.2766199111938477, "reward_std": 0.21593984961509705, "rewards/check_originality_func/mean": 0.5357142686843872, "rewards/check_originality_func/std": 0.5032362937927246, "rewards/perplexity_reward/mean": 0.8097098469734192, "rewards/perplexity_reward/std": 0.08912202715873718, "rewards/sentence_count_match_reward_logic/mean": 0.931195855140686, "rewards/sentence_count_match_reward_logic/std": 0.16521097719669342, "step": 1773 }, { "clip_ratio": 0.07292866706848145, "epoch": 0.6207137858642408, "grad_norm": 1.5143732624618575, "kl": 2.671875, "learning_rate": 8.714225342789799e-06, "loss": 0.036, "step": 1774 }, { "clip_ratio": 0.10670679062604904, "epoch": 0.6210636808957313, "grad_norm": 0.9742975570821415, "kl": 2.703125, "learning_rate": 8.712180345015789e-06, "loss": 0.0319, "step": 1775 }, { "clip_ratio": 0.13688382506370544, "epoch": 0.6214135759272218, "grad_norm": 2.25405679248316, "kl": 2.578125, "learning_rate": 8.710133962633644e-06, "loss": 0.0304, "step": 1776 }, { "clip_ratio": 0.007077476475387812, "completions/clipped_ratio": 0.0, "completions/max_length": 255.0, "completions/max_terminated_length": 255.0, "completions/mean_length": 82.78572082519531, "completions/mean_terminated_length": 82.78572082519531, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "epoch": 0.6217634709587124, "grad_norm": 0.8012036755149312, "kl": 5.46875, "learning_rate": 8.708086196406646e-06, "loss": 0.0053, "num_tokens": 7782727.0, "reward": 2.191897392272949, "reward_std": 0.0467224046587944, "rewards/check_originality_func/mean": 0.4285714328289032, "rewards/check_originality_func/std": 0.4993502199649811, "rewards/perplexity_reward/mean": 0.77392578125, "rewards/perplexity_reward/std": 0.09440191835165024, "rewards/sentence_count_match_reward_logic/mean": 0.9894002079963684, "rewards/sentence_count_match_reward_logic/std": 0.028395719826221466, "step": 1777 }, { "clip_ratio": 0.04900960251688957, "epoch": 0.622113365990203, "grad_norm": 0.4473298715269865, "kl": 5.53125, "learning_rate": 8.70603704709859e-06, "loss": 0.0025, "step": 1778 }, { "clip_ratio": 0.09947439283132553, "epoch": 0.6224632610216935, "grad_norm": 0.40432546588781815, "kl": 5.71875, "learning_rate": 8.70398651547379e-06, "loss": 0.0012, "step": 1779 }, { "clip_ratio": 0.13007156550884247, "epoch": 0.622813156053184, "grad_norm": 0.33008450237637016, "kl": 5.84375, "learning_rate": 8.701934602297077e-06, "loss": 0.0001, "step": 1780 }, { "clip_ratio": 0.014798053540289402, "completions/clipped_ratio": 0.0535714285714286, "completions/max_length": 256.0, "completions/max_terminated_length": 254.0, "completions/mean_length": 91.46428680419922, "completions/mean_terminated_length": 82.15094757080078, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "epoch": 0.6231630510846746, "grad_norm": 1.4630948613448194, "kl": 3.21875, "learning_rate": 8.699881308333794e-06, "loss": 0.0047, "num_tokens": 7798809.0, "reward": 2.0098226070404053, "reward_std": 0.24453450739383698, "rewards/check_originality_func/mean": 0.4107142984867096, "rewards/check_originality_func/std": 0.49641573429107666, "rewards/perplexity_reward/mean": 0.7902483344078064, "rewards/perplexity_reward/std": 0.06634113192558289, "rewards/sentence_count_match_reward_logic/mean": 0.8088598847389221, "rewards/sentence_count_match_reward_logic/std": 0.22148309648036957, "step": 1781 }, { "clip_ratio": 0.06129609793424606, "epoch": 0.6235129461161651, "grad_norm": 0.7973594676678932, "kl": 3.25, "learning_rate": 8.697826634349799e-06, "loss": 0.0004, "step": 1782 }, { "clip_ratio": 0.09295396506786346, "epoch": 0.6238628411476557, "grad_norm": 0.4646410307188732, "kl": 3.296875, "learning_rate": 8.695770581111467e-06, "loss": -0.0026, "step": 1783 }, { "clip_ratio": 0.12804265320301056, "epoch": 0.6242127361791463, "grad_norm": 0.4719099510717533, "kl": 3.390625, "learning_rate": 8.693713149385687e-06, "loss": -0.0044, "step": 1784 }, { "clip_ratio": 0.011892820708453655, "completions/clipped_ratio": 0.0714285714285714, "completions/max_length": 256.0, "completions/max_terminated_length": 243.0, "completions/mean_length": 85.01786041259766, "completions/mean_terminated_length": 71.86538696289062, "completions/min_length": 10.0, "completions/min_terminated_length": 10.0, "epoch": 0.6245626312106368, "grad_norm": 1.1324923455498785, "kl": 3.15625, "learning_rate": 8.69165433993986e-06, "loss": 0.0147, "num_tokens": 7813018.0, "reward": 2.332813262939453, "reward_std": 0.2541000247001648, "rewards/check_originality_func/mean": 0.5892857313156128, "rewards/check_originality_func/std": 0.49641573429107666, "rewards/perplexity_reward/mean": 0.86474609375, "rewards/perplexity_reward/std": 0.06517517566680908, "rewards/sentence_count_match_reward_logic/mean": 0.8787814974784851, "rewards/sentence_count_match_reward_logic/std": 0.13611090183258057, "step": 1785 }, { "clip_ratio": 0.08319657295942307, "epoch": 0.6249125262421273, "grad_norm": 0.8300194181620881, "kl": 3.484375, "learning_rate": 8.689594153541904e-06, "loss": 0.0107, "step": 1786 }, { "clip_ratio": 0.1196819469332695, "epoch": 0.6252624212736179, "grad_norm": 0.6445007257263325, "kl": 3.828125, "learning_rate": 8.687532590960247e-06, "loss": 0.0069, "step": 1787 }, { "clip_ratio": 0.18590934574604034, "epoch": 0.6256123163051085, "grad_norm": 0.3568739310406084, "kl": 3.953125, "learning_rate": 8.685469652963837e-06, "loss": 0.0042, "step": 1788 }, { "clip_ratio": 0.01311271172016859, "completions/clipped_ratio": 0.3035714285714286, "completions/max_length": 256.0, "completions/max_terminated_length": 250.0, "completions/mean_length": 144.9107208251953, "completions/mean_terminated_length": 96.4871826171875, "completions/min_length": 7.0, "completions/min_terminated_length": 7.0, "epoch": 0.625962211336599, "grad_norm": 2.144671097618565, "kl": 4.03125, "learning_rate": 8.683405340322123e-06, "loss": 0.0545, "num_tokens": 7834021.0, "reward": 2.1742639541625977, "reward_std": 0.20515117049217224, "rewards/check_originality_func/mean": 0.5178571343421936, "rewards/check_originality_func/std": 0.5042031407356262, "rewards/perplexity_reward/mean": 0.7981305718421936, "rewards/perplexity_reward/std": 0.12181923538446426, "rewards/sentence_count_match_reward_logic/mean": 0.8582760691642761, "rewards/sentence_count_match_reward_logic/std": 0.1910545825958252, "step": 1789 }, { "clip_ratio": 0.03731716424226761, "epoch": 0.6263121063680895, "grad_norm": 0.9301726582462685, "kl": 4.0625, "learning_rate": 8.68133965380508e-06, "loss": 0.0451, "step": 1790 }, { "clip_ratio": 0.06485121697187424, "epoch": 0.6266620013995802, "grad_norm": 2.7965235873198426, "kl": 4.15625, "learning_rate": 8.679272594183187e-06, "loss": 0.0442, "step": 1791 }, { "clip_ratio": 0.09812406450510025, "epoch": 0.6270118964310707, "grad_norm": 2.8489699189845132, "kl": 4.5625, "learning_rate": 8.67720416222744e-06, "loss": 0.0451, "step": 1792 }, { "clip_ratio": 0.01048736460506916, "completions/clipped_ratio": 0.0, "completions/max_length": 108.0, "completions/max_terminated_length": 108.0, "completions/mean_length": 37.94643020629883, "completions/mean_terminated_length": 37.94643020629883, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "epoch": 0.6273617914625612, "grad_norm": 2.1960814808209905, "kl": 6.84375, "learning_rate": 8.675134358709341e-06, "loss": 0.013, "num_tokens": 7841402.0, "reward": 2.041490077972412, "reward_std": 0.1525724232196808, "rewards/check_originality_func/mean": 0.3214285671710968, "rewards/check_originality_func/std": 0.4712514281272888, "rewards/perplexity_reward/mean": 0.7325614094734192, "rewards/perplexity_reward/std": 0.16425341367721558, "rewards/sentence_count_match_reward_logic/mean": 0.987500011920929, "rewards/sentence_count_match_reward_logic/std": 0.05330017954111099, "step": 1793 }, { "clip_ratio": 0.04007609561085701, "epoch": 0.6277116864940517, "grad_norm": 0.6197071464949776, "kl": 7.0625, "learning_rate": 8.67306318440091e-06, "loss": 0.0097, "step": 1794 }, { "clip_ratio": 0.15204580128192902, "epoch": 0.6280615815255424, "grad_norm": 0.5075760212315498, "kl": 7.25, "learning_rate": 8.670990640074675e-06, "loss": 0.0079, "step": 1795 }, { "clip_ratio": 0.21164846420288086, "epoch": 0.6284114765570329, "grad_norm": 0.39938860477292626, "kl": 7.28125, "learning_rate": 8.668916726503674e-06, "loss": 0.0064, "step": 1796 }, { "clip_ratio": 0.010953099466860294, "completions/clipped_ratio": 0.0, "completions/max_length": 253.0, "completions/max_terminated_length": 253.0, "completions/mean_length": 84.01786041259766, "completions/mean_terminated_length": 84.01786041259766, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "epoch": 0.6287613715885234, "grad_norm": 1.066641231304028, "kl": 4.5, "learning_rate": 8.666841444461456e-06, "loss": 0.0068, "num_tokens": 7854555.0, "reward": 2.4108598232269287, "reward_std": 0.1568652242422104, "rewards/check_originality_func/mean": 0.6428571343421936, "rewards/check_originality_func/std": 0.48349377512931824, "rewards/perplexity_reward/mean": 0.7877371907234192, "rewards/perplexity_reward/std": 0.09877702593803406, "rewards/sentence_count_match_reward_logic/mean": 0.9802656173706055, "rewards/sentence_count_match_reward_logic/std": 0.054421212524175644, "step": 1797 }, { "clip_ratio": 0.04614090546965599, "epoch": 0.629111266620014, "grad_norm": 0.6113088457856038, "kl": 4.5625, "learning_rate": 8.664764794722087e-06, "loss": 0.002, "step": 1798 }, { "clip_ratio": 0.08353400975465775, "epoch": 0.6294611616515046, "grad_norm": 0.4154945985530144, "kl": 4.65625, "learning_rate": 8.662686778060131e-06, "loss": 0.0002, "step": 1799 }, { "clip_ratio": 0.11190599948167801, "epoch": 0.6298110566829951, "grad_norm": 0.32956130501366804, "kl": 4.8125, "learning_rate": 8.660607395250673e-06, "loss": -0.0001, "step": 1800 }, { "clip_ratio": 0.014899051748216152, "completions/clipped_ratio": 0.017857142857142905, "completions/max_length": 256.0, "completions/max_terminated_length": 117.0, "completions/mean_length": 51.96428680419922, "completions/mean_terminated_length": 48.25454330444336, "completions/min_length": 7.0, "completions/min_terminated_length": 7.0, "epoch": 0.6301609517144856, "grad_norm": 5.020603848066533, "kl": 5.1875, "learning_rate": 8.658526647069303e-06, "loss": 0.0414, "num_tokens": 7863017.0, "reward": 1.9722083806991577, "reward_std": 0.15510223805904388, "rewards/check_originality_func/mean": 0.2321428507566452, "rewards/check_originality_func/std": 0.4260205328464508, "rewards/perplexity_reward/mean": 0.7803431749343872, "rewards/perplexity_reward/std": 0.13549470901489258, "rewards/sentence_count_match_reward_logic/mean": 0.9597222208976746, "rewards/sentence_count_match_reward_logic/std": 0.10726290196180344, "step": 1801 }, { "clip_ratio": 0.058224327862262726, "epoch": 0.6305108467459762, "grad_norm": 2.57268361301476, "kl": 6.28125, "learning_rate": 8.656444534292116e-06, "loss": 0.0405, "step": 1802 }, { "clip_ratio": 0.0818684846162796, "epoch": 0.6308607417774668, "grad_norm": 2.083010565023874, "kl": 5.9375, "learning_rate": 8.654361057695727e-06, "loss": 0.0342, "step": 1803 }, { "clip_ratio": 0.09353293478488922, "epoch": 0.6312106368089573, "grad_norm": 2.773136040798555, "kl": 5.1875, "learning_rate": 8.652276218057248e-06, "loss": 0.0287, "step": 1804 }, { "clip_ratio": 0.01821485348045826, "completions/clipped_ratio": 0.375, "completions/max_length": 256.0, "completions/max_terminated_length": 255.0, "completions/mean_length": 140.5357208251953, "completions/mean_terminated_length": 71.25714111328125, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "epoch": 0.6315605318404479, "grad_norm": 3.091343847036453, "kl": 3.453125, "learning_rate": 8.650190016154307e-06, "loss": 0.0082, "num_tokens": 7884215.0, "reward": 2.0652260780334473, "reward_std": 0.259190171957016, "rewards/check_originality_func/mean": 0.4285714328289032, "rewards/check_originality_func/std": 0.4993502199649811, "rewards/perplexity_reward/mean": 0.8214983344078064, "rewards/perplexity_reward/std": 0.12953191995620728, "rewards/sentence_count_match_reward_logic/mean": 0.8151562809944153, "rewards/sentence_count_match_reward_logic/std": 0.24477331340312958, "step": 1805 }, { "clip_ratio": 0.06411755830049515, "epoch": 0.6319104268719384, "grad_norm": 1.8015041630765354, "kl": 3.40625, "learning_rate": 8.648102452765036e-06, "loss": -0.0078, "step": 1806 }, { "clip_ratio": 0.13137216866016388, "epoch": 0.632260321903429, "grad_norm": 0.8129768176752781, "kl": 3.46875, "learning_rate": 8.646013528668078e-06, "loss": -0.014, "step": 1807 }, { "clip_ratio": 0.16938723623752594, "epoch": 0.6326102169349195, "grad_norm": 0.8318622099362754, "kl": 3.609375, "learning_rate": 8.64392324464258e-06, "loss": -0.0157, "step": 1808 }, { "clip_ratio": 0.01392687950283289, "completions/clipped_ratio": 0.0357142857142857, "completions/max_length": 256.0, "completions/max_terminated_length": 239.0, "completions/mean_length": 86.01786041259766, "completions/mean_terminated_length": 79.72222137451172, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "epoch": 0.6329601119664101, "grad_norm": 1.3395957855594516, "kl": 3.765625, "learning_rate": 8.641831601468198e-06, "loss": 0.0227, "num_tokens": 7896760.0, "reward": 2.236656904220581, "reward_std": 0.25224247574806213, "rewards/check_originality_func/mean": 0.4821428656578064, "rewards/check_originality_func/std": 0.5042031407356262, "rewards/perplexity_reward/mean": 0.8238002061843872, "rewards/perplexity_reward/std": 0.0756961926817894, "rewards/sentence_count_match_reward_logic/mean": 0.9307136535644531, "rewards/sentence_count_match_reward_logic/std": 0.16246569156646729, "step": 1809 }, { "clip_ratio": 0.030325694009661674, "epoch": 0.6333100069979006, "grad_norm": 0.6810495183041251, "kl": 3.859375, "learning_rate": 8.639738599925098e-06, "loss": 0.0179, "step": 1810 }, { "clip_ratio": 0.06283465772867203, "epoch": 0.6336599020293912, "grad_norm": 0.635995754842519, "kl": 3.890625, "learning_rate": 8.637644240793947e-06, "loss": 0.0149, "step": 1811 }, { "clip_ratio": 0.09774459898471832, "epoch": 0.6340097970608818, "grad_norm": 0.40548359991447946, "kl": 3.90625, "learning_rate": 8.635548524855924e-06, "loss": 0.012, "step": 1812 }, { "clip_ratio": 0.011584863997995853, "completions/clipped_ratio": 0.017857142857142905, "completions/max_length": 256.0, "completions/max_terminated_length": 254.0, "completions/mean_length": 126.55357360839844, "completions/mean_terminated_length": 124.19999694824219, "completions/min_length": 7.0, "completions/min_terminated_length": 7.0, "epoch": 0.6343596920923723, "grad_norm": 1.205738942613777, "kl": 3.359375, "learning_rate": 8.633451452892707e-06, "loss": 0.0141, "num_tokens": 7914615.0, "reward": 2.348172664642334, "reward_std": 0.22402401268482208, "rewards/check_originality_func/mean": 0.5714285969734192, "rewards/check_originality_func/std": 0.4993502199649811, "rewards/perplexity_reward/mean": 0.8601422905921936, "rewards/perplexity_reward/std": 0.05896153301000595, "rewards/sentence_count_match_reward_logic/mean": 0.9166017174720764, "rewards/sentence_count_match_reward_logic/std": 0.14672407507896423, "step": 1813 }, { "clip_ratio": 0.04535529762506485, "epoch": 0.6347095871238628, "grad_norm": 1.1959656782122183, "kl": 3.34375, "learning_rate": 8.631353025686487e-06, "loss": 0.0103, "step": 1814 }, { "clip_ratio": 0.06839443743228912, "epoch": 0.6350594821553533, "grad_norm": 6.662561626410496, "kl": 3.71875, "learning_rate": 8.629253244019957e-06, "loss": 0.0097, "step": 1815 }, { "clip_ratio": 0.10395845025777817, "epoch": 0.635409377186844, "grad_norm": 1.43360775070123, "kl": 3.546875, "learning_rate": 8.627152108676316e-06, "loss": 0.0072, "step": 1816 }, { "clip_ratio": 0.009933979250490665, "completions/clipped_ratio": 0.0, "completions/max_length": 216.0, "completions/max_terminated_length": 216.0, "completions/mean_length": 82.75, "completions/mean_terminated_length": 82.75, "completions/min_length": 7.0, "completions/min_terminated_length": 7.0, "epoch": 0.6357592722183345, "grad_norm": 1.2584131299784487, "kl": 4.46875, "learning_rate": 8.625049620439266e-06, "loss": 0.0191, "num_tokens": 7927689.0, "reward": 2.31623911857605, "reward_std": 0.12122257053852081, "rewards/check_originality_func/mean": 0.5357142686843872, "rewards/check_originality_func/std": 0.5032362937927246, "rewards/perplexity_reward/mean": 0.8077567219734192, "rewards/perplexity_reward/std": 0.08236948400735855, "rewards/sentence_count_match_reward_logic/mean": 0.9727678298950195, "rewards/sentence_count_match_reward_logic/std": 0.058246493339538574, "step": 1817 }, { "clip_ratio": 0.05634466931223869, "epoch": 0.636109167249825, "grad_norm": 0.5570300378615869, "kl": 4.5, "learning_rate": 8.622945780093017e-06, "loss": 0.0172, "step": 1818 }, { "clip_ratio": 0.10213716328144073, "epoch": 0.6364590622813157, "grad_norm": 0.48160814669660673, "kl": 4.53125, "learning_rate": 8.62084058842228e-06, "loss": 0.0156, "step": 1819 }, { "clip_ratio": 0.1241716668009758, "epoch": 0.6368089573128062, "grad_norm": 0.44874038191962884, "kl": 4.53125, "learning_rate": 8.618734046212275e-06, "loss": 0.0145, "step": 1820 }, { "clip_ratio": 0.013377645052969456, "completions/clipped_ratio": 0.1428571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 92.0, "completions/mean_length": 65.58928680419922, "completions/mean_terminated_length": 33.85416793823242, "completions/min_length": 6.0, "completions/min_terminated_length": 6.0, "epoch": 0.6371588523442967, "grad_norm": 1.6949845450686274, "kl": 4.8125, "learning_rate": 8.616626154248717e-06, "loss": 0.0114, "num_tokens": 7938866.0, "reward": 2.083803653717041, "reward_std": 0.15116900205612183, "rewards/check_originality_func/mean": 0.375, "rewards/check_originality_func/std": 0.48850420117378235, "rewards/perplexity_reward/mean": 0.7703682780265808, "rewards/perplexity_reward/std": 0.08076063543558121, "rewards/sentence_count_match_reward_logic/mean": 0.9384353756904602, "rewards/sentence_count_match_reward_logic/std": 0.09219760447740555, "step": 1821 }, { "clip_ratio": 0.020800843834877014, "epoch": 0.6375087473757872, "grad_norm": 0.5026458893575801, "kl": 4.84375, "learning_rate": 8.614516913317835e-06, "loss": 0.0031, "step": 1822 }, { "clip_ratio": 0.05230046436190605, "epoch": 0.6378586424072779, "grad_norm": 0.4156628776543931, "kl": 4.96875, "learning_rate": 8.61240632420635e-06, "loss": 0.0014, "step": 1823 }, { "clip_ratio": 0.09427648037672043, "epoch": 0.6382085374387684, "grad_norm": 0.41184695520650044, "kl": 5.0625, "learning_rate": 8.6102943877015e-06, "loss": 0.0017, "step": 1824 }, { "clip_ratio": 0.010482722893357277, "completions/clipped_ratio": 0.0, "completions/max_length": 69.0, "completions/max_terminated_length": 69.0, "completions/mean_length": 17.25, "completions/mean_terminated_length": 17.25, "completions/min_length": 6.0, "completions/min_terminated_length": 6.0, "epoch": 0.6385584324702589, "grad_norm": 0.31859494132706373, "kl": 8.4375, "learning_rate": 8.608181104591008e-06, "loss": 0.0045, "num_tokens": 7943472.0, "reward": 1.9023438692092896, "reward_std": 0.04407630115747452, "rewards/check_originality_func/mean": 0.1428571492433548, "rewards/check_originality_func/std": 0.3530939221382141, "rewards/perplexity_reward/mean": 0.7594866156578064, "rewards/perplexity_reward/std": 0.10532490909099579, "rewards/sentence_count_match_reward_logic/mean": 1.0, "rewards/sentence_count_match_reward_logic/std": 0.0, "step": 1825 }, { "clip_ratio": 0.03526432439684868, "epoch": 0.6389083275017495, "grad_norm": 0.26124952135135965, "kl": 8.5625, "learning_rate": 8.606066475663116e-06, "loss": 0.0041, "step": 1826 }, { "clip_ratio": 0.09989693015813828, "epoch": 0.63925822253324, "grad_norm": 0.17568459651985527, "kl": 8.625, "learning_rate": 8.603950501706555e-06, "loss": 0.0035, "step": 1827 }, { "clip_ratio": 0.16823358833789825, "epoch": 0.6396081175647306, "grad_norm": 0.14451334727237314, "kl": 8.625, "learning_rate": 8.601833183510565e-06, "loss": 0.0032, "step": 1828 }, { "clip_ratio": 0.025328483432531357, "completions/clipped_ratio": 0.0892857142857143, "completions/max_length": 256.0, "completions/max_terminated_length": 255.0, "completions/mean_length": 71.66072082519531, "completions/mean_terminated_length": 53.58823776245117, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "epoch": 0.6399580125962211, "grad_norm": 1.2610184144189698, "kl": 4.6875, "learning_rate": 8.59971452186489e-06, "loss": 0.0066, "num_tokens": 7955477.0, "reward": 2.1006884574890137, "reward_std": 0.17006216943264008, "rewards/check_originality_func/mean": 0.3571428656578064, "rewards/check_originality_func/std": 0.48349377512931824, "rewards/perplexity_reward/mean": 0.7798548936843872, "rewards/perplexity_reward/std": 0.16251711547374725, "rewards/sentence_count_match_reward_logic/mean": 0.963690459728241, "rewards/sentence_count_match_reward_logic/std": 0.09364165365695953, "step": 1829 }, { "clip_ratio": 0.06883184611797333, "epoch": 0.6403079076277117, "grad_norm": 0.6819363955301501, "kl": 4.6875, "learning_rate": 8.597594517559766e-06, "loss": 0.0037, "step": 1830 }, { "clip_ratio": 0.13335064053535461, "epoch": 0.6406578026592022, "grad_norm": 0.5165910748588479, "kl": 4.71875, "learning_rate": 8.595473171385936e-06, "loss": 0.0016, "step": 1831 }, { "clip_ratio": 0.1732528954744339, "epoch": 0.6410076976906928, "grad_norm": 0.32739715875171477, "kl": 4.71875, "learning_rate": 8.593350484134642e-06, "loss": -0.0003, "step": 1832 }, { "clip_ratio": 0.01896095834672451, "completions/clipped_ratio": 0.125, "completions/max_length": 256.0, "completions/max_terminated_length": 250.0, "completions/mean_length": 119.50000762939453, "completions/mean_terminated_length": 100.0, "completions/min_length": 6.0, "completions/min_terminated_length": 6.0, "epoch": 0.6413575927221833, "grad_norm": 2.9764604115927966, "kl": 3.09375, "learning_rate": 8.591226456597626e-06, "loss": 0.0189, "num_tokens": 7974089.0, "reward": 2.5440969467163086, "reward_std": 0.17913737893104553, "rewards/check_originality_func/mean": 0.7678571343421936, "rewards/check_originality_func/std": 0.4260205328464508, "rewards/perplexity_reward/mean": 0.8194754719734192, "rewards/perplexity_reward/std": 0.05736985802650452, "rewards/sentence_count_match_reward_logic/mean": 0.9567641019821167, "rewards/sentence_count_match_reward_logic/std": 0.06933669000864029, "step": 1833 }, { "clip_ratio": 0.07876507937908173, "epoch": 0.6417074877536739, "grad_norm": 1.0887602696783998, "kl": 2.84375, "learning_rate": 8.58910108956713e-06, "loss": 0.0152, "step": 1834 }, { "clip_ratio": 0.11462269723415375, "epoch": 0.6420573827851644, "grad_norm": 0.8855403062039127, "kl": 2.921875, "learning_rate": 8.586974383835897e-06, "loss": 0.0117, "step": 1835 }, { "clip_ratio": 0.18255221843719482, "epoch": 0.642407277816655, "grad_norm": 0.6438370777489445, "kl": 3.171875, "learning_rate": 8.584846340197167e-06, "loss": 0.008, "step": 1836 }, { "clip_ratio": 0.010236040689051151, "completions/clipped_ratio": 0.1071428571428571, "completions/max_length": 256.0, "completions/max_terminated_length": 254.0, "completions/mean_length": 100.67857360839844, "completions/mean_terminated_length": 82.04000091552734, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "epoch": 0.6427571728481456, "grad_norm": 2.899397890101731, "kl": 3.609375, "learning_rate": 8.582716959444679e-06, "loss": 0.0111, "num_tokens": 7990911.0, "reward": 2.383228063583374, "reward_std": 0.137113556265831, "rewards/check_originality_func/mean": 0.6785714030265808, "rewards/check_originality_func/std": 0.4712514281272888, "rewards/perplexity_reward/mean": 0.8406110405921936, "rewards/perplexity_reward/std": 0.12113937735557556, "rewards/sentence_count_match_reward_logic/mean": 0.8640455603599548, "rewards/sentence_count_match_reward_logic/std": 0.1926349699497223, "step": 1837 }, { "clip_ratio": 0.05989241600036621, "epoch": 0.6431070678796361, "grad_norm": 1.0032191030100175, "kl": 3.65625, "learning_rate": 8.580586242372675e-06, "loss": 0.0046, "step": 1838 }, { "clip_ratio": 0.1309569627046585, "epoch": 0.6434569629111266, "grad_norm": 0.7258953612740764, "kl": 3.78125, "learning_rate": 8.578454189775889e-06, "loss": -0.0002, "step": 1839 }, { "clip_ratio": 0.167197585105896, "epoch": 0.6438068579426172, "grad_norm": 0.46860506820090686, "kl": 4.03125, "learning_rate": 8.576320802449559e-06, "loss": -0.0014, "step": 1840 }, { "clip_ratio": 0.016208073124289513, "completions/clipped_ratio": 0.0, "completions/max_length": 244.0, "completions/max_terminated_length": 244.0, "completions/mean_length": 65.1964340209961, "completions/mean_terminated_length": 65.1964340209961, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "epoch": 0.6441567529741078, "grad_norm": 1.3479504675893257, "kl": 4.78125, "learning_rate": 8.574186081189416e-06, "loss": 0.0185, "num_tokens": 8001442.0, "reward": 2.0818817615509033, "reward_std": 0.19449658691883087, "rewards/check_originality_func/mean": 0.3571428656578064, "rewards/check_originality_func/std": 0.48349377512931824, "rewards/perplexity_reward/mean": 0.7746930718421936, "rewards/perplexity_reward/std": 0.1251111626625061, "rewards/sentence_count_match_reward_logic/mean": 0.9500457644462585, "rewards/sentence_count_match_reward_logic/std": 0.1353752464056015, "step": 1841 }, { "clip_ratio": 0.04921630024909973, "epoch": 0.6445066480055983, "grad_norm": 0.7011540903127628, "kl": 4.71875, "learning_rate": 8.572050026791693e-06, "loss": 0.0137, "step": 1842 }, { "clip_ratio": 0.12122644484043121, "epoch": 0.6448565430370888, "grad_norm": 0.519681570963606, "kl": 4.75, "learning_rate": 8.569912640053117e-06, "loss": 0.0119, "step": 1843 }, { "clip_ratio": 0.16120004653930664, "epoch": 0.6452064380685795, "grad_norm": 0.46814517756640583, "kl": 4.75, "learning_rate": 8.567773921770914e-06, "loss": 0.0106, "step": 1844 }, { "clip_ratio": 0.006181259639561176, "completions/clipped_ratio": 0.0, "completions/max_length": 126.0, "completions/max_terminated_length": 126.0, "completions/mean_length": 35.44643020629883, "completions/mean_terminated_length": 35.44643020629883, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "epoch": 0.64555633310007, "grad_norm": 1.4771530686948202, "kl": 6.84375, "learning_rate": 8.565633872742803e-06, "loss": 0.0031, "num_tokens": 8008619.0, "reward": 2.0579659938812256, "reward_std": 0.15600961446762085, "rewards/check_originality_func/mean": 0.375, "rewards/check_originality_func/std": 0.48850420117378235, "rewards/perplexity_reward/mean": 0.7365373969078064, "rewards/perplexity_reward/std": 0.08781181275844574, "rewards/sentence_count_match_reward_logic/mean": 0.9464285969734192, "rewards/sentence_count_match_reward_logic/std": 0.09902118146419525, "step": 1845 }, { "clip_ratio": 0.037033744156360626, "epoch": 0.6459062281315605, "grad_norm": 0.407767349873224, "kl": 6.84375, "learning_rate": 8.563492493767004e-06, "loss": 0.0006, "step": 1846 }, { "clip_ratio": 0.07360614836215973, "epoch": 0.646256123163051, "grad_norm": 0.35579027124928825, "kl": 6.9375, "learning_rate": 8.561349785642232e-06, "loss": -0.0001, "step": 1847 }, { "clip_ratio": 0.13024619221687317, "epoch": 0.6466060181945417, "grad_norm": 0.2970306155480428, "kl": 7.0, "learning_rate": 8.559205749167695e-06, "loss": -0.0011, "step": 1848 }, { "clip_ratio": 0.009256234392523766, "completions/clipped_ratio": 0.0, "completions/max_length": 185.0, "completions/max_terminated_length": 185.0, "completions/mean_length": 84.55357360839844, "completions/mean_terminated_length": 84.55357360839844, "completions/min_length": 7.0, "completions/min_terminated_length": 7.0, "epoch": 0.6469559132260322, "grad_norm": 1.9527009005138696, "kl": 4.5625, "learning_rate": 8.557060385143102e-06, "loss": 0.0117, "num_tokens": 8021946.0, "reward": 2.3100996017456055, "reward_std": 0.11883996427059174, "rewards/check_originality_func/mean": 0.5357142686843872, "rewards/check_originality_func/std": 0.5032362937927246, "rewards/perplexity_reward/mean": 0.8136160969734192, "rewards/perplexity_reward/std": 0.09633254259824753, "rewards/sentence_count_match_reward_logic/mean": 0.9607691168785095, "rewards/sentence_count_match_reward_logic/std": 0.07062959671020508, "step": 1849 }, { "clip_ratio": 0.06450448930263519, "epoch": 0.6473058082575227, "grad_norm": 0.7710573885008766, "kl": 4.65625, "learning_rate": 8.554913694368647e-06, "loss": 0.0014, "step": 1850 }, { "clip_ratio": 0.1144837960600853, "epoch": 0.6476557032890133, "grad_norm": 0.6076275730131262, "kl": 4.75, "learning_rate": 8.552765677645031e-06, "loss": -0.0018, "step": 1851 }, { "clip_ratio": 0.17743739485740662, "epoch": 0.6480055983205039, "grad_norm": 0.48583495962765577, "kl": 4.84375, "learning_rate": 8.550616335773441e-06, "loss": -0.004, "step": 1852 }, { "clip_ratio": 0.02143498882651329, "completions/clipped_ratio": 0.0, "completions/max_length": 148.0, "completions/max_terminated_length": 148.0, "completions/mean_length": 77.875, "completions/mean_terminated_length": 77.875, "completions/min_length": 7.0, "completions/min_terminated_length": 7.0, "epoch": 0.6483554933519944, "grad_norm": 0.7663593988284564, "kl": 2.921875, "learning_rate": 8.548465669555564e-06, "loss": 0.0062, "num_tokens": 8034443.0, "reward": 2.2292442321777344, "reward_std": 0.2881917655467987, "rewards/check_originality_func/mean": 0.4821428656578064, "rewards/check_originality_func/std": 0.5042031407356262, "rewards/perplexity_reward/mean": 0.7964564561843872, "rewards/perplexity_reward/std": 0.0720033347606659, "rewards/sentence_count_match_reward_logic/mean": 0.950644850730896, "rewards/sentence_count_match_reward_logic/std": 0.10961491614580154, "step": 1853 }, { "clip_ratio": 0.04066773131489754, "epoch": 0.6487053883834849, "grad_norm": 0.48417207916616045, "kl": 2.953125, "learning_rate": 8.546313679793573e-06, "loss": 0.0036, "step": 1854 }, { "clip_ratio": 0.0678919181227684, "epoch": 0.6490552834149755, "grad_norm": 0.3812931820712757, "kl": 2.953125, "learning_rate": 8.544160367290145e-06, "loss": 0.0014, "step": 1855 }, { "clip_ratio": 0.10820475965738297, "epoch": 0.6494051784464661, "grad_norm": 0.3058362235033121, "kl": 3.03125, "learning_rate": 8.542005732848441e-06, "loss": -0.0001, "step": 1856 }, { "clip_ratio": 0.005606341641396284, "completions/clipped_ratio": 0.0, "completions/max_length": 216.0, "completions/max_terminated_length": 216.0, "completions/mean_length": 67.39286041259766, "completions/mean_terminated_length": 67.39286041259766, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "epoch": 0.6497550734779566, "grad_norm": 1.2382013747440321, "kl": 5.25, "learning_rate": 8.539849777272125e-06, "loss": 0.0085, "num_tokens": 8046009.0, "reward": 2.153032064437866, "reward_std": 0.2868841886520386, "rewards/check_originality_func/mean": 0.375, "rewards/check_originality_func/std": 0.48850420117378235, "rewards/perplexity_reward/mean": 0.7982701063156128, "rewards/perplexity_reward/std": 0.09872687608003616, "rewards/sentence_count_match_reward_logic/mean": 0.9797619581222534, "rewards/sentence_count_match_reward_logic/std": 0.06440985202789307, "step": 1857 }, { "clip_ratio": 0.0633704885840416, "epoch": 0.6501049685094472, "grad_norm": 0.6617526396330544, "kl": 5.3125, "learning_rate": 8.537692501365342e-06, "loss": 0.0026, "step": 1858 }, { "clip_ratio": 0.12864886224269867, "epoch": 0.6504548635409377, "grad_norm": 0.3868340710571219, "kl": 5.4375, "learning_rate": 8.535533905932739e-06, "loss": -0.0, "step": 1859 }, { "clip_ratio": 0.1712072342634201, "epoch": 0.6508047585724283, "grad_norm": 0.37671643604007077, "kl": 5.53125, "learning_rate": 8.53337399177945e-06, "loss": -0.0022, "step": 1860 }, { "clip_ratio": 0.006601437926292419, "completions/clipped_ratio": 0.0, "completions/max_length": 84.0, "completions/max_terminated_length": 84.0, "completions/mean_length": 41.875, "completions/mean_terminated_length": 41.875, "completions/min_length": 7.0, "completions/min_terminated_length": 7.0, "epoch": 0.6511546536039188, "grad_norm": 1.390570205003018, "kl": 5.59375, "learning_rate": 8.531212759711103e-06, "loss": 0.0039, "num_tokens": 8053762.0, "reward": 2.105933904647827, "reward_std": 0.2498781830072403, "rewards/check_originality_func/mean": 0.2857142984867096, "rewards/check_originality_func/std": 0.4558423161506653, "rewards/perplexity_reward/mean": 0.8440290093421936, "rewards/perplexity_reward/std": 0.06659964472055435, "rewards/sentence_count_match_reward_logic/mean": 0.9761905074119568, "rewards/sentence_count_match_reward_logic/std": 0.07738511264324188, "step": 1861 }, { "clip_ratio": 0.04581482335925102, "epoch": 0.6515045486354094, "grad_norm": 0.4533770862662535, "kl": 5.59375, "learning_rate": 8.529050210533818e-06, "loss": 0.0019, "step": 1862 }, { "clip_ratio": 0.13039641082286835, "epoch": 0.6518544436668999, "grad_norm": 0.3759042367549056, "kl": 5.75, "learning_rate": 8.526886345054204e-06, "loss": 0.0002, "step": 1863 }, { "clip_ratio": 0.18118871748447418, "epoch": 0.6522043386983905, "grad_norm": 0.3342666191388675, "kl": 5.84375, "learning_rate": 8.524721164079363e-06, "loss": -0.0005, "step": 1864 }, { "clip_ratio": 0.015622534789144993, "completions/clipped_ratio": 0.0, "completions/max_length": 68.0, "completions/max_terminated_length": 68.0, "completions/mean_length": 21.767858505249023, "completions/mean_terminated_length": 21.767858505249023, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "epoch": 0.6525542337298811, "grad_norm": 0.6959579214126221, "kl": 8.1875, "learning_rate": 8.522554668416887e-06, "loss": 0.0068, "num_tokens": 8059429.0, "reward": 1.9164342880249023, "reward_std": 0.08846097439527512, "rewards/check_originality_func/mean": 0.2678571343421936, "rewards/check_originality_func/std": 0.44685041904449463, "rewards/perplexity_reward/mean": 0.6708984375, "rewards/perplexity_reward/std": 0.13239876925945282, "rewards/sentence_count_match_reward_logic/mean": 0.9776785969734192, "rewards/sentence_count_match_reward_logic/std": 0.07193408906459808, "step": 1865 }, { "clip_ratio": 0.03760523349046707, "epoch": 0.6529041287613716, "grad_norm": 0.3034187456395711, "kl": 8.3125, "learning_rate": 8.520386858874858e-06, "loss": 0.0055, "step": 1866 }, { "clip_ratio": 0.05058677867054939, "epoch": 0.6532540237928621, "grad_norm": 0.3108680814140486, "kl": 8.3125, "learning_rate": 8.518217736261848e-06, "loss": 0.0052, "step": 1867 }, { "clip_ratio": 0.12596642971038818, "epoch": 0.6536039188243526, "grad_norm": 0.19257603815803184, "kl": 8.375, "learning_rate": 8.516047301386923e-06, "loss": 0.0045, "step": 1868 }, { "clip_ratio": 0.00906759686768055, "completions/clipped_ratio": 0.0, "completions/max_length": 213.0, "completions/max_terminated_length": 213.0, "completions/mean_length": 82.89286041259766, "completions/mean_terminated_length": 82.89286041259766, "completions/min_length": 13.0, "completions/min_terminated_length": 13.0, "epoch": 0.6539538138558433, "grad_norm": 2.1536705114708936, "kl": 2.828125, "learning_rate": 8.51387555505963e-06, "loss": -0.0013, "num_tokens": 8072623.0, "reward": 2.415257692337036, "reward_std": 0.25095152854919434, "rewards/check_originality_func/mean": 0.6607142686843872, "rewards/check_originality_func/std": 0.477751761674881, "rewards/perplexity_reward/mean": 0.86376953125, "rewards/perplexity_reward/std": 0.0684574618935585, "rewards/sentence_count_match_reward_logic/mean": 0.8907737731933594, "rewards/sentence_count_match_reward_logic/std": 0.1764383763074875, "step": 1869 }, { "clip_ratio": 0.05325531214475632, "epoch": 0.6543037088873338, "grad_norm": 1.0731160292747666, "kl": 2.84375, "learning_rate": 8.51170249809001e-06, "loss": -0.0103, "step": 1870 }, { "clip_ratio": 0.13467420637607574, "epoch": 0.6546536039188243, "grad_norm": 0.9086655245562102, "kl": 3.03125, "learning_rate": 8.509528131288598e-06, "loss": -0.0135, "step": 1871 }, { "clip_ratio": 0.17750205099582672, "epoch": 0.655003498950315, "grad_norm": 1.330871643013282, "kl": 3.21875, "learning_rate": 8.507352455466405e-06, "loss": -0.0142, "step": 1872 }, { "clip_ratio": 0.013631242327392101, "completions/clipped_ratio": 0.0357142857142857, "completions/max_length": 256.0, "completions/max_terminated_length": 250.0, "completions/mean_length": 138.30357360839844, "completions/mean_terminated_length": 133.94444274902344, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "epoch": 0.6553533939818055, "grad_norm": 3.0975808952824826, "kl": 2.703125, "learning_rate": 8.505175471434943e-06, "loss": 0.0094, "num_tokens": 8092496.0, "reward": 2.517108917236328, "reward_std": 0.17331747710704803, "rewards/check_originality_func/mean": 0.6964285969734192, "rewards/check_originality_func/std": 0.4639609158039093, "rewards/perplexity_reward/mean": 0.8503069281578064, "rewards/perplexity_reward/std": 0.07743698358535767, "rewards/sentence_count_match_reward_logic/mean": 0.9703733325004578, "rewards/sentence_count_match_reward_logic/std": 0.06447494775056839, "step": 1873 }, { "clip_ratio": 0.05505579710006714, "epoch": 0.655703289013296, "grad_norm": 1.5907889358327374, "kl": 2.734375, "learning_rate": 8.502997180006202e-06, "loss": 0.0017, "step": 1874 }, { "clip_ratio": 0.10112524032592773, "epoch": 0.6560531840447865, "grad_norm": 0.9863603979932415, "kl": 2.984375, "learning_rate": 8.500817581992669e-06, "loss": -0.0018, "step": 1875 }, { "clip_ratio": 0.14288759231567383, "epoch": 0.6564030790762772, "grad_norm": 1.0326337238502599, "kl": 3.046875, "learning_rate": 8.498636678207311e-06, "loss": -0.0058, "step": 1876 }, { "clip_ratio": 0.014543270692229271, "completions/clipped_ratio": 0.0, "completions/max_length": 215.0, "completions/max_terminated_length": 215.0, "completions/mean_length": 66.73214721679688, "completions/mean_terminated_length": 66.73214721679688, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "epoch": 0.6567529741077677, "grad_norm": 5.017964653337753, "kl": 4.1875, "learning_rate": 8.496454469463583e-06, "loss": 0.0273, "num_tokens": 8103393.0, "reward": 2.147465705871582, "reward_std": 0.32674136757850647, "rewards/check_originality_func/mean": 0.3928571343421936, "rewards/check_originality_func/std": 0.4928053617477417, "rewards/perplexity_reward/mean": 0.7917829155921936, "rewards/perplexity_reward/std": 0.17584975063800812, "rewards/sentence_count_match_reward_logic/mean": 0.9628254175186157, "rewards/sentence_count_match_reward_logic/std": 0.09772463887929916, "step": 1877 }, { "clip_ratio": 0.08131847530603409, "epoch": 0.6571028691392582, "grad_norm": 1.0024646352049196, "kl": 4.375, "learning_rate": 8.494270956575428e-06, "loss": 0.0181, "step": 1878 }, { "clip_ratio": 0.14258158206939697, "epoch": 0.6574527641707488, "grad_norm": 1.5984752772379547, "kl": 4.78125, "learning_rate": 8.492086140357279e-06, "loss": 0.0134, "step": 1879 }, { "clip_ratio": 0.18020890653133392, "epoch": 0.6578026592022393, "grad_norm": 0.9733331600553515, "kl": 4.6875, "learning_rate": 8.48990002162405e-06, "loss": 0.011, "step": 1880 }, { "clip_ratio": 0.008232907392084599, "completions/clipped_ratio": 0.1607142857142857, "completions/max_length": 256.0, "completions/max_terminated_length": 100.0, "completions/mean_length": 83.17857360839844, "completions/mean_terminated_length": 50.085105895996094, "completions/min_length": 6.0, "completions/min_terminated_length": 6.0, "epoch": 0.6581525542337299, "grad_norm": 1.1457427687565656, "kl": 4.59375, "learning_rate": 8.487712601191143e-06, "loss": 0.0289, "num_tokens": 8117275.0, "reward": 2.3686118125915527, "reward_std": 0.21903860569000244, "rewards/check_originality_func/mean": 0.5535714030265808, "rewards/check_originality_func/std": 0.5016207695007324, "rewards/perplexity_reward/mean": 0.8292410969734192, "rewards/perplexity_reward/std": 0.0778358057141304, "rewards/sentence_count_match_reward_logic/mean": 0.9857993125915527, "rewards/sentence_count_match_reward_logic/std": 0.06326830387115479, "step": 1881 }, { "clip_ratio": 0.061142224818468094, "epoch": 0.6585024492652204, "grad_norm": 0.7637064875533175, "kl": 4.78125, "learning_rate": 8.485523879874444e-06, "loss": 0.023, "step": 1882 }, { "clip_ratio": 0.13920031487941742, "epoch": 0.658852344296711, "grad_norm": 0.459699128559105, "kl": 5.0, "learning_rate": 8.483333858490326e-06, "loss": 0.0201, "step": 1883 }, { "clip_ratio": 0.19193677604198456, "epoch": 0.6592022393282015, "grad_norm": 0.45357545389520093, "kl": 5.25, "learning_rate": 8.481142537855646e-06, "loss": 0.0193, "step": 1884 }, { "clip_ratio": 0.022702645510435104, "completions/clipped_ratio": 0.0, "completions/max_length": 130.0, "completions/max_terminated_length": 130.0, "completions/mean_length": 46.17857360839844, "completions/mean_terminated_length": 46.17857360839844, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "epoch": 0.6595521343596921, "grad_norm": 1.1175217199490939, "kl": 6.0, "learning_rate": 8.478949918787746e-06, "loss": 0.0086, "num_tokens": 8125653.0, "reward": 2.1427178382873535, "reward_std": 0.03764088451862335, "rewards/check_originality_func/mean": 0.4285714328289032, "rewards/check_originality_func/std": 0.4993502199649811, "rewards/perplexity_reward/mean": 0.7141461968421936, "rewards/perplexity_reward/std": 0.20833717286586761, "rewards/sentence_count_match_reward_logic/mean": 1.0, "rewards/sentence_count_match_reward_logic/std": 0.0, "step": 1885 }, { "clip_ratio": 0.03945712372660637, "epoch": 0.6599020293911827, "grad_norm": 0.7869188367177145, "kl": 6.0, "learning_rate": 8.476756002104453e-06, "loss": 0.0134, "step": 1886 }, { "clip_ratio": 0.07068627327680588, "epoch": 0.6602519244226732, "grad_norm": 0.4723215589716759, "kl": 6.09375, "learning_rate": 8.474560788624078e-06, "loss": 0.0111, "step": 1887 }, { "clip_ratio": 0.11575251072645187, "epoch": 0.6606018194541637, "grad_norm": 0.2787915919123678, "kl": 6.21875, "learning_rate": 8.47236427916541e-06, "loss": 0.0094, "step": 1888 }, { "clip_ratio": 0.011660324409604073, "completions/clipped_ratio": 0.1071428571428571, "completions/max_length": 256.0, "completions/max_terminated_length": 160.0, "completions/mean_length": 74.625, "completions/mean_terminated_length": 52.86000061035156, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "epoch": 0.6609517144856543, "grad_norm": 2.3355334461636703, "kl": 5.46875, "learning_rate": 8.470166474547731e-06, "loss": 0.0319, "num_tokens": 8137240.0, "reward": 2.0374183654785156, "reward_std": 0.250272274017334, "rewards/check_originality_func/mean": 0.4107142984867096, "rewards/check_originality_func/std": 0.49641573429107666, "rewards/perplexity_reward/mean": 0.76318359375, "rewards/perplexity_reward/std": 0.13932392001152039, "rewards/sentence_count_match_reward_logic/mean": 0.8635204434394836, "rewards/sentence_count_match_reward_logic/std": 0.24184507131576538, "step": 1889 }, { "clip_ratio": 0.08857059478759766, "epoch": 0.6613016095171449, "grad_norm": 1.203204172134564, "kl": 5.84375, "learning_rate": 8.467967375590801e-06, "loss": 0.024, "step": 1890 }, { "clip_ratio": 0.19947096705436707, "epoch": 0.6616515045486354, "grad_norm": 0.8600586156871948, "kl": 6.09375, "learning_rate": 8.46576698311486e-06, "loss": 0.0177, "step": 1891 }, { "clip_ratio": 0.21802322566509247, "epoch": 0.6620013995801259, "grad_norm": 1.2529317969094051, "kl": 6.0625, "learning_rate": 8.463565297940636e-06, "loss": 0.0187, "step": 1892 }, { "clip_ratio": 0.015361896716058254, "completions/clipped_ratio": 0.0714285714285714, "completions/max_length": 256.0, "completions/max_terminated_length": 248.0, "completions/mean_length": 111.37500762939453, "completions/mean_terminated_length": 100.25, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "epoch": 0.6623512946116166, "grad_norm": 4.517609611966857, "kl": 3.875, "learning_rate": 8.461362320889338e-06, "loss": -0.0038, "num_tokens": 8153429.0, "reward": 2.308596611022949, "reward_std": 0.2551356554031372, "rewards/check_originality_func/mean": 0.4821428656578064, "rewards/check_originality_func/std": 0.5042031407356262, "rewards/perplexity_reward/mean": 0.8866490125656128, "rewards/perplexity_reward/std": 0.0863577350974083, "rewards/sentence_count_match_reward_logic/mean": 0.9398044943809509, "rewards/sentence_count_match_reward_logic/std": 0.13360443711280823, "step": 1893 }, { "clip_ratio": 0.07882510125637054, "epoch": 0.6627011896431071, "grad_norm": 3.1684887823888754, "kl": 3.671875, "learning_rate": 8.459158052782652e-06, "loss": -0.0169, "step": 1894 }, { "clip_ratio": 0.11929120123386383, "epoch": 0.6630510846745976, "grad_norm": 1.2947149079039961, "kl": 3.84375, "learning_rate": 8.456952494442753e-06, "loss": -0.0209, "step": 1895 }, { "clip_ratio": 0.16083048284053802, "epoch": 0.6634009797060881, "grad_norm": 0.9998131572801567, "kl": 3.78125, "learning_rate": 8.454745646692291e-06, "loss": -0.0247, "step": 1896 }, { "clip_ratio": 0.05141642689704895, "completions/clipped_ratio": 0.2678571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 254.0, "completions/mean_length": 129.17857360839844, "completions/mean_terminated_length": 82.78048706054688, "completions/min_length": 16.0, "completions/min_terminated_length": 16.0, "epoch": 0.6637508747375788, "grad_norm": 12.294523153736558, "kl": 4.71875, "learning_rate": 8.452537510354397e-06, "loss": 0.0, "num_tokens": 8173647.0, "reward": 2.1811041831970215, "reward_std": 0.4421467185020447, "rewards/check_originality_func/mean": 0.5178571343421936, "rewards/check_originality_func/std": 0.5042031407356262, "rewards/perplexity_reward/mean": 0.8533760905265808, "rewards/perplexity_reward/std": 0.07531511038541794, "rewards/sentence_count_match_reward_logic/mean": 0.8098707795143127, "rewards/sentence_count_match_reward_logic/std": 0.2406345158815384, "step": 1897 }, { "clip_ratio": 0.14178870618343353, "epoch": 0.6641007697690693, "grad_norm": 6.565836629505576, "kl": 3.703125, "learning_rate": 8.45032808625269e-06, "loss": 0.0221, "step": 1898 }, { "clip_ratio": 0.21167370676994324, "epoch": 0.6644506648005598, "grad_norm": 2.3488735103821328, "kl": 3.515625, "learning_rate": 8.44811737521126e-06, "loss": 0.0094, "step": 1899 }, { "clip_ratio": 0.270100474357605, "epoch": 0.6648005598320503, "grad_norm": 8.649022948614691, "kl": 3.734375, "learning_rate": 8.445905378054686e-06, "loss": -0.0166, "step": 1900 }, { "clip_ratio": 0.018653396517038345, "completions/clipped_ratio": 0.0, "completions/max_length": 165.0, "completions/max_terminated_length": 165.0, "completions/mean_length": 57.607147216796875, "completions/mean_terminated_length": 57.607147216796875, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "epoch": 0.665150454863541, "grad_norm": 0.7509439879070603, "kl": 5.21875, "learning_rate": 8.443692095608019e-06, "loss": 0.0096, "num_tokens": 8183105.0, "reward": 2.036616802215576, "reward_std": 0.2000221610069275, "rewards/check_originality_func/mean": 0.2678571343421936, "rewards/check_originality_func/std": 0.44685041904449463, "rewards/perplexity_reward/mean": 0.8132672905921936, "rewards/perplexity_reward/std": 0.14895881712436676, "rewards/sentence_count_match_reward_logic/mean": 0.9554924368858337, "rewards/sentence_count_match_reward_logic/std": 0.1403612345457077, "step": 1901 }, { "clip_ratio": 0.08207858353853226, "epoch": 0.6655003498950315, "grad_norm": 0.5407381404185923, "kl": 5.3125, "learning_rate": 8.441477528696789e-06, "loss": 0.0079, "step": 1902 }, { "clip_ratio": 0.168746680021286, "epoch": 0.665850244926522, "grad_norm": 0.6708925424679215, "kl": 5.4375, "learning_rate": 8.439261678147014e-06, "loss": 0.0062, "step": 1903 }, { "clip_ratio": 0.20760169625282288, "epoch": 0.6662001399580126, "grad_norm": 0.4843394806065751, "kl": 5.46875, "learning_rate": 8.437044544785184e-06, "loss": 0.0048, "step": 1904 }, { "clip_ratio": 0.03443779796361923, "completions/clipped_ratio": 0.0535714285714286, "completions/max_length": 256.0, "completions/max_terminated_length": 247.0, "completions/mean_length": 102.1964340209961, "completions/mean_terminated_length": 93.49057006835938, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "epoch": 0.6665500349895032, "grad_norm": 5.080896817626412, "kl": 8.5625, "learning_rate": 8.43482612943827e-06, "loss": 0.0428, "num_tokens": 8198444.0, "reward": 2.1021108627319336, "reward_std": 0.1801789551973343, "rewards/check_originality_func/mean": 0.3928571343421936, "rewards/check_originality_func/std": 0.4928053617477417, "rewards/perplexity_reward/mean": 0.7364676594734192, "rewards/perplexity_reward/std": 0.16255900263786316, "rewards/sentence_count_match_reward_logic/mean": 0.9727858901023865, "rewards/sentence_count_match_reward_logic/std": 0.05614502355456352, "step": 1905 }, { "clip_ratio": 0.08086752891540527, "epoch": 0.6668999300209937, "grad_norm": 5.236923337100094, "kl": 8.9375, "learning_rate": 8.432606432933716e-06, "loss": 0.0332, "step": 1906 }, { "clip_ratio": 0.1528230756521225, "epoch": 0.6672498250524842, "grad_norm": 3.0332640774568693, "kl": 9.1875, "learning_rate": 8.43038545609945e-06, "loss": 0.022, "step": 1907 }, { "clip_ratio": 0.245840385556221, "epoch": 0.6675997200839748, "grad_norm": 1.349964854676416, "kl": 8.9375, "learning_rate": 8.428163199763876e-06, "loss": 0.0152, "step": 1908 }, { "clip_ratio": 0.016179382801055908, "completions/clipped_ratio": 0.0, "completions/max_length": 152.0, "completions/max_terminated_length": 152.0, "completions/mean_length": 35.57143020629883, "completions/mean_terminated_length": 35.57143020629883, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "epoch": 0.6679496151154654, "grad_norm": 1.195051091994818, "kl": 9.5625, "learning_rate": 8.425939664755874e-06, "loss": 0.0118, "num_tokens": 8205996.0, "reward": 1.9765626192092896, "reward_std": 0.15397164225578308, "rewards/check_originality_func/mean": 0.25, "rewards/check_originality_func/std": 0.43693143129348755, "rewards/perplexity_reward/mean": 0.7801339030265808, "rewards/perplexity_reward/std": 0.11429361253976822, "rewards/sentence_count_match_reward_logic/mean": 0.9464285969734192, "rewards/sentence_count_match_reward_logic/std": 0.1235303208231926, "step": 1909 }, { "clip_ratio": 0.1141883134841919, "epoch": 0.6682995101469559, "grad_norm": 0.4615748987137053, "kl": 9.8125, "learning_rate": 8.4237148519048e-06, "loss": 0.0105, "step": 1910 }, { "clip_ratio": 0.1904391199350357, "epoch": 0.6686494051784465, "grad_norm": 0.6266333028808263, "kl": 10.125, "learning_rate": 8.42148876204049e-06, "loss": 0.0112, "step": 1911 }, { "clip_ratio": 0.24482129514217377, "epoch": 0.668999300209937, "grad_norm": 0.4402879811560134, "kl": 10.125, "learning_rate": 8.419261395993255e-06, "loss": 0.0104, "step": 1912 }, { "clip_ratio": 0.052499886602163315, "completions/clipped_ratio": 0.0357142857142857, "completions/max_length": 256.0, "completions/max_terminated_length": 177.0, "completions/mean_length": 85.87500762939453, "completions/mean_terminated_length": 79.5740737915039, "completions/min_length": 6.0, "completions/min_terminated_length": 6.0, "epoch": 0.6693491952414276, "grad_norm": 5.097315159390665, "kl": 4.21875, "learning_rate": 8.417032754593879e-06, "loss": 0.0388, "num_tokens": 8218885.0, "reward": 2.266054630279541, "reward_std": 0.4220322072505951, "rewards/check_originality_func/mean": 0.5357142686843872, "rewards/check_originality_func/std": 0.5032362937927246, "rewards/perplexity_reward/mean": 0.8158482313156128, "rewards/perplexity_reward/std": 0.21018703281879425, "rewards/sentence_count_match_reward_logic/mean": 0.9144918322563171, "rewards/sentence_count_match_reward_logic/std": 0.140394389629364, "step": 1913 }, { "clip_ratio": 0.21931909024715424, "epoch": 0.6696990902729181, "grad_norm": 8.897064912497324, "kl": 3.828125, "learning_rate": 8.414802838673626e-06, "loss": 0.0869, "step": 1914 }, { "clip_ratio": 0.26187607645988464, "epoch": 0.6700489853044087, "grad_norm": 7.487926413583662, "kl": 3.875, "learning_rate": 8.412571649064233e-06, "loss": 0.0759, "step": 1915 }, { "clip_ratio": 0.28629565238952637, "epoch": 0.6703988803358992, "grad_norm": 7.299181811718248, "kl": 4.21875, "learning_rate": 8.410339186597914e-06, "loss": 0.0438, "step": 1916 }, { "clip_ratio": 0.03080095537006855, "completions/clipped_ratio": 0.1964285714285714, "completions/max_length": 256.0, "completions/max_terminated_length": 253.0, "completions/mean_length": 167.32144165039062, "completions/mean_terminated_length": 145.6444549560547, "completions/min_length": 99.0, "completions/min_terminated_length": 99.0, "epoch": 0.6707487753673897, "grad_norm": 4.211189413033035, "kl": 1.6640625, "learning_rate": 8.408105452107353e-06, "loss": 0.0164, "num_tokens": 8243295.0, "reward": 2.4143776893615723, "reward_std": 0.3398200571537018, "rewards/check_originality_func/mean": 0.625, "rewards/check_originality_func/std": 0.48850420117378235, "rewards/perplexity_reward/mean": 0.8577008843421936, "rewards/perplexity_reward/std": 0.02360096015036106, "rewards/sentence_count_match_reward_logic/mean": 0.9316768646240234, "rewards/sentence_count_match_reward_logic/std": 0.08248920738697052, "step": 1917 }, { "clip_ratio": 0.10456699877977371, "epoch": 0.6710986703988804, "grad_norm": 2.129734126526363, "kl": 1.6796875, "learning_rate": 8.405870446425715e-06, "loss": 0.0072, "step": 1918 }, { "clip_ratio": 0.13132871687412262, "epoch": 0.6714485654303709, "grad_norm": 1.0656245678152598, "kl": 1.6953125, "learning_rate": 8.403634170386634e-06, "loss": 0.0019, "step": 1919 }, { "clip_ratio": 0.1565399467945099, "epoch": 0.6717984604618614, "grad_norm": 1.381246873146109, "kl": 1.703125, "learning_rate": 8.40139662482422e-06, "loss": -0.0046, "step": 1920 }, { "clip_ratio": 0.04035665839910507, "completions/clipped_ratio": 0.017857142857142905, "completions/max_length": 256.0, "completions/max_terminated_length": 194.0, "completions/mean_length": 54.500003814697266, "completions/mean_terminated_length": 50.836360931396484, "completions/min_length": 6.0, "completions/min_terminated_length": 6.0, "epoch": 0.672148355493352, "grad_norm": 2.670054060275574, "kl": 8.4375, "learning_rate": 8.39915781057306e-06, "loss": -0.0144, "num_tokens": 8252299.0, "reward": 1.9089473485946655, "reward_std": 0.1600084900856018, "rewards/check_originality_func/mean": 0.125, "rewards/check_originality_func/std": 0.3337118923664093, "rewards/perplexity_reward/mean": 0.8367745280265808, "rewards/perplexity_reward/std": 0.0940062627196312, "rewards/sentence_count_match_reward_logic/mean": 0.9471725821495056, "rewards/sentence_count_match_reward_logic/std": 0.11165961623191833, "step": 1921 }, { "clip_ratio": 0.06976956129074097, "epoch": 0.6724982505248426, "grad_norm": 1.0614187927240972, "kl": 8.3125, "learning_rate": 8.396917728468203e-06, "loss": -0.0207, "step": 1922 }, { "clip_ratio": 0.12921002507209778, "epoch": 0.6728481455563331, "grad_norm": 0.5706954382659694, "kl": 8.25, "learning_rate": 8.394676379345187e-06, "loss": -0.0251, "step": 1923 }, { "clip_ratio": 0.1818186491727829, "epoch": 0.6731980405878236, "grad_norm": 0.6102120492116603, "kl": 8.375, "learning_rate": 8.392433764040008e-06, "loss": -0.0261, "step": 1924 }, { "clip_ratio": 0.048764076083898544, "completions/clipped_ratio": 0.1428571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 226.0, "completions/mean_length": 131.625, "completions/mean_terminated_length": 110.89583587646484, "completions/min_length": 42.0, "completions/min_terminated_length": 42.0, "epoch": 0.6735479356193143, "grad_norm": 7.442911877634559, "kl": 4.0625, "learning_rate": 8.390189883389143e-06, "loss": 0.0255, "num_tokens": 8271838.0, "reward": 2.4953925609588623, "reward_std": 0.4183788597583771, "rewards/check_originality_func/mean": 0.75, "rewards/check_originality_func/std": 0.43693143129348755, "rewards/perplexity_reward/mean": 0.8643973469734192, "rewards/perplexity_reward/std": 0.049694523215293884, "rewards/sentence_count_match_reward_logic/mean": 0.8809950947761536, "rewards/sentence_count_match_reward_logic/std": 0.12234984338283539, "step": 1925 }, { "clip_ratio": 0.1491992473602295, "epoch": 0.6738978306508048, "grad_norm": 3.9351695967362827, "kl": 3.53125, "learning_rate": 8.387944738229536e-06, "loss": 0.015, "step": 1926 }, { "clip_ratio": 0.20831622183322906, "epoch": 0.6742477256822953, "grad_norm": 2.029408343973316, "kl": 3.71875, "learning_rate": 8.385698329398609e-06, "loss": 0.0068, "step": 1927 }, { "clip_ratio": 0.26865026354789734, "epoch": 0.6745976207137858, "grad_norm": 1.5000578256528927, "kl": 3.59375, "learning_rate": 8.383450657734245e-06, "loss": 0.0026, "step": 1928 }, { "clip_ratio": 0.01896127499639988, "completions/clipped_ratio": 0.017857142857142905, "completions/max_length": 256.0, "completions/max_terminated_length": 135.0, "completions/mean_length": 65.0, "completions/mean_terminated_length": 61.52727127075195, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "epoch": 0.6749475157452764, "grad_norm": 2.646617098896068, "kl": 3.53125, "learning_rate": 8.38120172407481e-06, "loss": 0.0103, "num_tokens": 8282990.0, "reward": 2.2366771697998047, "reward_std": 0.39288052916526794, "rewards/check_originality_func/mean": 0.4642857015132904, "rewards/check_originality_func/std": 0.5032362341880798, "rewards/perplexity_reward/mean": 0.8839983344078064, "rewards/perplexity_reward/std": 0.09415431320667267, "rewards/sentence_count_match_reward_logic/mean": 0.8883928656578064, "rewards/sentence_count_match_reward_logic/std": 0.21395552158355713, "step": 1929 }, { "clip_ratio": 0.08449207991361618, "epoch": 0.675297410776767, "grad_norm": 1.6723957129497107, "kl": 3.859375, "learning_rate": 8.378951529259133e-06, "loss": 0.0059, "step": 1930 }, { "clip_ratio": 0.1786399781703949, "epoch": 0.6756473058082575, "grad_norm": 0.7443653131016599, "kl": 4.1875, "learning_rate": 8.376700074126515e-06, "loss": 0.001, "step": 1931 }, { "clip_ratio": 0.25531643629074097, "epoch": 0.6759972008397481, "grad_norm": 0.6655085025969081, "kl": 4.21875, "learning_rate": 8.374447359516727e-06, "loss": -0.0014, "step": 1932 }, { "clip_ratio": 0.023678584024310112, "completions/clipped_ratio": 0.1071428571428571, "completions/max_length": 256.0, "completions/max_terminated_length": 254.0, "completions/mean_length": 113.41072082519531, "completions/mean_terminated_length": 96.29999542236328, "completions/min_length": 3.0, "completions/min_terminated_length": 3.0, "epoch": 0.6763470958712386, "grad_norm": 3.3939136818233986, "kl": 3.96875, "learning_rate": 8.37219338627001e-06, "loss": -0.0032, "num_tokens": 8301037.0, "reward": 2.0910632610321045, "reward_std": 0.3207877576351166, "rewards/check_originality_func/mean": 0.4285714328289032, "rewards/check_originality_func/std": 0.4993502199649811, "rewards/perplexity_reward/mean": 0.7613002061843872, "rewards/perplexity_reward/std": 0.19463187456130981, "rewards/sentence_count_match_reward_logic/mean": 0.9011915922164917, "rewards/sentence_count_match_reward_logic/std": 0.13231271505355835, "step": 1933 }, { "clip_ratio": 0.07541804760694504, "epoch": 0.6766969909027292, "grad_norm": 1.2997218800479613, "kl": 4.3125, "learning_rate": 8.369938155227078e-06, "loss": -0.0068, "step": 1934 }, { "clip_ratio": 0.11540261656045914, "epoch": 0.6770468859342197, "grad_norm": 1.0547541973060164, "kl": 4.46875, "learning_rate": 8.367681667229107e-06, "loss": -0.012, "step": 1935 }, { "clip_ratio": 0.15480534732341766, "epoch": 0.6773967809657103, "grad_norm": 0.8787794447485113, "kl": 4.21875, "learning_rate": 8.365423923117747e-06, "loss": -0.018, "step": 1936 }, { "clip_ratio": 0.00978661235421896, "completions/clipped_ratio": 0.0, "completions/max_length": 168.0, "completions/max_terminated_length": 168.0, "completions/mean_length": 46.23214340209961, "completions/mean_terminated_length": 46.23214340209961, "completions/min_length": 6.0, "completions/min_terminated_length": 6.0, "epoch": 0.6777466759972008, "grad_norm": 0.8201511969769841, "kl": 6.4375, "learning_rate": 8.363164923735116e-06, "loss": 0.0407, "num_tokens": 8308922.0, "reward": 1.9211660623550415, "reward_std": 0.17848870158195496, "rewards/check_originality_func/mean": 0.2142857164144516, "rewards/check_originality_func/std": 0.4140393137931824, "rewards/perplexity_reward/mean": 0.7761579155921936, "rewards/perplexity_reward/std": 0.12133960425853729, "rewards/sentence_count_match_reward_logic/mean": 0.9307223558425903, "rewards/sentence_count_match_reward_logic/std": 0.20467333495616913, "step": 1937 }, { "clip_ratio": 0.0804450586438179, "epoch": 0.6780965710286914, "grad_norm": 0.45642941660906156, "kl": 6.4375, "learning_rate": 8.360904669923797e-06, "loss": 0.0379, "step": 1938 }, { "clip_ratio": 0.20846311748027802, "epoch": 0.678446466060182, "grad_norm": 0.4620022560936107, "kl": 6.4375, "learning_rate": 8.358643162526846e-06, "loss": 0.0356, "step": 1939 }, { "clip_ratio": 0.24630019068717957, "epoch": 0.6787963610916725, "grad_norm": 0.40910762375591875, "kl": 6.46875, "learning_rate": 8.356380402387783e-06, "loss": 0.0342, "step": 1940 }, { "clip_ratio": 0.02111140266060829, "completions/clipped_ratio": 0.0, "completions/max_length": 143.0, "completions/max_terminated_length": 143.0, "completions/mean_length": 36.05357360839844, "completions/mean_terminated_length": 36.05357360839844, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "epoch": 0.679146256123163, "grad_norm": 1.704906401393756, "kl": 8.125, "learning_rate": 8.354116390350594e-06, "loss": 0.0076, "num_tokens": 8316189.0, "reward": 1.8791887760162354, "reward_std": 0.22265928983688354, "rewards/check_originality_func/mean": 0.1964285671710968, "rewards/check_originality_func/std": 0.40089187026023865, "rewards/perplexity_reward/mean": 0.7271903157234192, "rewards/perplexity_reward/std": 0.1418248564004898, "rewards/sentence_count_match_reward_logic/mean": 0.9555697441101074, "rewards/sentence_count_match_reward_logic/std": 0.11722613126039505, "step": 1941 }, { "clip_ratio": 0.13974282145500183, "epoch": 0.6794961511546536, "grad_norm": 0.9527116927807935, "kl": 8.125, "learning_rate": 8.351851127259739e-06, "loss": 0.0023, "step": 1942 }, { "clip_ratio": 0.21273185312747955, "epoch": 0.6798460461861442, "grad_norm": 0.9735176201509284, "kl": 8.3125, "learning_rate": 8.349584613960136e-06, "loss": -0.0008, "step": 1943 }, { "clip_ratio": 0.30084577202796936, "epoch": 0.6801959412176347, "grad_norm": 1.5678158516017202, "kl": 8.1875, "learning_rate": 8.347316851297171e-06, "loss": -0.0015, "step": 1944 }, { "clip_ratio": 0.012454047799110413, "completions/clipped_ratio": 0.0, "completions/max_length": 110.0, "completions/max_terminated_length": 110.0, "completions/mean_length": 42.10714340209961, "completions/mean_terminated_length": 42.10714340209961, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "epoch": 0.6805458362491252, "grad_norm": 0.6974317990708887, "kl": 7.0625, "learning_rate": 8.345047840116704e-06, "loss": 0.0088, "num_tokens": 8324411.0, "reward": 2.015909433364868, "reward_std": 0.2294451892375946, "rewards/check_originality_func/mean": 0.3214285671710968, "rewards/check_originality_func/std": 0.4712514579296112, "rewards/perplexity_reward/mean": 0.759765625, "rewards/perplexity_reward/std": 0.13493458926677704, "rewards/sentence_count_match_reward_logic/mean": 0.9347151517868042, "rewards/sentence_count_match_reward_logic/std": 0.1379019320011139, "step": 1945 }, { "clip_ratio": 0.05983293801546097, "epoch": 0.6808957312806159, "grad_norm": 0.5038971222486144, "kl": 7.28125, "learning_rate": 8.342777581265048e-06, "loss": 0.0076, "step": 1946 }, { "clip_ratio": 0.13684919476509094, "epoch": 0.6812456263121064, "grad_norm": 0.5206494538953725, "kl": 7.34375, "learning_rate": 8.340506075588994e-06, "loss": 0.0056, "step": 1947 }, { "clip_ratio": 0.20727522671222687, "epoch": 0.6815955213435969, "grad_norm": 0.31865789877231604, "kl": 7.34375, "learning_rate": 8.33823332393579e-06, "loss": 0.0043, "step": 1948 }, { "clip_ratio": 0.016410483047366142, "completions/clipped_ratio": 0.1428571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 193.0, "completions/mean_length": 101.03572082519531, "completions/mean_terminated_length": 75.20833587646484, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "epoch": 0.6819454163750874, "grad_norm": 0.8327327652031868, "kl": 4.8125, "learning_rate": 8.335959327153148e-06, "loss": 0.0125, "num_tokens": 8341061.0, "reward": 2.2656564712524414, "reward_std": 0.19947868585586548, "rewards/check_originality_func/mean": 0.5357142686843872, "rewards/check_originality_func/std": 0.5032362937927246, "rewards/perplexity_reward/mean": 0.8062220811843872, "rewards/perplexity_reward/std": 0.09991473704576492, "rewards/sentence_count_match_reward_logic/mean": 0.9237200021743774, "rewards/sentence_count_match_reward_logic/std": 0.14276188611984253, "step": 1949 }, { "clip_ratio": 0.047087304294109344, "epoch": 0.6822953114065781, "grad_norm": 0.6533312226170634, "kl": 4.78125, "learning_rate": 8.333684086089251e-06, "loss": 0.0084, "step": 1950 }, { "clip_ratio": 0.07977380603551865, "epoch": 0.6826452064380686, "grad_norm": 0.5158753642318886, "kl": 4.78125, "learning_rate": 8.33140760159274e-06, "loss": 0.0051, "step": 1951 }, { "clip_ratio": 0.1698921024799347, "epoch": 0.6829951014695591, "grad_norm": 0.4565033800744894, "kl": 4.78125, "learning_rate": 8.329129874512724e-06, "loss": 0.0023, "step": 1952 }, { "clip_ratio": 0.010774536058306694, "completions/clipped_ratio": 0.0, "completions/max_length": 226.0, "completions/max_terminated_length": 226.0, "completions/mean_length": 64.75, "completions/mean_terminated_length": 64.75, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "epoch": 0.6833449965010497, "grad_norm": 0.7804062372096544, "kl": 4.6875, "learning_rate": 8.326850905698774e-06, "loss": 0.0269, "num_tokens": 8351463.0, "reward": 2.012364625930786, "reward_std": 0.2501603364944458, "rewards/check_originality_func/mean": 0.2857142984867096, "rewards/check_originality_func/std": 0.4558423161506653, "rewards/perplexity_reward/mean": 0.7737165093421936, "rewards/perplexity_reward/std": 0.1677379459142685, "rewards/sentence_count_match_reward_logic/mean": 0.9529337286949158, "rewards/sentence_count_match_reward_logic/std": 0.12174857407808304, "step": 1953 }, { "clip_ratio": 0.0398436114192009, "epoch": 0.6836948915325403, "grad_norm": 0.6865308064184779, "kl": 4.71875, "learning_rate": 8.32457069600092e-06, "loss": 0.024, "step": 1954 }, { "clip_ratio": 0.15122829377651215, "epoch": 0.6840447865640308, "grad_norm": 0.6064611916187104, "kl": 4.96875, "learning_rate": 8.322289246269662e-06, "loss": 0.0203, "step": 1955 }, { "clip_ratio": 0.1997169703245163, "epoch": 0.6843946815955213, "grad_norm": 0.4413133250893391, "kl": 5.1875, "learning_rate": 8.32000655735596e-06, "loss": 0.0177, "step": 1956 }, { "clip_ratio": 0.019417759031057358, "completions/clipped_ratio": 0.017857142857142905, "completions/max_length": 256.0, "completions/max_terminated_length": 203.0, "completions/mean_length": 60.55357360839844, "completions/mean_terminated_length": 57.0, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "epoch": 0.6847445766270119, "grad_norm": 0.5836934985592143, "kl": 7.59375, "learning_rate": 8.317722630111233e-06, "loss": 0.0263, "num_tokens": 8362558.0, "reward": 2.121152400970459, "reward_std": 0.17629842460155487, "rewards/check_originality_func/mean": 0.3571428656578064, "rewards/check_originality_func/std": 0.48349377512931824, "rewards/perplexity_reward/mean": 0.7879464030265808, "rewards/perplexity_reward/std": 0.12723000347614288, "rewards/sentence_count_match_reward_logic/mean": 0.9760628938674927, "rewards/sentence_count_match_reward_logic/std": 0.06973225623369217, "step": 1957 }, { "clip_ratio": 0.023468613624572754, "epoch": 0.6850944716585025, "grad_norm": 0.5510528941257404, "kl": 7.59375, "learning_rate": 8.315437465387367e-06, "loss": 0.025, "step": 1958 }, { "clip_ratio": 0.10435862094163895, "epoch": 0.685444366689993, "grad_norm": 0.5526252236274316, "kl": 7.59375, "learning_rate": 8.313151064036702e-06, "loss": 0.0215, "step": 1959 }, { "clip_ratio": 0.19789208471775055, "epoch": 0.6857942617214835, "grad_norm": 0.5542223513513093, "kl": 7.59375, "learning_rate": 8.310863426912047e-06, "loss": 0.0185, "step": 1960 }, { "clip_ratio": 0.021864721551537514, "completions/clipped_ratio": 0.0, "completions/max_length": 212.0, "completions/max_terminated_length": 212.0, "completions/mean_length": 53.33928680419922, "completions/mean_terminated_length": 53.33928680419922, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "epoch": 0.6861441567529741, "grad_norm": 1.0656664976277157, "kl": 6.71875, "learning_rate": 8.30857455486667e-06, "loss": 0.0071, "num_tokens": 8372849.0, "reward": 2.0027379989624023, "reward_std": 0.24272873997688293, "rewards/check_originality_func/mean": 0.3214285671710968, "rewards/check_originality_func/std": 0.4712514579296112, "rewards/perplexity_reward/mean": 0.748046875, "rewards/perplexity_reward/std": 0.13742007315158844, "rewards/sentence_count_match_reward_logic/mean": 0.9332624673843384, "rewards/sentence_count_match_reward_logic/std": 0.1248771995306015, "step": 1961 }, { "clip_ratio": 0.06572866439819336, "epoch": 0.6864940517844647, "grad_norm": 0.7225718258978028, "kl": 6.71875, "learning_rate": 8.306284448754298e-06, "loss": 0.0032, "step": 1962 }, { "clip_ratio": 0.1258017122745514, "epoch": 0.6868439468159552, "grad_norm": 0.42296391011810647, "kl": 6.6875, "learning_rate": 8.303993109429118e-06, "loss": 0.0011, "step": 1963 }, { "clip_ratio": 0.21814672648906708, "epoch": 0.6871938418474458, "grad_norm": 0.4304892123495933, "kl": 6.65625, "learning_rate": 8.301700537745777e-06, "loss": -0.0003, "step": 1964 }, { "clip_ratio": 0.02834133990108967, "completions/clipped_ratio": 0.0, "completions/max_length": 229.0, "completions/max_terminated_length": 229.0, "completions/mean_length": 63.16071701049805, "completions/mean_terminated_length": 63.16071701049805, "completions/min_length": 6.0, "completions/min_terminated_length": 6.0, "epoch": 0.6875437368789363, "grad_norm": 4.995948748117943, "kl": 5.78125, "learning_rate": 8.299406734559385e-06, "loss": 0.0116, "num_tokens": 8384210.0, "reward": 2.3110616207122803, "reward_std": 0.3268488943576813, "rewards/check_originality_func/mean": 0.5535714030265808, "rewards/check_originality_func/std": 0.5016207695007324, "rewards/perplexity_reward/mean": 0.8147321343421936, "rewards/perplexity_reward/std": 0.08493363112211227, "rewards/sentence_count_match_reward_logic/mean": 0.9427579045295715, "rewards/sentence_count_match_reward_logic/std": 0.14190681278705597, "step": 1965 }, { "clip_ratio": 0.06812293082475662, "epoch": 0.6878936319104269, "grad_norm": 2.8091122267468696, "kl": 6.1875, "learning_rate": 8.297111700725508e-06, "loss": 0.0089, "step": 1966 }, { "clip_ratio": 0.08612407743930817, "epoch": 0.6882435269419174, "grad_norm": 2.6904238776661553, "kl": 6.25, "learning_rate": 8.294815437100172e-06, "loss": 0.0064, "step": 1967 }, { "clip_ratio": 0.15666338801383972, "epoch": 0.688593421973408, "grad_norm": 0.9325802187807731, "kl": 6.0, "learning_rate": 8.29251794453986e-06, "loss": 0.0, "step": 1968 }, { "clip_ratio": 0.019642332568764687, "completions/clipped_ratio": 0.0, "completions/max_length": 61.0, "completions/max_terminated_length": 61.0, "completions/mean_length": 17.803571701049805, "completions/mean_terminated_length": 17.803571701049805, "completions/min_length": 3.0, "completions/min_terminated_length": 3.0, "epoch": 0.6889433170048985, "grad_norm": 0.5222553537099137, "kl": 9.6875, "learning_rate": 8.290219223901517e-06, "loss": 0.006, "num_tokens": 8389311.0, "reward": 1.794057011604309, "reward_std": 0.22566094994544983, "rewards/check_originality_func/mean": 0.125, "rewards/check_originality_func/std": 0.3337118923664093, "rewards/perplexity_reward/mean": 0.6895926594734192, "rewards/perplexity_reward/std": 0.16485530138015747, "rewards/sentence_count_match_reward_logic/mean": 0.9794642329216003, "rewards/sentence_count_match_reward_logic/std": 0.07673849910497665, "step": 1969 }, { "clip_ratio": 0.09498383104801178, "epoch": 0.689293212036389, "grad_norm": 0.506936002534068, "kl": 9.9375, "learning_rate": 8.287919276042541e-06, "loss": 0.0048, "step": 1970 }, { "clip_ratio": 0.1717005968093872, "epoch": 0.6896431070678797, "grad_norm": 0.35876389105035417, "kl": 10.125, "learning_rate": 8.285618101820799e-06, "loss": 0.004, "step": 1971 }, { "clip_ratio": 0.26473963260650635, "epoch": 0.6899930020993702, "grad_norm": 0.3922327427920564, "kl": 10.4375, "learning_rate": 8.283315702094597e-06, "loss": 0.0026, "step": 1972 }, { "clip_ratio": 0.03109542466700077, "completions/clipped_ratio": 0.0, "completions/max_length": 246.0, "completions/max_terminated_length": 246.0, "completions/mean_length": 60.92857360839844, "completions/mean_terminated_length": 60.92857360839844, "completions/min_length": 3.0, "completions/min_terminated_length": 3.0, "epoch": 0.6903428971308607, "grad_norm": 2.5499970540841748, "kl": 6.6875, "learning_rate": 8.281012077722712e-06, "loss": 0.0014, "num_tokens": 8399707.0, "reward": 1.9783356189727783, "reward_std": 0.292489230632782, "rewards/check_originality_func/mean": 0.2142857164144516, "rewards/check_originality_func/std": 0.4140393137931824, "rewards/perplexity_reward/mean": 0.8710240125656128, "rewards/perplexity_reward/std": 0.09984754770994186, "rewards/sentence_count_match_reward_logic/mean": 0.8930257558822632, "rewards/sentence_count_match_reward_logic/std": 0.1937408596277237, "step": 1973 }, { "clip_ratio": 0.1151624545454979, "epoch": 0.6906927921623512, "grad_norm": 1.7413396365807492, "kl": 6.59375, "learning_rate": 8.278707229564375e-06, "loss": -0.0082, "step": 1974 }, { "clip_ratio": 0.31713083386421204, "epoch": 0.6910426871938419, "grad_norm": 1.1750004342161742, "kl": 6.59375, "learning_rate": 8.276401158479272e-06, "loss": -0.0145, "step": 1975 }, { "clip_ratio": 0.3858749568462372, "epoch": 0.6913925822253324, "grad_norm": 1.397996909919831, "kl": 6.25, "learning_rate": 8.274093865327548e-06, "loss": -0.0142, "step": 1976 }, { "clip_ratio": 0.018666788935661316, "completions/clipped_ratio": 0.0, "completions/max_length": 151.0, "completions/max_terminated_length": 151.0, "completions/mean_length": 31.267858505249023, "completions/mean_terminated_length": 31.267858505249023, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "epoch": 0.6917424772568229, "grad_norm": 1.0042082377548838, "kl": 8.0, "learning_rate": 8.271785350969799e-06, "loss": 0.0051, "num_tokens": 8406346.0, "reward": 1.892624855041504, "reward_std": 0.14460158348083496, "rewards/check_originality_func/mean": 0.1607142835855484, "rewards/check_originality_func/std": 0.3705909848213196, "rewards/perplexity_reward/mean": 0.7467913031578064, "rewards/perplexity_reward/std": 0.12165743112564087, "rewards/sentence_count_match_reward_logic/mean": 0.9851189851760864, "rewards/sentence_count_match_reward_logic/std": 0.07066389173269272, "step": 1977 }, { "clip_ratio": 0.08630214631557465, "epoch": 0.6920923722883136, "grad_norm": 0.5354725728921983, "kl": 8.0, "learning_rate": 8.269475616267077e-06, "loss": 0.0015, "step": 1978 }, { "clip_ratio": 0.20903609693050385, "epoch": 0.6924422673198041, "grad_norm": 0.37754409447223136, "kl": 8.0, "learning_rate": 8.267164662080897e-06, "loss": -0.0001, "step": 1979 }, { "clip_ratio": 0.2852891981601715, "epoch": 0.6927921623512946, "grad_norm": 0.34440074290415457, "kl": 8.0625, "learning_rate": 8.264852489273217e-06, "loss": -0.0013, "step": 1980 }, { "clip_ratio": 0.03197992593050003, "completions/clipped_ratio": 0.0, "completions/max_length": 228.0, "completions/max_terminated_length": 228.0, "completions/mean_length": 57.142860412597656, "completions/mean_terminated_length": 57.142860412597656, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "epoch": 0.6931420573827851, "grad_norm": 4.967878750390193, "kl": 7.46875, "learning_rate": 8.26253909870646e-06, "loss": 0.0208, "num_tokens": 8416874.0, "reward": 2.0089259147644043, "reward_std": 0.2308911681175232, "rewards/check_originality_func/mean": 0.2857142984867096, "rewards/check_originality_func/std": 0.4558423161506653, "rewards/perplexity_reward/mean": 0.7840401530265808, "rewards/perplexity_reward/std": 0.1093742772936821, "rewards/sentence_count_match_reward_logic/mean": 0.9391712546348572, "rewards/sentence_count_match_reward_logic/std": 0.11518732458353043, "step": 1981 }, { "clip_ratio": 0.0691644698381424, "epoch": 0.6934919524142757, "grad_norm": 2.4198965942795745, "kl": 7.59375, "learning_rate": 8.260224491243497e-06, "loss": 0.0079, "step": 1982 }, { "clip_ratio": 0.10961413383483887, "epoch": 0.6938418474457663, "grad_norm": 1.2121838599874064, "kl": 7.59375, "learning_rate": 8.257908667747651e-06, "loss": 0.0034, "step": 1983 }, { "clip_ratio": 0.16840961575508118, "epoch": 0.6941917424772568, "grad_norm": 0.6803345329491526, "kl": 7.75, "learning_rate": 8.25559162908271e-06, "loss": 0.0003, "step": 1984 }, { "clip_ratio": 0.02048712968826294, "completions/clipped_ratio": 0.0, "completions/max_length": 192.0, "completions/max_terminated_length": 192.0, "completions/mean_length": 67.17857360839844, "completions/mean_terminated_length": 67.17857360839844, "completions/min_length": 3.0, "completions/min_terminated_length": 3.0, "epoch": 0.6945416375087474, "grad_norm": 1.9632780190744128, "kl": 6.28125, "learning_rate": 8.253273376112902e-06, "loss": -0.0025, "num_tokens": 8428452.0, "reward": 2.0400407314300537, "reward_std": 0.3298467993736267, "rewards/check_originality_func/mean": 0.3035714328289032, "rewards/check_originality_func/std": 0.4639609158039093, "rewards/perplexity_reward/mean": 0.7547432780265808, "rewards/perplexity_reward/std": 0.1545858085155487, "rewards/sentence_count_match_reward_logic/mean": 0.9817258715629578, "rewards/sentence_count_match_reward_logic/std": 0.04375702887773514, "step": 1985 }, { "clip_ratio": 0.06352771818637848, "epoch": 0.6948915325402379, "grad_norm": 0.8479635883119001, "kl": 6.3125, "learning_rate": 8.250953909702915e-06, "loss": -0.0112, "step": 1986 }, { "clip_ratio": 0.13627390563488007, "epoch": 0.6952414275717285, "grad_norm": 0.5743875852656647, "kl": 6.25, "learning_rate": 8.248633230717888e-06, "loss": -0.0147, "step": 1987 }, { "clip_ratio": 0.23026518523693085, "epoch": 0.695591322603219, "grad_norm": 0.41730692262124514, "kl": 6.25, "learning_rate": 8.246311340023412e-06, "loss": -0.0173, "step": 1988 }, { "clip_ratio": 0.014029751531779766, "completions/clipped_ratio": 0.0, "completions/max_length": 132.0, "completions/max_terminated_length": 132.0, "completions/mean_length": 35.17857360839844, "completions/mean_terminated_length": 35.17857360839844, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "epoch": 0.6959412176347096, "grad_norm": 2.8267832089791023, "kl": 8.5, "learning_rate": 8.24398823848553e-06, "loss": 0.0165, "num_tokens": 8436702.0, "reward": 2.039672374725342, "reward_std": 0.17817573249340057, "rewards/check_originality_func/mean": 0.3214285671710968, "rewards/check_originality_func/std": 0.4712514281272888, "rewards/perplexity_reward/mean": 0.8251953125, "rewards/perplexity_reward/std": 0.11796674877405167, "rewards/sentence_count_match_reward_logic/mean": 0.8930484652519226, "rewards/sentence_count_match_reward_logic/std": 0.1978413462638855, "step": 1989 }, { "clip_ratio": 0.09441357105970383, "epoch": 0.6962911126662001, "grad_norm": 0.9216321625770765, "kl": 8.625, "learning_rate": 8.241663926970738e-06, "loss": 0.0095, "step": 1990 }, { "clip_ratio": 0.2047935277223587, "epoch": 0.6966410076976907, "grad_norm": 1.0197372465539762, "kl": 9.0, "learning_rate": 8.23933840634598e-06, "loss": 0.0093, "step": 1991 }, { "clip_ratio": 0.29451480507850647, "epoch": 0.6969909027291813, "grad_norm": 1.0351420426122055, "kl": 9.125, "learning_rate": 8.237011677478657e-06, "loss": 0.0083, "step": 1992 }, { "clip_ratio": 0.01738528534770012, "completions/clipped_ratio": 0.0, "completions/max_length": 185.0, "completions/max_terminated_length": 185.0, "completions/mean_length": 76.80357360839844, "completions/mean_terminated_length": 76.80357360839844, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "epoch": 0.6973407977606718, "grad_norm": 0.6833457698069823, "kl": 5.84375, "learning_rate": 8.234683741236612e-06, "loss": 0.0126, "num_tokens": 8449179.0, "reward": 2.2418668270111084, "reward_std": 0.20070751011371613, "rewards/check_originality_func/mean": 0.4821428656578064, "rewards/check_originality_func/std": 0.5042031407356262, "rewards/perplexity_reward/mean": 0.7882952094078064, "rewards/perplexity_reward/std": 0.14305704832077026, "rewards/sentence_count_match_reward_logic/mean": 0.9714285731315613, "rewards/sentence_count_match_reward_logic/std": 0.06019448861479759, "step": 1993 }, { "clip_ratio": 0.03788737580180168, "epoch": 0.6976906927921623, "grad_norm": 0.5427981566004851, "kl": 5.84375, "learning_rate": 8.232354598488148e-06, "loss": 0.0099, "step": 1994 }, { "clip_ratio": 0.0740416869521141, "epoch": 0.6980405878236529, "grad_norm": 0.41047775964984606, "kl": 5.8125, "learning_rate": 8.230024250102009e-06, "loss": 0.0077, "step": 1995 }, { "clip_ratio": 0.10634622722864151, "epoch": 0.6983904828551435, "grad_norm": 0.31081534226175006, "kl": 5.84375, "learning_rate": 8.227692696947395e-06, "loss": 0.0063, "step": 1996 }, { "clip_ratio": 0.033547542989254, "completions/clipped_ratio": 0.0714285714285714, "completions/max_length": 256.0, "completions/max_terminated_length": 251.0, "completions/mean_length": 90.89286041259766, "completions/mean_terminated_length": 78.19231414794922, "completions/min_length": 3.0, "completions/min_terminated_length": 3.0, "epoch": 0.698740377886634, "grad_norm": 2.613976201800155, "kl": 7.28125, "learning_rate": 8.225359939893954e-06, "loss": 0.0076, "num_tokens": 8465869.0, "reward": 2.1976914405822754, "reward_std": 0.2055434286594391, "rewards/check_originality_func/mean": 0.4821428656578064, "rewards/check_originality_func/std": 0.5042031407356262, "rewards/perplexity_reward/mean": 0.8380301594734192, "rewards/perplexity_reward/std": 0.08241409063339233, "rewards/sentence_count_match_reward_logic/mean": 0.8775182962417603, "rewards/sentence_count_match_reward_logic/std": 0.15937377512454987, "step": 1997 }, { "clip_ratio": 0.06477127969264984, "epoch": 0.6990902729181245, "grad_norm": 1.5580538311741639, "kl": 7.3125, "learning_rate": 8.223025979811781e-06, "loss": -0.0011, "step": 1998 }, { "clip_ratio": 0.12920083105564117, "epoch": 0.6994401679496152, "grad_norm": 0.6918495743758396, "kl": 7.5, "learning_rate": 8.220690817571422e-06, "loss": -0.0057, "step": 1999 }, { "clip_ratio": 0.18895436823368073, "epoch": 0.6997900629811057, "grad_norm": 1.3482613460545163, "kl": 7.59375, "learning_rate": 8.218354454043868e-06, "loss": -0.0077, "step": 2000 }, { "clip_ratio": 0.02834419533610344, "completions/clipped_ratio": 0.0892857142857143, "completions/max_length": 256.0, "completions/max_terminated_length": 256.0, "completions/mean_length": 164.7857208251953, "completions/mean_terminated_length": 155.8431396484375, "completions/min_length": 49.0, "completions/min_terminated_length": 49.0, "epoch": 0.7001399580125962, "grad_norm": 3.479086089832393, "kl": 3.03125, "learning_rate": 8.216016890100564e-06, "loss": 0.018, "num_tokens": 8490697.0, "reward": 2.514065980911255, "reward_std": 0.30706989765167236, "rewards/check_originality_func/mean": 0.75, "rewards/check_originality_func/std": 0.43693143129348755, "rewards/check_winston_local_func/mean": 0.8322424292564392, "rewards/check_winston_local_func/std": 0.3450256884098053, "rewards/sentence_count_match_reward_logic/mean": 0.9318234324455261, "rewards/sentence_count_match_reward_logic/std": 0.06806223094463348, "step": 2001 }, { "clip_ratio": 0.08149178326129913, "epoch": 0.7004898530440867, "grad_norm": 2.4868671598945578, "kl": 3.0, "learning_rate": 8.213678126613398e-06, "loss": 0.0017, "step": 2002 }, { "clip_ratio": 0.1882130354642868, "epoch": 0.7008397480755774, "grad_norm": 2.136697746429734, "kl": 3.078125, "learning_rate": 8.211338164454706e-06, "loss": -0.0102, "step": 2003 }, { "clip_ratio": 0.27368760108947754, "epoch": 0.7011896431070679, "grad_norm": 2.779883133315294, "kl": 3.765625, "learning_rate": 8.20899700449727e-06, "loss": -0.0164, "step": 2004 }, { "clip_ratio": 0.026784615591168404, "completions/clipped_ratio": 0.3035714285714286, "completions/max_length": 256.0, "completions/max_terminated_length": 251.0, "completions/mean_length": 177.96429443359375, "completions/mean_terminated_length": 143.94871520996094, "completions/min_length": 43.0, "completions/min_terminated_length": 43.0, "epoch": 0.7015395381385584, "grad_norm": 4.241562835446423, "kl": 2.671875, "learning_rate": 8.206654647614323e-06, "loss": 0.0038, "num_tokens": 8517671.0, "reward": 2.469219923019409, "reward_std": 0.38260114192962646, "rewards/check_originality_func/mean": 0.6428571343421936, "rewards/check_originality_func/std": 0.48349377512931824, "rewards/check_winston_local_func/mean": 0.9717332124710083, "rewards/check_winston_local_func/std": 0.042218081653118134, "rewards/sentence_count_match_reward_logic/mean": 0.8546295166015625, "rewards/sentence_count_match_reward_logic/std": 0.14506499469280243, "step": 2005 }, { "clip_ratio": 0.11238840967416763, "epoch": 0.701889433170049, "grad_norm": 2.304484164924412, "kl": 2.75, "learning_rate": 8.204311094679543e-06, "loss": -0.0101, "step": 2006 }, { "clip_ratio": 0.20184186100959778, "epoch": 0.7022393282015396, "grad_norm": 1.3346851889759666, "kl": 3.0, "learning_rate": 8.20196634656705e-06, "loss": -0.0202, "step": 2007 }, { "clip_ratio": 0.2817560136318207, "epoch": 0.7025892232330301, "grad_norm": 2.391719614899981, "kl": 3.21875, "learning_rate": 8.199620404151413e-06, "loss": -0.0225, "step": 2008 }, { "clip_ratio": 0.01887264847755432, "completions/clipped_ratio": 0.1607142857142857, "completions/max_length": 256.0, "completions/max_terminated_length": 251.0, "completions/mean_length": 154.67857360839844, "completions/mean_terminated_length": 135.27659606933594, "completions/min_length": 47.0, "completions/min_terminated_length": 47.0, "epoch": 0.7029391182645206, "grad_norm": 7.566342009175401, "kl": 2.09375, "learning_rate": 8.19727326830765e-06, "loss": -0.0057, "num_tokens": 8541853.0, "reward": 2.717376232147217, "reward_std": 0.23122753202915192, "rewards/check_originality_func/mean": 0.8392857313156128, "rewards/check_originality_func/std": 0.3705909848213196, "rewards/check_winston_local_func/mean": 0.973173975944519, "rewards/check_winston_local_func/std": 0.04111328721046448, "rewards/sentence_count_match_reward_logic/mean": 0.9049164652824402, "rewards/sentence_count_match_reward_logic/std": 0.13733667135238647, "step": 2009 }, { "clip_ratio": 0.07348597049713135, "epoch": 0.7032890132960112, "grad_norm": 2.4624557695549183, "kl": 2.03125, "learning_rate": 8.194924939911217e-06, "loss": -0.0271, "step": 2010 }, { "clip_ratio": 0.1178620234131813, "epoch": 0.7036389083275018, "grad_norm": 1.4128892963148878, "kl": 2.0625, "learning_rate": 8.19257541983802e-06, "loss": -0.0374, "step": 2011 }, { "clip_ratio": 0.18470022082328796, "epoch": 0.7039888033589923, "grad_norm": 2.823670051043652, "kl": 2.171875, "learning_rate": 8.190224708964406e-06, "loss": -0.0354, "step": 2012 }, { "clip_ratio": 0.018194155767560005, "completions/clipped_ratio": 0.2678571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 255.0, "completions/mean_length": 150.55357360839844, "completions/mean_terminated_length": 111.9756088256836, "completions/min_length": 41.0, "completions/min_terminated_length": 41.0, "epoch": 0.7043386983904829, "grad_norm": 4.649271181579862, "kl": 2.609375, "learning_rate": 8.18787280816717e-06, "loss": 0.0126, "num_tokens": 8564788.0, "reward": 2.225192070007324, "reward_std": 0.41159161925315857, "rewards/check_originality_func/mean": 0.4107142984867096, "rewards/check_originality_func/std": 0.49641573429107666, "rewards/check_winston_local_func/mean": 0.9535320997238159, "rewards/check_winston_local_func/std": 0.06791657954454422, "rewards/sentence_count_match_reward_logic/mean": 0.8609456419944763, "rewards/sentence_count_match_reward_logic/std": 0.13672246038913727, "step": 2013 }, { "clip_ratio": 0.07127144187688828, "epoch": 0.7046885934219734, "grad_norm": 2.032653703824691, "kl": 2.671875, "learning_rate": 8.185519718323547e-06, "loss": -0.0037, "step": 2014 }, { "clip_ratio": 0.1659456491470337, "epoch": 0.705038488453464, "grad_norm": 1.3698271232095007, "kl": 2.953125, "learning_rate": 8.183165440311218e-06, "loss": -0.0078, "step": 2015 }, { "clip_ratio": 0.24121175706386566, "epoch": 0.7053883834849545, "grad_norm": 0.7969166650073889, "kl": 3.125, "learning_rate": 8.180809975008305e-06, "loss": -0.0156, "step": 2016 }, { "clip_ratio": 0.012577052228152752, "completions/clipped_ratio": 0.017857142857142905, "completions/max_length": 256.0, "completions/max_terminated_length": 255.0, "completions/mean_length": 151.6428680419922, "completions/mean_terminated_length": 149.74545288085938, "completions/min_length": 41.0, "completions/min_terminated_length": 41.0, "epoch": 0.7057382785164451, "grad_norm": 5.873392064382496, "kl": 2.59375, "learning_rate": 8.178453323293378e-06, "loss": 0.0086, "num_tokens": 8586504.0, "reward": 2.622021198272705, "reward_std": 0.38358792662620544, "rewards/check_originality_func/mean": 0.6785714030265808, "rewards/check_originality_func/std": 0.4712514281272888, "rewards/check_winston_local_func/mean": 0.9831398725509644, "rewards/check_winston_local_func/std": 0.023291584104299545, "rewards/sentence_count_match_reward_logic/mean": 0.9603096842765808, "rewards/sentence_count_match_reward_logic/std": 0.07966236025094986, "step": 2017 }, { "clip_ratio": 0.0400409959256649, "epoch": 0.7060881735479356, "grad_norm": 2.629694496144881, "kl": 2.625, "learning_rate": 8.176095486045443e-06, "loss": -0.0067, "step": 2018 }, { "clip_ratio": 0.10978733003139496, "epoch": 0.7064380685794261, "grad_norm": 1.910801908605383, "kl": 2.75, "learning_rate": 8.173736464143951e-06, "loss": -0.0162, "step": 2019 }, { "clip_ratio": 0.16922807693481445, "epoch": 0.7067879636109167, "grad_norm": 1.64117676005428, "kl": 2.625, "learning_rate": 8.171376258468796e-06, "loss": -0.0205, "step": 2020 }, { "clip_ratio": 0.02141737937927246, "completions/clipped_ratio": 0.0714285714285714, "completions/max_length": 256.0, "completions/max_terminated_length": 250.0, "completions/mean_length": 163.7678680419922, "completions/mean_terminated_length": 156.67308044433594, "completions/min_length": 41.0, "completions/min_terminated_length": 41.0, "epoch": 0.7071378586424073, "grad_norm": 6.910284484244164, "kl": 2.640625, "learning_rate": 8.169014869900308e-06, "loss": 0.0148, "num_tokens": 8610027.0, "reward": 2.7186176776885986, "reward_std": 0.33965954184532166, "rewards/check_originality_func/mean": 0.7857142686843872, "rewards/check_originality_func/std": 0.4140393137931824, "rewards/check_winston_local_func/mean": 0.9879809021949768, "rewards/check_winston_local_func/std": 0.004087504465132952, "rewards/sentence_count_match_reward_logic/mean": 0.9449224472045898, "rewards/sentence_count_match_reward_logic/std": 0.08238393068313599, "step": 2021 }, { "clip_ratio": 0.0718613788485527, "epoch": 0.7074877536738978, "grad_norm": 2.1983591515284133, "kl": 2.59375, "learning_rate": 8.16665229931927e-06, "loss": -0.0028, "step": 2022 }, { "clip_ratio": 0.156927227973938, "epoch": 0.7078376487053883, "grad_norm": 1.2544030818032976, "kl": 2.46875, "learning_rate": 8.164288547606893e-06, "loss": -0.0109, "step": 2023 }, { "clip_ratio": 0.23443596065044403, "epoch": 0.708187543736879, "grad_norm": 0.7271812280100921, "kl": 2.375, "learning_rate": 8.161923615644836e-06, "loss": -0.0167, "step": 2024 }, { "clip_ratio": 0.016765978187322617, "completions/clipped_ratio": 0.0535714285714286, "completions/max_length": 256.0, "completions/max_terminated_length": 246.0, "completions/mean_length": 168.2678680419922, "completions/mean_terminated_length": 163.30189514160156, "completions/min_length": 75.0, "completions/min_terminated_length": 75.0, "epoch": 0.7085374387683695, "grad_norm": 3.9270740025575543, "kl": 1.421875, "learning_rate": 8.159557504315197e-06, "loss": -0.0189, "num_tokens": 8635458.0, "reward": 2.48774790763855, "reward_std": 0.23840239644050598, "rewards/check_originality_func/mean": 0.5892857313156128, "rewards/check_originality_func/std": 0.49641573429107666, "rewards/check_winston_local_func/mean": 0.9843584895133972, "rewards/check_winston_local_func/std": 0.01633448712527752, "rewards/sentence_count_match_reward_logic/mean": 0.914103627204895, "rewards/sentence_count_match_reward_logic/std": 0.1149265244603157, "step": 2025 }, { "clip_ratio": 0.08014998584985733, "epoch": 0.70888733379986, "grad_norm": 2.852890719032929, "kl": 1.3515625, "learning_rate": 8.15719021450051e-06, "loss": -0.0308, "step": 2026 }, { "clip_ratio": 0.1423940658569336, "epoch": 0.7092372288313505, "grad_norm": 1.482734741565358, "kl": 1.375, "learning_rate": 8.154821747083755e-06, "loss": -0.0365, "step": 2027 }, { "clip_ratio": 0.18883046507835388, "epoch": 0.7095871238628412, "grad_norm": 0.8569099878513778, "kl": 1.4921875, "learning_rate": 8.152452102948348e-06, "loss": -0.0414, "step": 2028 }, { "clip_ratio": 0.013187120668590069, "completions/clipped_ratio": 0.125, "completions/max_length": 256.0, "completions/max_terminated_length": 250.0, "completions/mean_length": 105.5714340209961, "completions/mean_terminated_length": 84.08163452148438, "completions/min_length": 33.0, "completions/min_terminated_length": 33.0, "epoch": 0.7099370188943317, "grad_norm": 1.514578922863678, "kl": 1.875, "learning_rate": 8.150081282978139e-06, "loss": 0.001, "num_tokens": 8651698.0, "reward": 2.4462976455688477, "reward_std": 0.2815612256526947, "rewards/check_originality_func/mean": 0.5357142686843872, "rewards/check_originality_func/std": 0.5032362937927246, "rewards/check_winston_local_func/mean": 0.9729415774345398, "rewards/check_winston_local_func/std": 0.03651093319058418, "rewards/sentence_count_match_reward_logic/mean": 0.9376416802406311, "rewards/sentence_count_match_reward_logic/std": 0.10941674560308456, "step": 2029 }, { "clip_ratio": 0.034159161150455475, "epoch": 0.7102869139258222, "grad_norm": 1.2245080171074583, "kl": 1.890625, "learning_rate": 8.147709288057428e-06, "loss": -0.0055, "step": 2030 }, { "clip_ratio": 0.09749134629964828, "epoch": 0.7106368089573128, "grad_norm": 0.907496413892552, "kl": 1.9140625, "learning_rate": 8.145336119070942e-06, "loss": -0.0104, "step": 2031 }, { "clip_ratio": 0.11619359999895096, "epoch": 0.7109867039888034, "grad_norm": 0.5304629469432981, "kl": 1.890625, "learning_rate": 8.142961776903852e-06, "loss": -0.014, "step": 2032 }, { "clip_ratio": 0.012517849914729595, "completions/clipped_ratio": 0.2678571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 224.0, "completions/mean_length": 171.92857360839844, "completions/mean_terminated_length": 141.1707305908203, "completions/min_length": 77.0, "completions/min_terminated_length": 77.0, "epoch": 0.7113365990202939, "grad_norm": 2.8055759129776274, "kl": 1.5703125, "learning_rate": 8.140586262441767e-06, "loss": -0.0053, "num_tokens": 8677654.0, "reward": 2.5871963500976562, "reward_std": 0.3031530976295471, "rewards/check_originality_func/mean": 0.7321428656578064, "rewards/check_originality_func/std": 0.44685041904449463, "rewards/check_winston_local_func/mean": 0.9762702584266663, "rewards/check_winston_local_func/std": 0.04421885311603546, "rewards/sentence_count_match_reward_logic/mean": 0.8787833452224731, "rewards/sentence_count_match_reward_logic/std": 0.08748452365398407, "step": 2033 }, { "clip_ratio": 0.05486190319061279, "epoch": 0.7116864940517844, "grad_norm": 1.662997316532608, "kl": 1.515625, "learning_rate": 8.138209576570729e-06, "loss": -0.0169, "step": 2034 }, { "clip_ratio": 0.1119902953505516, "epoch": 0.712036389083275, "grad_norm": 0.9733467644637583, "kl": 1.5, "learning_rate": 8.135831720177221e-06, "loss": -0.0238, "step": 2035 }, { "clip_ratio": 0.17237186431884766, "epoch": 0.7123862841147656, "grad_norm": 0.6308894823706198, "kl": 1.5546875, "learning_rate": 8.133452694148159e-06, "loss": -0.029, "step": 2036 }, { "clip_ratio": 0.01450545433908701, "completions/clipped_ratio": 0.1428571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 230.0, "completions/mean_length": 160.08929443359375, "completions/mean_terminated_length": 144.1041717529297, "completions/min_length": 47.0, "completions/min_terminated_length": 47.0, "epoch": 0.7127361791462561, "grad_norm": 3.8258648492942053, "kl": 1.4375, "learning_rate": 8.131072499370897e-06, "loss": 0.0058, "num_tokens": 8702307.0, "reward": 2.872870445251465, "reward_std": 0.10680339485406876, "rewards/check_originality_func/mean": 0.9821428656578064, "rewards/check_originality_func/std": 0.13363061845302582, "rewards/check_winston_local_func/mean": 0.9735690951347351, "rewards/check_winston_local_func/std": 0.036222655326128006, "rewards/sentence_count_match_reward_logic/mean": 0.9171584844589233, "rewards/sentence_count_match_reward_logic/std": 0.10345593839883804, "step": 2037 }, { "clip_ratio": 0.04557033255696297, "epoch": 0.7130860741777467, "grad_norm": 1.7758501489713334, "kl": 1.4453125, "learning_rate": 8.128691136733228e-06, "loss": -0.0064, "step": 2038 }, { "clip_ratio": 0.1040593758225441, "epoch": 0.7134359692092372, "grad_norm": 0.9210214458061651, "kl": 1.4765625, "learning_rate": 8.126308607123375e-06, "loss": -0.013, "step": 2039 }, { "clip_ratio": 0.15610773861408234, "epoch": 0.7137858642407278, "grad_norm": 0.7101640013966838, "kl": 1.546875, "learning_rate": 8.12392491143e-06, "loss": -0.018, "step": 2040 }, { "clip_ratio": 0.011185462586581707, "completions/clipped_ratio": 0.1964285714285714, "completions/max_length": 256.0, "completions/max_terminated_length": 256.0, "completions/mean_length": 161.80357360839844, "completions/mean_terminated_length": 138.7777862548828, "completions/min_length": 35.0, "completions/min_terminated_length": 35.0, "epoch": 0.7141357592722183, "grad_norm": 3.1254863381174567, "kl": 1.5390625, "learning_rate": 8.121540050542198e-06, "loss": 0.0052, "num_tokens": 8725080.0, "reward": 2.585813522338867, "reward_std": 0.32420647144317627, "rewards/check_originality_func/mean": 0.6964285969734192, "rewards/check_originality_func/std": 0.4639609456062317, "rewards/check_winston_local_func/mean": 0.9836625456809998, "rewards/check_winston_local_func/std": 0.025807281956076622, "rewards/sentence_count_match_reward_logic/mean": 0.9057223200798035, "rewards/sentence_count_match_reward_logic/std": 0.13892126083374023, "step": 2041 }, { "clip_ratio": 0.0441264733672142, "epoch": 0.7144856543037089, "grad_norm": 1.9554205770614261, "kl": 1.5390625, "learning_rate": 8.119154025349503e-06, "loss": -0.0007, "step": 2042 }, { "clip_ratio": 0.1072867289185524, "epoch": 0.7148355493351994, "grad_norm": 1.4477155621954687, "kl": 1.5703125, "learning_rate": 8.116766836741877e-06, "loss": -0.0106, "step": 2043 }, { "clip_ratio": 0.163981094956398, "epoch": 0.71518544436669, "grad_norm": 0.99124712872577, "kl": 1.6953125, "learning_rate": 8.114378485609718e-06, "loss": -0.0152, "step": 2044 }, { "clip_ratio": 0.014261278323829174, "completions/clipped_ratio": 0.1071428571428571, "completions/max_length": 256.0, "completions/max_terminated_length": 252.0, "completions/mean_length": 142.25, "completions/mean_terminated_length": 128.59999084472656, "completions/min_length": 13.0, "completions/min_terminated_length": 13.0, "epoch": 0.7155353393981806, "grad_norm": 4.016263338419082, "kl": 2.078125, "learning_rate": 8.111988972843859e-06, "loss": 0.0074, "num_tokens": 8747118.0, "reward": 2.6086833477020264, "reward_std": 0.23445598781108856, "rewards/check_originality_func/mean": 0.7321428656578064, "rewards/check_originality_func/std": 0.44685041904449463, "rewards/check_winston_local_func/mean": 0.9728878140449524, "rewards/check_winston_local_func/std": 0.04192868247628212, "rewards/sentence_count_match_reward_logic/mean": 0.9036526083946228, "rewards/sentence_count_match_reward_logic/std": 0.115697480738163, "step": 2045 }, { "clip_ratio": 0.05940719693899155, "epoch": 0.7158852344296711, "grad_norm": 2.996968587348772, "kl": 2.078125, "learning_rate": 8.109598299335566e-06, "loss": -0.0062, "step": 2046 }, { "clip_ratio": 0.13558168709278107, "epoch": 0.7162351294611616, "grad_norm": 1.1486433183780145, "kl": 2.09375, "learning_rate": 8.107206465976538e-06, "loss": -0.015, "step": 2047 }, { "clip_ratio": 0.2006758600473404, "epoch": 0.7165850244926522, "grad_norm": 0.8539855533680816, "kl": 2.140625, "learning_rate": 8.104813473658908e-06, "loss": -0.0198, "step": 2048 }, { "clip_ratio": 0.007521037477999926, "completions/clipped_ratio": 0.3928571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 255.0, "completions/mean_length": 218.23214721679688, "completions/mean_terminated_length": 193.7941131591797, "completions/min_length": 172.0, "completions/min_terminated_length": 172.0, "epoch": 0.7169349195241428, "grad_norm": 2.6009905443471526, "kl": 0.98828125, "learning_rate": 8.102419323275234e-06, "loss": 0.0105, "num_tokens": 8779571.0, "reward": 2.7321925163269043, "reward_std": 0.28680217266082764, "rewards/check_originality_func/mean": 0.8392857313156128, "rewards/check_originality_func/std": 0.3705909848213196, "rewards/check_winston_local_func/mean": 0.9844992756843567, "rewards/check_winston_local_func/std": 0.011957438662648201, "rewards/sentence_count_match_reward_logic/mean": 0.9084075093269348, "rewards/sentence_count_match_reward_logic/std": 0.09207744896411896, "step": 2049 }, { "clip_ratio": 0.03246569633483887, "epoch": 0.7172848145556333, "grad_norm": 1.971225125559453, "kl": 0.98046875, "learning_rate": 8.100024015718517e-06, "loss": 0.0006, "step": 2050 }, { "clip_ratio": 0.09956305474042892, "epoch": 0.7176347095871238, "grad_norm": 1.3279531459347595, "kl": 1.015625, "learning_rate": 8.097627551882181e-06, "loss": -0.0125, "step": 2051 }, { "clip_ratio": 0.17342588305473328, "epoch": 0.7179846046186145, "grad_norm": 0.7845147705232182, "kl": 1.109375, "learning_rate": 8.095229932660087e-06, "loss": -0.022, "step": 2052 }, { "clip_ratio": 0.01208407897502184, "completions/clipped_ratio": 0.1964285714285714, "completions/max_length": 256.0, "completions/max_terminated_length": 255.0, "completions/mean_length": 151.1428680419922, "completions/mean_terminated_length": 125.51111602783203, "completions/min_length": 70.0, "completions/min_terminated_length": 70.0, "epoch": 0.718334499650105, "grad_norm": 3.7773022592345145, "kl": 1.8984375, "learning_rate": 8.09283115894652e-06, "loss": 0.0171, "num_tokens": 8802187.0, "reward": 2.4811525344848633, "reward_std": 0.3573780357837677, "rewards/check_originality_func/mean": 0.5535714030265808, "rewards/check_originality_func/std": 0.5016207695007324, "rewards/check_winston_local_func/mean": 0.9891870617866516, "rewards/check_winston_local_func/std": 0.003903393168002367, "rewards/sentence_count_match_reward_logic/mean": 0.9383940100669861, "rewards/sentence_count_match_reward_logic/std": 0.08590047806501389, "step": 2053 }, { "clip_ratio": 0.053768057376146317, "epoch": 0.7186843946815955, "grad_norm": 3.8067879326746437, "kl": 1.984375, "learning_rate": 8.090431231636204e-06, "loss": 0.0038, "step": 2054 }, { "clip_ratio": 0.1129576563835144, "epoch": 0.719034289713086, "grad_norm": 3.598087471466192, "kl": 2.015625, "learning_rate": 8.088030151624285e-06, "loss": -0.0092, "step": 2055 }, { "clip_ratio": 0.1963188797235489, "epoch": 0.7193841847445767, "grad_norm": 1.6554261187928145, "kl": 2.0, "learning_rate": 8.085627919806349e-06, "loss": -0.0203, "step": 2056 }, { "clip_ratio": 0.010182623751461506, "completions/clipped_ratio": 0.1607142857142857, "completions/max_length": 256.0, "completions/max_terminated_length": 252.0, "completions/mean_length": 198.73214721679688, "completions/mean_terminated_length": 187.7659454345703, "completions/min_length": 117.0, "completions/min_terminated_length": 117.0, "epoch": 0.7197340797760672, "grad_norm": 3.5067487183707278, "kl": 1.5546875, "learning_rate": 8.083224537078401e-06, "loss": 0.0064, "num_tokens": 8831164.0, "reward": 2.6043407917022705, "reward_std": 0.3909221887588501, "rewards/check_originality_func/mean": 0.7142857313156128, "rewards/check_originality_func/std": 0.4558422863483429, "rewards/check_winston_local_func/mean": 0.9572440981864929, "rewards/check_winston_local_func/std": 0.06800051778554916, "rewards/sentence_count_match_reward_logic/mean": 0.9328107237815857, "rewards/sentence_count_match_reward_logic/std": 0.08071253448724747, "step": 2057 }, { "clip_ratio": 0.046522025018930435, "epoch": 0.7200839748075577, "grad_norm": 3.00029820710176, "kl": 1.5, "learning_rate": 8.08082000433688e-06, "loss": -0.0044, "step": 2058 }, { "clip_ratio": 0.11173032969236374, "epoch": 0.7204338698390483, "grad_norm": 1.7144171060494895, "kl": 1.4453125, "learning_rate": 8.078414322478657e-06, "loss": -0.0171, "step": 2059 }, { "clip_ratio": 0.19766326248645782, "epoch": 0.7207837648705389, "grad_norm": 0.9387134877474846, "kl": 1.4140625, "learning_rate": 8.076007492401026e-06, "loss": -0.027, "step": 2060 }, { "clip_ratio": 0.013568262569606304, "completions/clipped_ratio": 0.2857142857142857, "completions/max_length": 256.0, "completions/max_terminated_length": 185.0, "completions/mean_length": 159.1607208251953, "completions/mean_terminated_length": 120.42500305175781, "completions/min_length": 73.0, "completions/min_terminated_length": 73.0, "epoch": 0.7211336599020294, "grad_norm": 2.6144886528557154, "kl": 1.4609375, "learning_rate": 8.073599515001713e-06, "loss": 0.0027, "num_tokens": 8856405.0, "reward": 2.6600327491760254, "reward_std": 0.27857381105422974, "rewards/check_originality_func/mean": 0.7857142686843872, "rewards/check_originality_func/std": 0.4140393137931824, "rewards/check_winston_local_func/mean": 0.9364921450614929, "rewards/check_winston_local_func/std": 0.1166558563709259, "rewards/sentence_count_match_reward_logic/mean": 0.9378260374069214, "rewards/sentence_count_match_reward_logic/std": 0.08514602482318878, "step": 2061 }, { "clip_ratio": 0.044155362993478775, "epoch": 0.7214835549335199, "grad_norm": 2.2123969774366756, "kl": 1.4453125, "learning_rate": 8.07119039117887e-06, "loss": -0.0034, "step": 2062 }, { "clip_ratio": 0.11643365770578384, "epoch": 0.7218334499650105, "grad_norm": 1.5126006323680627, "kl": 1.4609375, "learning_rate": 8.06878012183108e-06, "loss": -0.0139, "step": 2063 }, { "clip_ratio": 0.20218387246131897, "epoch": 0.722183344996501, "grad_norm": 0.8809356808543471, "kl": 1.515625, "learning_rate": 8.066368707857347e-06, "loss": -0.0218, "step": 2064 }, { "clip_ratio": 0.006888160947710276, "completions/clipped_ratio": 0.1428571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 220.0, "completions/mean_length": 122.48214721679688, "completions/mean_terminated_length": 100.22917175292969, "completions/min_length": 25.0, "completions/min_terminated_length": 25.0, "epoch": 0.7225332400279916, "grad_norm": 2.4866193879829392, "kl": 2.65625, "learning_rate": 8.063956150157107e-06, "loss": -0.0058, "num_tokens": 8875184.0, "reward": 2.4035842418670654, "reward_std": 0.47872403264045715, "rewards/check_originality_func/mean": 0.5, "rewards/check_originality_func/std": 0.5045249462127686, "rewards/check_winston_local_func/mean": 0.9792341589927673, "rewards/check_winston_local_func/std": 0.030184978619217873, "rewards/sentence_count_match_reward_logic/mean": 0.9243499636650085, "rewards/sentence_count_match_reward_logic/std": 0.10601525008678436, "step": 2065 }, { "clip_ratio": 0.045465681701898575, "epoch": 0.7228831350594822, "grad_norm": 1.9189690578469698, "kl": 2.703125, "learning_rate": 8.061542449630222e-06, "loss": -0.018, "step": 2066 }, { "clip_ratio": 0.10180281102657318, "epoch": 0.7232330300909727, "grad_norm": 1.14729555389925, "kl": 2.703125, "learning_rate": 8.05912760717698e-06, "loss": -0.0261, "step": 2067 }, { "clip_ratio": 0.1937001496553421, "epoch": 0.7235829251224632, "grad_norm": 1.3793865105388723, "kl": 2.703125, "learning_rate": 8.05671162369809e-06, "loss": -0.0322, "step": 2068 }, { "clip_ratio": 0.006567354779690504, "completions/clipped_ratio": 0.2142857142857143, "completions/max_length": 256.0, "completions/max_terminated_length": 254.0, "completions/mean_length": 152.32144165039062, "completions/mean_terminated_length": 124.04545593261719, "completions/min_length": 52.0, "completions/min_terminated_length": 52.0, "epoch": 0.7239328201539538, "grad_norm": 2.178329963140257, "kl": 1.3984375, "learning_rate": 8.054294500094697e-06, "loss": 0.0014, "num_tokens": 8897818.0, "reward": 2.4017553329467773, "reward_std": 0.4056221544742584, "rewards/check_originality_func/mean": 0.5178571343421936, "rewards/check_originality_func/std": 0.5042031407356262, "rewards/check_winston_local_func/mean": 0.9634178876876831, "rewards/check_winston_local_func/std": 0.06795614957809448, "rewards/sentence_count_match_reward_logic/mean": 0.9204801321029663, "rewards/sentence_count_match_reward_logic/std": 0.09974642843008041, "step": 2069 }, { "clip_ratio": 0.029686113819479942, "epoch": 0.7242827151854444, "grad_norm": 1.2195725435813571, "kl": 1.390625, "learning_rate": 8.05187623726836e-06, "loss": -0.0069, "step": 2070 }, { "clip_ratio": 0.07284052670001984, "epoch": 0.7246326102169349, "grad_norm": 0.8551685278418419, "kl": 1.390625, "learning_rate": 8.049456836121072e-06, "loss": -0.0117, "step": 2071 }, { "clip_ratio": 0.12103170901536942, "epoch": 0.7249825052484254, "grad_norm": 0.6581405302738587, "kl": 1.390625, "learning_rate": 8.047036297555247e-06, "loss": -0.0169, "step": 2072 }, { "clip_ratio": 0.006500171963125467, "completions/clipped_ratio": 0.3392857142857143, "completions/max_length": 256.0, "completions/max_terminated_length": 256.0, "completions/mean_length": 210.5357208251953, "completions/mean_terminated_length": 187.18919372558594, "completions/min_length": 46.0, "completions/min_terminated_length": 46.0, "epoch": 0.7253324002799161, "grad_norm": 2.682708367027251, "kl": 1.03125, "learning_rate": 8.044614622473717e-06, "loss": 0.0061, "num_tokens": 8929264.0, "reward": 2.643139600753784, "reward_std": 0.29655715823173523, "rewards/check_originality_func/mean": 0.8035714030265808, "rewards/check_originality_func/std": 0.40089184045791626, "rewards/check_winston_local_func/mean": 0.9812751412391663, "rewards/check_winston_local_func/std": 0.01564428210258484, "rewards/sentence_count_match_reward_logic/mean": 0.8582929968833923, "rewards/sentence_count_match_reward_logic/std": 0.20243683457374573, "step": 2073 }, { "clip_ratio": 0.035904090851545334, "epoch": 0.7256822953114066, "grad_norm": 2.1850735083644173, "kl": 1.0390625, "learning_rate": 8.042191811779752e-06, "loss": -0.0038, "step": 2074 }, { "clip_ratio": 0.11271816492080688, "epoch": 0.7260321903428971, "grad_norm": 1.2704021132970873, "kl": 1.0703125, "learning_rate": 8.03976786637703e-06, "loss": -0.0162, "step": 2075 }, { "clip_ratio": 0.17268137633800507, "epoch": 0.7263820853743876, "grad_norm": 0.9699574664849976, "kl": 1.1171875, "learning_rate": 8.037342787169666e-06, "loss": -0.0239, "step": 2076 }, { "clip_ratio": 0.007569573819637299, "completions/clipped_ratio": 0.5178571428571428, "completions/max_length": 256.0, "completions/max_terminated_length": 255.0, "completions/mean_length": 241.07144165039062, "completions/mean_terminated_length": 225.0370330810547, "completions/min_length": 183.0, "completions/min_terminated_length": 183.0, "epoch": 0.7267319804058783, "grad_norm": 2.7839789078091566, "kl": 0.87890625, "learning_rate": 8.034916575062188e-06, "loss": 0.0158, "num_tokens": 8964236.0, "reward": 2.666931390762329, "reward_std": 0.2933032214641571, "rewards/check_originality_func/mean": 0.8035714030265808, "rewards/check_originality_func/std": 0.40089184045791626, "rewards/check_winston_local_func/mean": 0.9516452550888062, "rewards/check_winston_local_func/std": 0.06271722167730331, "rewards/sentence_count_match_reward_logic/mean": 0.9117146134376526, "rewards/sentence_count_match_reward_logic/std": 0.07589399814605713, "step": 2077 }, { "clip_ratio": 0.02418726682662964, "epoch": 0.7270818754373688, "grad_norm": 2.0044026016553884, "kl": 0.8984375, "learning_rate": 8.03248923095955e-06, "loss": -0.0041, "step": 2078 }, { "clip_ratio": 0.07218316942453384, "epoch": 0.7274317704688593, "grad_norm": 1.4944791994337105, "kl": 0.91015625, "learning_rate": 8.030060755767131e-06, "loss": -0.0121, "step": 2079 }, { "clip_ratio": 0.12585200369358063, "epoch": 0.72778166550035, "grad_norm": 1.3561110909102727, "kl": 0.89453125, "learning_rate": 8.027631150390728e-06, "loss": -0.0201, "step": 2080 }, { "clip_ratio": 0.007285294588655233, "completions/clipped_ratio": 0.125, "completions/max_length": 256.0, "completions/max_terminated_length": 252.0, "completions/mean_length": 147.57144165039062, "completions/mean_terminated_length": 132.08163452148438, "completions/min_length": 68.0, "completions/min_terminated_length": 68.0, "epoch": 0.7281315605318405, "grad_norm": 2.2923731204082056, "kl": 1.3671875, "learning_rate": 8.02520041573656e-06, "loss": -0.0092, "num_tokens": 8986484.0, "reward": 2.500863790512085, "reward_std": 0.35358259081840515, "rewards/check_originality_func/mean": 0.5714285969734192, "rewards/check_originality_func/std": 0.4993502199649811, "rewards/check_winston_local_func/mean": 0.9854946136474609, "rewards/check_winston_local_func/std": 0.011973317712545395, "rewards/sentence_count_match_reward_logic/mean": 0.9439404606819153, "rewards/sentence_count_match_reward_logic/std": 0.09100401401519775, "step": 2081 }, { "clip_ratio": 0.02462742105126381, "epoch": 0.728481455563331, "grad_norm": 1.4337934474214393, "kl": 1.3671875, "learning_rate": 8.022768552711268e-06, "loss": -0.0158, "step": 2082 }, { "clip_ratio": 0.05466243252158165, "epoch": 0.7288313505948215, "grad_norm": 1.1998148160038882, "kl": 1.3828125, "learning_rate": 8.020335562221914e-06, "loss": -0.021, "step": 2083 }, { "clip_ratio": 0.10532122105360031, "epoch": 0.7291812456263121, "grad_norm": 0.8408982331758545, "kl": 1.390625, "learning_rate": 8.017901445175981e-06, "loss": -0.027, "step": 2084 }, { "clip_ratio": 0.007551539223641157, "completions/clipped_ratio": 0.1071428571428571, "completions/max_length": 256.0, "completions/max_terminated_length": 227.0, "completions/mean_length": 145.0357208251953, "completions/mean_terminated_length": 131.72000122070312, "completions/min_length": 55.0, "completions/min_terminated_length": 55.0, "epoch": 0.7295311406578027, "grad_norm": 4.133996157320523, "kl": 1.578125, "learning_rate": 8.015466202481371e-06, "loss": 0.0123, "num_tokens": 9008174.0, "reward": 2.5804290771484375, "reward_std": 0.27501094341278076, "rewards/check_originality_func/mean": 0.6785714030265808, "rewards/check_originality_func/std": 0.4712514281272888, "rewards/check_winston_local_func/mean": 0.9768494367599487, "rewards/check_winston_local_func/std": 0.034525495022535324, "rewards/sentence_count_match_reward_logic/mean": 0.9250084161758423, "rewards/sentence_count_match_reward_logic/std": 0.09152408689260483, "step": 2085 }, { "clip_ratio": 0.03823931887745857, "epoch": 0.7298810356892932, "grad_norm": 1.5814041145831348, "kl": 1.609375, "learning_rate": 8.013029835046408e-06, "loss": 0.0039, "step": 2086 }, { "clip_ratio": 0.09447644650936127, "epoch": 0.7302309307207837, "grad_norm": 0.9976870485615048, "kl": 1.609375, "learning_rate": 8.010592343779833e-06, "loss": -0.0021, "step": 2087 }, { "clip_ratio": 0.14373953640460968, "epoch": 0.7305808257522743, "grad_norm": 0.6977275570032493, "kl": 1.609375, "learning_rate": 8.008153729590806e-06, "loss": -0.0081, "step": 2088 }, { "clip_ratio": 0.010246066376566887, "completions/clipped_ratio": 0.2857142857142857, "completions/max_length": 256.0, "completions/max_terminated_length": 256.0, "completions/mean_length": 200.83929443359375, "completions/mean_terminated_length": 178.77500915527344, "completions/min_length": 58.0, "completions/min_terminated_length": 58.0, "epoch": 0.7309307207837649, "grad_norm": 4.880054708801226, "kl": 1.1796875, "learning_rate": 8.005713993388908e-06, "loss": 0.0025, "num_tokens": 9037933.0, "reward": 2.617938995361328, "reward_std": 0.3320016860961914, "rewards/check_originality_func/mean": 0.75, "rewards/check_originality_func/std": 0.43693143129348755, "rewards/check_winston_local_func/mean": 0.976317286491394, "rewards/check_winston_local_func/std": 0.03573204204440117, "rewards/sentence_count_match_reward_logic/mean": 0.8916215896606445, "rewards/sentence_count_match_reward_logic/std": 0.095183365046978, "step": 2089 }, { "clip_ratio": 0.03513098135590553, "epoch": 0.7312806158152554, "grad_norm": 3.130873568429079, "kl": 1.1875, "learning_rate": 8.00327313608414e-06, "loss": -0.0059, "step": 2090 }, { "clip_ratio": 0.09361396729946136, "epoch": 0.731630510846746, "grad_norm": 1.9722923902463712, "kl": 1.2265625, "learning_rate": 8.000831158586915e-06, "loss": -0.0119, "step": 2091 }, { "clip_ratio": 0.15100474655628204, "epoch": 0.7319804058782365, "grad_norm": 1.5308180967472773, "kl": 1.2734375, "learning_rate": 7.998388061808067e-06, "loss": -0.0203, "step": 2092 }, { "clip_ratio": 0.010815792717039585, "completions/clipped_ratio": 0.3392857142857143, "completions/max_length": 256.0, "completions/max_terminated_length": 248.0, "completions/mean_length": 205.87501525878906, "completions/mean_terminated_length": 180.13514709472656, "completions/min_length": 109.0, "completions/min_terminated_length": 109.0, "epoch": 0.7323303009097271, "grad_norm": 4.3974799600871375, "kl": 1.7109375, "learning_rate": 7.995943846658852e-06, "loss": 0.0041, "num_tokens": 9067518.0, "reward": 2.8805735111236572, "reward_std": 0.1367570161819458, "rewards/check_originality_func/mean": 0.9821428656578064, "rewards/check_originality_func/std": 0.13363061845302582, "rewards/check_winston_local_func/mean": 0.9881356954574585, "rewards/check_winston_local_func/std": 0.008676106110215187, "rewards/sentence_count_match_reward_logic/mean": 0.9102945327758789, "rewards/sentence_count_match_reward_logic/std": 0.12227718532085419, "step": 2093 }, { "clip_ratio": 0.044201355427503586, "epoch": 0.7326801959412176, "grad_norm": 3.1843138330438197, "kl": 1.75, "learning_rate": 7.993498514050935e-06, "loss": -0.0125, "step": 2094 }, { "clip_ratio": 0.11979924887418747, "epoch": 0.7330300909727082, "grad_norm": 2.4927544938386603, "kl": 1.828125, "learning_rate": 7.991052064896406e-06, "loss": -0.0263, "step": 2095 }, { "clip_ratio": 0.18600189685821533, "epoch": 0.7333799860041987, "grad_norm": 2.565419172221815, "kl": 1.8828125, "learning_rate": 7.988604500107764e-06, "loss": -0.0312, "step": 2096 }, { "clip_ratio": 0.024549713358283043, "completions/clipped_ratio": 0.1071428571428571, "completions/max_length": 256.0, "completions/max_terminated_length": 228.0, "completions/mean_length": 189.32144165039062, "completions/mean_terminated_length": 181.3199920654297, "completions/min_length": 131.0, "completions/min_terminated_length": 131.0, "epoch": 0.7337298810356893, "grad_norm": 4.83374699174438, "kl": 1.234375, "learning_rate": 7.986155820597927e-06, "loss": 0.0046, "num_tokens": 9095000.0, "reward": 2.6384265422821045, "reward_std": 0.4014017879962921, "rewards/check_originality_func/mean": 0.75, "rewards/check_originality_func/std": 0.43693143129348755, "rewards/check_winston_local_func/mean": 0.9871391654014587, "rewards/check_winston_local_func/std": 0.010513568297028542, "rewards/sentence_count_match_reward_logic/mean": 0.9012871980667114, "rewards/sentence_count_match_reward_logic/std": 0.08340079337358475, "step": 2097 }, { "clip_ratio": 0.04389538988471031, "epoch": 0.7340797760671799, "grad_norm": 3.1528837192944494, "kl": 1.2578125, "learning_rate": 7.983706027280232e-06, "loss": -0.0041, "step": 2098 }, { "clip_ratio": 0.12550893425941467, "epoch": 0.7344296710986704, "grad_norm": 3.0238868291673833, "kl": 1.328125, "learning_rate": 7.981255121068428e-06, "loss": -0.0138, "step": 2099 }, { "clip_ratio": 0.1574120819568634, "epoch": 0.7347795661301609, "grad_norm": 1.855314110398461, "kl": 1.359375, "learning_rate": 7.97880310287668e-06, "loss": -0.0222, "step": 2100 }, { "clip_ratio": 0.009243912994861603, "completions/clipped_ratio": 0.3392857142857143, "completions/max_length": 256.0, "completions/max_terminated_length": 255.0, "completions/mean_length": 224.87501525878906, "completions/mean_terminated_length": 208.8918914794922, "completions/min_length": 159.0, "completions/min_terminated_length": 159.0, "epoch": 0.7351294611616515, "grad_norm": 2.7501867695898365, "kl": 1.046875, "learning_rate": 7.976349973619567e-06, "loss": 0.0079, "num_tokens": 9128441.0, "reward": 2.616645336151123, "reward_std": 0.3150328993797302, "rewards/check_originality_func/mean": 0.7857142686843872, "rewards/check_originality_func/std": 0.4140393137931824, "rewards/check_winston_local_func/mean": 0.9833932518959045, "rewards/check_winston_local_func/std": 0.01921728439629078, "rewards/sentence_count_match_reward_logic/mean": 0.8475378751754761, "rewards/sentence_count_match_reward_logic/std": 0.15697452425956726, "step": 2101 }, { "clip_ratio": 0.019717911258339882, "epoch": 0.7354793561931421, "grad_norm": 2.5211049031421613, "kl": 1.03125, "learning_rate": 7.973895734212082e-06, "loss": -0.0024, "step": 2102 }, { "clip_ratio": 0.071398064494133, "epoch": 0.7358292512246326, "grad_norm": 1.3901804198876282, "kl": 1.0, "learning_rate": 7.971440385569635e-06, "loss": -0.0113, "step": 2103 }, { "clip_ratio": 0.1351730227470398, "epoch": 0.7361791462561231, "grad_norm": 1.083251051739554, "kl": 1.0234375, "learning_rate": 7.968983928608047e-06, "loss": -0.018, "step": 2104 }, { "clip_ratio": 0.01066523790359497, "completions/clipped_ratio": 0.2142857142857143, "completions/max_length": 256.0, "completions/max_terminated_length": 255.0, "completions/mean_length": 200.42857360839844, "completions/mean_terminated_length": 185.27273559570312, "completions/min_length": 120.0, "completions/min_terminated_length": 120.0, "epoch": 0.7365290412876138, "grad_norm": 5.514672735274368, "kl": 1.1484375, "learning_rate": 7.966526364243553e-06, "loss": 0.0179, "num_tokens": 9157801.0, "reward": 2.568223237991333, "reward_std": 0.30370670557022095, "rewards/check_originality_func/mean": 0.6785714030265808, "rewards/check_originality_func/std": 0.4712514281272888, "rewards/check_winston_local_func/mean": 0.9868224263191223, "rewards/check_winston_local_func/std": 0.013299710117280483, "rewards/sentence_count_match_reward_logic/mean": 0.9028292894363403, "rewards/sentence_count_match_reward_logic/std": 0.1146521270275116, "step": 2105 }, { "clip_ratio": 0.04199964925646782, "epoch": 0.7368789363191043, "grad_norm": 3.098842752350656, "kl": 1.1640625, "learning_rate": 7.9640676933928e-06, "loss": -0.0002, "step": 2106 }, { "clip_ratio": 0.10670782625675201, "epoch": 0.7372288313505948, "grad_norm": 2.046429540385848, "kl": 1.171875, "learning_rate": 7.961607916972853e-06, "loss": -0.0087, "step": 2107 }, { "clip_ratio": 0.14950302243232727, "epoch": 0.7375787263820853, "grad_norm": 1.0245899948902362, "kl": 1.1328125, "learning_rate": 7.959147035901181e-06, "loss": -0.0143, "step": 2108 }, { "clip_ratio": 0.010296591557562351, "completions/clipped_ratio": 0.1964285714285714, "completions/max_length": 256.0, "completions/max_terminated_length": 256.0, "completions/mean_length": 183.96429443359375, "completions/mean_terminated_length": 166.35556030273438, "completions/min_length": 56.0, "completions/min_terminated_length": 56.0, "epoch": 0.737928621413576, "grad_norm": 4.403075631084002, "kl": 1.2421875, "learning_rate": 7.956685051095672e-06, "loss": 0.0008, "num_tokens": 9184447.0, "reward": 2.670360803604126, "reward_std": 0.35104358196258545, "rewards/check_originality_func/mean": 0.7321428656578064, "rewards/check_originality_func/std": 0.44685041904449463, "rewards/check_winston_local_func/mean": 0.9854849576950073, "rewards/check_winston_local_func/std": 0.009685038588941097, "rewards/sentence_count_match_reward_logic/mean": 0.9527326822280884, "rewards/sentence_count_match_reward_logic/std": 0.06827964633703232, "step": 2109 }, { "clip_ratio": 0.08260070532560349, "epoch": 0.7382785164450665, "grad_norm": 1.5781051337124943, "kl": 1.2421875, "learning_rate": 7.95422196347462e-06, "loss": -0.0109, "step": 2110 }, { "clip_ratio": 0.1368333101272583, "epoch": 0.738628411476557, "grad_norm": 1.1145272889745201, "kl": 1.25, "learning_rate": 7.951757773956738e-06, "loss": -0.0162, "step": 2111 }, { "clip_ratio": 0.1788545697927475, "epoch": 0.7389783065080476, "grad_norm": 0.964634192300305, "kl": 1.296875, "learning_rate": 7.949292483461141e-06, "loss": -0.021, "step": 2112 }, { "clip_ratio": 0.015435287728905678, "completions/clipped_ratio": 0.25, "completions/max_length": 256.0, "completions/max_terminated_length": 248.0, "completions/mean_length": 189.7857208251953, "completions/mean_terminated_length": 167.71429443359375, "completions/min_length": 57.0, "completions/min_terminated_length": 57.0, "epoch": 0.7393282015395382, "grad_norm": 2.9905123176792583, "kl": 1.1953125, "learning_rate": 7.946826092907362e-06, "loss": -0.0082, "num_tokens": 9212307.0, "reward": 2.4765942096710205, "reward_std": 0.48577016592025757, "rewards/check_originality_func/mean": 0.5892857313156128, "rewards/check_originality_func/std": 0.49641573429107666, "rewards/check_winston_local_func/mean": 0.9812982678413391, "rewards/check_winston_local_func/std": 0.022118711844086647, "rewards/sentence_count_match_reward_logic/mean": 0.9060102105140686, "rewards/sentence_count_match_reward_logic/std": 0.142809197306633, "step": 2113 }, { "clip_ratio": 0.05037607625126839, "epoch": 0.7396780965710287, "grad_norm": 1.8058702252044752, "kl": 1.1953125, "learning_rate": 7.94435860321534e-06, "loss": -0.02, "step": 2114 }, { "clip_ratio": 0.08491295576095581, "epoch": 0.7400279916025192, "grad_norm": 1.8907651325682944, "kl": 1.2109375, "learning_rate": 7.941890015305425e-06, "loss": -0.0257, "step": 2115 }, { "clip_ratio": 0.13617852330207825, "epoch": 0.7403778866340098, "grad_norm": 2.1471341877107726, "kl": 1.2109375, "learning_rate": 7.939420330098376e-06, "loss": -0.0307, "step": 2116 }, { "clip_ratio": 0.010014116764068604, "completions/clipped_ratio": 0.2857142857142857, "completions/max_length": 256.0, "completions/max_terminated_length": 221.0, "completions/mean_length": 166.2857208251953, "completions/mean_terminated_length": 130.40000915527344, "completions/min_length": 39.0, "completions/min_terminated_length": 39.0, "epoch": 0.7407277816655004, "grad_norm": 1.8853439698454961, "kl": 1.4140625, "learning_rate": 7.936949548515364e-06, "loss": 0.0128, "num_tokens": 9237819.0, "reward": 2.6359989643096924, "reward_std": 0.25013768672943115, "rewards/check_originality_func/mean": 0.7678571343421936, "rewards/check_originality_func/std": 0.4260205328464508, "rewards/check_winston_local_func/mean": 0.9569735527038574, "rewards/check_winston_local_func/std": 0.07081453502178192, "rewards/sentence_count_match_reward_logic/mean": 0.9111679792404175, "rewards/sentence_count_match_reward_logic/std": 0.1107606366276741, "step": 2117 }, { "clip_ratio": 0.023580705747008324, "epoch": 0.7410776766969909, "grad_norm": 1.5008809271332921, "kl": 1.421875, "learning_rate": 7.934477671477968e-06, "loss": 0.0062, "step": 2118 }, { "clip_ratio": 0.07803497463464737, "epoch": 0.7414275717284815, "grad_norm": 0.856004446603552, "kl": 1.4296875, "learning_rate": 7.932004699908173e-06, "loss": -0.0015, "step": 2119 }, { "clip_ratio": 0.12370114028453827, "epoch": 0.741777466759972, "grad_norm": 0.8168372419003269, "kl": 1.4453125, "learning_rate": 7.929530634728376e-06, "loss": -0.0058, "step": 2120 }, { "clip_ratio": 0.009186724200844765, "completions/clipped_ratio": 0.0892857142857143, "completions/max_length": 256.0, "completions/max_terminated_length": 254.0, "completions/mean_length": 178.71429443359375, "completions/mean_terminated_length": 171.13726806640625, "completions/min_length": 66.0, "completions/min_terminated_length": 66.0, "epoch": 0.7421273617914625, "grad_norm": 4.3300443890467655, "kl": 1.3828125, "learning_rate": 7.927055476861376e-06, "loss": 0.0047, "num_tokens": 9265619.0, "reward": 2.718733310699463, "reward_std": 0.33825770020484924, "rewards/check_originality_func/mean": 0.8214285969734192, "rewards/check_originality_func/std": 0.3864591419696808, "rewards/check_winston_local_func/mean": 0.982514500617981, "rewards/check_winston_local_func/std": 0.01855611428618431, "rewards/sentence_count_match_reward_logic/mean": 0.9147900342941284, "rewards/sentence_count_match_reward_logic/std": 0.09130743891000748, "step": 2121 }, { "clip_ratio": 0.031038101762533188, "epoch": 0.7424772568229531, "grad_norm": 3.215222031093704, "kl": 1.4375, "learning_rate": 7.924579227230387e-06, "loss": -0.0081, "step": 2122 }, { "clip_ratio": 0.0919000580906868, "epoch": 0.7428271518544437, "grad_norm": 2.375862625259057, "kl": 1.5, "learning_rate": 7.922101886759024e-06, "loss": -0.0207, "step": 2123 }, { "clip_ratio": 0.14092738926410675, "epoch": 0.7431770468859342, "grad_norm": 2.2934517278327577, "kl": 1.5625, "learning_rate": 7.919623456371315e-06, "loss": -0.0286, "step": 2124 }, { "clip_ratio": 0.027591990306973457, "completions/clipped_ratio": 0.6071428571428572, "completions/max_length": 256.0, "completions/max_terminated_length": 251.0, "completions/mean_length": 207.80357360839844, "completions/mean_terminated_length": 133.3181915283203, "completions/min_length": 92.0, "completions/min_terminated_length": 92.0, "epoch": 0.7435269419174247, "grad_norm": 13.577552435547204, "kl": 2.359375, "learning_rate": 7.917143936991688e-06, "loss": 0.0222, "num_tokens": 9299192.0, "reward": 2.5989599227905273, "reward_std": 0.3657134175300598, "rewards/check_originality_func/mean": 0.7321428656578064, "rewards/check_originality_func/std": 0.44685041904449463, "rewards/check_winston_local_func/mean": 0.9843548536300659, "rewards/check_winston_local_func/std": 0.01264265924692154, "rewards/sentence_count_match_reward_logic/mean": 0.882462203502655, "rewards/sentence_count_match_reward_logic/std": 0.11494461447000504, "step": 2125 }, { "clip_ratio": 0.06929879635572433, "epoch": 0.7438768369489154, "grad_norm": 120.70080129946864, "kl": 1.515625, "learning_rate": 7.914663329544983e-06, "loss": 0.1521, "step": 2126 }, { "clip_ratio": 0.11668096482753754, "epoch": 0.7442267319804059, "grad_norm": 13.471159025178023, "kl": 1.5, "learning_rate": 7.91218163495644e-06, "loss": 0.0037, "step": 2127 }, { "clip_ratio": 0.1350736767053604, "epoch": 0.7445766270118964, "grad_norm": 19.662641036460546, "kl": 3.25, "learning_rate": 7.909698854151708e-06, "loss": 0.0044, "step": 2128 }, { "clip_ratio": 0.02331198938190937, "completions/clipped_ratio": 0.375, "completions/max_length": 256.0, "completions/max_terminated_length": 252.0, "completions/mean_length": 184.21429443359375, "completions/mean_terminated_length": 141.14285278320312, "completions/min_length": 39.0, "completions/min_terminated_length": 39.0, "epoch": 0.7449265220433869, "grad_norm": 8.968741232730055, "kl": 2.421875, "learning_rate": 7.907214988056844e-06, "loss": 0.021, "num_tokens": 9327620.0, "reward": 2.509275436401367, "reward_std": 0.3446023762226105, "rewards/check_originality_func/mean": 0.6428571343421936, "rewards/check_originality_func/std": 0.48349377512931824, "rewards/check_winston_local_func/mean": 0.9828205108642578, "rewards/check_winston_local_func/std": 0.019438499584794044, "rewards/sentence_count_match_reward_logic/mean": 0.8835976719856262, "rewards/sentence_count_match_reward_logic/std": 0.11661939322948456, "step": 2129 }, { "clip_ratio": 0.03665489703416824, "epoch": 0.7452764170748776, "grad_norm": 3.0678006762221885, "kl": 2.546875, "learning_rate": 7.904730037598303e-06, "loss": 0.0075, "step": 2130 }, { "clip_ratio": 0.07824007421731949, "epoch": 0.7456263121063681, "grad_norm": 1.5532360144595587, "kl": 2.546875, "learning_rate": 7.902244003702951e-06, "loss": -0.0005, "step": 2131 }, { "clip_ratio": 0.14311255514621735, "epoch": 0.7459762071378586, "grad_norm": 1.1925721257130328, "kl": 2.59375, "learning_rate": 7.89975688729805e-06, "loss": -0.0057, "step": 2132 }, { "clip_ratio": 0.01830432191491127, "completions/clipped_ratio": 0.2857142857142857, "completions/max_length": 256.0, "completions/max_terminated_length": 246.0, "completions/mean_length": 182.98214721679688, "completions/mean_terminated_length": 153.77500915527344, "completions/min_length": 40.0, "completions/min_terminated_length": 40.0, "epoch": 0.7463261021693492, "grad_norm": 5.233811642059753, "kl": 1.9375, "learning_rate": 7.897268689311278e-06, "loss": 0.0278, "num_tokens": 9354251.0, "reward": 2.4832119941711426, "reward_std": 0.4157294034957886, "rewards/check_originality_func/mean": 0.6071428656578064, "rewards/check_originality_func/std": 0.4928053617477417, "rewards/check_winston_local_func/mean": 0.9882098436355591, "rewards/check_winston_local_func/std": 0.0060817585326731205, "rewards/sentence_count_match_reward_logic/mean": 0.8878589272499084, "rewards/sentence_count_match_reward_logic/std": 0.11236348748207092, "step": 2133 }, { "clip_ratio": 0.04213395714759827, "epoch": 0.7466759972008398, "grad_norm": 3.763641858455588, "kl": 1.9453125, "learning_rate": 7.894779410670704e-06, "loss": 0.0101, "step": 2134 }, { "clip_ratio": 0.12040141969919205, "epoch": 0.7470258922323303, "grad_norm": 2.1304395462811057, "kl": 1.9765625, "learning_rate": 7.892289052304807e-06, "loss": 0.0087, "step": 2135 }, { "clip_ratio": 0.2114437073469162, "epoch": 0.7473757872638208, "grad_norm": 1.8404868650097077, "kl": 1.953125, "learning_rate": 7.889797615142467e-06, "loss": -0.006, "step": 2136 }, { "clip_ratio": 0.008401838131248951, "completions/clipped_ratio": 0.3214285714285714, "completions/max_length": 256.0, "completions/max_terminated_length": 249.0, "completions/mean_length": 192.32144165039062, "completions/mean_terminated_length": 162.15789794921875, "completions/min_length": 55.0, "completions/min_terminated_length": 55.0, "epoch": 0.7477256822953114, "grad_norm": 2.7133529685160753, "kl": 2.03125, "learning_rate": 7.887305100112967e-06, "loss": 0.0159, "num_tokens": 9383101.0, "reward": 2.8041958808898926, "reward_std": 0.26495248079299927, "rewards/check_originality_func/mean": 0.8928571343421936, "rewards/check_originality_func/std": 0.3120938837528229, "rewards/check_winston_local_func/mean": 0.9866042733192444, "rewards/check_winston_local_func/std": 0.014547840692102909, "rewards/sentence_count_match_reward_logic/mean": 0.9247344136238098, "rewards/sentence_count_match_reward_logic/std": 0.082570880651474, "step": 2137 }, { "clip_ratio": 0.05364837870001793, "epoch": 0.748075577326802, "grad_norm": 1.9520489084626695, "kl": 1.8671875, "learning_rate": 7.884811508145993e-06, "loss": 0.0066, "step": 2138 }, { "clip_ratio": 0.11355232447385788, "epoch": 0.7484254723582925, "grad_norm": 1.5412051183812536, "kl": 1.7578125, "learning_rate": 7.882316840171626e-06, "loss": -0.0037, "step": 2139 }, { "clip_ratio": 0.17719995975494385, "epoch": 0.7487753673897831, "grad_norm": 1.6414196239060654, "kl": 1.6953125, "learning_rate": 7.879821097120358e-06, "loss": -0.0098, "step": 2140 }, { "clip_ratio": 0.00729231396690011, "completions/clipped_ratio": 0.2142857142857143, "completions/max_length": 256.0, "completions/max_terminated_length": 253.0, "completions/mean_length": 193.7857208251953, "completions/mean_terminated_length": 176.8181915283203, "completions/min_length": 50.0, "completions/min_terminated_length": 50.0, "epoch": 0.7491252624212736, "grad_norm": 1.4258509157480972, "kl": 1.1796875, "learning_rate": 7.877324279923078e-06, "loss": 0.0054, "num_tokens": 9411361.0, "reward": 2.67287540435791, "reward_std": 0.24957087635993958, "rewards/check_originality_func/mean": 0.75, "rewards/check_originality_func/std": 0.43693143129348755, "rewards/check_winston_local_func/mean": 0.9869065880775452, "rewards/check_winston_local_func/std": 0.004938574507832527, "rewards/sentence_count_match_reward_logic/mean": 0.935968816280365, "rewards/sentence_count_match_reward_logic/std": 0.10220713913440704, "step": 2141 }, { "clip_ratio": 0.020145174115896225, "epoch": 0.7494751574527642, "grad_norm": 1.0801602548098903, "kl": 1.1796875, "learning_rate": 7.874826389511073e-06, "loss": 0.0009, "step": 2142 }, { "clip_ratio": 0.051303036510944366, "epoch": 0.7498250524842547, "grad_norm": 0.7953747110256156, "kl": 1.171875, "learning_rate": 7.872327426816031e-06, "loss": -0.0051, "step": 2143 }, { "clip_ratio": 0.07895483076572418, "epoch": 0.7501749475157453, "grad_norm": 0.609074832894136, "kl": 1.1640625, "learning_rate": 7.869827392770046e-06, "loss": -0.0098, "step": 2144 }, { "clip_ratio": 0.009038022719323635, "completions/clipped_ratio": 0.0714285714285714, "completions/max_length": 256.0, "completions/max_terminated_length": 252.0, "completions/mean_length": 176.19644165039062, "completions/mean_terminated_length": 170.0576934814453, "completions/min_length": 52.0, "completions/min_terminated_length": 52.0, "epoch": 0.7505248425472358, "grad_norm": 3.3049512689439493, "kl": 1.3046875, "learning_rate": 7.867326288305603e-06, "loss": 0.0044, "num_tokens": 9436484.0, "reward": 2.655747652053833, "reward_std": 0.28322455286979675, "rewards/check_originality_func/mean": 0.75, "rewards/check_originality_func/std": 0.43693143129348755, "rewards/check_winston_local_func/mean": 0.9841887354850769, "rewards/check_winston_local_func/std": 0.024882102385163307, "rewards/sentence_count_match_reward_logic/mean": 0.9215587973594666, "rewards/sentence_count_match_reward_logic/std": 0.10530741512775421, "step": 2145 }, { "clip_ratio": 0.03930291160941124, "epoch": 0.7508747375787264, "grad_norm": 1.647422884564362, "kl": 1.2734375, "learning_rate": 7.864824114355591e-06, "loss": -0.0014, "step": 2146 }, { "clip_ratio": 0.06726961582899094, "epoch": 0.7512246326102169, "grad_norm": 0.7341171063973528, "kl": 1.2734375, "learning_rate": 7.8623208718533e-06, "loss": -0.0048, "step": 2147 }, { "clip_ratio": 0.08873240649700165, "epoch": 0.7515745276417075, "grad_norm": 0.7447212380237466, "kl": 1.2734375, "learning_rate": 7.859816561732414e-06, "loss": -0.008, "step": 2148 }, { "clip_ratio": 0.005772545468062162, "completions/clipped_ratio": 0.1428571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 252.0, "completions/mean_length": 156.71429443359375, "completions/mean_terminated_length": 140.1666717529297, "completions/min_length": 38.0, "completions/min_terminated_length": 38.0, "epoch": 0.751924422673198, "grad_norm": 2.4812568207997727, "kl": 1.390625, "learning_rate": 7.857311184927015e-06, "loss": 0.0148, "num_tokens": 9460236.0, "reward": 2.5799224376678467, "reward_std": 0.39010894298553467, "rewards/check_originality_func/mean": 0.7142857313156128, "rewards/check_originality_func/std": 0.4558422863483429, "rewards/check_winston_local_func/mean": 0.98729407787323, "rewards/check_winston_local_func/std": 0.008542174473404884, "rewards/sentence_count_match_reward_logic/mean": 0.8783427476882935, "rewards/sentence_count_match_reward_logic/std": 0.1141805350780487, "step": 2149 }, { "clip_ratio": 0.012322924099862576, "epoch": 0.7522743177046886, "grad_norm": 1.998510105544124, "kl": 1.3984375, "learning_rate": 7.854804742371588e-06, "loss": 0.0067, "step": 2150 }, { "clip_ratio": 0.056206244975328445, "epoch": 0.7526242127361792, "grad_norm": 2.058111638697551, "kl": 1.4140625, "learning_rate": 7.85229723500101e-06, "loss": 0.0031, "step": 2151 }, { "clip_ratio": 0.08041717112064362, "epoch": 0.7529741077676697, "grad_norm": 0.9626733028292886, "kl": 1.4140625, "learning_rate": 7.84978866375056e-06, "loss": -0.0044, "step": 2152 }, { "clip_ratio": 0.00529991602525115, "completions/clipped_ratio": 0.2321428571428571, "completions/max_length": 256.0, "completions/max_terminated_length": 255.0, "completions/mean_length": 200.6607208251953, "completions/mean_terminated_length": 183.93023681640625, "completions/min_length": 119.0, "completions/min_terminated_length": 119.0, "epoch": 0.7533240027991602, "grad_norm": 3.587996838186113, "kl": 1.2578125, "learning_rate": 7.847279029555908e-06, "loss": -0.0084, "num_tokens": 9489705.0, "reward": 2.5287673473358154, "reward_std": 0.1705343872308731, "rewards/check_originality_func/mean": 0.6428571343421936, "rewards/check_originality_func/std": 0.48349377512931824, "rewards/check_winston_local_func/mean": 0.981205403804779, "rewards/check_winston_local_func/std": 0.0329505018889904, "rewards/sentence_count_match_reward_logic/mean": 0.9047044515609741, "rewards/sentence_count_match_reward_logic/std": 0.07078342139720917, "step": 2153 }, { "clip_ratio": 0.027630461379885674, "epoch": 0.7536738978306508, "grad_norm": 1.7859650230959052, "kl": 1.234375, "learning_rate": 7.844768333353127e-06, "loss": -0.0198, "step": 2154 }, { "clip_ratio": 0.0627666488289833, "epoch": 0.7540237928621414, "grad_norm": 1.6909826399556471, "kl": 1.1875, "learning_rate": 7.842256576078682e-06, "loss": -0.0315, "step": 2155 }, { "clip_ratio": 0.14344774186611176, "epoch": 0.7543736878936319, "grad_norm": 1.4496094283597238, "kl": 1.171875, "learning_rate": 7.839743758669433e-06, "loss": -0.0383, "step": 2156 }, { "clip_ratio": 0.011644698679447174, "completions/clipped_ratio": 0.125, "completions/max_length": 256.0, "completions/max_terminated_length": 253.0, "completions/mean_length": 123.16072082519531, "completions/mean_terminated_length": 104.18367004394531, "completions/min_length": 24.0, "completions/min_terminated_length": 24.0, "epoch": 0.7547235829251224, "grad_norm": 1.8089275820400639, "kl": 2.078125, "learning_rate": 7.837229882062638e-06, "loss": -0.0003, "num_tokens": 9508530.0, "reward": 2.3880162239074707, "reward_std": 0.37421920895576477, "rewards/check_originality_func/mean": 0.4642857015132904, "rewards/check_originality_func/std": 0.5032362341880798, "rewards/check_winston_local_func/mean": 0.9812345504760742, "rewards/check_winston_local_func/std": 0.03822438791394234, "rewards/sentence_count_match_reward_logic/mean": 0.9424957633018494, "rewards/sentence_count_match_reward_logic/std": 0.08870209008455276, "step": 2157 }, { "clip_ratio": 0.032357968389987946, "epoch": 0.7550734779566131, "grad_norm": 1.1460374673807012, "kl": 2.078125, "learning_rate": 7.83471494719595e-06, "loss": -0.0068, "step": 2158 }, { "clip_ratio": 0.08822914212942123, "epoch": 0.7554233729881036, "grad_norm": 0.7113766329971365, "kl": 2.09375, "learning_rate": 7.832198955007416e-06, "loss": -0.0129, "step": 2159 }, { "clip_ratio": 0.13177916407585144, "epoch": 0.7557732680195941, "grad_norm": 0.6160455281097519, "kl": 2.109375, "learning_rate": 7.829681906435475e-06, "loss": -0.017, "step": 2160 }, { "clip_ratio": 0.004348939750343561, "completions/clipped_ratio": 0.1071428571428571, "completions/max_length": 256.0, "completions/max_terminated_length": 255.0, "completions/mean_length": 191.07144165039062, "completions/mean_terminated_length": 183.27999877929688, "completions/min_length": 67.0, "completions/min_terminated_length": 67.0, "epoch": 0.7561231630510846, "grad_norm": 3.0785712995890466, "kl": 1.359375, "learning_rate": 7.827163802418967e-06, "loss": 0.0105, "num_tokens": 9534846.0, "reward": 2.665756940841675, "reward_std": 0.3538864850997925, "rewards/check_originality_func/mean": 0.75, "rewards/check_originality_func/std": 0.43693143129348755, "rewards/check_winston_local_func/mean": 0.9751632809638977, "rewards/check_winston_local_func/std": 0.05598682910203934, "rewards/sentence_count_match_reward_logic/mean": 0.940593421459198, "rewards/sentence_count_match_reward_logic/std": 0.07376660406589508, "step": 2161 }, { "clip_ratio": 0.04320605844259262, "epoch": 0.7564730580825753, "grad_norm": 2.4094600006388114, "kl": 1.3515625, "learning_rate": 7.824644643897116e-06, "loss": -0.0009, "step": 2162 }, { "clip_ratio": 0.09783897548913956, "epoch": 0.7568229531140658, "grad_norm": 1.6332233487545003, "kl": 1.328125, "learning_rate": 7.822124431809546e-06, "loss": -0.0152, "step": 2163 }, { "clip_ratio": 0.17668724060058594, "epoch": 0.7571728481455563, "grad_norm": 1.3078120309854084, "kl": 1.3125, "learning_rate": 7.819603167096272e-06, "loss": -0.0256, "step": 2164 }, { "clip_ratio": 0.0037288819439709187, "completions/clipped_ratio": 0.2857142857142857, "completions/max_length": 256.0, "completions/max_terminated_length": 180.0, "completions/mean_length": 166.125, "completions/mean_terminated_length": 130.1750030517578, "completions/min_length": 90.0, "completions/min_terminated_length": 90.0, "epoch": 0.7575227431770469, "grad_norm": 2.4271987198544225, "kl": 1.1484375, "learning_rate": 7.817080850697705e-06, "loss": 0.008, "num_tokens": 9559605.0, "reward": 2.7046613693237305, "reward_std": 0.30501577258110046, "rewards/check_originality_func/mean": 0.8214285969734192, "rewards/check_originality_func/std": 0.3864591419696808, "rewards/check_winston_local_func/mean": 0.973307728767395, "rewards/check_winston_local_func/std": 0.04051772877573967, "rewards/sentence_count_match_reward_logic/mean": 0.9099251627922058, "rewards/sentence_count_match_reward_logic/std": 0.09889595210552216, "step": 2165 }, { "clip_ratio": 0.021914701908826828, "epoch": 0.7578726382085375, "grad_norm": 1.7711895593386227, "kl": 1.1328125, "learning_rate": 7.81455748355464e-06, "loss": -0.0021, "step": 2166 }, { "clip_ratio": 0.09262580424547195, "epoch": 0.758222533240028, "grad_norm": 1.0711309841807855, "kl": 1.1171875, "learning_rate": 7.812033066608273e-06, "loss": -0.0109, "step": 2167 }, { "clip_ratio": 0.1536041498184204, "epoch": 0.7585724282715185, "grad_norm": 1.0707593061220766, "kl": 1.109375, "learning_rate": 7.809507600800186e-06, "loss": -0.0159, "step": 2168 }, { "clip_ratio": 0.004545513074845076, "completions/clipped_ratio": 0.3214285714285714, "completions/max_length": 256.0, "completions/max_terminated_length": 256.0, "completions/mean_length": 172.1428680419922, "completions/mean_terminated_length": 132.42105102539062, "completions/min_length": 47.0, "completions/min_terminated_length": 47.0, "epoch": 0.7589223233030091, "grad_norm": 2.1472670977726738, "kl": 1.203125, "learning_rate": 7.806981087072354e-06, "loss": 0.0067, "num_tokens": 9585181.0, "reward": 2.5002007484436035, "reward_std": 0.36401936411857605, "rewards/check_originality_func/mean": 0.6785714030265808, "rewards/check_originality_func/std": 0.4712514281272888, "rewards/check_winston_local_func/mean": 0.9740623235702515, "rewards/check_winston_local_func/std": 0.04762289673089981, "rewards/sentence_count_match_reward_logic/mean": 0.8475667238235474, "rewards/sentence_count_match_reward_logic/std": 0.1600535809993744, "step": 2169 }, { "clip_ratio": 0.024864360690116882, "epoch": 0.7592722183344996, "grad_norm": 1.6024489010230993, "kl": 1.203125, "learning_rate": 7.804453526367142e-06, "loss": 0.0037, "step": 2170 }, { "clip_ratio": 0.053475745022296906, "epoch": 0.7596221133659902, "grad_norm": 1.4624756689349228, "kl": 1.203125, "learning_rate": 7.801924919627308e-06, "loss": -0.0042, "step": 2171 }, { "clip_ratio": 0.09953618049621582, "epoch": 0.7599720083974808, "grad_norm": 1.2076846427540135, "kl": 1.234375, "learning_rate": 7.799395267795997e-06, "loss": -0.0105, "step": 2172 }, { "clip_ratio": 0.003395227249711752, "completions/clipped_ratio": 0.125, "completions/max_length": 256.0, "completions/max_terminated_length": 256.0, "completions/mean_length": 160.58929443359375, "completions/mean_terminated_length": 146.9591827392578, "completions/min_length": 55.0, "completions/min_terminated_length": 55.0, "epoch": 0.7603219034289713, "grad_norm": 2.1246461382047155, "kl": 1.4609375, "learning_rate": 7.796864571816745e-06, "loss": 0.011, "num_tokens": 9607670.0, "reward": 2.5154788494110107, "reward_std": 0.4379109740257263, "rewards/check_originality_func/mean": 0.6428571343421936, "rewards/check_originality_func/std": 0.48349377512931824, "rewards/check_winston_local_func/mean": 0.9897643327713013, "rewards/check_winston_local_func/std": 0.003436259925365448, "rewards/sentence_count_match_reward_logic/mean": 0.8828572034835815, "rewards/sentence_count_match_reward_logic/std": 0.11735150963068008, "step": 2173 }, { "clip_ratio": 0.022078145295381546, "epoch": 0.7606717984604618, "grad_norm": 1.7264248433419247, "kl": 1.4765625, "learning_rate": 7.794332832633479e-06, "loss": 0.0027, "step": 2174 }, { "clip_ratio": 0.07303136587142944, "epoch": 0.7610216934919524, "grad_norm": 1.186669515141657, "kl": 1.4921875, "learning_rate": 7.791800051190513e-06, "loss": -0.0062, "step": 2175 }, { "clip_ratio": 0.14210180938243866, "epoch": 0.761371588523443, "grad_norm": 1.4142949298424974, "kl": 1.5234375, "learning_rate": 7.78926622843255e-06, "loss": -0.0132, "step": 2176 }, { "clip_ratio": 0.0031786151230335236, "completions/clipped_ratio": 0.0, "completions/max_length": 228.0, "completions/max_terminated_length": 228.0, "completions/mean_length": 128.5357208251953, "completions/mean_terminated_length": 128.5357208251953, "completions/min_length": 58.0, "completions/min_terminated_length": 58.0, "epoch": 0.7617214835549335, "grad_norm": 2.039867743214715, "kl": 1.5390625, "learning_rate": 7.786731365304682e-06, "loss": 0.0001, "num_tokens": 9627084.0, "reward": 2.472219944000244, "reward_std": 0.25490739941596985, "rewards/check_originality_func/mean": 0.5357142686843872, "rewards/check_originality_func/std": 0.5032362937927246, "rewards/check_winston_local_func/mean": 0.9803491830825806, "rewards/check_winston_local_func/std": 0.04339771345257759, "rewards/sentence_count_match_reward_logic/mean": 0.9561562538146973, "rewards/sentence_count_match_reward_logic/std": 0.07006101310253143, "step": 2177 }, { "clip_ratio": 0.019371164962649345, "epoch": 0.762071378586424, "grad_norm": 1.4471602741948244, "kl": 1.546875, "learning_rate": 7.78419546275239e-06, "loss": -0.0061, "step": 2178 }, { "clip_ratio": 0.0842612162232399, "epoch": 0.7624212736179147, "grad_norm": 1.0856101219084366, "kl": 1.5546875, "learning_rate": 7.781658521721542e-06, "loss": -0.0151, "step": 2179 }, { "clip_ratio": 0.1355629712343216, "epoch": 0.7627711686494052, "grad_norm": 0.6940262921998606, "kl": 1.5625, "learning_rate": 7.77912054315839e-06, "loss": -0.0214, "step": 2180 }, { "clip_ratio": 0.004360164050012827, "completions/clipped_ratio": 0.1428571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 250.0, "completions/mean_length": 191.94644165039062, "completions/mean_terminated_length": 181.27084350585938, "completions/min_length": 97.0, "completions/min_terminated_length": 97.0, "epoch": 0.7631210636808957, "grad_norm": 1.3720811075814436, "kl": 1.1171875, "learning_rate": 7.77658152800958e-06, "loss": -0.0064, "num_tokens": 9654945.0, "reward": 2.7809340953826904, "reward_std": 0.32467612624168396, "rewards/check_originality_func/mean": 0.8571428656578064, "rewards/check_originality_func/std": 0.3530939221382141, "rewards/check_winston_local_func/mean": 0.9897170662879944, "rewards/check_winston_local_func/std": 0.003332756692543626, "rewards/sentence_count_match_reward_logic/mean": 0.9340740442276001, "rewards/sentence_count_match_reward_logic/std": 0.08381291478872299, "step": 2181 }, { "clip_ratio": 0.01407172717154026, "epoch": 0.7634709587123862, "grad_norm": 1.4387937883122959, "kl": 1.109375, "learning_rate": 7.774041477222136e-06, "loss": -0.0114, "step": 2182 }, { "clip_ratio": 0.030122052878141403, "epoch": 0.7638208537438769, "grad_norm": 1.2938464236249605, "kl": 1.109375, "learning_rate": 7.771500391743478e-06, "loss": -0.0206, "step": 2183 }, { "clip_ratio": 0.08252425491809845, "epoch": 0.7641707487753674, "grad_norm": 0.952434933802757, "kl": 1.125, "learning_rate": 7.768958272521404e-06, "loss": -0.031, "step": 2184 }, { "clip_ratio": 0.006354030221700668, "completions/clipped_ratio": 0.4285714285714286, "completions/max_length": 256.0, "completions/max_terminated_length": 254.0, "completions/mean_length": 211.48214721679688, "completions/mean_terminated_length": 178.09375, "completions/min_length": 98.0, "completions/min_terminated_length": 98.0, "epoch": 0.7645206438068579, "grad_norm": 3.5269807614953486, "kl": 1.0703125, "learning_rate": 7.766415120504098e-06, "loss": 0.0151, "num_tokens": 9685572.0, "reward": 2.7593586444854736, "reward_std": 0.26423755288124084, "rewards/check_originality_func/mean": 0.8571428656578064, "rewards/check_originality_func/std": 0.3530939221382141, "rewards/check_winston_local_func/mean": 0.987498939037323, "rewards/check_winston_local_func/std": 0.008862697519361973, "rewards/sentence_count_match_reward_logic/mean": 0.9147165417671204, "rewards/sentence_count_match_reward_logic/std": 0.0749981477856636, "step": 2185 }, { "clip_ratio": 0.029261963441967964, "epoch": 0.7648705388383485, "grad_norm": 2.1659219113968775, "kl": 1.09375, "learning_rate": 7.763870936640137e-06, "loss": 0.0042, "step": 2186 }, { "clip_ratio": 0.07231487333774567, "epoch": 0.7652204338698391, "grad_norm": 1.3453087556954448, "kl": 1.125, "learning_rate": 7.761325721878474e-06, "loss": -0.0022, "step": 2187 }, { "clip_ratio": 0.11067313700914383, "epoch": 0.7655703289013296, "grad_norm": 1.2859797830957531, "kl": 1.1640625, "learning_rate": 7.758779477168452e-06, "loss": -0.0075, "step": 2188 }, { "clip_ratio": 0.006471028085798025, "completions/clipped_ratio": 0.1964285714285714, "completions/max_length": 256.0, "completions/max_terminated_length": 255.0, "completions/mean_length": 162.71429443359375, "completions/mean_terminated_length": 139.91111755371094, "completions/min_length": 73.0, "completions/min_terminated_length": 73.0, "epoch": 0.7659202239328201, "grad_norm": 1.4443966648346809, "kl": 1.4375, "learning_rate": 7.756232203459794e-06, "loss": 0.0087, "num_tokens": 9709236.0, "reward": 2.6279408931732178, "reward_std": 0.46264350414276123, "rewards/check_originality_func/mean": 0.6964285969734192, "rewards/check_originality_func/std": 0.4639609158039093, "rewards/check_winston_local_func/mean": 0.9865401387214661, "rewards/check_winston_local_func/std": 0.00798175297677517, "rewards/sentence_count_match_reward_logic/mean": 0.944972038269043, "rewards/sentence_count_match_reward_logic/std": 0.09981350600719452, "step": 2189 }, { "clip_ratio": 0.013499220833182335, "epoch": 0.7662701189643107, "grad_norm": 1.3485675633804195, "kl": 1.4296875, "learning_rate": 7.75368390170261e-06, "loss": 0.0042, "step": 2190 }, { "clip_ratio": 0.06615624576807022, "epoch": 0.7666200139958013, "grad_norm": 1.2972925537859783, "kl": 1.421875, "learning_rate": 7.751134572847394e-06, "loss": -0.0044, "step": 2191 }, { "clip_ratio": 0.12388921529054642, "epoch": 0.7669699090272918, "grad_norm": 1.3602095681798851, "kl": 1.4296875, "learning_rate": 7.74858421784502e-06, "loss": -0.0116, "step": 2192 }, { "clip_ratio": 0.009366417303681374, "completions/clipped_ratio": 0.0, "completions/max_length": 241.0, "completions/max_terminated_length": 241.0, "completions/mean_length": 125.96429443359375, "completions/mean_terminated_length": 125.96429443359375, "completions/min_length": 20.0, "completions/min_terminated_length": 20.0, "epoch": 0.7673198040587824, "grad_norm": 2.4401841073359214, "kl": 2.203125, "learning_rate": 7.746032837646742e-06, "loss": 0.0108, "num_tokens": 9726914.0, "reward": 2.49743390083313, "reward_std": 0.21794536709785461, "rewards/check_originality_func/mean": 0.5892857313156128, "rewards/check_originality_func/std": 0.49641573429107666, "rewards/check_winston_local_func/mean": 0.9635710120201111, "rewards/check_winston_local_func/std": 0.06226015463471413, "rewards/sentence_count_match_reward_logic/mean": 0.9445772171020508, "rewards/sentence_count_match_reward_logic/std": 0.1112762913107872, "step": 2193 }, { "clip_ratio": 0.026472726836800575, "epoch": 0.7676696990902729, "grad_norm": 2.15716482406296, "kl": 2.203125, "learning_rate": 7.74348043320421e-06, "loss": 0.0008, "step": 2194 }, { "clip_ratio": 0.1158338338136673, "epoch": 0.7680195941217635, "grad_norm": 1.5373315999157682, "kl": 2.171875, "learning_rate": 7.74092700546944e-06, "loss": -0.0073, "step": 2195 }, { "clip_ratio": 0.21280279755592346, "epoch": 0.768369489153254, "grad_norm": 0.9622340967772834, "kl": 2.171875, "learning_rate": 7.738372555394837e-06, "loss": -0.0144, "step": 2196 }, { "clip_ratio": 0.0047065988183021545, "completions/clipped_ratio": 0.0892857142857143, "completions/max_length": 256.0, "completions/max_terminated_length": 244.0, "completions/mean_length": 141.5178680419922, "completions/mean_terminated_length": 130.29412841796875, "completions/min_length": 52.0, "completions/min_terminated_length": 52.0, "epoch": 0.7687193841847446, "grad_norm": 2.640072742045888, "kl": 1.625, "learning_rate": 7.735817083933189e-06, "loss": 0.0051, "num_tokens": 9747559.0, "reward": 2.685037136077881, "reward_std": 0.31801411509513855, "rewards/check_originality_func/mean": 0.7678571343421936, "rewards/check_originality_func/std": 0.4260205328464508, "rewards/check_winston_local_func/mean": 0.9771997332572937, "rewards/check_winston_local_func/std": 0.03690180554986, "rewards/sentence_count_match_reward_logic/mean": 0.939980149269104, "rewards/sentence_count_match_reward_logic/std": 0.11307545751333237, "step": 2197 }, { "clip_ratio": 0.026428887620568275, "epoch": 0.7690692792162351, "grad_norm": 2.0211711682939963, "kl": 1.640625, "learning_rate": 7.733260592037661e-06, "loss": -0.0048, "step": 2198 }, { "clip_ratio": 0.09972604364156723, "epoch": 0.7694191742477257, "grad_norm": 1.5524146962938439, "kl": 1.6640625, "learning_rate": 7.7307030806618e-06, "loss": -0.0123, "step": 2199 }, { "clip_ratio": 0.1613694578409195, "epoch": 0.7697690692792163, "grad_norm": 1.310070312620791, "kl": 1.6875, "learning_rate": 7.728144550759535e-06, "loss": -0.0182, "step": 2200 }, { "clip_ratio": 0.00609177490696311, "completions/clipped_ratio": 0.0714285714285714, "completions/max_length": 256.0, "completions/max_terminated_length": 253.0, "completions/mean_length": 161.8928680419922, "completions/mean_terminated_length": 154.6538543701172, "completions/min_length": 72.0, "completions/min_terminated_length": 72.0, "epoch": 0.7701189643107068, "grad_norm": 3.9691837295422876, "kl": 1.4375, "learning_rate": 7.725585003285175e-06, "loss": -0.0131, "num_tokens": 9770361.0, "reward": 2.710193157196045, "reward_std": 0.2279442697763443, "rewards/check_originality_func/mean": 0.8035714030265808, "rewards/check_originality_func/std": 0.40089184045791626, "rewards/check_winston_local_func/mean": 0.985541045665741, "rewards/check_winston_local_func/std": 0.01851566694676876, "rewards/sentence_count_match_reward_logic/mean": 0.9210807085037231, "rewards/sentence_count_match_reward_logic/std": 0.09233293682336807, "step": 2201 }, { "clip_ratio": 0.03749777749180794, "epoch": 0.7704688593421973, "grad_norm": 3.1593742600430543, "kl": 1.4765625, "learning_rate": 7.723024439193401e-06, "loss": -0.0229, "step": 2202 }, { "clip_ratio": 0.07899674028158188, "epoch": 0.7708187543736879, "grad_norm": 2.3866331549112614, "kl": 1.4921875, "learning_rate": 7.720462859439284e-06, "loss": -0.0324, "step": 2203 }, { "clip_ratio": 0.13802890479564667, "epoch": 0.7711686494051785, "grad_norm": 0.8875012192535301, "kl": 1.515625, "learning_rate": 7.717900264978267e-06, "loss": -0.0395, "step": 2204 }, { "clip_ratio": 0.012124977074563503, "completions/clipped_ratio": 0.25, "completions/max_length": 256.0, "completions/max_terminated_length": 256.0, "completions/mean_length": 179.8928680419922, "completions/mean_terminated_length": 154.52381896972656, "completions/min_length": 97.0, "completions/min_terminated_length": 97.0, "epoch": 0.771518544436669, "grad_norm": 2.1507639444287294, "kl": 1.3203125, "learning_rate": 7.715336656766176e-06, "loss": 0.0037, "num_tokens": 9795915.0, "reward": 2.75156307220459, "reward_std": 0.25562936067581177, "rewards/check_originality_func/mean": 0.8571428656578064, "rewards/check_originality_func/std": 0.3530939221382141, "rewards/check_winston_local_func/mean": 0.9896439909934998, "rewards/check_winston_local_func/std": 0.003711374942213297, "rewards/sentence_count_match_reward_logic/mean": 0.9047760367393494, "rewards/sentence_count_match_reward_logic/std": 0.0942518338561058, "step": 2205 }, { "clip_ratio": 0.02811644971370697, "epoch": 0.7718684394681595, "grad_norm": 1.9770213924956914, "kl": 1.3125, "learning_rate": 7.71277203575921e-06, "loss": -0.0087, "step": 2206 }, { "clip_ratio": 0.08839934319257736, "epoch": 0.77221833449965, "grad_norm": 1.5512222953665478, "kl": 1.3046875, "learning_rate": 7.71020640291395e-06, "loss": -0.0167, "step": 2207 }, { "clip_ratio": 0.14020106196403503, "epoch": 0.7725682295311407, "grad_norm": 1.126968594490254, "kl": 1.3125, "learning_rate": 7.70763975918735e-06, "loss": -0.0256, "step": 2208 }, { "clip_ratio": 0.005849718116223812, "completions/clipped_ratio": 0.1428571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 209.0, "completions/mean_length": 148.5, "completions/mean_terminated_length": 130.58334350585938, "completions/min_length": 66.0, "completions/min_terminated_length": 66.0, "epoch": 0.7729181245626312, "grad_norm": 1.9445863643855898, "kl": 1.4296875, "learning_rate": 7.705072105536748e-06, "loss": 0.0043, "num_tokens": 9817239.0, "reward": 2.6878244876861572, "reward_std": 0.19927768409252167, "rewards/check_originality_func/mean": 0.7857142686843872, "rewards/check_originality_func/std": 0.41403928399086, "rewards/check_winston_local_func/mean": 0.989421010017395, "rewards/check_winston_local_func/std": 0.006548275705426931, "rewards/sentence_count_match_reward_logic/mean": 0.9126887321472168, "rewards/sentence_count_match_reward_logic/std": 0.09531959146261215, "step": 2209 }, { "clip_ratio": 0.025860771536827087, "epoch": 0.7732680195941217, "grad_norm": 1.725990427028286, "kl": 1.4296875, "learning_rate": 7.702503442919853e-06, "loss": -0.0047, "step": 2210 }, { "clip_ratio": 0.08107620477676392, "epoch": 0.7736179146256124, "grad_norm": 1.0686487856423894, "kl": 1.421875, "learning_rate": 7.69993377229475e-06, "loss": -0.0123, "step": 2211 }, { "clip_ratio": 0.1277000904083252, "epoch": 0.7739678096571029, "grad_norm": 0.7491491432146866, "kl": 1.4375, "learning_rate": 7.697363094619903e-06, "loss": -0.0177, "step": 2212 }, { "clip_ratio": 0.0066116368398070335, "completions/clipped_ratio": 0.25, "completions/max_length": 256.0, "completions/max_terminated_length": 241.0, "completions/mean_length": 185.6607208251953, "completions/mean_terminated_length": 162.21429443359375, "completions/min_length": 58.0, "completions/min_terminated_length": 58.0, "epoch": 0.7743177046885934, "grad_norm": 3.3521097987871737, "kl": 1.3359375, "learning_rate": 7.69479141085415e-06, "loss": 0.0056, "num_tokens": 9843148.0, "reward": 2.730358362197876, "reward_std": 0.19677403569221497, "rewards/check_originality_func/mean": 0.8392857313156128, "rewards/check_originality_func/std": 0.3705909848213196, "rewards/check_winston_local_func/mean": 0.9875723123550415, "rewards/check_winston_local_func/std": 0.00828735250979662, "rewards/sentence_count_match_reward_logic/mean": 0.9035002589225769, "rewards/sentence_count_match_reward_logic/std": 0.10182515531778336, "step": 2213 }, { "clip_ratio": 0.026296546682715416, "epoch": 0.7746675997200839, "grad_norm": 2.9512497198881653, "kl": 1.3359375, "learning_rate": 7.692218721956706e-06, "loss": -0.0033, "step": 2214 }, { "clip_ratio": 0.09981407970190048, "epoch": 0.7750174947515746, "grad_norm": 2.7333654320858, "kl": 1.3515625, "learning_rate": 7.689645028887159e-06, "loss": -0.0187, "step": 2215 }, { "clip_ratio": 0.15983986854553223, "epoch": 0.7753673897830651, "grad_norm": 2.1619197541267465, "kl": 1.390625, "learning_rate": 7.687070332605474e-06, "loss": -0.0303, "step": 2216 }, { "clip_ratio": 0.008454442955553532, "completions/clipped_ratio": 0.2857142857142857, "completions/max_length": 256.0, "completions/max_terminated_length": 255.0, "completions/mean_length": 216.1607208251953, "completions/mean_terminated_length": 200.22500610351562, "completions/min_length": 87.0, "completions/min_terminated_length": 87.0, "epoch": 0.7757172848145556, "grad_norm": 2.612785310937482, "kl": 1.0859375, "learning_rate": 7.684494634071982e-06, "loss": 0.0099, "num_tokens": 9874237.0, "reward": 2.8147332668304443, "reward_std": 0.17048224806785583, "rewards/check_originality_func/mean": 0.9464285969734192, "rewards/check_originality_func/std": 0.22720777988433838, "rewards/check_winston_local_func/mean": 0.9837613701820374, "rewards/check_winston_local_func/std": 0.029443060979247093, "rewards/sentence_count_match_reward_logic/mean": 0.8845432996749878, "rewards/sentence_count_match_reward_logic/std": 0.10516150295734406, "step": 2217 }, { "clip_ratio": 0.034882668405771255, "epoch": 0.7760671798460462, "grad_norm": 1.9091419063585278, "kl": 1.09375, "learning_rate": 7.6819179342474e-06, "loss": 0.0012, "step": 2218 }, { "clip_ratio": 0.08611135929822922, "epoch": 0.7764170748775368, "grad_norm": 1.1374412893172006, "kl": 1.1015625, "learning_rate": 7.679340234092812e-06, "loss": -0.0054, "step": 2219 }, { "clip_ratio": 0.13253627717494965, "epoch": 0.7767669699090273, "grad_norm": 1.151734227244294, "kl": 1.109375, "learning_rate": 7.676761534569675e-06, "loss": -0.0102, "step": 2220 }, { "clip_ratio": 0.011530433781445026, "completions/clipped_ratio": 0.2142857142857143, "completions/max_length": 256.0, "completions/max_terminated_length": 255.0, "completions/mean_length": 202.6607208251953, "completions/mean_terminated_length": 188.1136474609375, "completions/min_length": 72.0, "completions/min_terminated_length": 72.0, "epoch": 0.7771168649405178, "grad_norm": 4.410082811417794, "kl": 1.1953125, "learning_rate": 7.674181836639819e-06, "loss": 0.0221, "num_tokens": 9902538.0, "reward": 2.790733814239502, "reward_std": 0.2255198359489441, "rewards/check_originality_func/mean": 0.875, "rewards/check_originality_func/std": 0.3337118923664093, "rewards/check_winston_local_func/mean": 0.9899013638496399, "rewards/check_winston_local_func/std": 0.0034135174937546253, "rewards/sentence_count_match_reward_logic/mean": 0.9258323311805725, "rewards/sentence_count_match_reward_logic/std": 0.08334045112133026, "step": 2221 }, { "clip_ratio": 0.04774842783808708, "epoch": 0.7774667599720084, "grad_norm": 2.6036003189905594, "kl": 1.2109375, "learning_rate": 7.671601141265447e-06, "loss": 0.0096, "step": 2222 }, { "clip_ratio": 0.09202117472887039, "epoch": 0.777816655003499, "grad_norm": 2.0605668340849803, "kl": 1.2109375, "learning_rate": 7.669019449409133e-06, "loss": -0.0005, "step": 2223 }, { "clip_ratio": 0.18652987480163574, "epoch": 0.7781665500349895, "grad_norm": 1.5286394843421138, "kl": 1.25, "learning_rate": 7.666436762033828e-06, "loss": -0.0147, "step": 2224 }, { "clip_ratio": 0.010855472646653652, "completions/clipped_ratio": 0.1428571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 255.0, "completions/mean_length": 181.57144165039062, "completions/mean_terminated_length": 169.1666717529297, "completions/min_length": 113.0, "completions/min_terminated_length": 113.0, "epoch": 0.7785164450664801, "grad_norm": 4.957765740151751, "kl": 1.2734375, "learning_rate": 7.663853080102845e-06, "loss": 0.0009, "num_tokens": 9927986.0, "reward": 2.7959554195404053, "reward_std": 0.26743632555007935, "rewards/check_originality_func/mean": 0.8571428656578064, "rewards/check_originality_func/std": 0.3530939221382141, "rewards/check_winston_local_func/mean": 0.9872046709060669, "rewards/check_winston_local_func/std": 0.01398618146777153, "rewards/sentence_count_match_reward_logic/mean": 0.9516080021858215, "rewards/sentence_count_match_reward_logic/std": 0.07722657173871994, "step": 2225 }, { "clip_ratio": 0.03283597156405449, "epoch": 0.7788663400979706, "grad_norm": 2.4099376187829207, "kl": 1.2890625, "learning_rate": 7.661268404579876e-06, "loss": -0.0056, "step": 2226 }, { "clip_ratio": 0.055786412209272385, "epoch": 0.7792162351294611, "grad_norm": 1.5319021321992168, "kl": 1.296875, "learning_rate": 7.658682736428977e-06, "loss": -0.01, "step": 2227 }, { "clip_ratio": 0.09655939042568207, "epoch": 0.7795661301609517, "grad_norm": 0.8676287516573535, "kl": 1.3046875, "learning_rate": 7.656096076614581e-06, "loss": -0.0167, "step": 2228 }, { "clip_ratio": 0.0179483313113451, "completions/clipped_ratio": 0.1607142857142857, "completions/max_length": 256.0, "completions/max_terminated_length": 256.0, "completions/mean_length": 147.6607208251953, "completions/mean_terminated_length": 126.91488647460938, "completions/min_length": 59.0, "completions/min_terminated_length": 59.0, "epoch": 0.7799160251924423, "grad_norm": 3.1677326705286317, "kl": 2.46875, "learning_rate": 7.653508426101488e-06, "loss": 0.0305, "num_tokens": 9949623.0, "reward": 2.5624330043792725, "reward_std": 0.39881810545921326, "rewards/check_originality_func/mean": 0.6607142686843872, "rewards/check_originality_func/std": 0.477751761674881, "rewards/check_winston_local_func/mean": 0.9892166256904602, "rewards/check_winston_local_func/std": 0.006180686876177788, "rewards/sentence_count_match_reward_logic/mean": 0.9125021696090698, "rewards/sentence_count_match_reward_logic/std": 0.11508551239967346, "step": 2229 }, { "clip_ratio": 0.05965755134820938, "epoch": 0.7802659202239328, "grad_norm": 2.9339354415686936, "kl": 2.46875, "learning_rate": 7.650919785854865e-06, "loss": 0.017, "step": 2230 }, { "clip_ratio": 0.130947545170784, "epoch": 0.7806158152554233, "grad_norm": 1.9256095273998415, "kl": 2.53125, "learning_rate": 7.64833015684025e-06, "loss": 0.0067, "step": 2231 }, { "clip_ratio": 0.19801728427410126, "epoch": 0.780965710286914, "grad_norm": 1.5602774938918509, "kl": 2.703125, "learning_rate": 7.645739540023552e-06, "loss": -0.001, "step": 2232 }, { "clip_ratio": 0.013361240737140179, "completions/clipped_ratio": 0.017857142857142905, "completions/max_length": 256.0, "completions/max_terminated_length": 253.0, "completions/mean_length": 158.60714721679688, "completions/mean_terminated_length": 156.83636474609375, "completions/min_length": 78.0, "completions/min_terminated_length": 78.0, "epoch": 0.7813156053184045, "grad_norm": 4.488141282192667, "kl": 1.484375, "learning_rate": 7.643147936371047e-06, "loss": -0.0008, "num_tokens": 9971625.0, "reward": 2.5730292797088623, "reward_std": 0.28272750973701477, "rewards/check_originality_func/mean": 0.6607142686843872, "rewards/check_originality_func/std": 0.477751761674881, "rewards/check_winston_local_func/mean": 0.9839953184127808, "rewards/check_winston_local_func/std": 0.018814347684383392, "rewards/sentence_count_match_reward_logic/mean": 0.9283195734024048, "rewards/sentence_count_match_reward_logic/std": 0.09186284989118576, "step": 2233 }, { "clip_ratio": 0.0671687051653862, "epoch": 0.781665500349895, "grad_norm": 2.468351703152532, "kl": 1.4453125, "learning_rate": 7.640555346849376e-06, "loss": -0.0098, "step": 2234 }, { "clip_ratio": 0.09887521713972092, "epoch": 0.7820153953813855, "grad_norm": 1.6915055594052018, "kl": 1.421875, "learning_rate": 7.63796177242555e-06, "loss": -0.0127, "step": 2235 }, { "clip_ratio": 0.12615637481212616, "epoch": 0.7823652904128762, "grad_norm": 1.0616712485495223, "kl": 1.375, "learning_rate": 7.635367214066953e-06, "loss": -0.0186, "step": 2236 }, { "clip_ratio": 0.008154957555234432, "completions/clipped_ratio": 0.1964285714285714, "completions/max_length": 256.0, "completions/max_terminated_length": 254.0, "completions/mean_length": 185.58929443359375, "completions/mean_terminated_length": 168.37777709960938, "completions/min_length": 76.0, "completions/min_terminated_length": 76.0, "epoch": 0.7827151854443667, "grad_norm": 2.3200604639982085, "kl": 1.5078125, "learning_rate": 7.632771672741326e-06, "loss": -0.0001, "num_tokens": 9997522.0, "reward": 2.7074930667877197, "reward_std": 0.2094484269618988, "rewards/check_originality_func/mean": 0.7678571343421936, "rewards/check_originality_func/std": 0.4260205328464508, "rewards/check_winston_local_func/mean": 0.989928126335144, "rewards/check_winston_local_func/std": 0.0048869019374251366, "rewards/sentence_count_match_reward_logic/mean": 0.9497074484825134, "rewards/sentence_count_match_reward_logic/std": 0.062081653624773026, "step": 2237 }, { "clip_ratio": 0.0347776934504509, "epoch": 0.7830650804758572, "grad_norm": 1.5274021369481963, "kl": 1.4453125, "learning_rate": 7.630175149416783e-06, "loss": -0.0078, "step": 2238 }, { "clip_ratio": 0.09452975541353226, "epoch": 0.7834149755073478, "grad_norm": 1.0540984944612717, "kl": 1.4140625, "learning_rate": 7.627577645061801e-06, "loss": -0.0163, "step": 2239 }, { "clip_ratio": 0.14522159099578857, "epoch": 0.7837648705388384, "grad_norm": 0.7187111282053682, "kl": 1.3828125, "learning_rate": 7.624979160645225e-06, "loss": -0.0214, "step": 2240 }, { "clip_ratio": 0.005536025855690241, "completions/clipped_ratio": 0.3571428571428571, "completions/max_length": 256.0, "completions/max_terminated_length": 256.0, "completions/mean_length": 194.5178680419922, "completions/mean_terminated_length": 160.36111450195312, "completions/min_length": 23.0, "completions/min_terminated_length": 23.0, "epoch": 0.7841147655703289, "grad_norm": 3.179851454417174, "kl": 1.625, "learning_rate": 7.6223796971362685e-06, "loss": 0.0115, "num_tokens": 10024143.0, "reward": 2.593635082244873, "reward_std": 0.2726803123950958, "rewards/check_originality_func/mean": 0.7142857313156128, "rewards/check_originality_func/std": 0.4558422863483429, "rewards/check_winston_local_func/mean": 0.9806774258613586, "rewards/check_winston_local_func/std": 0.03353649750351906, "rewards/sentence_count_match_reward_logic/mean": 0.8986716866493225, "rewards/sentence_count_match_reward_logic/std": 0.10780616104602814, "step": 2241 }, { "clip_ratio": 0.01275672111660242, "epoch": 0.7844646606018194, "grad_norm": 2.5432140190172876, "kl": 1.6328125, "learning_rate": 7.619779255504502e-06, "loss": 0.0075, "step": 2242 }, { "clip_ratio": 0.06258870661258698, "epoch": 0.78481455563331, "grad_norm": 1.0120734494791903, "kl": 1.7109375, "learning_rate": 7.617177836719869e-06, "loss": 0.0023, "step": 2243 }, { "clip_ratio": 0.10200650244951248, "epoch": 0.7851644506648006, "grad_norm": 0.8452542436582832, "kl": 1.8046875, "learning_rate": 7.614575441752672e-06, "loss": -0.0028, "step": 2244 }, { "clip_ratio": 0.009320421144366264, "completions/clipped_ratio": 0.6071428571428572, "completions/max_length": 256.0, "completions/max_terminated_length": 255.0, "completions/mean_length": 230.7857208251953, "completions/mean_terminated_length": 191.8181915283203, "completions/min_length": 115.0, "completions/min_terminated_length": 115.0, "epoch": 0.7855143456962911, "grad_norm": 4.279946386863906, "kl": 1.0078125, "learning_rate": 7.611972071573579e-06, "loss": 0.0202, "num_tokens": 10058643.0, "reward": 2.856940984725952, "reward_std": 0.1464061290025711, "rewards/check_originality_func/mean": 0.9642857313156128, "rewards/check_originality_func/std": 0.187256321310997, "rewards/check_winston_local_func/mean": 0.9883952140808105, "rewards/check_winston_local_func/std": 0.007024173624813557, "rewards/sentence_count_match_reward_logic/mean": 0.9042600989341736, "rewards/sentence_count_match_reward_logic/std": 0.09645906090736389, "step": 2245 }, { "clip_ratio": 0.043288666754961014, "epoch": 0.7858642407277817, "grad_norm": 3.0162004107361176, "kl": 1.0234375, "learning_rate": 7.6093677271536245e-06, "loss": -0.0003, "step": 2246 }, { "clip_ratio": 0.1111360639333725, "epoch": 0.7862141357592722, "grad_norm": 1.9007710011041483, "kl": 1.046875, "learning_rate": 7.6067624094642055e-06, "loss": -0.0135, "step": 2247 }, { "clip_ratio": 0.17982296645641327, "epoch": 0.7865640307907628, "grad_norm": 1.283066058415585, "kl": 1.0859375, "learning_rate": 7.60415611947708e-06, "loss": -0.0249, "step": 2248 }, { "clip_ratio": 0.006667276378720999, "completions/clipped_ratio": 0.0535714285714286, "completions/max_length": 256.0, "completions/max_terminated_length": 256.0, "completions/mean_length": 178.19644165039062, "completions/mean_terminated_length": 173.79244995117188, "completions/min_length": 55.0, "completions/min_terminated_length": 55.0, "epoch": 0.7869139258222533, "grad_norm": 4.102881113121063, "kl": 1.5859375, "learning_rate": 7.601548858164366e-06, "loss": 0.0141, "num_tokens": 10083118.0, "reward": 2.772270679473877, "reward_std": 0.23936304450035095, "rewards/check_originality_func/mean": 0.8392857313156128, "rewards/check_originality_func/std": 0.3705909848213196, "rewards/check_winston_local_func/mean": 0.9845709800720215, "rewards/check_winston_local_func/std": 0.019724983721971512, "rewards/sentence_count_match_reward_logic/mean": 0.9484137892723083, "rewards/sentence_count_match_reward_logic/std": 0.08922921121120453, "step": 2249 }, { "clip_ratio": 0.02462579309940338, "epoch": 0.7872638208537439, "grad_norm": 2.897516146036742, "kl": 1.640625, "learning_rate": 7.59894062649855e-06, "loss": 0.0047, "step": 2250 }, { "clip_ratio": 0.0799042135477066, "epoch": 0.7876137158852344, "grad_norm": 2.3319693001636623, "kl": 1.7265625, "learning_rate": 7.596331425452479e-06, "loss": -0.0071, "step": 2251 }, { "clip_ratio": 0.14575913548469543, "epoch": 0.787963610916725, "grad_norm": 1.4356502460268012, "kl": 1.7734375, "learning_rate": 7.593721255999358e-06, "loss": -0.0135, "step": 2252 }, { "clip_ratio": 0.012187047861516476, "completions/clipped_ratio": 0.6428571428571428, "completions/max_length": 256.0, "completions/max_terminated_length": 255.0, "completions/mean_length": 221.96429443359375, "completions/mean_terminated_length": 160.6999969482422, "completions/min_length": 126.0, "completions/min_terminated_length": 126.0, "epoch": 0.7883135059482156, "grad_norm": 4.6754984654430425, "kl": 1.234375, "learning_rate": 7.591110119112757e-06, "loss": 0.0182, "num_tokens": 10115300.0, "reward": 2.6475343704223633, "reward_std": 0.3090335428714752, "rewards/check_originality_func/mean": 0.8035714030265808, "rewards/check_originality_func/std": 0.40089187026023865, "rewards/check_winston_local_func/mean": 0.9867726564407349, "rewards/check_winston_local_func/std": 0.011396595276892185, "rewards/sentence_count_match_reward_logic/mean": 0.8571901917457581, "rewards/sentence_count_match_reward_logic/std": 0.09857548773288727, "step": 2253 }, { "clip_ratio": 0.024967432022094727, "epoch": 0.7886634009797061, "grad_norm": 3.6406915821726478, "kl": 1.2734375, "learning_rate": 7.588498015766604e-06, "loss": 0.0046, "step": 2254 }, { "clip_ratio": 0.07770909368991852, "epoch": 0.7890132960111966, "grad_norm": 1.773300328111227, "kl": 1.3515625, "learning_rate": 7.5858849469351895e-06, "loss": -0.0059, "step": 2255 }, { "clip_ratio": 0.14686863124370575, "epoch": 0.7893631910426872, "grad_norm": 1.0827035222408068, "kl": 1.453125, "learning_rate": 7.583270913593163e-06, "loss": -0.0151, "step": 2256 }, { "clip_ratio": 0.014506646431982517, "completions/clipped_ratio": 0.2857142857142857, "completions/max_length": 256.0, "completions/max_terminated_length": 255.0, "completions/mean_length": 181.85714721679688, "completions/mean_terminated_length": 152.1999969482422, "completions/min_length": 113.0, "completions/min_terminated_length": 113.0, "epoch": 0.7897130860741778, "grad_norm": 4.9139583339670745, "kl": 1.203125, "learning_rate": 7.580655916715537e-06, "loss": -0.0074, "num_tokens": 10140516.0, "reward": 2.7738606929779053, "reward_std": 0.14688366651535034, "rewards/check_originality_func/mean": 0.8571428656578064, "rewards/check_originality_func/std": 0.3530939221382141, "rewards/check_winston_local_func/mean": 0.9698733687400818, "rewards/check_winston_local_func/std": 0.056519754230976105, "rewards/sentence_count_match_reward_logic/mean": 0.9468445181846619, "rewards/sentence_count_match_reward_logic/std": 0.0731915533542633, "step": 2257 }, { "clip_ratio": 0.052603691816329956, "epoch": 0.7900629811056683, "grad_norm": 3.0651479840333553, "kl": 1.2578125, "learning_rate": 7.578039957277678e-06, "loss": -0.015, "step": 2258 }, { "clip_ratio": 0.0965602919459343, "epoch": 0.7904128761371588, "grad_norm": 1.3059109395919788, "kl": 1.3359375, "learning_rate": 7.575423036255312e-06, "loss": -0.0263, "step": 2259 }, { "clip_ratio": 0.15543386340141296, "epoch": 0.7907627711686495, "grad_norm": 0.8940155209971272, "kl": 1.390625, "learning_rate": 7.57280515462453e-06, "loss": -0.0325, "step": 2260 }, { "clip_ratio": 0.02632203884422779, "completions/clipped_ratio": 0.3928571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 255.0, "completions/mean_length": 220.3035888671875, "completions/mean_terminated_length": 197.2058868408203, "completions/min_length": 104.0, "completions/min_terminated_length": 104.0, "epoch": 0.79111266620014, "grad_norm": 5.67967532278498, "kl": 2.328125, "learning_rate": 7.5701863133617735e-06, "loss": 0.029, "num_tokens": 10170413.0, "reward": 2.7740907669067383, "reward_std": 0.2763034403324127, "rewards/check_originality_func/mean": 0.8392857313156128, "rewards/check_originality_func/std": 0.3705909848213196, "rewards/check_winston_local_func/mean": 0.9855499267578125, "rewards/check_winston_local_func/std": 0.015025891363620758, "rewards/sentence_count_match_reward_logic/mean": 0.9492548704147339, "rewards/sentence_count_match_reward_logic/std": 0.10552500933408737, "step": 2261 }, { "clip_ratio": 0.03254774585366249, "epoch": 0.7914625612316305, "grad_norm": 5.039366032368212, "kl": 2.375, "learning_rate": 7.567566513443849e-06, "loss": 0.0187, "step": 2262 }, { "clip_ratio": 0.13170906901359558, "epoch": 0.791812456263121, "grad_norm": 2.6413824979935274, "kl": 2.453125, "learning_rate": 7.564945755847917e-06, "loss": -0.0015, "step": 2263 }, { "clip_ratio": 0.22485879063606262, "epoch": 0.7921623512946117, "grad_norm": 1.4771707095272841, "kl": 2.640625, "learning_rate": 7.5623240415514895e-06, "loss": -0.0102, "step": 2264 }, { "clip_ratio": 0.009273948147892952, "completions/clipped_ratio": 0.1607142857142857, "completions/max_length": 256.0, "completions/max_terminated_length": 252.0, "completions/mean_length": 216.7857208251953, "completions/mean_terminated_length": 209.27659606933594, "completions/min_length": 137.0, "completions/min_terminated_length": 137.0, "epoch": 0.7925122463261022, "grad_norm": 12.20259331961761, "kl": 2.234375, "learning_rate": 7.559701371532449e-06, "loss": 0.0334, "num_tokens": 10199009.0, "reward": 2.8728187084198, "reward_std": 0.16346199810504913, "rewards/check_originality_func/mean": 0.9642857313156128, "rewards/check_originality_func/std": 0.187256321310997, "rewards/check_winston_local_func/mean": 0.9863495826721191, "rewards/check_winston_local_func/std": 0.01130951102823019, "rewards/sentence_count_match_reward_logic/mean": 0.9221834540367126, "rewards/sentence_count_match_reward_logic/std": 0.10647861659526825, "step": 2265 }, { "clip_ratio": 0.055988017469644547, "epoch": 0.7928621413575927, "grad_norm": 4.011380370848819, "kl": 2.484375, "learning_rate": 7.557077746769019e-06, "loss": 0.0114, "step": 2266 }, { "clip_ratio": 0.10198584944009781, "epoch": 0.7932120363890833, "grad_norm": 2.7648197561492123, "kl": 2.890625, "learning_rate": 7.554453168239794e-06, "loss": 0.0034, "step": 2267 }, { "clip_ratio": 0.15152767300605774, "epoch": 0.7935619314205739, "grad_norm": 2.2007156270728845, "kl": 3.28125, "learning_rate": 7.551827636923712e-06, "loss": 0.0014, "step": 2268 }, { "clip_ratio": 0.017019374296069145, "completions/clipped_ratio": 0.5714285714285714, "completions/max_length": 256.0, "completions/max_terminated_length": 254.0, "completions/mean_length": 194.1607208251953, "completions/mean_terminated_length": 111.70833587646484, "completions/min_length": 35.0, "completions/min_terminated_length": 35.0, "epoch": 0.7939118264520644, "grad_norm": 6.85745415369804, "kl": 3.09375, "learning_rate": 7.549201153800073e-06, "loss": 0.025, "num_tokens": 10226346.0, "reward": 2.3932433128356934, "reward_std": 0.32587918639183044, "rewards/check_originality_func/mean": 0.5535714030265808, "rewards/check_originality_func/std": 0.5016207098960876, "rewards/check_winston_local_func/mean": 0.985698401927948, "rewards/check_winston_local_func/std": 0.018434274941682816, "rewards/sentence_count_match_reward_logic/mean": 0.853973388671875, "rewards/sentence_count_match_reward_logic/std": 0.16826270520687103, "step": 2269 }, { "clip_ratio": 0.04438909515738487, "epoch": 0.7942617214835549, "grad_norm": 3.900491682963429, "kl": 3.21875, "learning_rate": 7.546573719848529e-06, "loss": 0.0073, "step": 2270 }, { "clip_ratio": 0.14096413552761078, "epoch": 0.7946116165150455, "grad_norm": 2.990769513289856, "kl": 3.453125, "learning_rate": 7.543945336049092e-06, "loss": -0.0011, "step": 2271 }, { "clip_ratio": 0.202264204621315, "epoch": 0.794961511546536, "grad_norm": 2.138343784077799, "kl": 3.765625, "learning_rate": 7.541316003382118e-06, "loss": -0.0043, "step": 2272 }, { "clip_ratio": 0.017327910289168358, "completions/clipped_ratio": 0.2678571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 245.0, "completions/mean_length": 198.96429443359375, "completions/mean_terminated_length": 178.09754943847656, "completions/min_length": 127.0, "completions/min_terminated_length": 127.0, "epoch": 0.7953114065780266, "grad_norm": 5.2051325199791885, "kl": 2.625, "learning_rate": 7.53868572282833e-06, "loss": 0.0272, "num_tokens": 10253064.0, "reward": 2.7245070934295654, "reward_std": 0.2912927269935608, "rewards/check_originality_func/mean": 0.8571428656578064, "rewards/check_originality_func/std": 0.3530939221382141, "rewards/check_winston_local_func/mean": 0.9868748784065247, "rewards/check_winston_local_func/std": 0.01398521475493908, "rewards/sentence_count_match_reward_logic/mean": 0.8804892301559448, "rewards/sentence_count_match_reward_logic/std": 0.11501865833997726, "step": 2273 }, { "clip_ratio": 0.05091169849038124, "epoch": 0.7956613016095171, "grad_norm": 2.936779998933911, "kl": 2.75, "learning_rate": 7.536054495368792e-06, "loss": 0.0155, "step": 2274 }, { "clip_ratio": 0.1252208799123764, "epoch": 0.7960111966410077, "grad_norm": 1.7119897759294356, "kl": 3.03125, "learning_rate": 7.5334223219849305e-06, "loss": 0.0071, "step": 2275 }, { "clip_ratio": 0.17976713180541992, "epoch": 0.7963610916724982, "grad_norm": 1.5787273973318838, "kl": 3.171875, "learning_rate": 7.53078920365852e-06, "loss": 0.0021, "step": 2276 }, { "clip_ratio": 0.010003956966102123, "completions/clipped_ratio": 0.4285714285714286, "completions/max_length": 256.0, "completions/max_terminated_length": 253.0, "completions/mean_length": 209.19644165039062, "completions/mean_terminated_length": 174.09375, "completions/min_length": 99.0, "completions/min_terminated_length": 99.0, "epoch": 0.7967109867039888, "grad_norm": 3.843239170670923, "kl": 1.859375, "learning_rate": 7.528155141371688e-06, "loss": 0.0288, "num_tokens": 10280627.0, "reward": 2.540706157684326, "reward_std": 0.3529549539089203, "rewards/check_originality_func/mean": 0.6785714030265808, "rewards/check_originality_func/std": 0.4712514281272888, "rewards/check_winston_local_func/mean": 0.9827927350997925, "rewards/check_winston_local_func/std": 0.0342363603413105, "rewards/sentence_count_match_reward_logic/mean": 0.8793419599533081, "rewards/sentence_count_match_reward_logic/std": 0.14186638593673706, "step": 2277 }, { "clip_ratio": 0.03886999562382698, "epoch": 0.7970608817354794, "grad_norm": 1.6210828482056994, "kl": 1.9453125, "learning_rate": 7.5255201361069154e-06, "loss": 0.0181, "step": 2278 }, { "clip_ratio": 0.07196886092424393, "epoch": 0.7974107767669699, "grad_norm": 1.4214809949112597, "kl": 2.15625, "learning_rate": 7.522884188847035e-06, "loss": 0.0157, "step": 2279 }, { "clip_ratio": 0.10379577428102493, "epoch": 0.7977606717984604, "grad_norm": 1.1121638531423634, "kl": 2.234375, "learning_rate": 7.52024730057523e-06, "loss": 0.0106, "step": 2280 }, { "clip_ratio": 0.02662082016468048, "completions/clipped_ratio": 0.4285714285714286, "completions/max_length": 256.0, "completions/max_terminated_length": 191.0, "completions/mean_length": 170.32144165039062, "completions/mean_terminated_length": 106.0625, "completions/min_length": 46.0, "completions/min_terminated_length": 46.0, "epoch": 0.798110566829951, "grad_norm": 9.40993896041416, "kl": 2.234375, "learning_rate": 7.5176094722750344e-06, "loss": 0.0191, "num_tokens": 10305501.0, "reward": 2.60031795501709, "reward_std": 0.33188581466674805, "rewards/check_originality_func/mean": 0.75, "rewards/check_originality_func/std": 0.43693143129348755, "rewards/check_winston_local_func/mean": 0.989582359790802, "rewards/check_winston_local_func/std": 0.005458058323711157, "rewards/sentence_count_match_reward_logic/mean": 0.8607354760169983, "rewards/sentence_count_match_reward_logic/std": 0.1302364319562912, "step": 2281 }, { "clip_ratio": 0.08740045875310898, "epoch": 0.7984604618614416, "grad_norm": 3.6074128392016527, "kl": 2.796875, "learning_rate": 7.514970704930333e-06, "loss": 0.0018, "step": 2282 }, { "clip_ratio": 0.13691695034503937, "epoch": 0.7988103568929321, "grad_norm": 3.1460576006323553, "kl": 3.453125, "learning_rate": 7.5123309995253616e-06, "loss": -0.0015, "step": 2283 }, { "clip_ratio": 0.15269555151462555, "epoch": 0.7991602519244226, "grad_norm": 3.6081710717884157, "kl": 3.796875, "learning_rate": 7.509690357044705e-06, "loss": 0.0008, "step": 2284 }, { "clip_ratio": 0.03902921825647354, "completions/clipped_ratio": 0.1428571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 194.0, "completions/mean_length": 152.46429443359375, "completions/mean_terminated_length": 135.20834350585938, "completions/min_length": 49.0, "completions/min_terminated_length": 49.0, "epoch": 0.7995101469559133, "grad_norm": 4.1564120338665385, "kl": 2.21875, "learning_rate": 7.507048778473296e-06, "loss": 0.0134, "num_tokens": 10325999.0, "reward": 2.622997283935547, "reward_std": 0.3700047433376312, "rewards/check_originality_func/mean": 0.7321428656578064, "rewards/check_originality_func/std": 0.44685041904449463, "rewards/check_winston_local_func/mean": 0.9837541580200195, "rewards/check_winston_local_func/std": 0.018494801595807076, "rewards/sentence_count_match_reward_logic/mean": 0.9071003198623657, "rewards/sentence_count_match_reward_logic/std": 0.10611722618341446, "step": 2285 }, { "clip_ratio": 0.08090399205684662, "epoch": 0.7998600419874038, "grad_norm": 3.18142195031944, "kl": 2.296875, "learning_rate": 7.504406264796422e-06, "loss": 0.0019, "step": 2286 }, { "clip_ratio": 0.15250621736049652, "epoch": 0.8002099370188943, "grad_norm": 1.402467239122517, "kl": 2.265625, "learning_rate": 7.501762816999714e-06, "loss": -0.0074, "step": 2287 }, { "clip_ratio": 0.18993647396564484, "epoch": 0.8005598320503848, "grad_norm": 0.9126062455728505, "kl": 2.046875, "learning_rate": 7.499118436069151e-06, "loss": -0.0127, "step": 2288 }, { "clip_ratio": 0.033788640052080154, "completions/clipped_ratio": 0.5535714285714286, "completions/max_length": 256.0, "completions/max_terminated_length": 249.0, "completions/mean_length": 221.23214721679688, "completions/mean_terminated_length": 178.1199951171875, "completions/min_length": 142.0, "completions/min_terminated_length": 142.0, "epoch": 0.8009097270818755, "grad_norm": 9.541807917445903, "kl": 2.359375, "learning_rate": 7.496473122991066e-06, "loss": 0.0166, "num_tokens": 10356988.0, "reward": 2.544959306716919, "reward_std": 0.23686566948890686, "rewards/check_originality_func/mean": 0.6607142686843872, "rewards/check_originality_func/std": 0.477751761674881, "rewards/check_winston_local_func/mean": 0.9847763776779175, "rewards/check_winston_local_func/std": 0.00820600800216198, "rewards/sentence_count_match_reward_logic/mean": 0.8994685411453247, "rewards/sentence_count_match_reward_logic/std": 0.11867358535528183, "step": 2289 }, { "clip_ratio": 0.09564834088087082, "epoch": 0.801259622113366, "grad_norm": 4.841899264756931, "kl": 2.09375, "learning_rate": 7.493826878752132e-06, "loss": 0.0011, "step": 2290 }, { "clip_ratio": 0.1329750269651413, "epoch": 0.8016095171448565, "grad_norm": 2.979590776611706, "kl": 2.015625, "learning_rate": 7.491179704339376e-06, "loss": -0.0064, "step": 2291 }, { "clip_ratio": 0.17480938136577606, "epoch": 0.8019594121763471, "grad_norm": 3.5973187004539144, "kl": 1.9453125, "learning_rate": 7.488531600740167e-06, "loss": -0.0134, "step": 2292 }, { "clip_ratio": 0.025134596973657608, "completions/clipped_ratio": 0.6785714285714286, "completions/max_length": 256.0, "completions/max_terminated_length": 255.0, "completions/mean_length": 230.4107208251953, "completions/mean_terminated_length": 176.38888549804688, "completions/min_length": 83.0, "completions/min_terminated_length": 83.0, "epoch": 0.8023093072078377, "grad_norm": 7.819760675488213, "kl": 2.65625, "learning_rate": 7.485882568942222e-06, "loss": 0.0339, "num_tokens": 10390323.0, "reward": 2.597254753112793, "reward_std": 0.2846475839614868, "rewards/check_originality_func/mean": 0.7857142686843872, "rewards/check_originality_func/std": 0.4140393137931824, "rewards/check_winston_local_func/mean": 0.9890750646591187, "rewards/check_winston_local_func/std": 0.009862786158919334, "rewards/sentence_count_match_reward_logic/mean": 0.8224653601646423, "rewards/sentence_count_match_reward_logic/std": 0.13675013184547424, "step": 2293 }, { "clip_ratio": 0.06767397373914719, "epoch": 0.8026592022393282, "grad_norm": 3.279554994272924, "kl": 2.953125, "learning_rate": 7.483232609933608e-06, "loss": 0.0068, "step": 2294 }, { "clip_ratio": 0.1137784868478775, "epoch": 0.8030090972708187, "grad_norm": 1.508040231582116, "kl": 2.96875, "learning_rate": 7.4805817247027335e-06, "loss": 0.0023, "step": 2295 }, { "clip_ratio": 0.13026578724384308, "epoch": 0.8033589923023093, "grad_norm": 4.406680253258925, "kl": 2.765625, "learning_rate": 7.477929914238351e-06, "loss": -0.0007, "step": 2296 }, { "clip_ratio": 0.026233088225126266, "completions/clipped_ratio": 0.7857142857142857, "completions/max_length": 256.0, "completions/max_terminated_length": 250.0, "completions/mean_length": 231.7678680419922, "completions/mean_terminated_length": 142.9166717529297, "completions/min_length": 84.0, "completions/min_terminated_length": 84.0, "epoch": 0.8037088873337999, "grad_norm": 3.803290399153618, "kl": 3.390625, "learning_rate": 7.475277179529562e-06, "loss": 0.0291, "num_tokens": 10423710.0, "reward": 2.6387734413146973, "reward_std": 0.2203185111284256, "rewards/check_originality_func/mean": 0.8035714030265808, "rewards/check_originality_func/std": 0.40089184045791626, "rewards/check_winston_local_func/mean": 0.9859246015548706, "rewards/check_winston_local_func/std": 0.015422673895955086, "rewards/sentence_count_match_reward_logic/mean": 0.8492771983146667, "rewards/sentence_count_match_reward_logic/std": 0.10205701738595963, "step": 2297 }, { "clip_ratio": 0.03733932599425316, "epoch": 0.8040587823652904, "grad_norm": 2.2839087975502887, "kl": 3.71875, "learning_rate": 7.4726235215658116e-06, "loss": 0.0219, "step": 2298 }, { "clip_ratio": 0.08810727298259735, "epoch": 0.804408677396781, "grad_norm": 1.6383908448688271, "kl": 3.40625, "learning_rate": 7.469968941336892e-06, "loss": 0.0184, "step": 2299 }, { "clip_ratio": 0.11349426209926605, "epoch": 0.8047585724282715, "grad_norm": 1.5307504896272965, "kl": 3.15625, "learning_rate": 7.467313439832933e-06, "loss": 0.014, "step": 2300 }, { "clip_ratio": 0.035020675510168076, "completions/clipped_ratio": 0.4285714285714286, "completions/max_length": 256.0, "completions/max_terminated_length": 250.0, "completions/mean_length": 229.60714721679688, "completions/mean_terminated_length": 209.8125, "completions/min_length": 137.0, "completions/min_terminated_length": 137.0, "epoch": 0.8051084674597621, "grad_norm": 6.0330408819803765, "kl": 1.703125, "learning_rate": 7.464657018044411e-06, "loss": 0.0192, "num_tokens": 10455064.0, "reward": 2.6864068508148193, "reward_std": 0.3615921139717102, "rewards/check_originality_func/mean": 0.7678571343421936, "rewards/check_originality_func/std": 0.4260205626487732, "rewards/check_winston_local_func/mean": 0.9886890053749084, "rewards/check_winston_local_func/std": 0.005773965269327164, "rewards/sentence_count_match_reward_logic/mean": 0.9298604726791382, "rewards/sentence_count_match_reward_logic/std": 0.07496728003025055, "step": 2301 }, { "clip_ratio": 0.066364586353302, "epoch": 0.8054583624912526, "grad_norm": 7.319950518539721, "kl": 2.109375, "learning_rate": 7.461999676962148e-06, "loss": 0.0207, "step": 2302 }, { "clip_ratio": 0.05580645427107811, "epoch": 0.8058082575227432, "grad_norm": 6.036509553858156, "kl": 1.765625, "learning_rate": 7.4593414175773085e-06, "loss": 0.0117, "step": 2303 }, { "clip_ratio": 0.07322466373443604, "epoch": 0.8061581525542337, "grad_norm": 29.634669259749373, "kl": 1.375, "learning_rate": 7.456682240881395e-06, "loss": 0.0454, "step": 2304 }, { "clip_ratio": 0.026340510696172714, "completions/clipped_ratio": 0.4464285714285714, "completions/max_length": 256.0, "completions/max_terminated_length": 250.0, "completions/mean_length": 195.0178680419922, "completions/mean_terminated_length": 145.8386993408203, "completions/min_length": 31.0, "completions/min_terminated_length": 31.0, "epoch": 0.8065080475857243, "grad_norm": 3.7313008599220856, "kl": 1.796875, "learning_rate": 7.4540221478662565e-06, "loss": 0.0099, "num_tokens": 10481353.0, "reward": 2.5179696083068848, "reward_std": 0.14386211335659027, "rewards/check_originality_func/mean": 0.5714285969734192, "rewards/check_originality_func/std": 0.4993502199649811, "rewards/check_winston_local_func/mean": 0.9813135862350464, "rewards/check_winston_local_func/std": 0.02302693948149681, "rewards/sentence_count_match_reward_logic/mean": 0.9652276635169983, "rewards/sentence_count_match_reward_logic/std": 0.061886366456747055, "step": 2305 }, { "clip_ratio": 0.05443096160888672, "epoch": 0.8068579426172149, "grad_norm": 2.006249967219232, "kl": 1.796875, "learning_rate": 7.451361139524082e-06, "loss": -0.0001, "step": 2306 }, { "clip_ratio": 0.10687299817800522, "epoch": 0.8072078376487054, "grad_norm": 1.2451120712041703, "kl": 1.9140625, "learning_rate": 7.448699216847404e-06, "loss": -0.0037, "step": 2307 }, { "clip_ratio": 0.14011497795581818, "epoch": 0.8075577326801959, "grad_norm": 1.4228059295919548, "kl": 1.9296875, "learning_rate": 7.446036380829093e-06, "loss": -0.007, "step": 2308 }, { "clip_ratio": 0.0148063525557518, "completions/clipped_ratio": 0.4285714285714286, "completions/max_length": 256.0, "completions/max_terminated_length": 247.0, "completions/mean_length": 221.19644165039062, "completions/mean_terminated_length": 195.09375, "completions/min_length": 158.0, "completions/min_terminated_length": 158.0, "epoch": 0.8079076277116864, "grad_norm": 7.403665453763828, "kl": 0.9609375, "learning_rate": 7.443372632462363e-06, "loss": -0.0037, "num_tokens": 10512012.0, "reward": 2.713524103164673, "reward_std": 0.3046710193157196, "rewards/check_originality_func/mean": 0.8035714030265808, "rewards/check_originality_func/std": 0.40089187026023865, "rewards/check_winston_local_func/mean": 0.9801796674728394, "rewards/check_winston_local_func/std": 0.038190022110939026, "rewards/sentence_count_match_reward_logic/mean": 0.9297728538513184, "rewards/sentence_count_match_reward_logic/std": 0.080776646733284, "step": 2309 }, { "clip_ratio": 0.05271121487021446, "epoch": 0.8082575227431771, "grad_norm": 3.5051025044766972, "kl": 0.94921875, "learning_rate": 7.440707972740766e-06, "loss": -0.0135, "step": 2310 }, { "clip_ratio": 0.0905427485704422, "epoch": 0.8086074177746676, "grad_norm": 1.142065602242468, "kl": 0.98046875, "learning_rate": 7.438042402658195e-06, "loss": -0.0211, "step": 2311 }, { "clip_ratio": 0.11151977628469467, "epoch": 0.8089573128061581, "grad_norm": 0.9445948796201649, "kl": 1.078125, "learning_rate": 7.435375923208879e-06, "loss": -0.0243, "step": 2312 }, { "clip_ratio": 0.01304649282246828, "completions/clipped_ratio": 0.4285714285714286, "completions/max_length": 256.0, "completions/max_terminated_length": 244.0, "completions/mean_length": 197.8928680419922, "completions/mean_terminated_length": 154.3125, "completions/min_length": 73.0, "completions/min_terminated_length": 73.0, "epoch": 0.8093072078376488, "grad_norm": 4.941752434615134, "kl": 1.234375, "learning_rate": 7.432708535387397e-06, "loss": 0.0088, "num_tokens": 10540958.0, "reward": 2.6875863075256348, "reward_std": 0.28142035007476807, "rewards/check_originality_func/mean": 0.8392857313156128, "rewards/check_originality_func/std": 0.3705909848213196, "rewards/check_winston_local_func/mean": 0.9877066612243652, "rewards/check_winston_local_func/std": 0.021436337381601334, "rewards/sentence_count_match_reward_logic/mean": 0.8605937957763672, "rewards/sentence_count_match_reward_logic/std": 0.13091033697128296, "step": 2313 }, { "clip_ratio": 0.02716033346951008, "epoch": 0.8096571028691393, "grad_norm": 2.112066091867651, "kl": 1.2890625, "learning_rate": 7.430040240188653e-06, "loss": 0.0019, "step": 2314 }, { "clip_ratio": 0.0441734604537487, "epoch": 0.8100069979006298, "grad_norm": 0.9423369925116468, "kl": 1.4453125, "learning_rate": 7.427371038607899e-06, "loss": -0.0024, "step": 2315 }, { "clip_ratio": 0.06819414347410202, "epoch": 0.8103568929321203, "grad_norm": 0.8616268966432299, "kl": 1.5625, "learning_rate": 7.4247009316407215e-06, "loss": -0.0076, "step": 2316 }, { "clip_ratio": 0.030214708298444748, "completions/clipped_ratio": 0.4642857142857143, "completions/max_length": 256.0, "completions/max_terminated_length": 254.0, "completions/mean_length": 206.5357208251953, "completions/mean_terminated_length": 163.6666717529297, "completions/min_length": 57.0, "completions/min_terminated_length": 57.0, "epoch": 0.810706787963611, "grad_norm": 3.272646204291984, "kl": 2.109375, "learning_rate": 7.422029920283044e-06, "loss": 0.0133, "num_tokens": 10570852.0, "reward": 2.6290900707244873, "reward_std": 0.23258449137210846, "rewards/check_originality_func/mean": 0.7857142686843872, "rewards/check_originality_func/std": 0.4140393137931824, "rewards/check_winston_local_func/mean": 0.9886234998703003, "rewards/check_winston_local_func/std": 0.00746278278529644, "rewards/sentence_count_match_reward_logic/mean": 0.854752242565155, "rewards/sentence_count_match_reward_logic/std": 0.12496347725391388, "step": 2317 }, { "clip_ratio": 0.05989088490605354, "epoch": 0.8110566829951015, "grad_norm": 3.2417200224383893, "kl": 2.125, "learning_rate": 7.419358005531129e-06, "loss": 0.0059, "step": 2318 }, { "clip_ratio": 0.11575792729854584, "epoch": 0.811406578026592, "grad_norm": 1.4874369395415945, "kl": 2.03125, "learning_rate": 7.416685188381576e-06, "loss": -0.0036, "step": 2319 }, { "clip_ratio": 0.1622866839170456, "epoch": 0.8117564730580826, "grad_norm": 1.3835987171097737, "kl": 2.078125, "learning_rate": 7.414011469831319e-06, "loss": -0.012, "step": 2320 }, { "clip_ratio": 0.010592175647616386, "completions/clipped_ratio": 0.2857142857142857, "completions/max_length": 256.0, "completions/max_terminated_length": 247.0, "completions/mean_length": 176.1428680419922, "completions/mean_terminated_length": 144.1999969482422, "completions/min_length": 69.0, "completions/min_terminated_length": 69.0, "epoch": 0.8121063680895731, "grad_norm": 4.055299793646812, "kl": 1.4140625, "learning_rate": 7.411336850877633e-06, "loss": 0.0195, "num_tokens": 10596748.0, "reward": 2.6667370796203613, "reward_std": 0.305709570646286, "rewards/check_originality_func/mean": 0.7857142686843872, "rewards/check_originality_func/std": 0.4140393137931824, "rewards/check_winston_local_func/mean": 0.9888386130332947, "rewards/check_winston_local_func/std": 0.01022789254784584, "rewards/sentence_count_match_reward_logic/mean": 0.8921839594841003, "rewards/sentence_count_match_reward_logic/std": 0.11801266670227051, "step": 2321 }, { "clip_ratio": 0.04942096024751663, "epoch": 0.8124562631210637, "grad_norm": 1.5850235686485148, "kl": 1.4453125, "learning_rate": 7.408661332518122e-06, "loss": 0.0088, "step": 2322 }, { "clip_ratio": 0.08703946322202682, "epoch": 0.8128061581525542, "grad_norm": 1.3989956894908833, "kl": 1.546875, "learning_rate": 7.40598491575073e-06, "loss": 0.0039, "step": 2323 }, { "clip_ratio": 0.13785628974437714, "epoch": 0.8131560531840448, "grad_norm": 1.0769311282598897, "kl": 1.703125, "learning_rate": 7.403307601573737e-06, "loss": -0.0002, "step": 2324 }, { "clip_ratio": 0.028897715732455254, "completions/clipped_ratio": 0.5714285714285714, "completions/max_length": 256.0, "completions/max_terminated_length": 256.0, "completions/mean_length": 237.19644165039062, "completions/mean_terminated_length": 212.125, "completions/min_length": 152.0, "completions/min_terminated_length": 152.0, "epoch": 0.8135059482155353, "grad_norm": 9.133381626420624, "kl": 1.9453125, "learning_rate": 7.400629390985753e-06, "loss": 0.0252, "num_tokens": 10630543.0, "reward": 2.8024179935455322, "reward_std": 0.07994672656059265, "rewards/check_originality_func/mean": 1.0, "rewards/check_originality_func/std": 0.0, "rewards/check_winston_local_func/mean": 0.9843754768371582, "rewards/check_winston_local_func/std": 0.022920899093151093, "rewards/sentence_count_match_reward_logic/mean": 0.8180422186851501, "rewards/sentence_count_match_reward_logic/std": 0.12118315696716309, "step": 2325 }, { "clip_ratio": 0.07919464260339737, "epoch": 0.8138558432470259, "grad_norm": 4.882429042243005, "kl": 2.1875, "learning_rate": 7.397950284985728e-06, "loss": 0.012, "step": 2326 }, { "clip_ratio": 0.12222526967525482, "epoch": 0.8142057382785165, "grad_norm": 2.9069281591685714, "kl": 2.328125, "learning_rate": 7.395270284572939e-06, "loss": 0.0042, "step": 2327 }, { "clip_ratio": 0.1528492420911789, "epoch": 0.814555633310007, "grad_norm": 3.139211047232303, "kl": 2.40625, "learning_rate": 7.392589390747006e-06, "loss": -0.0031, "step": 2328 }, { "clip_ratio": 0.009007819928228855, "completions/clipped_ratio": 0.5714285714285714, "completions/max_length": 256.0, "completions/max_terminated_length": 194.0, "completions/mean_length": 217.1428680419922, "completions/mean_terminated_length": 165.33334350585938, "completions/min_length": 130.0, "completions/min_terminated_length": 130.0, "epoch": 0.8149055283414975, "grad_norm": 4.35614020778445, "kl": 0.97265625, "learning_rate": 7.389907604507874e-06, "loss": 0.0113, "num_tokens": 10660103.0, "reward": 2.8532514572143555, "reward_std": 0.15840260684490204, "rewards/check_originality_func/mean": 0.9464285969734192, "rewards/check_originality_func/std": 0.22720777988433838, "rewards/check_winston_local_func/mean": 0.9790388941764832, "rewards/check_winston_local_func/std": 0.0381169393658638, "rewards/sentence_count_match_reward_logic/mean": 0.9277839064598083, "rewards/sentence_count_match_reward_logic/std": 0.07779709994792938, "step": 2329 }, { "clip_ratio": 0.028353359550237656, "epoch": 0.8152554233729881, "grad_norm": 1.3936889325670412, "kl": 0.96875, "learning_rate": 7.387224926855824e-06, "loss": 0.0031, "step": 2330 }, { "clip_ratio": 0.047890208661556244, "epoch": 0.8156053184044787, "grad_norm": 1.784776968555401, "kl": 0.96484375, "learning_rate": 7.384541358791471e-06, "loss": -0.0018, "step": 2331 }, { "clip_ratio": 0.06573735177516937, "epoch": 0.8159552134359692, "grad_norm": 1.61946686783958, "kl": 0.96875, "learning_rate": 7.3818569013157585e-06, "loss": -0.0065, "step": 2332 }, { "clip_ratio": 0.008013056591153145, "completions/clipped_ratio": 0.25, "completions/max_length": 256.0, "completions/max_terminated_length": 252.0, "completions/mean_length": 196.07144165039062, "completions/mean_terminated_length": 176.09524536132812, "completions/min_length": 75.0, "completions/min_terminated_length": 75.0, "epoch": 0.8163051084674597, "grad_norm": 3.714843577577829, "kl": 1.1640625, "learning_rate": 7.379171555429965e-06, "loss": 0.011, "num_tokens": 10687443.0, "reward": 2.5743165016174316, "reward_std": 0.2148856669664383, "rewards/check_originality_func/mean": 0.6607142686843872, "rewards/check_originality_func/std": 0.477751761674881, "rewards/check_winston_local_func/mean": 0.9902417063713074, "rewards/check_winston_local_func/std": 0.009753740392625332, "rewards/sentence_count_match_reward_logic/mean": 0.923360288143158, "rewards/sentence_count_match_reward_logic/std": 0.10486602038145065, "step": 2333 }, { "clip_ratio": 0.06205228716135025, "epoch": 0.8166550034989503, "grad_norm": 1.7677663185138215, "kl": 1.1484375, "learning_rate": 7.376485322135697e-06, "loss": 0.0004, "step": 2334 }, { "clip_ratio": 0.12219363451004028, "epoch": 0.8170048985304409, "grad_norm": 0.879519794470073, "kl": 1.1484375, "learning_rate": 7.373798202434896e-06, "loss": -0.005, "step": 2335 }, { "clip_ratio": 0.14740967750549316, "epoch": 0.8173547935619314, "grad_norm": 0.7821592561813016, "kl": 1.1796875, "learning_rate": 7.371110197329834e-06, "loss": -0.0084, "step": 2336 }, { "clip_ratio": 0.008446133695542812, "completions/clipped_ratio": 0.1428571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 231.0, "completions/mean_length": 122.76786041259766, "completions/mean_terminated_length": 100.5625, "completions/min_length": 46.0, "completions/min_terminated_length": 46.0, "epoch": 0.8177046885934219, "grad_norm": 4.082587844173457, "kl": 2.09375, "learning_rate": 7.3684213078231084e-06, "loss": 0.0038, "num_tokens": 10704950.0, "reward": 2.5611023902893066, "reward_std": 0.33991751074790955, "rewards/check_originality_func/mean": 0.6071428656578064, "rewards/check_originality_func/std": 0.4928053617477417, "rewards/check_winston_local_func/mean": 0.9874368906021118, "rewards/check_winston_local_func/std": 0.006983400322496891, "rewards/sentence_count_match_reward_logic/mean": 0.9665223360061646, "rewards/sentence_count_match_reward_logic/std": 0.07164443284273148, "step": 2337 }, { "clip_ratio": 0.019894029945135117, "epoch": 0.8180545836249126, "grad_norm": 3.0435762057124367, "kl": 2.078125, "learning_rate": 7.365731534917651e-06, "loss": -0.0063, "step": 2338 }, { "clip_ratio": 0.10895616561174393, "epoch": 0.8184044786564031, "grad_norm": 1.05007192736092, "kl": 2.109375, "learning_rate": 7.36304087961672e-06, "loss": -0.0141, "step": 2339 }, { "clip_ratio": 0.17973443865776062, "epoch": 0.8187543736878936, "grad_norm": 1.5354617647962199, "kl": 2.125, "learning_rate": 7.360349342923905e-06, "loss": -0.0158, "step": 2340 }, { "clip_ratio": 0.009233607910573483, "completions/clipped_ratio": 0.3928571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 247.0, "completions/mean_length": 183.6428680419922, "completions/mean_terminated_length": 136.8235321044922, "completions/min_length": 51.0, "completions/min_terminated_length": 51.0, "epoch": 0.8191042687193841, "grad_norm": 3.096104550926685, "kl": 1.8203125, "learning_rate": 7.357656925843125e-06, "loss": 0.013, "num_tokens": 10730386.0, "reward": 2.68153977394104, "reward_std": 0.3474389314651489, "rewards/check_originality_func/mean": 0.75, "rewards/check_originality_func/std": 0.43693143129348755, "rewards/check_winston_local_func/mean": 0.9824420213699341, "rewards/check_winston_local_func/std": 0.016509974375367165, "rewards/sentence_count_match_reward_logic/mean": 0.9490973353385925, "rewards/sentence_count_match_reward_logic/std": 0.10087907314300537, "step": 2341 }, { "clip_ratio": 0.06813081353902817, "epoch": 0.8194541637508748, "grad_norm": 1.704140680791407, "kl": 1.6015625, "learning_rate": 7.354963629378626e-06, "loss": 0.0035, "step": 2342 }, { "clip_ratio": 0.10805808007717133, "epoch": 0.8198040587823653, "grad_norm": 1.7120632026633555, "kl": 1.4765625, "learning_rate": 7.352269454534981e-06, "loss": -0.0048, "step": 2343 }, { "clip_ratio": 0.13801942765712738, "epoch": 0.8201539538138558, "grad_norm": 0.9430607013325729, "kl": 1.3984375, "learning_rate": 7.3495744023170936e-06, "loss": -0.0109, "step": 2344 }, { "clip_ratio": 0.009606166742742062, "completions/clipped_ratio": 0.5714285714285714, "completions/max_length": 256.0, "completions/max_terminated_length": 248.0, "completions/mean_length": 224.00001525878906, "completions/mean_terminated_length": 181.33334350585938, "completions/min_length": 72.0, "completions/min_terminated_length": 72.0, "epoch": 0.8205038488453464, "grad_norm": 4.196146879219149, "kl": 0.97265625, "learning_rate": 7.346878473730189e-06, "loss": 0.0126, "num_tokens": 10762394.0, "reward": 2.7034313678741455, "reward_std": 0.23316609859466553, "rewards/check_originality_func/mean": 0.8035714030265808, "rewards/check_originality_func/std": 0.40089187026023865, "rewards/check_winston_local_func/mean": 0.9837914705276489, "rewards/check_winston_local_func/std": 0.015991047024726868, "rewards/sentence_count_match_reward_logic/mean": 0.9160684943199158, "rewards/sentence_count_match_reward_logic/std": 0.10437313467264175, "step": 2345 }, { "clip_ratio": 0.04122444614768028, "epoch": 0.820853743876837, "grad_norm": 2.3028471062618365, "kl": 0.96484375, "learning_rate": 7.344181669779826e-06, "loss": -0.0009, "step": 2346 }, { "clip_ratio": 0.08688979595899582, "epoch": 0.8212036389083275, "grad_norm": 2.226168421801502, "kl": 0.97265625, "learning_rate": 7.341483991471887e-06, "loss": -0.0082, "step": 2347 }, { "clip_ratio": 0.11226452887058258, "epoch": 0.821553533939818, "grad_norm": 1.302027067908255, "kl": 0.98046875, "learning_rate": 7.33878543981258e-06, "loss": -0.0162, "step": 2348 }, { "clip_ratio": 0.007598350290209055, "completions/clipped_ratio": 0.3571428571428571, "completions/max_length": 256.0, "completions/max_terminated_length": 251.0, "completions/mean_length": 190.85714721679688, "completions/mean_terminated_length": 154.6666717529297, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "epoch": 0.8219034289713086, "grad_norm": 1.3072666934075716, "kl": 1.296875, "learning_rate": 7.336086015808439e-06, "loss": 0.0087, "num_tokens": 10787290.0, "reward": 2.655980348587036, "reward_std": 0.18855591118335724, "rewards/check_originality_func/mean": 0.7321428656578064, "rewards/check_originality_func/std": 0.44685041904449463, "rewards/check_winston_local_func/mean": 0.9749531149864197, "rewards/check_winston_local_func/std": 0.03870873898267746, "rewards/sentence_count_match_reward_logic/mean": 0.9488844275474548, "rewards/sentence_count_match_reward_logic/std": 0.07405932247638702, "step": 2349 }, { "clip_ratio": 0.016074951738119125, "epoch": 0.8222533240027992, "grad_norm": 1.0456869922742977, "kl": 1.296875, "learning_rate": 7.333385720466323e-06, "loss": 0.0046, "step": 2350 }, { "clip_ratio": 0.03619750216603279, "epoch": 0.8226032190342897, "grad_norm": 0.7722040743572254, "kl": 1.3046875, "learning_rate": 7.330684554793417e-06, "loss": 0.001, "step": 2351 }, { "clip_ratio": 0.05808711051940918, "epoch": 0.8229531140657803, "grad_norm": 0.7134355172845133, "kl": 1.3125, "learning_rate": 7.3279825197972306e-06, "loss": -0.0016, "step": 2352 }, { "clip_ratio": 0.007874681614339352, "completions/clipped_ratio": 0.2857142857142857, "completions/max_length": 256.0, "completions/max_terminated_length": 252.0, "completions/mean_length": 170.71429443359375, "completions/mean_terminated_length": 136.60000610351562, "completions/min_length": 85.0, "completions/min_terminated_length": 85.0, "epoch": 0.8233030090972708, "grad_norm": 1.3111378067048525, "kl": 1.171875, "learning_rate": 7.325279616485599e-06, "loss": 0.0078, "num_tokens": 10809754.0, "reward": 2.671137809753418, "reward_std": 0.22175277769565582, "rewards/check_originality_func/mean": 0.7321428656578064, "rewards/check_originality_func/std": 0.44685041904449463, "rewards/check_winston_local_func/mean": 0.9895438551902771, "rewards/check_winston_local_func/std": 0.004497200716286898, "rewards/sentence_count_match_reward_logic/mean": 0.9494511485099792, "rewards/sentence_count_match_reward_logic/std": 0.06816883385181427, "step": 2353 }, { "clip_ratio": 0.02225000597536564, "epoch": 0.8236529041287614, "grad_norm": 0.8104235082291139, "kl": 1.171875, "learning_rate": 7.322575845866675e-06, "loss": 0.0055, "step": 2354 }, { "clip_ratio": 0.030349010601639748, "epoch": 0.8240027991602519, "grad_norm": 0.8804973236205421, "kl": 1.1640625, "learning_rate": 7.3198712089489435e-06, "loss": 0.0015, "step": 2355 }, { "clip_ratio": 0.05323222279548645, "epoch": 0.8243526941917425, "grad_norm": 0.756681897199883, "kl": 1.1640625, "learning_rate": 7.317165706741207e-06, "loss": -0.0028, "step": 2356 }, { "clip_ratio": 0.01794593222439289, "completions/clipped_ratio": 0.4285714285714286, "completions/max_length": 256.0, "completions/max_terminated_length": 256.0, "completions/mean_length": 207.19644165039062, "completions/mean_terminated_length": 170.59375, "completions/min_length": 113.0, "completions/min_terminated_length": 113.0, "epoch": 0.824702589223233, "grad_norm": 2.752861234571921, "kl": 0.98046875, "learning_rate": 7.314459340252593e-06, "loss": 0.0038, "num_tokens": 10838581.0, "reward": 2.6597113609313965, "reward_std": 0.2257414013147354, "rewards/check_originality_func/mean": 0.8035714030265808, "rewards/check_originality_func/std": 0.40089187026023865, "rewards/check_winston_local_func/mean": 0.984869122505188, "rewards/check_winston_local_func/std": 0.018981998786330223, "rewards/sentence_count_match_reward_logic/mean": 0.8712707161903381, "rewards/sentence_count_match_reward_logic/std": 0.13925427198410034, "step": 2357 }, { "clip_ratio": 0.051133401691913605, "epoch": 0.8250524842547236, "grad_norm": 2.6411856280202772, "kl": 1.0, "learning_rate": 7.311752110492549e-06, "loss": 0.0021, "step": 2358 }, { "clip_ratio": 0.05941963940858841, "epoch": 0.8254023792862142, "grad_norm": 2.2936112337160974, "kl": 0.98046875, "learning_rate": 7.309044018470848e-06, "loss": -0.0052, "step": 2359 }, { "clip_ratio": 0.08975464850664139, "epoch": 0.8257522743177047, "grad_norm": 4.1654980603851595, "kl": 0.95703125, "learning_rate": 7.306335065197581e-06, "loss": -0.0077, "step": 2360 }, { "clip_ratio": 0.006539177615195513, "completions/clipped_ratio": 0.4642857142857143, "completions/max_length": 256.0, "completions/max_terminated_length": 256.0, "completions/mean_length": 201.2678680419922, "completions/mean_terminated_length": 153.83334350585938, "completions/min_length": 61.0, "completions/min_terminated_length": 61.0, "epoch": 0.8261021693491952, "grad_norm": 2.589600982132617, "kl": 0.9765625, "learning_rate": 7.303625251683162e-06, "loss": 0.0007, "num_tokens": 10865044.0, "reward": 2.767583131790161, "reward_std": 0.18974243104457855, "rewards/check_originality_func/mean": 0.8214285969734192, "rewards/check_originality_func/std": 0.3864591419696808, "rewards/check_winston_local_func/mean": 0.9890440702438354, "rewards/check_winston_local_func/std": 0.0051937103271484375, "rewards/sentence_count_match_reward_logic/mean": 0.9571101069450378, "rewards/sentence_count_match_reward_logic/std": 0.07349196076393127, "step": 2361 }, { "clip_ratio": 0.01603439263999462, "epoch": 0.8264520643806857, "grad_norm": 2.165692921959618, "kl": 0.9765625, "learning_rate": 7.300914578938327e-06, "loss": -0.0032, "step": 2362 }, { "clip_ratio": 0.043379735201597214, "epoch": 0.8268019594121764, "grad_norm": 1.2340033237027908, "kl": 0.9765625, "learning_rate": 7.2982030479741315e-06, "loss": -0.013, "step": 2363 }, { "clip_ratio": 0.06659281998872757, "epoch": 0.8271518544436669, "grad_norm": 0.8267907582551625, "kl": 1.0, "learning_rate": 7.29549065980195e-06, "loss": -0.0192, "step": 2364 }, { "clip_ratio": 0.008157732896506786, "completions/clipped_ratio": 0.7321428571428572, "completions/max_length": 256.0, "completions/max_terminated_length": 250.0, "completions/mean_length": 248.4285888671875, "completions/mean_terminated_length": 227.7333526611328, "completions/min_length": 177.0, "completions/min_terminated_length": 177.0, "epoch": 0.8275017494751574, "grad_norm": 4.433730428486719, "kl": 0.8359375, "learning_rate": 7.2927774154334765e-06, "loss": 0.0094, "num_tokens": 10898940.0, "reward": 2.8391306400299072, "reward_std": 0.09361329674720764, "rewards/check_originality_func/mean": 0.9821428656578064, "rewards/check_originality_func/std": 0.13363061845302582, "rewards/check_winston_local_func/mean": 0.9785416722297668, "rewards/check_winston_local_func/std": 0.03216804936528206, "rewards/sentence_count_match_reward_logic/mean": 0.8784458041191101, "rewards/sentence_count_match_reward_logic/std": 0.1496991515159607, "step": 2365 }, { "clip_ratio": 0.016551213338971138, "epoch": 0.827851644506648, "grad_norm": 2.983216080996931, "kl": 0.83984375, "learning_rate": 7.290063315880727e-06, "loss": -0.0023, "step": 2366 }, { "clip_ratio": 0.05432254821062088, "epoch": 0.8282015395381386, "grad_norm": 2.6257132288877734, "kl": 0.8671875, "learning_rate": 7.287348362156035e-06, "loss": -0.0124, "step": 2367 }, { "clip_ratio": 0.10720915347337723, "epoch": 0.8285514345696291, "grad_norm": 1.6145622692940598, "kl": 0.87890625, "learning_rate": 7.2846325552720536e-06, "loss": -0.0219, "step": 2368 }, { "clip_ratio": 0.004432950634509325, "completions/clipped_ratio": 0.2857142857142857, "completions/max_length": 256.0, "completions/max_terminated_length": 224.0, "completions/mean_length": 173.4107208251953, "completions/mean_terminated_length": 140.375, "completions/min_length": 86.0, "completions/min_terminated_length": 86.0, "epoch": 0.8289013296011196, "grad_norm": 2.5438408539766493, "kl": 1.1484375, "learning_rate": 7.281915896241749e-06, "loss": 0.0086, "num_tokens": 10921819.0, "reward": 2.8263559341430664, "reward_std": 0.21749450266361237, "rewards/check_originality_func/mean": 0.9285714030265808, "rewards/check_originality_func/std": 0.25987008213996887, "rewards/check_winston_local_func/mean": 0.9815183877944946, "rewards/check_winston_local_func/std": 0.028955401852726936, "rewards/sentence_count_match_reward_logic/mean": 0.9162660241127014, "rewards/sentence_count_match_reward_logic/std": 0.11734078824520111, "step": 2369 }, { "clip_ratio": 0.011907701380550861, "epoch": 0.8292512246326103, "grad_norm": 2.0474678924105927, "kl": 1.171875, "learning_rate": 7.279198386078414e-06, "loss": 0.001, "step": 2370 }, { "clip_ratio": 0.05817355588078499, "epoch": 0.8296011196641008, "grad_norm": 1.1595011219965603, "kl": 1.2265625, "learning_rate": 7.276480025795649e-06, "loss": -0.0054, "step": 2371 }, { "clip_ratio": 0.0906866192817688, "epoch": 0.8299510146955913, "grad_norm": 1.1030647901747441, "kl": 1.3046875, "learning_rate": 7.273760816407384e-06, "loss": -0.009, "step": 2372 }, { "clip_ratio": 0.012931959703564644, "completions/clipped_ratio": 0.4285714285714286, "completions/max_length": 256.0, "completions/max_terminated_length": 199.0, "completions/mean_length": 165.08929443359375, "completions/mean_terminated_length": 96.90625, "completions/min_length": 55.0, "completions/min_terminated_length": 55.0, "epoch": 0.8303009097270819, "grad_norm": 3.278136104595984, "kl": 1.140625, "learning_rate": 7.271040758927852e-06, "loss": 0.0049, "num_tokens": 10944712.0, "reward": 2.4407474994659424, "reward_std": 0.31116706132888794, "rewards/check_originality_func/mean": 0.5535714030265808, "rewards/check_originality_func/std": 0.5016207695007324, "rewards/check_winston_local_func/mean": 0.9784704446792603, "rewards/check_winston_local_func/std": 0.043044563382864, "rewards/sentence_count_match_reward_logic/mean": 0.9087055921554565, "rewards/sentence_count_match_reward_logic/std": 0.10184082388877869, "step": 2373 }, { "clip_ratio": 0.029099520295858383, "epoch": 0.8306508047585724, "grad_norm": 1.3950093598072002, "kl": 1.1875, "learning_rate": 7.268319854371611e-06, "loss": 0.0011, "step": 2374 }, { "clip_ratio": 0.04035413637757301, "epoch": 0.831000699790063, "grad_norm": 1.0327355240897258, "kl": 1.2109375, "learning_rate": 7.265598103753533e-06, "loss": -0.0041, "step": 2375 }, { "clip_ratio": 0.0639076977968216, "epoch": 0.8313505948215535, "grad_norm": 0.850504458312545, "kl": 1.2421875, "learning_rate": 7.262875508088805e-06, "loss": -0.0094, "step": 2376 }, { "clip_ratio": 0.00819365493953228, "completions/clipped_ratio": 0.7142857142857143, "completions/max_length": 256.0, "completions/max_terminated_length": 256.0, "completions/mean_length": 242.00001525878906, "completions/mean_terminated_length": 207.0, "completions/min_length": 169.0, "completions/min_terminated_length": 169.0, "epoch": 0.8317004898530441, "grad_norm": 3.675805679619358, "kl": 0.98828125, "learning_rate": 7.260152068392927e-06, "loss": 0.006, "num_tokens": 10977944.0, "reward": 2.7958779335021973, "reward_std": 0.11804372817277908, "rewards/check_originality_func/mean": 0.9107142686843872, "rewards/check_originality_func/std": 0.28773635625839233, "rewards/check_winston_local_func/mean": 0.9846764206886292, "rewards/check_winston_local_func/std": 0.02078547701239586, "rewards/sentence_count_match_reward_logic/mean": 0.9004870057106018, "rewards/sentence_count_match_reward_logic/std": 0.11147835850715637, "step": 2377 }, { "clip_ratio": 0.025994261726737022, "epoch": 0.8320503848845346, "grad_norm": 3.1951768105110943, "kl": 0.97265625, "learning_rate": 7.25742778568172e-06, "loss": -0.0023, "step": 2378 }, { "clip_ratio": 0.07328449189662933, "epoch": 0.8324002799160252, "grad_norm": 1.8677816021790017, "kl": 0.96484375, "learning_rate": 7.254702660971313e-06, "loss": -0.0159, "step": 2379 }, { "clip_ratio": 0.1053536906838417, "epoch": 0.8327501749475158, "grad_norm": 1.1188806974783536, "kl": 0.97265625, "learning_rate": 7.251976695278155e-06, "loss": -0.0225, "step": 2380 }, { "clip_ratio": 0.016017528250813484, "completions/clipped_ratio": 0.5, "completions/max_length": 256.0, "completions/max_terminated_length": 256.0, "completions/mean_length": 221.82144165039062, "completions/mean_terminated_length": 187.6428680419922, "completions/min_length": 115.0, "completions/min_terminated_length": 115.0, "epoch": 0.8331000699790063, "grad_norm": 2.705076499971609, "kl": 1.0234375, "learning_rate": 7.2492498896190015e-06, "loss": 0.0018, "num_tokens": 11008326.0, "reward": 2.8755064010620117, "reward_std": 0.09147533774375916, "rewards/check_originality_func/mean": 0.9821428656578064, "rewards/check_originality_func/std": 0.13363061845302582, "rewards/check_winston_local_func/mean": 0.9749167561531067, "rewards/check_winston_local_func/std": 0.060509517788887024, "rewards/sentence_count_match_reward_logic/mean": 0.9184465408325195, "rewards/sentence_count_match_reward_logic/std": 0.0958322137594223, "step": 2381 }, { "clip_ratio": 0.04522637277841568, "epoch": 0.8334499650104968, "grad_norm": 1.6039974447831986, "kl": 1.0078125, "learning_rate": 7.246522245010926e-06, "loss": -0.0081, "step": 2382 }, { "clip_ratio": 0.08295433968305588, "epoch": 0.8337998600419874, "grad_norm": 1.2885912855478452, "kl": 1.0078125, "learning_rate": 7.243793762471316e-06, "loss": -0.012, "step": 2383 }, { "clip_ratio": 0.12130969017744064, "epoch": 0.834149755073478, "grad_norm": 1.1629129190995522, "kl": 1.0078125, "learning_rate": 7.24106444301787e-06, "loss": -0.0171, "step": 2384 }, { "clip_ratio": 0.005515242926776409, "completions/clipped_ratio": 0.5, "completions/max_length": 256.0, "completions/max_terminated_length": 254.0, "completions/mean_length": 200.2678680419922, "completions/mean_terminated_length": 144.5357208251953, "completions/min_length": 60.0, "completions/min_terminated_length": 60.0, "epoch": 0.8344996501049685, "grad_norm": 5.0226622429965975, "kl": 1.1875, "learning_rate": 7.238334287668595e-06, "loss": 0.0102, "num_tokens": 11036925.0, "reward": 2.683830976486206, "reward_std": 0.30742669105529785, "rewards/check_originality_func/mean": 0.8214285969734192, "rewards/check_originality_func/std": 0.3864591419696808, "rewards/check_winston_local_func/mean": 0.980274498462677, "rewards/check_winston_local_func/std": 0.03647006303071976, "rewards/sentence_count_match_reward_logic/mean": 0.8821280598640442, "rewards/sentence_count_match_reward_logic/std": 0.12255989015102386, "step": 2385 }, { "clip_ratio": 0.015449591912329197, "epoch": 0.834849545136459, "grad_norm": 4.446336891349071, "kl": 1.1875, "learning_rate": 7.235603297441817e-06, "loss": 0.0047, "step": 2386 }, { "clip_ratio": 0.05226381868124008, "epoch": 0.8351994401679497, "grad_norm": 1.8024819492536774, "kl": 1.25, "learning_rate": 7.2328714733561655e-06, "loss": -0.0048, "step": 2387 }, { "clip_ratio": 0.09565799683332443, "epoch": 0.8355493351994402, "grad_norm": 1.378466370249309, "kl": 1.3125, "learning_rate": 7.230138816430587e-06, "loss": -0.0133, "step": 2388 }, { "clip_ratio": 0.013696512207388878, "completions/clipped_ratio": 0.4285714285714286, "completions/max_length": 256.0, "completions/max_terminated_length": 179.0, "completions/mean_length": 193.6607208251953, "completions/mean_terminated_length": 146.90625, "completions/min_length": 122.0, "completions/min_terminated_length": 122.0, "epoch": 0.8358992302309307, "grad_norm": 3.9397392148562194, "kl": 1.0078125, "learning_rate": 7.227405327684339e-06, "loss": 0.0016, "num_tokens": 11063474.0, "reward": 2.7257754802703857, "reward_std": 0.22435559332370758, "rewards/check_originality_func/mean": 0.8571428656578064, "rewards/check_originality_func/std": 0.3530939221382141, "rewards/check_winston_local_func/mean": 0.9854041337966919, "rewards/check_winston_local_func/std": 0.013990257866680622, "rewards/sentence_count_match_reward_logic/mean": 0.8832283020019531, "rewards/sentence_count_match_reward_logic/std": 0.09956391900777817, "step": 2389 }, { "clip_ratio": 0.0258707907050848, "epoch": 0.8362491252624212, "grad_norm": 2.51167478201519, "kl": 1.015625, "learning_rate": 7.224671008136983e-06, "loss": -0.0059, "step": 2390 }, { "clip_ratio": 0.07333756983280182, "epoch": 0.8365990202939119, "grad_norm": 2.1263156151766056, "kl": 1.0390625, "learning_rate": 7.221935858808398e-06, "loss": -0.0132, "step": 2391 }, { "clip_ratio": 0.12281568348407745, "epoch": 0.8369489153254024, "grad_norm": 1.5832968449826512, "kl": 1.0625, "learning_rate": 7.219199880718763e-06, "loss": -0.0205, "step": 2392 }, { "clip_ratio": 0.005769271869212389, "completions/clipped_ratio": 0.4285714285714286, "completions/max_length": 256.0, "completions/max_terminated_length": 252.0, "completions/mean_length": 185.46429443359375, "completions/mean_terminated_length": 132.5625, "completions/min_length": 87.0, "completions/min_terminated_length": 87.0, "epoch": 0.8372988103568929, "grad_norm": 2.6124340690935908, "kl": 1.1640625, "learning_rate": 7.216463074888579e-06, "loss": 0.0121, "num_tokens": 11089212.0, "reward": 2.417848587036133, "reward_std": 0.18810705840587616, "rewards/check_originality_func/mean": 0.5892857313156128, "rewards/check_originality_func/std": 0.49641573429107666, "rewards/check_winston_local_func/mean": 0.968913197517395, "rewards/check_winston_local_func/std": 0.06436502933502197, "rewards/sentence_count_match_reward_logic/mean": 0.8596495389938354, "rewards/sentence_count_match_reward_logic/std": 0.11033087223768234, "step": 2393 }, { "clip_ratio": 0.02087363414466381, "epoch": 0.8376487053883834, "grad_norm": 1.70374373200439, "kl": 1.140625, "learning_rate": 7.213725442338645e-06, "loss": 0.0049, "step": 2394 }, { "clip_ratio": 0.0491635836660862, "epoch": 0.8379986004198741, "grad_norm": 0.8490758558315141, "kl": 1.140625, "learning_rate": 7.210986984090073e-06, "loss": -0.0002, "step": 2395 }, { "clip_ratio": 0.07552889734506607, "epoch": 0.8383484954513646, "grad_norm": 0.7817438482864726, "kl": 1.140625, "learning_rate": 7.2082477011642795e-06, "loss": -0.0029, "step": 2396 }, { "clip_ratio": 0.00431107496842742, "completions/clipped_ratio": 0.4642857142857143, "completions/max_length": 256.0, "completions/max_terminated_length": 256.0, "completions/mean_length": 209.62501525878906, "completions/mean_terminated_length": 169.433349609375, "completions/min_length": 81.0, "completions/min_terminated_length": 81.0, "epoch": 0.8386983904828551, "grad_norm": 3.1243194799535297, "kl": 1.078125, "learning_rate": 7.205507594582994e-06, "loss": 0.01, "num_tokens": 11116791.0, "reward": 2.632453203201294, "reward_std": 0.3223627805709839, "rewards/check_originality_func/mean": 0.7857142686843872, "rewards/check_originality_func/std": 0.4140393137931824, "rewards/check_winston_local_func/mean": 0.9477173686027527, "rewards/check_winston_local_func/std": 0.09141925722360611, "rewards/sentence_count_match_reward_logic/mean": 0.8990212678909302, "rewards/sentence_count_match_reward_logic/std": 0.12004131078720093, "step": 2397 }, { "clip_ratio": 0.030005017295479774, "epoch": 0.8390482855143457, "grad_norm": 2.213104967896378, "kl": 1.046875, "learning_rate": 7.20276666536825e-06, "loss": 0.0035, "step": 2398 }, { "clip_ratio": 0.07583402097225189, "epoch": 0.8393981805458363, "grad_norm": 1.363231724151875, "kl": 1.0234375, "learning_rate": 7.2000249145423874e-06, "loss": -0.0055, "step": 2399 }, { "clip_ratio": 0.10467435419559479, "epoch": 0.8397480755773268, "grad_norm": 0.9776991566398787, "kl": 1.0078125, "learning_rate": 7.197282343128053e-06, "loss": -0.0103, "step": 2400 }, { "clip_ratio": 0.0012025845935568213, "completions/clipped_ratio": 0.4285714285714286, "completions/max_length": 256.0, "completions/max_terminated_length": 223.0, "completions/mean_length": 194.35714721679688, "completions/mean_terminated_length": 148.125, "completions/min_length": 96.0, "completions/min_terminated_length": 96.0, "epoch": 0.8400979706088173, "grad_norm": 1.1026706362072387, "kl": 1.0625, "learning_rate": 7.194538952148201e-06, "loss": 0.0084, "num_tokens": 11142851.0, "reward": 0.7321428656578064, "reward_std": 0.26450884342193604, "rewards/check_originality_func/mean": 0.7321428656578064, "rewards/check_originality_func/std": 0.44685041904449463, "step": 2401 }, { "clip_ratio": 0.01328342966735363, "epoch": 0.8404478656403079, "grad_norm": 0.8728237333647846, "kl": 1.0546875, "learning_rate": 7.19179474262609e-06, "loss": 0.0062, "step": 2402 }, { "clip_ratio": 0.029023628681898117, "epoch": 0.8407977606717985, "grad_norm": 0.7322872727787106, "kl": 1.0703125, "learning_rate": 7.189049715585281e-06, "loss": 0.002, "step": 2403 }, { "clip_ratio": 0.05232435464859009, "epoch": 0.841147655703289, "grad_norm": 1.0932231385347453, "kl": 1.078125, "learning_rate": 7.1863038720496495e-06, "loss": -0.0033, "step": 2404 }, { "clip_ratio": 0.003571566194295883, "completions/clipped_ratio": 0.6785714285714286, "completions/max_length": 256.0, "completions/max_terminated_length": 252.0, "completions/mean_length": 219.85714721679688, "completions/mean_terminated_length": 143.55555725097656, "completions/min_length": 70.0, "completions/min_terminated_length": 70.0, "epoch": 0.8414975507347796, "grad_norm": 0.7049385402906766, "kl": 0.87890625, "learning_rate": 7.183557213043365e-06, "loss": 0.0051, "num_tokens": 11173795.0, "reward": 0.7142857313156128, "reward_std": 0.24888646602630615, "rewards/check_originality_func/mean": 0.7142857313156128, "rewards/check_originality_func/std": 0.4558422863483429, "step": 2405 }, { "clip_ratio": 0.0030621339101344347, "epoch": 0.8418474457662701, "grad_norm": 0.6988561146500324, "kl": 0.87890625, "learning_rate": 7.180809739590906e-06, "loss": 0.0048, "step": 2406 }, { "clip_ratio": 0.010546623729169369, "epoch": 0.8421973407977607, "grad_norm": 0.528008935669335, "kl": 0.87890625, "learning_rate": 7.1780614527170535e-06, "loss": 0.0021, "step": 2407 }, { "clip_ratio": 0.024392860010266304, "epoch": 0.8425472358292512, "grad_norm": 0.4261347883106438, "kl": 0.87890625, "learning_rate": 7.175312353446893e-06, "loss": -0.0006, "step": 2408 }, { "clip_ratio": 0.0026506695430725813, "completions/clipped_ratio": 0.4642857142857143, "completions/max_length": 256.0, "completions/max_terminated_length": 256.0, "completions/mean_length": 209.7857208251953, "completions/mean_terminated_length": 169.73333740234375, "completions/min_length": 118.0, "completions/min_terminated_length": 118.0, "epoch": 0.8428971308607418, "grad_norm": 0.6033805027191274, "kl": 0.7109375, "learning_rate": 7.172562442805812e-06, "loss": 0.0052, "num_tokens": 11203767.0, "reward": 0.8750000596046448, "reward_std": 0.11663764715194702, "rewards/check_originality_func/mean": 0.875, "rewards/check_originality_func/std": 0.3337118923664093, "step": 2409 }, { "clip_ratio": 0.001953125, "epoch": 0.8432470258922323, "grad_norm": 0.5255091001125477, "kl": 0.70703125, "learning_rate": 7.169811721819503e-06, "loss": 0.0047, "step": 2410 }, { "clip_ratio": 0.012904576025903225, "epoch": 0.8435969209237228, "grad_norm": 0.48333192028663025, "kl": 0.70703125, "learning_rate": 7.167060191513957e-06, "loss": 0.002, "step": 2411 }, { "clip_ratio": 0.02532087080180645, "epoch": 0.8439468159552135, "grad_norm": 0.7703019509073941, "kl": 0.69921875, "learning_rate": 7.16430785291547e-06, "loss": 0.0016, "step": 2412 }, { "clip_ratio": 0.0024054679088294506, "completions/clipped_ratio": 0.0, "completions/max_length": 193.0, "completions/max_terminated_length": 193.0, "completions/mean_length": 147.10714721679688, "completions/mean_terminated_length": 147.10714721679688, "completions/min_length": 56.0, "completions/min_terminated_length": 56.0, "epoch": 0.844296710986704, "grad_norm": 1.0971739101014542, "kl": 0.99609375, "learning_rate": 7.161554707050637e-06, "loss": 0.0092, "num_tokens": 11222869.0, "reward": 0.8035714626312256, "reward_std": 0.3174411952495575, "rewards/check_originality_func/mean": 0.8035714030265808, "rewards/check_originality_func/std": 0.40089187026023865, "step": 2413 }, { "clip_ratio": 0.008926447480916977, "epoch": 0.8446466060181945, "grad_norm": 0.9783384768129679, "kl": 0.99609375, "learning_rate": 7.158800754946356e-06, "loss": 0.0057, "step": 2414 }, { "clip_ratio": 0.03517744317650795, "epoch": 0.844996501049685, "grad_norm": 0.6318181012039569, "kl": 0.99609375, "learning_rate": 7.1560459976298245e-06, "loss": 0.0013, "step": 2415 }, { "clip_ratio": 0.06082535162568092, "epoch": 0.8453463960811757, "grad_norm": 0.474971733775438, "kl": 1.0078125, "learning_rate": 7.1532904361285426e-06, "loss": -0.0021, "step": 2416 }, { "clip_ratio": 0.0018269026186317205, "completions/clipped_ratio": 0.6071428571428572, "completions/max_length": 256.0, "completions/max_terminated_length": 243.0, "completions/mean_length": 228.4285888671875, "completions/mean_terminated_length": 185.8181915283203, "completions/min_length": 136.0, "completions/min_terminated_length": 136.0, "epoch": 0.8456962911126662, "grad_norm": 2.2243542004506205, "kl": 0.98046875, "learning_rate": 7.1505340714703074e-06, "loss": 0.0084, "num_tokens": 11253877.0, "reward": 0.8928571939468384, "reward_std": 0.19057324528694153, "rewards/check_originality_func/mean": 0.8928571343421936, "rewards/check_originality_func/std": 0.31209391355514526, "step": 2417 }, { "clip_ratio": 0.014275305904448032, "epoch": 0.8460461861441567, "grad_norm": 1.781233988472017, "kl": 0.9765625, "learning_rate": 7.1477769046832156e-06, "loss": -0.0001, "step": 2418 }, { "clip_ratio": 0.05118279904127121, "epoch": 0.8463960811756474, "grad_norm": 1.276438418650061, "kl": 0.97265625, "learning_rate": 7.145018936795667e-06, "loss": -0.0074, "step": 2419 }, { "clip_ratio": 0.09235860407352448, "epoch": 0.8467459762071379, "grad_norm": 0.9250081333127046, "kl": 0.9609375, "learning_rate": 7.142260168836356e-06, "loss": -0.013, "step": 2420 }, { "clip_ratio": 0.0012947812210768461, "completions/clipped_ratio": 0.5, "completions/max_length": 256.0, "completions/max_terminated_length": 256.0, "completions/mean_length": 189.1428680419922, "completions/mean_terminated_length": 122.28572082519531, "completions/min_length": 81.0, "completions/min_terminated_length": 81.0, "epoch": 0.8470958712386284, "grad_norm": 0.5734395907210383, "kl": 0.9765625, "learning_rate": 7.1395006018342774e-06, "loss": 0.0043, "num_tokens": 11280101.0, "reward": 0.8750000596046448, "reward_std": 0.2061956226825714, "rewards/check_originality_func/mean": 0.875, "rewards/check_originality_func/std": 0.3337118923664093, "step": 2421 }, { "clip_ratio": 0.0050550648011267185, "epoch": 0.8474457662701189, "grad_norm": 0.429882939207183, "kl": 0.9765625, "learning_rate": 7.136740236818722e-06, "loss": 0.0025, "step": 2422 }, { "clip_ratio": 0.013142263516783714, "epoch": 0.8477956613016095, "grad_norm": 0.3182429301744994, "kl": 0.98046875, "learning_rate": 7.1339790748192815e-06, "loss": 0.0017, "step": 2423 }, { "clip_ratio": 0.020457444712519646, "epoch": 0.8481455563331001, "grad_norm": 0.28349448216495227, "kl": 0.9921875, "learning_rate": 7.131217116865845e-06, "loss": 0.0006, "step": 2424 }, { "clip_ratio": 0.0028168819844722748, "completions/clipped_ratio": 0.2857142857142857, "completions/max_length": 256.0, "completions/max_terminated_length": 218.0, "completions/mean_length": 151.80357360839844, "completions/mean_terminated_length": 110.125, "completions/min_length": 50.0, "completions/min_terminated_length": 50.0, "epoch": 0.8484954513645906, "grad_norm": 3.8797960906769338, "kl": 1.375, "learning_rate": 7.128454363988592e-06, "loss": 0.0125, "num_tokens": 11301618.0, "reward": 0.785714328289032, "reward_std": 0.22545847296714783, "rewards/check_originality_func/mean": 0.7857142686843872, "rewards/check_originality_func/std": 0.4140393137931824, "step": 2425 }, { "clip_ratio": 0.016091031953692436, "epoch": 0.8488453463960812, "grad_norm": 0.8377143327952686, "kl": 1.3828125, "learning_rate": 7.125690817218007e-06, "loss": 0.0007, "step": 2426 }, { "clip_ratio": 0.03192908689379692, "epoch": 0.8491952414275717, "grad_norm": 0.4859020615792367, "kl": 1.390625, "learning_rate": 7.122926477584864e-06, "loss": -0.001, "step": 2427 }, { "clip_ratio": 0.043988171964883804, "epoch": 0.8495451364590623, "grad_norm": 0.3491956920967602, "kl": 1.3984375, "learning_rate": 7.120161346120238e-06, "loss": -0.0018, "step": 2428 }, { "clip_ratio": 0.0007932098233141005, "completions/clipped_ratio": 0.4642857142857143, "completions/max_length": 256.0, "completions/max_terminated_length": 256.0, "completions/mean_length": 223.5535888671875, "completions/mean_terminated_length": 195.433349609375, "completions/min_length": 86.0, "completions/min_terminated_length": 86.0, "epoch": 0.8498950314905528, "grad_norm": 0.5201167122756871, "kl": 0.94140625, "learning_rate": 7.117395423855496e-06, "loss": 0.0084, "num_tokens": 11332049.0, "reward": 0.7142857313156128, "reward_std": 0.147871196269989, "rewards/check_originality_func/mean": 0.7142857313156128, "rewards/check_originality_func/std": 0.4558422863483429, "step": 2429 }, { "clip_ratio": 0.0011485759168863297, "epoch": 0.8502449265220434, "grad_norm": 0.5209339788210198, "kl": 0.9453125, "learning_rate": 7.1146287118223e-06, "loss": 0.008, "step": 2430 }, { "clip_ratio": 0.013921498320996761, "epoch": 0.8505948215535339, "grad_norm": 0.31026441170510494, "kl": 0.9296875, "learning_rate": 7.11186121105261e-06, "loss": 0.006, "step": 2431 }, { "clip_ratio": 0.01983371190726757, "epoch": 0.8509447165850245, "grad_norm": 0.23132152007023032, "kl": 0.9296875, "learning_rate": 7.109092922578673e-06, "loss": 0.0054, "step": 2432 }, { "clip_ratio": 0.0037166716065257788, "completions/clipped_ratio": 0.2857142857142857, "completions/max_length": 256.0, "completions/max_terminated_length": 242.0, "completions/mean_length": 181.67857360839844, "completions/mean_terminated_length": 151.9499969482422, "completions/min_length": 90.0, "completions/min_terminated_length": 90.0, "epoch": 0.8512946116165151, "grad_norm": 2.5777887673543463, "kl": 1.09375, "learning_rate": 7.106323847433039e-06, "loss": 0.0083, "num_tokens": 11357311.0, "reward": 0.8214285969734192, "reward_std": 0.30181884765625, "rewards/check_originality_func/mean": 0.8214285969734192, "rewards/check_originality_func/std": 0.3864591419696808, "step": 2433 }, { "clip_ratio": 0.014486836269497871, "epoch": 0.8516445066480056, "grad_norm": 1.464085379698731, "kl": 1.1015625, "learning_rate": 7.103553986648543e-06, "loss": 0.0012, "step": 2434 }, { "clip_ratio": 0.030780797824263573, "epoch": 0.8519944016794961, "grad_norm": 1.3336076870741056, "kl": 1.109375, "learning_rate": 7.100783341258321e-06, "loss": -0.0016, "step": 2435 }, { "clip_ratio": 0.05421718955039978, "epoch": 0.8523442967109867, "grad_norm": 0.47874818629345683, "kl": 1.09375, "learning_rate": 7.0980119122957946e-06, "loss": -0.0042, "step": 2436 }, { "clip_ratio": 0.004074595402926207, "completions/clipped_ratio": 0.2678571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 252.0, "completions/mean_length": 145.80357360839844, "completions/mean_terminated_length": 105.48780059814453, "completions/min_length": 30.0, "completions/min_terminated_length": 30.0, "epoch": 0.8526941917424773, "grad_norm": 0.9326530668208143, "kl": 1.453125, "learning_rate": 7.09523970079468e-06, "loss": 0.0076, "num_tokens": 11377508.0, "reward": 0.625, "reward_std": 0.287936806678772, "rewards/check_originality_func/mean": 0.625, "rewards/check_originality_func/std": 0.48850420117378235, "step": 2437 }, { "clip_ratio": 0.013325219042599201, "epoch": 0.8530440867739678, "grad_norm": 0.7904658080849551, "kl": 1.453125, "learning_rate": 7.092466707788988e-06, "loss": 0.005, "step": 2438 }, { "clip_ratio": 0.03482639417052269, "epoch": 0.8533939818054583, "grad_norm": 0.4491911802735163, "kl": 1.4609375, "learning_rate": 7.0896929343130185e-06, "loss": 0.0023, "step": 2439 }, { "clip_ratio": 0.05314372852444649, "epoch": 0.853743876836949, "grad_norm": 0.32729333646018244, "kl": 1.4765625, "learning_rate": 7.08691838140136e-06, "loss": 0.0014, "step": 2440 }, { "clip_ratio": 0.0011310249101370573, "completions/clipped_ratio": 0.1428571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 232.0, "completions/mean_length": 181.6428680419922, "completions/mean_terminated_length": 169.25, "completions/min_length": 112.0, "completions/min_terminated_length": 112.0, "epoch": 0.8540937718684395, "grad_norm": 0.4201075874780051, "kl": 1.0078125, "learning_rate": 7.084143050088898e-06, "loss": 0.0086, "num_tokens": 11401848.0, "reward": 0.9464285969734192, "reward_std": 0.11663764715194702, "rewards/check_originality_func/mean": 0.9464285969734192, "rewards/check_originality_func/std": 0.22720777988433838, "step": 2441 }, { "clip_ratio": 0.001305028679780662, "epoch": 0.85444366689993, "grad_norm": 0.4134440932062591, "kl": 1.0078125, "learning_rate": 7.081366941410803e-06, "loss": 0.0075, "step": 2442 }, { "clip_ratio": 0.005247124470770359, "epoch": 0.8547935619314205, "grad_norm": 0.32033553718444463, "kl": 1.0078125, "learning_rate": 7.078590056402539e-06, "loss": 0.0059, "step": 2443 }, { "clip_ratio": 0.013740329071879387, "epoch": 0.8551434569629112, "grad_norm": 0.282033484911736, "kl": 1.0078125, "learning_rate": 7.075812396099855e-06, "loss": 0.005, "step": 2444 }, { "clip_ratio": 0.005150181706994772, "completions/clipped_ratio": 0.4821428571428571, "completions/max_length": 256.0, "completions/max_terminated_length": 255.0, "completions/mean_length": 213.10714721679688, "completions/mean_terminated_length": 173.1724090576172, "completions/min_length": 128.0, "completions/min_terminated_length": 128.0, "epoch": 0.8554933519944017, "grad_norm": 1.4689156452521355, "kl": 0.9296875, "learning_rate": 7.073033961538793e-06, "loss": 0.0097, "num_tokens": 11431374.0, "reward": 0.8392857313156128, "reward_std": 0.29401323199272156, "rewards/check_originality_func/mean": 0.8392857313156128, "rewards/check_originality_func/std": 0.3705909848213196, "step": 2445 }, { "clip_ratio": 0.011286432854831219, "epoch": 0.8558432470258922, "grad_norm": 1.67813208478643, "kl": 0.9296875, "learning_rate": 7.070254753755684e-06, "loss": 0.0044, "step": 2446 }, { "clip_ratio": 0.028210435062646866, "epoch": 0.8561931420573828, "grad_norm": 0.8944562293934657, "kl": 0.92578125, "learning_rate": 7.067474773787146e-06, "loss": -0.0029, "step": 2447 }, { "clip_ratio": 0.05652981996536255, "epoch": 0.8565430370888734, "grad_norm": 0.8818591054263467, "kl": 0.9296875, "learning_rate": 7.064694022670085e-06, "loss": -0.007, "step": 2448 }, { "clip_ratio": 0.0033377003856003284, "completions/clipped_ratio": 0.2857142857142857, "completions/max_length": 256.0, "completions/max_terminated_length": 245.0, "completions/mean_length": 206.3928680419922, "completions/mean_terminated_length": 186.5500030517578, "completions/min_length": 146.0, "completions/min_terminated_length": 146.0, "epoch": 0.8568929321203639, "grad_norm": 1.6561381781550404, "kl": 0.87890625, "learning_rate": 7.061912501441694e-06, "loss": 0.0045, "num_tokens": 11458564.0, "reward": 0.9285714626312256, "reward_std": 0.1671452820301056, "rewards/check_originality_func/mean": 0.9285714030265808, "rewards/check_originality_func/std": 0.25987008213996887, "step": 2449 }, { "clip_ratio": 0.008758735843002796, "epoch": 0.8572428271518544, "grad_norm": 1.1943239380065354, "kl": 0.87890625, "learning_rate": 7.059130211139453e-06, "loss": 0.0017, "step": 2450 }, { "clip_ratio": 0.03248555213212967, "epoch": 0.857592722183345, "grad_norm": 0.5555529787821555, "kl": 0.8984375, "learning_rate": 7.056347152801133e-06, "loss": -0.0009, "step": 2451 }, { "clip_ratio": 0.03967589512467384, "epoch": 0.8579426172148356, "grad_norm": 0.48835482903269484, "kl": 0.9453125, "learning_rate": 7.0535633274647875e-06, "loss": -0.0005, "step": 2452 }, { "clip_ratio": 0.0029317140579223633, "completions/clipped_ratio": 0.2678571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 254.0, "completions/mean_length": 210.67857360839844, "completions/mean_terminated_length": 194.09754943847656, "completions/min_length": 158.0, "completions/min_terminated_length": 158.0, "epoch": 0.8582925122463261, "grad_norm": 0.8894152607276236, "kl": 0.95703125, "learning_rate": 7.050778736168757e-06, "loss": 0.0057, "num_tokens": 11487274.0, "reward": 0.9642857313156128, "reward_std": 0.10101525485515594, "rewards/check_originality_func/mean": 0.9642857313156128, "rewards/check_originality_func/std": 0.187256321310997, "step": 2453 }, { "clip_ratio": 0.005781138781458139, "epoch": 0.8586424072778167, "grad_norm": 0.9312738119042621, "kl": 0.9609375, "learning_rate": 7.047993379951665e-06, "loss": 0.0035, "step": 2454 }, { "clip_ratio": 0.014723834581673145, "epoch": 0.8589923023093072, "grad_norm": 0.4940333634850184, "kl": 0.95703125, "learning_rate": 7.045207259852427e-06, "loss": -0.0004, "step": 2455 }, { "clip_ratio": 0.024021457880735397, "epoch": 0.8593421973407978, "grad_norm": 0.40995301762306635, "kl": 0.9609375, "learning_rate": 7.042420376910236e-06, "loss": -0.0027, "step": 2456 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4285714285714286, "completions/max_length": 256.0, "completions/max_terminated_length": 217.0, "completions/mean_length": 209.2857208251953, "completions/mean_terminated_length": 174.25, "completions/min_length": 106.0, "completions/min_terminated_length": 106.0, "epoch": 0.8596920923722883, "grad_norm": 0.03293778965746558, "kl": 0.84765625, "learning_rate": 7.039632732164576e-06, "loss": 0.0067, "num_tokens": 11516186.0, "reward": 0.8571429252624512, "reward_std": 0.0, "rewards/check_originality_func/mean": 0.8571428656578064, "rewards/check_originality_func/std": 0.3530939221382141, "step": 2457 }, { "clip_ratio": 0.0, "epoch": 0.8600419874037789, "grad_norm": 0.030190192590716796, "kl": 0.84765625, "learning_rate": 7.036844326655212e-06, "loss": 0.0066, "step": 2458 }, { "clip_ratio": 0.0, "epoch": 0.8603918824352694, "grad_norm": 0.030442053903961614, "kl": 0.84765625, "learning_rate": 7.034055161422189e-06, "loss": 0.0066, "step": 2459 }, { "clip_ratio": 0.0, "epoch": 0.86074177746676, "grad_norm": 0.026268958100257266, "kl": 0.84375, "learning_rate": 7.031265237505842e-06, "loss": 0.0066, "step": 2460 }, { "clip_ratio": 0.004789560101926327, "completions/clipped_ratio": 0.2857142857142857, "completions/max_length": 256.0, "completions/max_terminated_length": 222.0, "completions/mean_length": 187.5178680419922, "completions/mean_terminated_length": 160.125, "completions/min_length": 77.0, "completions/min_terminated_length": 77.0, "epoch": 0.8610916724982505, "grad_norm": 2.6421727971954967, "kl": 1.2109375, "learning_rate": 7.028474555946787e-06, "loss": 0.0105, "num_tokens": 11541127.0, "reward": 0.8392857313156128, "reward_std": 0.18276765942573547, "rewards/check_originality_func/mean": 0.8392857313156128, "rewards/check_originality_func/std": 0.3705909848213196, "step": 2461 }, { "clip_ratio": 0.0238325297832489, "epoch": 0.8614415675297411, "grad_norm": 1.8642506548829183, "kl": 1.203125, "learning_rate": 7.025683117785921e-06, "loss": 0.0039, "step": 2462 }, { "clip_ratio": 0.045373182743787766, "epoch": 0.8617914625612316, "grad_norm": 1.1512464543305732, "kl": 1.1953125, "learning_rate": 7.0228909240644235e-06, "loss": -0.0025, "step": 2463 }, { "clip_ratio": 0.0841718539595604, "epoch": 0.8621413575927221, "grad_norm": 0.56452733698512, "kl": 1.203125, "learning_rate": 7.020097975823756e-06, "loss": -0.0077, "step": 2464 }, { "clip_ratio": 0.0021678046323359013, "completions/clipped_ratio": 0.25, "completions/max_length": 256.0, "completions/max_terminated_length": 256.0, "completions/mean_length": 165.60714721679688, "completions/mean_terminated_length": 135.4761962890625, "completions/min_length": 50.0, "completions/min_terminated_length": 50.0, "epoch": 0.8624912526242128, "grad_norm": 0.6317322139542109, "kl": 1.1796875, "learning_rate": 7.017304274105663e-06, "loss": 0.0073, "num_tokens": 11563193.0, "reward": 0.8214285969734192, "reward_std": 0.20080357789993286, "rewards/check_originality_func/mean": 0.8214285969734192, "rewards/check_originality_func/std": 0.3864591419696808, "step": 2465 }, { "clip_ratio": 0.0031774421222507954, "epoch": 0.8628411476557033, "grad_norm": 0.5680712554173584, "kl": 1.1796875, "learning_rate": 7.014509819952168e-06, "loss": 0.0058, "step": 2466 }, { "clip_ratio": 0.022360576316714287, "epoch": 0.8631910426871938, "grad_norm": 0.40279492869990513, "kl": 1.1953125, "learning_rate": 7.011714614405577e-06, "loss": 0.0044, "step": 2467 }, { "clip_ratio": 0.0404832549393177, "epoch": 0.8635409377186843, "grad_norm": 0.2669459766968935, "kl": 1.203125, "learning_rate": 7.008918658508474e-06, "loss": 0.0032, "step": 2468 }, { "clip_ratio": 0.0007672990905120969, "completions/clipped_ratio": 0.6964285714285714, "completions/max_length": 256.0, "completions/max_terminated_length": 253.0, "completions/mean_length": 227.62501525878906, "completions/mean_terminated_length": 162.5294189453125, "completions/min_length": 137.0, "completions/min_terminated_length": 137.0, "epoch": 0.863890832750175, "grad_norm": 0.3356765364796091, "kl": 0.8125, "learning_rate": 7.006121953303724e-06, "loss": 0.0072, "num_tokens": 11594836.0, "reward": 0.9642857313156128, "reward_std": 0.06613001227378845, "rewards/check_originality_func/mean": 0.9642857313156128, "rewards/check_originality_func/std": 0.187256321310997, "step": 2469 }, { "clip_ratio": 0.00146484375, "epoch": 0.8642407277816655, "grad_norm": 0.278870686257356, "kl": 0.8125, "learning_rate": 7.003324499834472e-06, "loss": 0.0054, "step": 2470 }, { "clip_ratio": 0.0020228794310241938, "epoch": 0.864590622813156, "grad_norm": 0.27183150676058726, "kl": 0.8125, "learning_rate": 7.000526299144141e-06, "loss": 0.0056, "step": 2471 }, { "clip_ratio": 0.0055106026120483875, "epoch": 0.8649405178446467, "grad_norm": 0.31911999219041054, "kl": 0.8125, "learning_rate": 6.997727352276437e-06, "loss": 0.0043, "step": 2472 }, { "clip_ratio": 0.0029972705524414778, "completions/clipped_ratio": 0.5535714285714286, "completions/max_length": 256.0, "completions/max_terminated_length": 220.0, "completions/mean_length": 206.0178680419922, "completions/mean_terminated_length": 144.0399932861328, "completions/min_length": 85.0, "completions/min_terminated_length": 85.0, "epoch": 0.8652904128761372, "grad_norm": 0.9008199513145473, "kl": 0.92578125, "learning_rate": 6.9949276602753345e-06, "loss": 0.0078, "num_tokens": 11624045.0, "reward": 0.8750000596046448, "reward_std": 0.2061956226825714, "rewards/check_originality_func/mean": 0.875, "rewards/check_originality_func/std": 0.3337118923664093, "step": 2473 }, { "clip_ratio": 0.010740016587078571, "epoch": 0.8656403079076277, "grad_norm": 0.5877733994472863, "kl": 0.91796875, "learning_rate": 6.992127224185098e-06, "loss": 0.0051, "step": 2474 }, { "clip_ratio": 0.01687406562268734, "epoch": 0.8659902029391182, "grad_norm": 0.4647926669211897, "kl": 0.91796875, "learning_rate": 6.98932604505026e-06, "loss": 0.0037, "step": 2475 }, { "clip_ratio": 0.02514652907848358, "epoch": 0.8663400979706088, "grad_norm": 0.37957336521540636, "kl": 0.91796875, "learning_rate": 6.9865241239156345e-06, "loss": 0.0019, "step": 2476 }, { "clip_ratio": 0.0010625564027577639, "completions/clipped_ratio": 0.3392857142857143, "completions/max_length": 256.0, "completions/max_terminated_length": 255.0, "completions/mean_length": 184.87501525878906, "completions/mean_terminated_length": 148.35134887695312, "completions/min_length": 60.0, "completions/min_terminated_length": 60.0, "epoch": 0.8666899930020994, "grad_norm": 0.5924986616011353, "kl": 1.171875, "learning_rate": 6.983721461826312e-06, "loss": 0.0053, "num_tokens": 11648270.0, "reward": 0.8750000596046448, "reward_std": 0.17495085299015045, "rewards/check_originality_func/mean": 0.875, "rewards/check_originality_func/std": 0.3337118923664093, "step": 2477 }, { "clip_ratio": 0.0026076510548591614, "epoch": 0.8670398880335899, "grad_norm": 0.59122066026559, "kl": 1.1640625, "learning_rate": 6.9809180598276595e-06, "loss": 0.0042, "step": 2478 }, { "clip_ratio": 0.02093871310353279, "epoch": 0.8673897830650805, "grad_norm": 0.44007817953274764, "kl": 1.1640625, "learning_rate": 6.9781139189653194e-06, "loss": 0.0024, "step": 2479 }, { "clip_ratio": 0.03475317731499672, "epoch": 0.867739678096571, "grad_norm": 0.2638879283036362, "kl": 1.1484375, "learning_rate": 6.9753090402852076e-06, "loss": 0.0009, "step": 2480 }, { "clip_ratio": 0.00034877232974395156, "completions/clipped_ratio": 0.5714285714285714, "completions/max_length": 256.0, "completions/max_terminated_length": 207.0, "completions/mean_length": 198.94644165039062, "completions/mean_terminated_length": 122.875, "completions/min_length": 73.0, "completions/min_terminated_length": 73.0, "epoch": 0.8680895731280616, "grad_norm": 0.5956011884953206, "kl": 0.98828125, "learning_rate": 6.972503424833519e-06, "loss": 0.0067, "num_tokens": 11676699.0, "reward": 0.9642857313156128, "reward_std": 0.06613001227378845, "rewards/check_originality_func/mean": 0.9642857313156128, "rewards/check_originality_func/std": 0.187256321310997, "step": 2481 }, { "clip_ratio": 0.0012555803405120969, "epoch": 0.8684394681595521, "grad_norm": 0.47497644536062394, "kl": 0.9921875, "learning_rate": 6.969697073656722e-06, "loss": 0.0065, "step": 2482 }, { "clip_ratio": 0.006277901586145163, "epoch": 0.8687893631910427, "grad_norm": 0.34674606615314685, "kl": 0.9921875, "learning_rate": 6.966889987801557e-06, "loss": 0.0043, "step": 2483 }, { "clip_ratio": 0.01018415205180645, "epoch": 0.8691392582225332, "grad_norm": 0.18635219050097376, "kl": 0.9921875, "learning_rate": 6.964082168315044e-06, "loss": 0.0039, "step": 2484 }, { "clip_ratio": 0.0010045453673228621, "completions/clipped_ratio": 0.2857142857142857, "completions/max_length": 256.0, "completions/max_terminated_length": 252.0, "completions/mean_length": 158.7678680419922, "completions/mean_terminated_length": 119.875, "completions/min_length": 28.0, "completions/min_terminated_length": 28.0, "epoch": 0.8694891532540238, "grad_norm": 0.2797228392300324, "kl": 1.359375, "learning_rate": 6.9612736162444695e-06, "loss": 0.0085, "num_tokens": 11698526.0, "reward": 0.4285714626312256, "reward_std": 0.10101525485515594, "rewards/check_originality_func/mean": 0.4285714328289032, "rewards/check_originality_func/std": 0.4993502199649811, "step": 2485 }, { "clip_ratio": 0.001263378537259996, "epoch": 0.8698390482855144, "grad_norm": 0.26915443230462016, "kl": 1.359375, "learning_rate": 6.958464332637399e-06, "loss": 0.008, "step": 2486 }, { "clip_ratio": 0.0033182422630488873, "epoch": 0.8701889433170049, "grad_norm": 0.23173918002699656, "kl": 1.3515625, "learning_rate": 6.955654318541668e-06, "loss": 0.0074, "step": 2487 }, { "clip_ratio": 0.004277014639228582, "epoch": 0.8705388383484954, "grad_norm": 0.20632808819358914, "kl": 1.3515625, "learning_rate": 6.9528435750053845e-06, "loss": 0.007, "step": 2488 }, { "clip_ratio": 0.002738175680860877, "completions/clipped_ratio": 0.125, "completions/max_length": 256.0, "completions/max_terminated_length": 255.0, "completions/mean_length": 164.125, "completions/mean_terminated_length": 151.0, "completions/min_length": 83.0, "completions/min_terminated_length": 83.0, "epoch": 0.870888733379986, "grad_norm": 0.5610225678904116, "kl": 1.1640625, "learning_rate": 6.950032103076931e-06, "loss": 0.008, "num_tokens": 11720541.0, "reward": 0.8035714626312256, "reward_std": 0.17495086789131165, "rewards/check_originality_func/mean": 0.8035714030265808, "rewards/check_originality_func/std": 0.40089184045791626, "step": 2489 }, { "clip_ratio": 0.003416634863242507, "epoch": 0.8712386284114766, "grad_norm": 0.5818561661004821, "kl": 1.1640625, "learning_rate": 6.947219903804958e-06, "loss": 0.0072, "step": 2490 }, { "clip_ratio": 0.015103059820830822, "epoch": 0.8715885234429671, "grad_norm": 0.4488398831058607, "kl": 1.1640625, "learning_rate": 6.94440697823839e-06, "loss": 0.0052, "step": 2491 }, { "clip_ratio": 0.024671005085110664, "epoch": 0.8719384184744576, "grad_norm": 0.39850105665188507, "kl": 1.1484375, "learning_rate": 6.9415933274264224e-06, "loss": 0.0039, "step": 2492 }, { "clip_ratio": 0.0014482049737125635, "completions/clipped_ratio": 0.4285714285714286, "completions/max_length": 256.0, "completions/max_terminated_length": 235.0, "completions/mean_length": 214.69644165039062, "completions/mean_terminated_length": 183.71875, "completions/min_length": 98.0, "completions/min_terminated_length": 98.0, "epoch": 0.8722883135059483, "grad_norm": 0.7312984890434288, "kl": 0.86328125, "learning_rate": 6.938778952418519e-06, "loss": 0.0057, "num_tokens": 11750260.0, "reward": 0.9285714626312256, "reward_std": 0.1671452671289444, "rewards/check_originality_func/mean": 0.9285714030265808, "rewards/check_originality_func/std": 0.25987008213996887, "step": 2493 }, { "clip_ratio": 0.00828531477600336, "epoch": 0.8726382085374388, "grad_norm": 0.5314444942702717, "kl": 0.85546875, "learning_rate": 6.935963854264416e-06, "loss": 0.0043, "step": 2494 }, { "clip_ratio": 0.014057631604373455, "epoch": 0.8729881035689293, "grad_norm": 0.46531171866205406, "kl": 0.85546875, "learning_rate": 6.9331480340141185e-06, "loss": 0.0027, "step": 2495 }, { "clip_ratio": 0.02332291007041931, "epoch": 0.8733379986004198, "grad_norm": 0.4287859186631894, "kl": 0.85546875, "learning_rate": 6.930331492717899e-06, "loss": 0.0007, "step": 2496 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.2678571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 254.0, "completions/mean_length": 204.37501525878906, "completions/mean_terminated_length": 185.48779296875, "completions/min_length": 122.0, "completions/min_terminated_length": 122.0, "epoch": 0.8736878936319105, "grad_norm": 0.2453804636311413, "kl": 0.75, "learning_rate": 6.927514231426302e-06, "loss": 0.0062, "num_tokens": 11776857.0, "reward": 0.9821429252624512, "reward_std": 0.05050762742757797, "rewards/check_originality_func/mean": 0.9821428656578064, "rewards/check_originality_func/std": 0.13363061845302582, "step": 2497 }, { "clip_ratio": 0.0003990422992501408, "epoch": 0.874037788663401, "grad_norm": 0.284209465723415, "kl": 0.74609375, "learning_rate": 6.92469625119014e-06, "loss": 0.0057, "step": 2498 }, { "clip_ratio": 0.0031923383940011263, "epoch": 0.8743876836948915, "grad_norm": 0.26935675946253723, "kl": 0.74609375, "learning_rate": 6.921877553060493e-06, "loss": 0.0055, "step": 2499 }, { "clip_ratio": 0.007980845868587494, "epoch": 0.8747375787263821, "grad_norm": 0.18564920060821255, "kl": 0.7421875, "learning_rate": 6.919058138088708e-06, "loss": 0.0047, "step": 2500 }, { "clip_ratio": 0.004451253917068243, "completions/clipped_ratio": 0.0, "completions/max_length": 197.0, "completions/max_terminated_length": 197.0, "completions/mean_length": 144.44644165039062, "completions/mean_terminated_length": 144.44644165039062, "completions/min_length": 103.0, "completions/min_terminated_length": 103.0, "epoch": 0.7597205346294046, "grad_norm": 0.657387789954185, "kl": 1.546875, "learning_rate": 6.916238007326399e-06, "loss": 0.009, "num_tokens": 11796482.0, "reward": 1.9050122499465942, "reward_std": 0.24488183856010437, "rewards/check_gptzero_func/mean": 0.7857142686843872, "rewards/check_gptzero_func/std": 0.4140393137931824, "rewards/check_originality_func/mean": 0.1428571492433548, "rewards/check_originality_func/std": 0.3530939221382141, "rewards/check_winston_local_func/mean": 0.9764407277107239, "rewards/check_winston_local_func/std": 0.03759937360882759, "step": 2501 }, { "clip_ratio": 0.0063996645621955395, "epoch": 0.7600243013365735, "grad_norm": 0.6077137007810649, "kl": 1.546875, "learning_rate": 7.798894336925713e-06, "loss": 0.0078, "step": 2502 }, { "clip_ratio": 0.015464112162590027, "epoch": 0.7603280680437424, "grad_norm": 0.5052905635069729, "kl": 1.546875, "learning_rate": 7.796697103313633e-06, "loss": 0.0056, "step": 2503 }, { "clip_ratio": 0.036349985748529434, "epoch": 0.7606318347509113, "grad_norm": 0.3602929738315295, "kl": 1.546875, "learning_rate": 7.794499083437433e-06, "loss": 0.0036, "step": 2504 }, { "clip_ratio": 0.006664055399596691, "completions/clipped_ratio": 0.0, "completions/max_length": 185.0, "completions/max_terminated_length": 185.0, "completions/mean_length": 151.69644165039062, "completions/mean_terminated_length": 151.69644165039062, "completions/min_length": 117.0, "completions/min_terminated_length": 117.0, "epoch": 0.7609356014580801, "grad_norm": 1.002491600034576, "kl": 1.5703125, "learning_rate": 7.792300277915062e-06, "loss": 0.0058, "num_tokens": 11817017.0, "reward": 1.8526548147201538, "reward_std": 0.38264161348342896, "rewards/check_gptzero_func/mean": 0.5357142686843872, "rewards/check_gptzero_func/std": 0.5032362937927246, "rewards/check_originality_func/mean": 0.3392857015132904, "rewards/check_originality_func/std": 0.4777517318725586, "rewards/check_winston_local_func/mean": 0.9776546359062195, "rewards/check_winston_local_func/std": 0.04020502418279648, "step": 2505 }, { "clip_ratio": 0.012843897566199303, "epoch": 0.7612393681652491, "grad_norm": 1.014486325726026, "kl": 1.5703125, "learning_rate": 7.790100687364694e-06, "loss": 0.0016, "step": 2506 }, { "clip_ratio": 0.03799566254019737, "epoch": 0.761543134872418, "grad_norm": 0.7771752005333771, "kl": 1.5703125, "learning_rate": 7.787900312404725e-06, "loss": -0.003, "step": 2507 }, { "clip_ratio": 0.07210918515920639, "epoch": 0.7618469015795869, "grad_norm": 0.572770167812967, "kl": 1.5703125, "learning_rate": 7.785699153653765e-06, "loss": -0.0067, "step": 2508 }, { "clip_ratio": 0.0045629204250872135, "completions/clipped_ratio": 0.0, "completions/max_length": 197.0, "completions/max_terminated_length": 197.0, "completions/mean_length": 138.69644165039062, "completions/mean_terminated_length": 138.69644165039062, "completions/min_length": 112.0, "completions/min_terminated_length": 112.0, "epoch": 0.7621506682867558, "grad_norm": 1.0412792954972565, "kl": 1.5, "learning_rate": 7.78349721173065e-06, "loss": 0.0126, "num_tokens": 11836072.0, "reward": 1.2313852310180664, "reward_std": 0.45405977964401245, "rewards/check_gptzero_func/mean": 0.25, "rewards/check_gptzero_func/std": 0.43693143129348755, "rewards/check_originality_func/mean": 0.1071428582072258, "rewards/check_originality_func/std": 0.3120938837528229, "rewards/check_winston_local_func/mean": 0.8742424845695496, "rewards/check_winston_local_func/std": 0.15102887153625488, "step": 2509 }, { "clip_ratio": 0.009931693784892559, "epoch": 0.7624544349939246, "grad_norm": 1.029709488524691, "kl": 1.5078125, "learning_rate": 7.781294487254436e-06, "loss": 0.0102, "step": 2510 }, { "clip_ratio": 0.042107660323381424, "epoch": 0.7627582017010935, "grad_norm": 0.7405957781136979, "kl": 1.5390625, "learning_rate": 7.779090980844394e-06, "loss": 0.0058, "step": 2511 }, { "clip_ratio": 0.08548353612422943, "epoch": 0.7630619684082625, "grad_norm": 0.9834950234116442, "kl": 1.6953125, "learning_rate": 7.776886693120022e-06, "loss": 0.0035, "step": 2512 }, { "clip_ratio": 0.005055877845734358, "completions/clipped_ratio": 0.0, "completions/max_length": 165.0, "completions/max_terminated_length": 165.0, "completions/mean_length": 142.08929443359375, "completions/mean_terminated_length": 142.08929443359375, "completions/min_length": 99.0, "completions/min_terminated_length": 99.0, "epoch": 0.7633657351154314, "grad_norm": 0.8487766175453301, "kl": 1.4921875, "learning_rate": 7.774681624701032e-06, "loss": 0.0085, "num_tokens": 11855389.0, "reward": 1.6887232065200806, "reward_std": 0.449515700340271, "rewards/check_gptzero_func/mean": 0.5535714030265808, "rewards/check_gptzero_func/std": 0.5016207695007324, "rewards/check_originality_func/mean": 0.1785714328289032, "rewards/check_originality_func/std": 0.3864591121673584, "rewards/check_winston_local_func/mean": 0.9565802812576294, "rewards/check_winston_local_func/std": 0.06930767744779587, "step": 2513 }, { "clip_ratio": 0.008286120370030403, "epoch": 0.7636695018226003, "grad_norm": 0.8006097547383308, "kl": 1.4921875, "learning_rate": 7.772475776207358e-06, "loss": 0.0046, "step": 2514 }, { "clip_ratio": 0.036533188074827194, "epoch": 0.7639732685297691, "grad_norm": 0.6476918522276155, "kl": 1.4921875, "learning_rate": 7.770269148259153e-06, "loss": 0.0011, "step": 2515 }, { "clip_ratio": 0.058234501630067825, "epoch": 0.764277035236938, "grad_norm": 0.552133498697129, "kl": 1.4921875, "learning_rate": 7.768061741476788e-06, "loss": -0.0019, "step": 2516 }, { "clip_ratio": 0.00535136042162776, "completions/clipped_ratio": 0.0, "completions/max_length": 164.0, "completions/max_terminated_length": 164.0, "completions/mean_length": 136.83929443359375, "completions/mean_terminated_length": 136.83929443359375, "completions/min_length": 104.0, "completions/min_terminated_length": 104.0, "epoch": 0.7645808019441069, "grad_norm": 1.1984268811231806, "kl": 1.734375, "learning_rate": 7.765853556480854e-06, "loss": 0.0081, "num_tokens": 11874572.0, "reward": 2.2533035278320312, "reward_std": 0.5635105967521667, "rewards/check_gptzero_func/mean": 0.7321428656578064, "rewards/check_gptzero_func/std": 0.446850448846817, "rewards/check_originality_func/mean": 0.5714285969734192, "rewards/check_originality_func/std": 0.4993502199649811, "rewards/check_winston_local_func/mean": 0.9497321248054504, "rewards/check_winston_local_func/std": 0.10326797515153885, "step": 2517 }, { "clip_ratio": 0.01379047054797411, "epoch": 0.7648845686512759, "grad_norm": 1.0740633725515558, "kl": 1.734375, "learning_rate": 7.763644593892162e-06, "loss": 0.0033, "step": 2518 }, { "clip_ratio": 0.04689256101846695, "epoch": 0.7651883353584447, "grad_norm": 0.755951648589451, "kl": 1.7421875, "learning_rate": 7.761434854331738e-06, "loss": -0.0018, "step": 2519 }, { "clip_ratio": 0.0877455323934555, "epoch": 0.7654921020656136, "grad_norm": 0.8468752740877067, "kl": 1.75, "learning_rate": 7.759224338420832e-06, "loss": -0.0055, "step": 2520 }, { "clip_ratio": 0.004811789374798536, "completions/clipped_ratio": 0.0, "completions/max_length": 191.0, "completions/max_terminated_length": 191.0, "completions/mean_length": 152.85714721679688, "completions/mean_terminated_length": 152.85714721679688, "completions/min_length": 109.0, "completions/min_terminated_length": 109.0, "epoch": 0.7657958687727825, "grad_norm": 1.746521688130738, "kl": 1.4453125, "learning_rate": 7.757013046780906e-06, "loss": 0.0103, "num_tokens": 11895164.0, "reward": 1.9331622123718262, "reward_std": 0.6283038258552551, "rewards/check_gptzero_func/mean": 0.6428571343421936, "rewards/check_gptzero_func/std": 0.48349377512931824, "rewards/check_originality_func/mean": 0.3035714328289032, "rewards/check_originality_func/std": 0.4639609158039093, "rewards/check_winston_local_func/mean": 0.9867334961891174, "rewards/check_winston_local_func/std": 0.02724437229335308, "step": 2521 }, { "clip_ratio": 0.010974795557558537, "epoch": 0.7660996354799514, "grad_norm": 1.3542542882306967, "kl": 1.4453125, "learning_rate": 7.754800980033646e-06, "loss": 0.0055, "step": 2522 }, { "clip_ratio": 0.03286687657237053, "epoch": 0.7664034021871203, "grad_norm": 0.9604612820834708, "kl": 1.4453125, "learning_rate": 7.75258813880095e-06, "loss": 0.0013, "step": 2523 }, { "clip_ratio": 0.06061823293566704, "epoch": 0.7667071688942891, "grad_norm": 0.6846052180039627, "kl": 1.453125, "learning_rate": 7.750374523704937e-06, "loss": -0.0025, "step": 2524 }, { "clip_ratio": 0.007803735788911581, "completions/clipped_ratio": 0.0, "completions/max_length": 166.0, "completions/max_terminated_length": 166.0, "completions/mean_length": 122.87500762939453, "completions/mean_terminated_length": 122.87500762939453, "completions/min_length": 90.0, "completions/min_terminated_length": 90.0, "epoch": 0.767010935601458, "grad_norm": 1.5442952245991632, "kl": 1.8203125, "learning_rate": 7.748160135367946e-06, "loss": 0.0155, "num_tokens": 11912885.0, "reward": 2.0694217681884766, "reward_std": 0.6030719876289368, "rewards/check_gptzero_func/mean": 0.6607142686843872, "rewards/check_gptzero_func/std": 0.477751761674881, "rewards/check_originality_func/mean": 0.4642857015132904, "rewards/check_originality_func/std": 0.5032362341880798, "rewards/check_winston_local_func/mean": 0.9444217681884766, "rewards/check_winston_local_func/std": 0.08359844982624054, "step": 2525 }, { "clip_ratio": 0.01795497164130211, "epoch": 0.767314702308627, "grad_norm": 1.2121400588903273, "kl": 1.828125, "learning_rate": 7.745944974412527e-06, "loss": 0.0117, "step": 2526 }, { "clip_ratio": 0.06296025216579437, "epoch": 0.7676184690157959, "grad_norm": 0.9143714321209971, "kl": 1.828125, "learning_rate": 7.743729041461454e-06, "loss": 0.006, "step": 2527 }, { "clip_ratio": 0.11606572568416595, "epoch": 0.7679222357229648, "grad_norm": 0.6758772399014114, "kl": 1.8359375, "learning_rate": 7.741512337137715e-06, "loss": -0.0, "step": 2528 }, { "clip_ratio": 0.010822183452546597, "completions/clipped_ratio": 0.0, "completions/max_length": 176.0, "completions/max_terminated_length": 176.0, "completions/mean_length": 138.625, "completions/mean_terminated_length": 138.625, "completions/min_length": 103.0, "completions/min_terminated_length": 103.0, "epoch": 0.7682260024301336, "grad_norm": 1.0510431066470753, "kl": 1.828125, "learning_rate": 7.739294862064513e-06, "loss": 0.0149, "num_tokens": 11931952.0, "reward": 2.3026909828186035, "reward_std": 0.39348849654197693, "rewards/check_gptzero_func/mean": 0.7857142686843872, "rewards/check_gptzero_func/std": 0.4140393137931824, "rewards/check_originality_func/mean": 0.5535714030265808, "rewards/check_originality_func/std": 0.5016207695007324, "rewards/check_winston_local_func/mean": 0.9634049534797668, "rewards/check_winston_local_func/std": 0.05688372626900673, "step": 2529 }, { "clip_ratio": 0.013518265448510647, "epoch": 0.7685297691373025, "grad_norm": 0.9332048617632401, "kl": 1.828125, "learning_rate": 7.73707661686527e-06, "loss": 0.0137, "step": 2530 }, { "clip_ratio": 0.05078430101275444, "epoch": 0.7688335358444714, "grad_norm": 0.8300947044037512, "kl": 1.8359375, "learning_rate": 7.734857602163623e-06, "loss": 0.0088, "step": 2531 }, { "clip_ratio": 0.09284964948892593, "epoch": 0.7691373025516404, "grad_norm": 0.7883432853096871, "kl": 1.8515625, "learning_rate": 7.732637818583429e-06, "loss": 0.0038, "step": 2532 }, { "clip_ratio": 0.00671658618375659, "completions/clipped_ratio": 0.0, "completions/max_length": 181.0, "completions/max_terminated_length": 181.0, "completions/mean_length": 144.69644165039062, "completions/mean_terminated_length": 144.69644165039062, "completions/min_length": 95.0, "completions/min_terminated_length": 95.0, "epoch": 0.7694410692588093, "grad_norm": 1.6686828124246325, "kl": 1.7265625, "learning_rate": 7.730417266748755e-06, "loss": 0.0134, "num_tokens": 11951783.0, "reward": 2.4193522930145264, "reward_std": 0.4391181766986847, "rewards/check_gptzero_func/mean": 0.6964285969734192, "rewards/check_gptzero_func/std": 0.4639609158039093, "rewards/check_originality_func/mean": 0.75, "rewards/check_originality_func/std": 0.43693143129348755, "rewards/check_winston_local_func/mean": 0.9729233980178833, "rewards/check_winston_local_func/std": 0.05289791151881218, "step": 2533 }, { "clip_ratio": 0.02471369132399559, "epoch": 0.7697448359659781, "grad_norm": 1.0029039513775029, "kl": 1.7265625, "learning_rate": 7.728195947283892e-06, "loss": 0.0079, "step": 2534 }, { "clip_ratio": 0.058056823909282684, "epoch": 0.770048602673147, "grad_norm": 0.8750894127065861, "kl": 1.734375, "learning_rate": 7.725973860813338e-06, "loss": 0.0038, "step": 2535 }, { "clip_ratio": 0.10083218663930893, "epoch": 0.7703523693803159, "grad_norm": 0.7504102229414124, "kl": 1.734375, "learning_rate": 7.723751007961815e-06, "loss": -0.0015, "step": 2536 }, { "clip_ratio": 0.007285316940397024, "completions/clipped_ratio": 0.0, "completions/max_length": 181.0, "completions/max_terminated_length": 181.0, "completions/mean_length": 130.21429443359375, "completions/mean_terminated_length": 130.21429443359375, "completions/min_length": 101.0, "completions/min_terminated_length": 101.0, "epoch": 0.7706561360874848, "grad_norm": 1.5421202621719656, "kl": 2.078125, "learning_rate": 7.721527389354254e-06, "loss": 0.0133, "num_tokens": 11970043.0, "reward": 2.4408998489379883, "reward_std": 0.5055117607116699, "rewards/check_gptzero_func/mean": 0.8214285969734192, "rewards/check_gptzero_func/std": 0.3864591419696808, "rewards/check_originality_func/mean": 0.6785714030265808, "rewards/check_originality_func/std": 0.4712514281272888, "rewards/check_winston_local_func/mean": 0.9408997893333435, "rewards/check_winston_local_func/std": 0.1048685684800148, "step": 2537 }, { "clip_ratio": 0.018889207392930984, "epoch": 0.7709599027946537, "grad_norm": 1.3276984923887738, "kl": 2.078125, "learning_rate": 7.719303005615802e-06, "loss": 0.0087, "step": 2538 }, { "clip_ratio": 0.04761987924575806, "epoch": 0.7712636695018226, "grad_norm": 1.132304895967724, "kl": 2.09375, "learning_rate": 7.717077857371824e-06, "loss": 0.0019, "step": 2539 }, { "clip_ratio": 0.10683081299066544, "epoch": 0.7715674362089915, "grad_norm": 0.8191923407082641, "kl": 2.125, "learning_rate": 7.7148519452479e-06, "loss": -0.0034, "step": 2540 }, { "clip_ratio": 0.011198783293366432, "completions/clipped_ratio": 0.0, "completions/max_length": 170.0, "completions/max_terminated_length": 170.0, "completions/mean_length": 129.67857360839844, "completions/mean_terminated_length": 129.67857360839844, "completions/min_length": 69.0, "completions/min_terminated_length": 69.0, "epoch": 0.7718712029161604, "grad_norm": 1.4604048631313127, "kl": 2.015625, "learning_rate": 7.712625269869825e-06, "loss": 0.0126, "num_tokens": 11988345.0, "reward": 2.5542428493499756, "reward_std": 0.4074919819831848, "rewards/check_gptzero_func/mean": 0.7678571343421936, "rewards/check_gptzero_func/std": 0.4260205328464508, "rewards/check_originality_func/mean": 0.8035714030265808, "rewards/check_originality_func/std": 0.40089187026023865, "rewards/check_winston_local_func/mean": 0.9828141331672668, "rewards/check_winston_local_func/std": 0.021683646366000175, "step": 2541 }, { "clip_ratio": 0.027532413601875305, "epoch": 0.7721749696233293, "grad_norm": 1.2119407363135835, "kl": 2.015625, "learning_rate": 7.710397831863604e-06, "loss": 0.0074, "step": 2542 }, { "clip_ratio": 0.06527095288038254, "epoch": 0.7724787363304981, "grad_norm": 0.9562037475742858, "kl": 2.015625, "learning_rate": 7.708169631855457e-06, "loss": 0.0009, "step": 2543 }, { "clip_ratio": 0.12933000922203064, "epoch": 0.772782503037667, "grad_norm": 0.7631316508349213, "kl": 2.015625, "learning_rate": 7.705940670471828e-06, "loss": -0.0046, "step": 2544 }, { "clip_ratio": 0.012885714881122112, "completions/clipped_ratio": 0.0, "completions/max_length": 201.0, "completions/max_terminated_length": 201.0, "completions/mean_length": 136.30357360839844, "completions/mean_terminated_length": 136.30357360839844, "completions/min_length": 98.0, "completions/min_terminated_length": 98.0, "epoch": 0.773086269744836, "grad_norm": 2.1196623297115624, "kl": 2.109375, "learning_rate": 7.703710948339363e-06, "loss": 0.0021, "num_tokens": 12007234.0, "reward": 2.436691999435425, "reward_std": 0.48536452651023865, "rewards/check_gptzero_func/mean": 0.9107142686843872, "rewards/check_gptzero_func/std": 0.28773635625839233, "rewards/check_originality_func/mean": 0.5357142686843872, "rewards/check_originality_func/std": 0.5032362341880798, "rewards/check_winston_local_func/mean": 0.9902634620666504, "rewards/check_winston_local_func/std": 0.010784112848341465, "step": 2545 }, { "clip_ratio": 0.03493214398622513, "epoch": 0.7733900364520049, "grad_norm": 1.5614230496061106, "kl": 2.109375, "learning_rate": 7.701480466084927e-06, "loss": -0.0039, "step": 2546 }, { "clip_ratio": 0.08390764147043228, "epoch": 0.7736938031591738, "grad_norm": 1.1813571629829722, "kl": 2.09375, "learning_rate": 7.699249224335598e-06, "loss": -0.0101, "step": 2547 }, { "clip_ratio": 0.1415046900510788, "epoch": 0.7739975698663426, "grad_norm": 1.0312044183213807, "kl": 2.109375, "learning_rate": 7.697017223718672e-06, "loss": -0.0148, "step": 2548 }, { "clip_ratio": 0.008240378461778164, "completions/clipped_ratio": 0.0, "completions/max_length": 158.0, "completions/max_terminated_length": 158.0, "completions/mean_length": 122.76786041259766, "completions/mean_terminated_length": 122.76786041259766, "completions/min_length": 91.0, "completions/min_terminated_length": 91.0, "epoch": 0.7743013365735115, "grad_norm": 1.9536261111426354, "kl": 2.171875, "learning_rate": 7.694784464861648e-06, "loss": 0.0044, "num_tokens": 12024853.0, "reward": 2.6618404388427734, "reward_std": 0.4749709665775299, "rewards/check_gptzero_func/mean": 0.9107142686843872, "rewards/check_gptzero_func/std": 0.28773635625839233, "rewards/check_originality_func/mean": 0.7857142686843872, "rewards/check_originality_func/std": 0.4140393137931824, "rewards/check_winston_local_func/mean": 0.9654118418693542, "rewards/check_winston_local_func/std": 0.06462220847606659, "step": 2549 }, { "clip_ratio": 0.02527369000017643, "epoch": 0.7746051032806804, "grad_norm": 1.4961402515587972, "kl": 2.171875, "learning_rate": 7.692550948392248e-06, "loss": -0.0019, "step": 2550 }, { "clip_ratio": 0.08138333261013031, "epoch": 0.7749088699878494, "grad_norm": 1.0608409749005772, "kl": 2.171875, "learning_rate": 7.690316674938402e-06, "loss": -0.0079, "step": 2551 }, { "clip_ratio": 0.15153487026691437, "epoch": 0.7752126366950183, "grad_norm": 0.7461780574281464, "kl": 2.203125, "learning_rate": 7.688081645128258e-06, "loss": -0.0124, "step": 2552 }, { "clip_ratio": 0.006485645659267902, "completions/clipped_ratio": 0.0, "completions/max_length": 183.0, "completions/max_terminated_length": 183.0, "completions/mean_length": 138.71429443359375, "completions/mean_terminated_length": 138.71429443359375, "completions/min_length": 102.0, "completions/min_terminated_length": 102.0, "epoch": 0.7755164034021871, "grad_norm": 1.4936605752775356, "kl": 1.9375, "learning_rate": 7.685845859590167e-06, "loss": 0.0101, "num_tokens": 12044277.0, "reward": 2.491255760192871, "reward_std": 0.4578903317451477, "rewards/check_gptzero_func/mean": 0.7678571343421936, "rewards/check_gptzero_func/std": 0.4260205328464508, "rewards/check_originality_func/mean": 0.75, "rewards/check_originality_func/std": 0.43693143129348755, "rewards/check_winston_local_func/mean": 0.9733983278274536, "rewards/check_winston_local_func/std": 0.042767398059368134, "step": 2553 }, { "clip_ratio": 0.012337096966803074, "epoch": 0.775820170109356, "grad_norm": 1.4384333790268082, "kl": 1.9296875, "learning_rate": 7.683609318952701e-06, "loss": 0.0048, "step": 2554 }, { "clip_ratio": 0.051044855266809464, "epoch": 0.7761239368165249, "grad_norm": 0.9754214593965097, "kl": 1.921875, "learning_rate": 7.681372023844642e-06, "loss": -0.0016, "step": 2555 }, { "clip_ratio": 0.09758269786834717, "epoch": 0.7764277035236938, "grad_norm": 0.8883199335132623, "kl": 1.921875, "learning_rate": 7.679133974894984e-06, "loss": -0.0076, "step": 2556 }, { "clip_ratio": 0.0063069164752960205, "completions/clipped_ratio": 0.0, "completions/max_length": 191.0, "completions/max_terminated_length": 191.0, "completions/mean_length": 147.0, "completions/mean_terminated_length": 147.0, "completions/min_length": 89.0, "completions/min_terminated_length": 89.0, "epoch": 0.7767314702308628, "grad_norm": 1.8842082949113719, "kl": 1.859375, "learning_rate": 7.67689517273293e-06, "loss": 0.0123, "num_tokens": 12064549.0, "reward": 2.890275716781616, "reward_std": 0.19828929007053375, "rewards/check_gptzero_func/mean": 0.9642857313156128, "rewards/check_gptzero_func/std": 0.187256321310997, "rewards/check_originality_func/mean": 0.9464285969734192, "rewards/check_originality_func/std": 0.22720777988433838, "rewards/check_winston_local_func/mean": 0.9795612692832947, "rewards/check_winston_local_func/std": 0.05040336400270462, "step": 2557 }, { "clip_ratio": 0.010683001019060612, "epoch": 0.7770352369380316, "grad_norm": 1.6765304555370344, "kl": 1.859375, "learning_rate": 7.674655617987901e-06, "loss": 0.008, "step": 2558 }, { "clip_ratio": 0.04152679443359375, "epoch": 0.7773390036452005, "grad_norm": 1.003622098061125, "kl": 1.859375, "learning_rate": 7.672415311289525e-06, "loss": 0.0016, "step": 2559 }, { "clip_ratio": 0.08300624042749405, "epoch": 0.7776427703523694, "grad_norm": 0.8501902272750753, "kl": 1.875, "learning_rate": 7.67017425326764e-06, "loss": -0.003, "step": 2560 }, { "clip_ratio": 0.006857627537101507, "completions/clipped_ratio": 0.0, "completions/max_length": 227.0, "completions/max_terminated_length": 227.0, "completions/mean_length": 135.125, "completions/mean_terminated_length": 135.125, "completions/min_length": 96.0, "completions/min_terminated_length": 96.0, "epoch": 0.7779465370595383, "grad_norm": 1.2041747174542305, "kl": 2.15625, "learning_rate": 7.667932444552302e-06, "loss": 0.0019, "num_tokens": 12083620.0, "reward": 2.8792710304260254, "reward_std": 0.22877588868141174, "rewards/check_gptzero_func/mean": 0.9464285969734192, "rewards/check_gptzero_func/std": 0.22720777988433838, "rewards/check_originality_func/mean": 0.9464285969734192, "rewards/check_originality_func/std": 0.22720777988433838, "rewards/check_winston_local_func/mean": 0.9864134192466736, "rewards/check_winston_local_func/std": 0.019117476418614388, "step": 2561 }, { "clip_ratio": 0.014762476086616516, "epoch": 0.7782503037667071, "grad_norm": 1.1234135633226359, "kl": 2.171875, "learning_rate": 7.665689885773768e-06, "loss": -0.0019, "step": 2562 }, { "clip_ratio": 0.05013540759682655, "epoch": 0.778554070473876, "grad_norm": 0.8667307864640493, "kl": 2.171875, "learning_rate": 7.663446577562516e-06, "loss": -0.007, "step": 2563 }, { "clip_ratio": 0.10473233461380005, "epoch": 0.778857837181045, "grad_norm": 0.9594728112014683, "kl": 2.1875, "learning_rate": 7.661202520549228e-06, "loss": -0.0119, "step": 2564 }, { "clip_ratio": 0.007925333455204964, "completions/clipped_ratio": 0.0, "completions/max_length": 190.0, "completions/max_terminated_length": 190.0, "completions/mean_length": 140.69644165039062, "completions/mean_terminated_length": 140.69644165039062, "completions/min_length": 106.0, "completions/min_terminated_length": 106.0, "epoch": 0.7791616038882139, "grad_norm": 1.9870711439335347, "kl": 2.03125, "learning_rate": 7.658957715364801e-06, "loss": 0.0122, "num_tokens": 12103267.0, "reward": 2.752067804336548, "reward_std": 0.31427323818206787, "rewards/check_gptzero_func/mean": 0.9821428656578064, "rewards/check_gptzero_func/std": 0.13363061845302582, "rewards/check_originality_func/mean": 0.7857142686843872, "rewards/check_originality_func/std": 0.4140393137931824, "rewards/check_winston_local_func/mean": 0.9842103123664856, "rewards/check_winston_local_func/std": 0.02704361453652382, "step": 2565 }, { "clip_ratio": 0.018413901329040527, "epoch": 0.7794653705953828, "grad_norm": 1.757194155334824, "kl": 2.03125, "learning_rate": 7.656712162640338e-06, "loss": 0.0048, "step": 2566 }, { "clip_ratio": 0.09067726135253906, "epoch": 0.7797691373025516, "grad_norm": 1.0261155868248644, "kl": 2.046875, "learning_rate": 7.654465863007154e-06, "loss": -0.002, "step": 2567 }, { "clip_ratio": 0.16807906329631805, "epoch": 0.7800729040097205, "grad_norm": 1.0883406136960203, "kl": 2.046875, "learning_rate": 7.652218817096775e-06, "loss": -0.0073, "step": 2568 }, { "clip_ratio": 0.00541040999814868, "completions/clipped_ratio": 0.0, "completions/max_length": 191.0, "completions/max_terminated_length": 191.0, "completions/mean_length": 141.5357208251953, "completions/mean_terminated_length": 141.5357208251953, "completions/min_length": 84.0, "completions/min_terminated_length": 84.0, "epoch": 0.7803766707168894, "grad_norm": 2.3417643785773055, "kl": 2.109375, "learning_rate": 7.649971025540937e-06, "loss": 0.0049, "num_tokens": 12122729.0, "reward": 2.928393602371216, "reward_std": 0.12540128827095032, "rewards/check_gptzero_func/mean": 0.9821428656578064, "rewards/check_gptzero_func/std": 0.13363061845302582, "rewards/check_originality_func/mean": 0.9642857313156128, "rewards/check_originality_func/std": 0.187256321310997, "rewards/check_winston_local_func/mean": 0.9819649457931519, "rewards/check_winston_local_func/std": 0.022216688841581345, "step": 2569 }, { "clip_ratio": 0.030530890449881554, "epoch": 0.7806804374240583, "grad_norm": 1.973348837935604, "kl": 2.140625, "learning_rate": 7.647722488971585e-06, "loss": -0.004, "step": 2570 }, { "clip_ratio": 0.09946811944246292, "epoch": 0.7809842041312273, "grad_norm": 1.5993938551804079, "kl": 2.140625, "learning_rate": 7.645473208020871e-06, "loss": -0.0143, "step": 2571 }, { "clip_ratio": 0.18408067524433136, "epoch": 0.7812879708383961, "grad_norm": 1.075610861426703, "kl": 2.140625, "learning_rate": 7.643223183321158e-06, "loss": -0.0228, "step": 2572 }, { "clip_ratio": 0.008704775013029575, "completions/clipped_ratio": 0.017857142857142905, "completions/max_length": 256.0, "completions/max_terminated_length": 182.0, "completions/mean_length": 151.8928680419922, "completions/mean_terminated_length": 150.0, "completions/min_length": 124.0, "completions/min_terminated_length": 124.0, "epoch": 0.781591737545565, "grad_norm": 2.397433039363698, "kl": 1.7890625, "learning_rate": 7.64097241550502e-06, "loss": 0.0255, "num_tokens": 12143107.0, "reward": 2.6108081340789795, "reward_std": 0.17869678139686584, "rewards/check_gptzero_func/mean": 0.9285714030265808, "rewards/check_gptzero_func/std": 0.25987008213996887, "rewards/check_originality_func/mean": 0.6964285969734192, "rewards/check_originality_func/std": 0.4639609158039093, "rewards/check_winston_local_func/mean": 0.9858080148696899, "rewards/check_winston_local_func/std": 0.024525385349988937, "step": 2573 }, { "clip_ratio": 0.014982199296355247, "epoch": 0.7818955042527339, "grad_norm": 2.258308962099699, "kl": 1.7890625, "learning_rate": 7.638720905205239e-06, "loss": 0.0174, "step": 2574 }, { "clip_ratio": 0.05921706557273865, "epoch": 0.7821992709599028, "grad_norm": 1.6833446119023296, "kl": 1.7890625, "learning_rate": 7.636468653054804e-06, "loss": 0.0095, "step": 2575 }, { "clip_ratio": 0.1320498287677765, "epoch": 0.7825030376670717, "grad_norm": 0.9572908112647621, "kl": 1.8046875, "learning_rate": 7.634215659686916e-06, "loss": 0.0006, "step": 2576 }, { "clip_ratio": 0.007773370482027531, "completions/clipped_ratio": 0.0, "completions/max_length": 166.0, "completions/max_terminated_length": 166.0, "completions/mean_length": 121.3214340209961, "completions/mean_terminated_length": 121.3214340209961, "completions/min_length": 93.0, "completions/min_terminated_length": 93.0, "epoch": 0.7828068043742406, "grad_norm": 1.1941682945115426, "kl": 2.15625, "learning_rate": 7.631961925734979e-06, "loss": 0.002, "num_tokens": 12160565.0, "reward": 2.859583616256714, "reward_std": 0.28861746191978455, "rewards/check_gptzero_func/mean": 0.9821428656578064, "rewards/check_gptzero_func/std": 0.13363061845302582, "rewards/check_originality_func/mean": 0.8928571343421936, "rewards/check_originality_func/std": 0.31209391355514526, "rewards/check_winston_local_func/mean": 0.9845834374427795, "rewards/check_winston_local_func/std": 0.01798386126756668, "step": 2577 }, { "clip_ratio": 0.015373344533145428, "epoch": 0.7831105710814095, "grad_norm": 1.127880742980355, "kl": 2.15625, "learning_rate": 7.62970745183261e-06, "loss": 0.0015, "step": 2578 }, { "clip_ratio": 0.05074479058384895, "epoch": 0.7834143377885784, "grad_norm": 0.8115842910143622, "kl": 2.140625, "learning_rate": 7.6274522386136305e-06, "loss": -0.0048, "step": 2579 }, { "clip_ratio": 0.10754823684692383, "epoch": 0.7837181044957473, "grad_norm": 0.5330188676210407, "kl": 2.140625, "learning_rate": 7.625196286712076e-06, "loss": -0.0078, "step": 2580 }, { "clip_ratio": 0.009823343716561794, "completions/clipped_ratio": 0.0, "completions/max_length": 164.0, "completions/max_terminated_length": 164.0, "completions/mean_length": 136.58929443359375, "completions/mean_terminated_length": 136.58929443359375, "completions/min_length": 100.0, "completions/min_terminated_length": 100.0, "epoch": 0.7840218712029161, "grad_norm": 1.997333299642003, "kl": 1.9765625, "learning_rate": 7.62293959676218e-06, "loss": 0.0044, "num_tokens": 12179710.0, "reward": 2.8955578804016113, "reward_std": 0.18205170333385468, "rewards/check_gptzero_func/mean": 0.9642857313156128, "rewards/check_gptzero_func/std": 0.187256321310997, "rewards/check_originality_func/mean": 0.9464285969734192, "rewards/check_originality_func/std": 0.22720777988433838, "rewards/check_winston_local_func/mean": 0.9848434329032898, "rewards/check_winston_local_func/std": 0.018532760441303253, "step": 2581 }, { "clip_ratio": 0.033973898738622665, "epoch": 0.784325637910085, "grad_norm": 1.148417772159937, "kl": 1.9609375, "learning_rate": 7.620682169398392e-06, "loss": -0.001, "step": 2582 }, { "clip_ratio": 0.08872251957654953, "epoch": 0.784629404617254, "grad_norm": 1.283274923141922, "kl": 1.9609375, "learning_rate": 7.618424005255366e-06, "loss": -0.005, "step": 2583 }, { "clip_ratio": 0.11588367074728012, "epoch": 0.7849331713244229, "grad_norm": 1.0905732799931587, "kl": 1.9765625, "learning_rate": 7.616165104967961e-06, "loss": -0.0091, "step": 2584 }, { "clip_ratio": 0.008854215033352375, "completions/clipped_ratio": 0.0, "completions/max_length": 200.0, "completions/max_terminated_length": 200.0, "completions/mean_length": 147.08929443359375, "completions/mean_terminated_length": 147.08929443359375, "completions/min_length": 116.0, "completions/min_terminated_length": 116.0, "epoch": 0.7852369380315918, "grad_norm": 1.51878112027937, "kl": 1.9765625, "learning_rate": 7.613905469171247e-06, "loss": 0.0127, "num_tokens": 12199667.0, "reward": 2.751786947250366, "reward_std": 0.36114728450775146, "rewards/check_gptzero_func/mean": 0.9464285969734192, "rewards/check_gptzero_func/std": 0.22720777988433838, "rewards/check_originality_func/mean": 0.8392857313156128, "rewards/check_originality_func/std": 0.3705909848213196, "rewards/check_winston_local_func/mean": 0.9660728573799133, "rewards/check_winston_local_func/std": 0.07241517305374146, "step": 2585 }, { "clip_ratio": 0.011670393869280815, "epoch": 0.7855407047387606, "grad_norm": 1.2321418740522267, "kl": 1.984375, "learning_rate": 7.6116450985004965e-06, "loss": 0.0075, "step": 2586 }, { "clip_ratio": 0.03952303156256676, "epoch": 0.7858444714459295, "grad_norm": 0.9495772054348062, "kl": 1.984375, "learning_rate": 7.609383993591192e-06, "loss": 0.0017, "step": 2587 }, { "clip_ratio": 0.09185515344142914, "epoch": 0.7861482381530984, "grad_norm": 0.7566216156372132, "kl": 1.984375, "learning_rate": 7.607122155079021e-06, "loss": -0.0041, "step": 2588 }, { "clip_ratio": 0.009218933060765266, "completions/clipped_ratio": 0.0, "completions/max_length": 155.0, "completions/max_terminated_length": 155.0, "completions/mean_length": 124.17857360839844, "completions/mean_terminated_length": 124.17857360839844, "completions/min_length": 88.0, "completions/min_terminated_length": 88.0, "epoch": 0.7864520048602673, "grad_norm": 1.2364023949704261, "kl": 2.40625, "learning_rate": 7.604859583599875e-06, "loss": 0.0176, "num_tokens": 12217245.0, "reward": 2.8626275062561035, "reward_std": 0.1782088726758957, "rewards/check_gptzero_func/mean": 1.0, "rewards/check_gptzero_func/std": 0.0, "rewards/check_originality_func/mean": 0.8928571343421936, "rewards/check_originality_func/std": 0.31209391355514526, "rewards/check_winston_local_func/mean": 0.9697701334953308, "rewards/check_winston_local_func/std": 0.060758043080568314, "step": 2589 }, { "clip_ratio": 0.019554797559976578, "epoch": 0.7867557715674363, "grad_norm": 1.1243892225773107, "kl": 2.40625, "learning_rate": 7.602596279789858e-06, "loss": 0.014, "step": 2590 }, { "clip_ratio": 0.04565010592341423, "epoch": 0.7870595382746051, "grad_norm": 0.874130946694307, "kl": 2.40625, "learning_rate": 7.600332244285271e-06, "loss": 0.0085, "step": 2591 }, { "clip_ratio": 0.08215729147195816, "epoch": 0.787363304981774, "grad_norm": 0.7352301535462451, "kl": 2.421875, "learning_rate": 7.598067477722631e-06, "loss": 0.0046, "step": 2592 }, { "clip_ratio": 0.011615946888923645, "completions/clipped_ratio": 0.0, "completions/max_length": 178.0, "completions/max_terminated_length": 178.0, "completions/mean_length": 145.10714721679688, "completions/mean_terminated_length": 145.10714721679688, "completions/min_length": 115.0, "completions/min_terminated_length": 115.0, "epoch": 0.7876670716889429, "grad_norm": 1.124273514361565, "kl": 2.078125, "learning_rate": 7.595801980738651e-06, "loss": 0.0117, "num_tokens": 12236867.0, "reward": 2.932929754257202, "reward_std": 0.12384478002786636, "rewards/check_gptzero_func/mean": 1.0, "rewards/check_gptzero_func/std": 0.0, "rewards/check_originality_func/mean": 0.9464285969734192, "rewards/check_originality_func/std": 0.22720777988433838, "rewards/check_winston_local_func/mean": 0.9865010380744934, "rewards/check_winston_local_func/std": 0.014940010383725166, "step": 2593 }, { "clip_ratio": 0.01941766031086445, "epoch": 0.7879708383961118, "grad_norm": 1.0500241204944953, "kl": 2.078125, "learning_rate": 7.593535753970254e-06, "loss": 0.0069, "step": 2594 }, { "clip_ratio": 0.06013374775648117, "epoch": 0.7882746051032807, "grad_norm": 2.087170263292237, "kl": 2.078125, "learning_rate": 7.591268798054569e-06, "loss": 0.0063, "step": 2595 }, { "clip_ratio": 0.07563751935958862, "epoch": 0.7885783718104495, "grad_norm": 0.973089959880087, "kl": 2.078125, "learning_rate": 7.589001113628928e-06, "loss": 0.0006, "step": 2596 }, { "clip_ratio": 0.008377564139664173, "completions/clipped_ratio": 0.0, "completions/max_length": 167.0, "completions/max_terminated_length": 167.0, "completions/mean_length": 130.08929443359375, "completions/mean_terminated_length": 130.08929443359375, "completions/min_length": 89.0, "completions/min_terminated_length": 89.0, "epoch": 0.7888821385176185, "grad_norm": 2.091856211003089, "kl": 2.15625, "learning_rate": 7.586732701330868e-06, "loss": 0.0156, "num_tokens": 12255320.0, "reward": 2.8104348182678223, "reward_std": 0.28069186210632324, "rewards/check_gptzero_func/mean": 0.9107142686843872, "rewards/check_gptzero_func/std": 0.28773635625839233, "rewards/check_originality_func/mean": 0.9107142686843872, "rewards/check_originality_func/std": 0.28773635625839233, "rewards/check_winston_local_func/mean": 0.9890060424804688, "rewards/check_winston_local_func/std": 0.008102682419121265, "step": 2597 }, { "clip_ratio": 0.03953031823039055, "epoch": 0.7891859052247874, "grad_norm": 1.5563245715268903, "kl": 2.15625, "learning_rate": 7.58446356179813e-06, "loss": 0.0064, "step": 2598 }, { "clip_ratio": 0.11089088022708893, "epoch": 0.7894896719319563, "grad_norm": 1.4205809673060925, "kl": 2.15625, "learning_rate": 7.582193695668663e-06, "loss": 0.0003, "step": 2599 }, { "clip_ratio": 0.16274991631507874, "epoch": 0.7897934386391251, "grad_norm": 0.9717762089449302, "kl": 2.171875, "learning_rate": 7.579923103580615e-06, "loss": -0.005, "step": 2600 }, { "clip_ratio": 0.010476936586201191, "completions/clipped_ratio": 0.017857142857142905, "completions/max_length": 256.0, "completions/max_terminated_length": 209.0, "completions/mean_length": 151.71429443359375, "completions/mean_terminated_length": 149.81817626953125, "completions/min_length": 104.0, "completions/min_terminated_length": 104.0, "epoch": 0.790097205346294, "grad_norm": 1.9501011566721136, "kl": 2.03125, "learning_rate": 7.577651786172343e-06, "loss": 0.0239, "num_tokens": 12275288.0, "reward": 2.736722230911255, "reward_std": 0.34744495153427124, "rewards/check_gptzero_func/mean": 0.8928571343421936, "rewards/check_gptzero_func/std": 0.3120938837528229, "rewards/check_originality_func/mean": 0.8571428656578064, "rewards/check_originality_func/std": 0.3530939221382141, "rewards/check_winston_local_func/mean": 0.9867221117019653, "rewards/check_winston_local_func/std": 0.02102796360850334, "step": 2601 }, { "clip_ratio": 0.02935647964477539, "epoch": 0.7904009720534629, "grad_norm": 1.3471154376573564, "kl": 2.03125, "learning_rate": 7.575379744082405e-06, "loss": 0.0189, "step": 2602 }, { "clip_ratio": 0.07907260954380035, "epoch": 0.7907047387606319, "grad_norm": 1.005662554458308, "kl": 2.03125, "learning_rate": 7.57310697794956e-06, "loss": 0.0146, "step": 2603 }, { "clip_ratio": 0.11443553119897842, "epoch": 0.7910085054678008, "grad_norm": 1.2233322959398831, "kl": 2.046875, "learning_rate": 7.570833488412781e-06, "loss": 0.0112, "step": 2604 }, { "clip_ratio": 0.014453091658651829, "completions/clipped_ratio": 0.0, "completions/max_length": 183.0, "completions/max_terminated_length": 183.0, "completions/mean_length": 142.5, "completions/mean_terminated_length": 142.5, "completions/min_length": 96.0, "completions/min_terminated_length": 96.0, "epoch": 0.7913122721749696, "grad_norm": 1.8727485962733876, "kl": 2.09375, "learning_rate": 7.568559276111234e-06, "loss": 0.0163, "num_tokens": 12294572.0, "reward": 2.9605653285980225, "reward_std": 0.07697727531194687, "rewards/check_gptzero_func/mean": 1.0, "rewards/check_gptzero_func/std": 0.0, "rewards/check_originality_func/mean": 0.9821428656578064, "rewards/check_originality_func/std": 0.13363061845302582, "rewards/check_winston_local_func/mean": 0.9784224629402161, "rewards/check_winston_local_func/std": 0.03773580119013786, "step": 2605 }, { "clip_ratio": 0.02053484320640564, "epoch": 0.7916160388821385, "grad_norm": 1.7726044540337673, "kl": 2.09375, "learning_rate": 7.566284341684289e-06, "loss": 0.0112, "step": 2606 }, { "clip_ratio": 0.06215304508805275, "epoch": 0.7919198055893074, "grad_norm": 1.2358360977595968, "kl": 2.09375, "learning_rate": 7.564008685771526e-06, "loss": 0.0029, "step": 2607 }, { "clip_ratio": 0.12086344510316849, "epoch": 0.7922235722964763, "grad_norm": 0.8692706158709105, "kl": 2.09375, "learning_rate": 7.56173230901272e-06, "loss": -0.0051, "step": 2608 }, { "clip_ratio": 0.017575737088918686, "completions/clipped_ratio": 0.017857142857142905, "completions/max_length": 256.0, "completions/max_terminated_length": 192.0, "completions/mean_length": 151.3928680419922, "completions/mean_terminated_length": 149.49090576171875, "completions/min_length": 100.0, "completions/min_terminated_length": 100.0, "epoch": 0.7925273390036452, "grad_norm": 2.557423197970072, "kl": 2.046875, "learning_rate": 7.559455212047855e-06, "loss": 0.03, "num_tokens": 12314890.0, "reward": 2.7725090980529785, "reward_std": 0.2556931972503662, "rewards/check_gptzero_func/mean": 0.8214285969734192, "rewards/check_gptzero_func/std": 0.3864591419696808, "rewards/check_originality_func/mean": 0.9642857313156128, "rewards/check_originality_func/std": 0.187256321310997, "rewards/check_winston_local_func/mean": 0.9867945909500122, "rewards/check_winston_local_func/std": 0.025935286656022072, "step": 2609 }, { "clip_ratio": 0.035616468638181686, "epoch": 0.792831105710814, "grad_norm": 1.9793879045447056, "kl": 2.03125, "learning_rate": 7.5571773955171124e-06, "loss": 0.0212, "step": 2610 }, { "clip_ratio": 0.10734807699918747, "epoch": 0.793134872417983, "grad_norm": 1.4511204188750875, "kl": 2.03125, "learning_rate": 7.554898860060879e-06, "loss": 0.0112, "step": 2611 }, { "clip_ratio": 0.1862730234861374, "epoch": 0.7934386391251519, "grad_norm": 1.2943406644779467, "kl": 2.0625, "learning_rate": 7.552619606319742e-06, "loss": 0.0053, "step": 2612 }, { "clip_ratio": 0.009470516815781593, "completions/clipped_ratio": 0.0, "completions/max_length": 194.0, "completions/max_terminated_length": 194.0, "completions/mean_length": 154.5178680419922, "completions/mean_terminated_length": 154.5178680419922, "completions/min_length": 102.0, "completions/min_terminated_length": 102.0, "epoch": 0.7937424058323208, "grad_norm": 1.5656384861279653, "kl": 1.8515625, "learning_rate": 7.550339634934494e-06, "loss": 0.0138, "num_tokens": 12335599.0, "reward": 2.9147555828094482, "reward_std": 0.090358667075634, "rewards/check_gptzero_func/mean": 0.9285714030265808, "rewards/check_gptzero_func/std": 0.25987008213996887, "rewards/check_originality_func/mean": 1.0, "rewards/check_originality_func/std": 0.0, "rewards/check_winston_local_func/mean": 0.9861840009689331, "rewards/check_winston_local_func/std": 0.02450570836663246, "step": 2613 }, { "clip_ratio": 0.019478796049952507, "epoch": 0.7940461725394897, "grad_norm": 1.4049890502630114, "kl": 1.859375, "learning_rate": 7.5480589465461225e-06, "loss": 0.0093, "step": 2614 }, { "clip_ratio": 0.06539983302354813, "epoch": 0.7943499392466585, "grad_norm": 0.9235572423237555, "kl": 1.8671875, "learning_rate": 7.5457775417958234e-06, "loss": 0.0033, "step": 2615 }, { "clip_ratio": 0.10167434066534042, "epoch": 0.7946537059538274, "grad_norm": 0.9009181067428143, "kl": 1.875, "learning_rate": 7.543495421324992e-06, "loss": -0.0017, "step": 2616 }, { "clip_ratio": 0.0097753731533885, "completions/clipped_ratio": 0.017857142857142905, "completions/max_length": 256.0, "completions/max_terminated_length": 208.0, "completions/mean_length": 150.48214721679688, "completions/mean_terminated_length": 148.56362915039062, "completions/min_length": 82.0, "completions/min_terminated_length": 82.0, "epoch": 0.7949574726609964, "grad_norm": 2.903624374233779, "kl": 2.015625, "learning_rate": 7.541212585775224e-06, "loss": 0.0301, "num_tokens": 12355770.0, "reward": 2.955101728439331, "reward_std": 0.10673456639051437, "rewards/check_gptzero_func/mean": 0.9642857313156128, "rewards/check_gptzero_func/std": 0.187256321310997, "rewards/check_originality_func/mean": 1.0, "rewards/check_originality_func/std": 0.0, "rewards/check_winston_local_func/mean": 0.9908156394958496, "rewards/check_winston_local_func/std": 0.008390103466808796, "step": 2617 }, { "clip_ratio": 0.0238319244235754, "epoch": 0.7952612393681653, "grad_norm": 1.3773182674724158, "kl": 2.03125, "learning_rate": 7.538929035788315e-06, "loss": 0.0203, "step": 2618 }, { "clip_ratio": 0.04432518035173416, "epoch": 0.7955650060753341, "grad_norm": 1.1112456031221578, "kl": 2.046875, "learning_rate": 7.536644772006263e-06, "loss": 0.0155, "step": 2619 }, { "clip_ratio": 0.07964398711919785, "epoch": 0.795868772782503, "grad_norm": 1.0510862029957595, "kl": 2.078125, "learning_rate": 7.534359795071269e-06, "loss": 0.0105, "step": 2620 }, { "clip_ratio": 0.008753376081585884, "completions/clipped_ratio": 0.0, "completions/max_length": 166.0, "completions/max_terminated_length": 166.0, "completions/mean_length": 133.5, "completions/mean_terminated_length": 133.5, "completions/min_length": 95.0, "completions/min_terminated_length": 95.0, "epoch": 0.7961725394896719, "grad_norm": 1.7649434618644382, "kl": 2.28125, "learning_rate": 7.5320741056257285e-06, "loss": 0.0205, "num_tokens": 12374366.0, "reward": 2.9864907264709473, "reward_std": 0.00906800851225853, "rewards/check_gptzero_func/mean": 1.0, "rewards/check_gptzero_func/std": 0.0, "rewards/check_originality_func/mean": 1.0, "rewards/check_originality_func/std": 0.0, "rewards/check_winston_local_func/mean": 0.9864905476570129, "rewards/check_winston_local_func/std": 0.020763520151376724, "step": 2621 }, { "clip_ratio": 0.03199036046862602, "epoch": 0.7964763061968408, "grad_norm": 1.4079278162834785, "kl": 2.28125, "learning_rate": 7.5297877043122415e-06, "loss": 0.0151, "step": 2622 }, { "clip_ratio": 0.07241082191467285, "epoch": 0.7967800729040098, "grad_norm": 1.6067817322002267, "kl": 2.28125, "learning_rate": 7.527500591773609e-06, "loss": 0.0096, "step": 2623 }, { "clip_ratio": 0.09753699600696564, "epoch": 0.7970838396111786, "grad_norm": 1.3562658940461194, "kl": 2.28125, "learning_rate": 7.52521276865283e-06, "loss": 0.0046, "step": 2624 }, { "clip_ratio": 0.007382513489574194, "completions/clipped_ratio": 0.0, "completions/max_length": 191.0, "completions/max_terminated_length": 191.0, "completions/mean_length": 136.21429443359375, "completions/mean_terminated_length": 136.21429443359375, "completions/min_length": 96.0, "completions/min_terminated_length": 96.0, "epoch": 0.7973876063183475, "grad_norm": 2.7422379664116634, "kl": 2.25, "learning_rate": 7.522924235593103e-06, "loss": 0.0007, "num_tokens": 12393242.0, "reward": 2.9152398109436035, "reward_std": 0.1788322627544403, "rewards/check_gptzero_func/mean": 0.9285714030265808, "rewards/check_gptzero_func/std": 0.25987008213996887, "rewards/check_originality_func/mean": 1.0, "rewards/check_originality_func/std": 0.0, "rewards/check_winston_local_func/mean": 0.986668050289154, "rewards/check_winston_local_func/std": 0.02509036660194397, "step": 2625 }, { "clip_ratio": 0.029767245054244995, "epoch": 0.7976913730255164, "grad_norm": 2.2080649999749675, "kl": 2.25, "learning_rate": 7.520634993237827e-06, "loss": -0.0063, "step": 2626 }, { "clip_ratio": 0.08505354821681976, "epoch": 0.7979951397326853, "grad_norm": 1.6883517410217002, "kl": 2.25, "learning_rate": 7.518345042230601e-06, "loss": -0.0124, "step": 2627 }, { "clip_ratio": 0.17175190150737762, "epoch": 0.7982989064398542, "grad_norm": 0.9061505528161028, "kl": 2.265625, "learning_rate": 7.51605438321522e-06, "loss": -0.0183, "step": 2628 }, { "clip_ratio": 0.008025518618524075, "completions/clipped_ratio": 0.0, "completions/max_length": 195.0, "completions/max_terminated_length": 195.0, "completions/mean_length": 159.44644165039062, "completions/mean_terminated_length": 159.44644165039062, "completions/min_length": 98.0, "completions/min_terminated_length": 98.0, "epoch": 0.798602673147023, "grad_norm": 1.6706659152656564, "kl": 1.8125, "learning_rate": 7.513763016835682e-06, "loss": 0.0147, "num_tokens": 12414787.0, "reward": 2.9739620685577393, "reward_std": 0.052949678152799606, "rewards/check_gptzero_func/mean": 1.0, "rewards/check_gptzero_func/std": 0.0, "rewards/check_originality_func/mean": 0.9821428656578064, "rewards/check_originality_func/std": 0.13363061845302582, "rewards/check_winston_local_func/mean": 0.9918188452720642, "rewards/check_winston_local_func/std": 0.0045213960111141205, "step": 2629 }, { "clip_ratio": 0.019909799098968506, "epoch": 0.798906439854192, "grad_norm": 1.5304419103764468, "kl": 1.8125, "learning_rate": 7.5114709437361844e-06, "loss": 0.0096, "step": 2630 }, { "clip_ratio": 0.06995701044797897, "epoch": 0.7992102065613609, "grad_norm": 1.2561618470176832, "kl": 1.8125, "learning_rate": 7.509178164561119e-06, "loss": 0.001, "step": 2631 }, { "clip_ratio": 0.11699115484952927, "epoch": 0.7995139732685298, "grad_norm": 1.0764735033171815, "kl": 1.8125, "learning_rate": 7.506884679955078e-06, "loss": -0.0032, "step": 2632 }, { "clip_ratio": 0.011267201974987984, "completions/clipped_ratio": 0.0, "completions/max_length": 194.0, "completions/max_terminated_length": 194.0, "completions/mean_length": 154.6607208251953, "completions/mean_terminated_length": 154.6607208251953, "completions/min_length": 117.0, "completions/min_terminated_length": 117.0, "epoch": 0.7998177399756987, "grad_norm": 1.7602133502249375, "kl": 2.15625, "learning_rate": 7.504590490562851e-06, "loss": 0.0083, "num_tokens": 12435144.0, "reward": 2.956373691558838, "reward_std": 0.06850367039442062, "rewards/check_gptzero_func/mean": 1.0, "rewards/check_gptzero_func/std": 0.0, "rewards/check_originality_func/mean": 0.9642857313156128, "rewards/check_originality_func/std": 0.187256321310997, "rewards/check_winston_local_func/mean": 0.9920875430107117, "rewards/check_winston_local_func/std": 0.003533373586833477, "step": 2633 }, { "clip_ratio": 0.02100287936627865, "epoch": 0.8001215066828675, "grad_norm": 1.5924801587615611, "kl": 2.15625, "learning_rate": 7.5022955970294295e-06, "loss": 0.0032, "step": 2634 }, { "clip_ratio": 0.05323025584220886, "epoch": 0.8004252733900364, "grad_norm": 1.2270268546344645, "kl": 2.171875, "learning_rate": 7.500000000000001e-06, "loss": -0.0037, "step": 2635 }, { "clip_ratio": 0.1067885160446167, "epoch": 0.8007290400972054, "grad_norm": 1.0309490795984724, "kl": 2.171875, "learning_rate": 7.497703700119948e-06, "loss": -0.0102, "step": 2636 }, { "clip_ratio": 0.011661357246339321, "completions/clipped_ratio": 0.017857142857142905, "completions/max_length": 256.0, "completions/max_terminated_length": 226.0, "completions/mean_length": 153.05357360839844, "completions/mean_terminated_length": 151.1818084716797, "completions/min_length": 108.0, "completions/min_terminated_length": 108.0, "epoch": 0.8010328068043743, "grad_norm": 1.6190818591803433, "kl": 2.25, "learning_rate": 7.4954066980348535e-06, "loss": 0.0005, "num_tokens": 12455019.0, "reward": 2.9545328617095947, "reward_std": 0.10297774523496628, "rewards/check_gptzero_func/mean": 1.0, "rewards/check_gptzero_func/std": 0.0, "rewards/check_originality_func/mean": 0.9642857313156128, "rewards/check_originality_func/std": 0.187256321310997, "rewards/check_winston_local_func/mean": 0.9902471303939819, "rewards/check_winston_local_func/std": 0.008775725960731506, "step": 2637 }, { "clip_ratio": 0.019900349900126457, "epoch": 0.8013365735115431, "grad_norm": 1.416114548958318, "kl": 2.25, "learning_rate": 7.493108994390496e-06, "loss": -0.0025, "step": 2638 }, { "clip_ratio": 0.04970303922891617, "epoch": 0.801640340218712, "grad_norm": 1.0068442874985364, "kl": 2.25, "learning_rate": 7.490810589832857e-06, "loss": -0.0098, "step": 2639 }, { "clip_ratio": 0.08477471768856049, "epoch": 0.8019441069258809, "grad_norm": 0.9366970855286404, "kl": 2.265625, "learning_rate": 7.488511485008106e-06, "loss": -0.0126, "step": 2640 }, { "clip_ratio": 0.012751678004860878, "completions/clipped_ratio": 0.0357142857142857, "completions/max_length": 256.0, "completions/max_terminated_length": 211.0, "completions/mean_length": 152.60714721679688, "completions/mean_terminated_length": 148.7777862548828, "completions/min_length": 101.0, "completions/min_terminated_length": 101.0, "epoch": 0.8022478736330498, "grad_norm": 2.4434944146151327, "kl": 2.078125, "learning_rate": 7.486211680562615e-06, "loss": 0.0553, "num_tokens": 12475077.0, "reward": 2.955043315887451, "reward_std": 0.10534431785345078, "rewards/check_gptzero_func/mean": 1.0, "rewards/check_gptzero_func/std": 0.0, "rewards/check_originality_func/mean": 0.9642857313156128, "rewards/check_originality_func/std": 0.187256321310997, "rewards/check_winston_local_func/mean": 0.9907575249671936, "rewards/check_winston_local_func/std": 0.007320513483136892, "step": 2641 }, { "clip_ratio": 0.019945796579122543, "epoch": 0.8025516403402188, "grad_norm": 2.2147160368538845, "kl": 2.078125, "learning_rate": 7.483911177142952e-06, "loss": 0.0484, "step": 2642 }, { "clip_ratio": 0.045745689421892166, "epoch": 0.8028554070473876, "grad_norm": 1.232675971104131, "kl": 2.078125, "learning_rate": 7.481609975395881e-06, "loss": 0.0444, "step": 2643 }, { "clip_ratio": 0.07562315464019775, "epoch": 0.8031591737545565, "grad_norm": 1.0994730984517551, "kl": 2.078125, "learning_rate": 7.479308075968363e-06, "loss": 0.0386, "step": 2644 }, { "clip_ratio": 0.011357598938047886, "completions/clipped_ratio": 0.0, "completions/max_length": 214.0, "completions/max_terminated_length": 214.0, "completions/mean_length": 145.6607208251953, "completions/mean_terminated_length": 145.6607208251953, "completions/min_length": 89.0, "completions/min_terminated_length": 89.0, "epoch": 0.8034629404617254, "grad_norm": 2.018541990422239, "kl": 2.234375, "learning_rate": 7.477005479507553e-06, "loss": 0.016, "num_tokens": 12494618.0, "reward": 2.974076986312866, "reward_std": 0.054210249334573746, "rewards/check_gptzero_func/mean": 1.0, "rewards/check_gptzero_func/std": 0.0, "rewards/check_originality_func/mean": 0.9821428656578064, "rewards/check_originality_func/std": 0.13363061845302582, "rewards/check_winston_local_func/mean": 0.9919341802597046, "rewards/check_winston_local_func/std": 0.007373161148279905, "step": 2645 }, { "clip_ratio": 0.02472064271569252, "epoch": 0.8037667071688943, "grad_norm": 1.765233910560697, "kl": 2.234375, "learning_rate": 7.474702186660806e-06, "loss": 0.0108, "step": 2646 }, { "clip_ratio": 0.07253807783126831, "epoch": 0.8040704738760632, "grad_norm": 1.2269230084688985, "kl": 2.234375, "learning_rate": 7.472398198075669e-06, "loss": 0.0074, "step": 2647 }, { "clip_ratio": 0.10049581527709961, "epoch": 0.804374240583232, "grad_norm": 1.387077045510112, "kl": 2.234375, "learning_rate": 7.470093514399884e-06, "loss": 0.0027, "step": 2648 }, { "clip_ratio": 0.012111223302781582, "completions/clipped_ratio": 0.0, "completions/max_length": 210.0, "completions/max_terminated_length": 210.0, "completions/mean_length": 157.67857360839844, "completions/mean_terminated_length": 157.67857360839844, "completions/min_length": 111.0, "completions/min_terminated_length": 111.0, "epoch": 0.804678007290401, "grad_norm": 1.6391302487414274, "kl": 1.890625, "learning_rate": 7.467788136281393e-06, "loss": 0.0107, "num_tokens": 12515544.0, "reward": 2.938621759414673, "reward_std": 0.11896640062332153, "rewards/check_gptzero_func/mean": 0.9464285969734192, "rewards/check_gptzero_func/std": 0.22720777988433838, "rewards/check_originality_func/mean": 1.0, "rewards/check_originality_func/std": 0.0, "rewards/check_winston_local_func/mean": 0.9921929240226746, "rewards/check_winston_local_func/std": 0.0032128707971423864, "step": 2649 }, { "clip_ratio": 0.02846689522266388, "epoch": 0.8049817739975699, "grad_norm": 1.3270906437583265, "kl": 1.890625, "learning_rate": 7.46548206436833e-06, "loss": 0.0059, "step": 2650 }, { "clip_ratio": 0.07321693003177643, "epoch": 0.8052855407047388, "grad_norm": 0.9062065986658538, "kl": 1.8828125, "learning_rate": 7.463175299309024e-06, "loss": -0.0005, "step": 2651 }, { "clip_ratio": 0.10527505725622177, "epoch": 0.8055893074119077, "grad_norm": 0.7041917773525841, "kl": 1.890625, "learning_rate": 7.460867841752001e-06, "loss": -0.0046, "step": 2652 }, { "clip_ratio": 0.00896956492215395, "completions/clipped_ratio": 0.0, "completions/max_length": 176.0, "completions/max_terminated_length": 176.0, "completions/mean_length": 139.875, "completions/mean_terminated_length": 139.875, "completions/min_length": 104.0, "completions/min_terminated_length": 104.0, "epoch": 0.8058930741190765, "grad_norm": 1.7261621869200312, "kl": 2.15625, "learning_rate": 7.458559692345978e-06, "loss": 0.018, "num_tokens": 12534617.0, "reward": 2.8119733333587646, "reward_std": 0.35770031809806824, "rewards/check_gptzero_func/mean": 0.9107142686843872, "rewards/check_gptzero_func/std": 0.28773635625839233, "rewards/check_originality_func/mean": 0.9107142686843872, "rewards/check_originality_func/std": 0.28773635625839233, "rewards/check_winston_local_func/mean": 0.9905446767807007, "rewards/check_winston_local_func/std": 0.0057556284591555595, "step": 2653 }, { "clip_ratio": 0.014180862344801426, "epoch": 0.8061968408262454, "grad_norm": 1.662832934914725, "kl": 2.15625, "learning_rate": 7.45625085173987e-06, "loss": 0.0107, "step": 2654 }, { "clip_ratio": 0.05075320973992348, "epoch": 0.8065006075334143, "grad_norm": 1.1827721322752536, "kl": 2.15625, "learning_rate": 7.453941320582785e-06, "loss": 0.0051, "step": 2655 }, { "clip_ratio": 0.09777295589447021, "epoch": 0.8068043742405833, "grad_norm": 1.1862931213683117, "kl": 2.171875, "learning_rate": 7.451631099524025e-06, "loss": -0.0006, "step": 2656 }, { "clip_ratio": 0.010484954342246056, "completions/clipped_ratio": 0.0, "completions/max_length": 194.0, "completions/max_terminated_length": 194.0, "completions/mean_length": 158.7678680419922, "completions/mean_terminated_length": 158.7678680419922, "completions/min_length": 110.0, "completions/min_terminated_length": 110.0, "epoch": 0.8071081409477521, "grad_norm": 2.253505676362007, "kl": 1.9140625, "learning_rate": 7.449320189213085e-06, "loss": 0.0026, "num_tokens": 12556044.0, "reward": 2.9925570487976074, "reward_std": 0.002632706891745329, "rewards/check_gptzero_func/mean": 1.0, "rewards/check_gptzero_func/std": 0.0, "rewards/check_originality_func/mean": 1.0, "rewards/check_originality_func/std": 0.0, "rewards/check_winston_local_func/mean": 0.9925571084022522, "rewards/check_winston_local_func/std": 0.0029900060035288334, "step": 2657 }, { "clip_ratio": 0.021026700735092163, "epoch": 0.807411907654921, "grad_norm": 2.0621234411391884, "kl": 1.9140625, "learning_rate": 7.447008590299658e-06, "loss": -0.0038, "step": 2658 }, { "clip_ratio": 0.056320417672395706, "epoch": 0.8077156743620899, "grad_norm": 1.3416242818486372, "kl": 1.9140625, "learning_rate": 7.444696303433625e-06, "loss": -0.0107, "step": 2659 }, { "clip_ratio": 0.10611327737569809, "epoch": 0.8080194410692588, "grad_norm": 0.9177010582071101, "kl": 1.8984375, "learning_rate": 7.442383329265063e-06, "loss": -0.0177, "step": 2660 }, { "clip_ratio": 0.010790948756039143, "completions/clipped_ratio": 0.0, "completions/max_length": 191.0, "completions/max_terminated_length": 191.0, "completions/mean_length": 147.875, "completions/mean_terminated_length": 147.875, "completions/min_length": 100.0, "completions/min_terminated_length": 100.0, "epoch": 0.8083232077764277, "grad_norm": 2.092297627552312, "kl": 2.375, "learning_rate": 7.440069668444242e-06, "loss": 0.0132, "num_tokens": 12575493.0, "reward": 2.974158763885498, "reward_std": 0.05159468948841095, "rewards/check_gptzero_func/mean": 0.9821428656578064, "rewards/check_gptzero_func/std": 0.13363061845302582, "rewards/check_originality_func/mean": 1.0, "rewards/check_originality_func/std": 0.0, "rewards/check_winston_local_func/mean": 0.9920158982276917, "rewards/check_winston_local_func/std": 0.0030836351215839386, "step": 2661 }, { "clip_ratio": 0.020624475553631783, "epoch": 0.8086269744835966, "grad_norm": 1.6973769745444136, "kl": 2.375, "learning_rate": 7.437755321621625e-06, "loss": 0.0081, "step": 2662 }, { "clip_ratio": 0.05957182124257088, "epoch": 0.8089307411907655, "grad_norm": 1.1486885797521824, "kl": 2.390625, "learning_rate": 7.43544028944787e-06, "loss": -0.0013, "step": 2663 }, { "clip_ratio": 0.09544117003679276, "epoch": 0.8092345078979344, "grad_norm": 0.8974534026825854, "kl": 2.390625, "learning_rate": 7.433124572573824e-06, "loss": -0.007, "step": 2664 }, { "clip_ratio": 0.013461804017424583, "completions/clipped_ratio": 0.0357142857142857, "completions/max_length": 256.0, "completions/max_terminated_length": 235.0, "completions/mean_length": 160.23214721679688, "completions/mean_terminated_length": 156.6851806640625, "completions/min_length": 108.0, "completions/min_terminated_length": 108.0, "epoch": 0.8095382746051033, "grad_norm": 2.4813882030140557, "kl": 2.34375, "learning_rate": 7.43080817165053e-06, "loss": 0.0108, "num_tokens": 12595642.0, "reward": 2.957200050354004, "reward_std": 0.10222074389457703, "rewards/check_gptzero_func/mean": 1.0, "rewards/check_gptzero_func/std": 0.0, "rewards/check_originality_func/mean": 0.9642857313156128, "rewards/check_originality_func/std": 0.187256321310997, "rewards/check_winston_local_func/mean": 0.9929141402244568, "rewards/check_winston_local_func/std": 0.00210722628980875, "step": 2665 }, { "clip_ratio": 0.02367057465016842, "epoch": 0.8098420413122722, "grad_norm": 2.1396403424816475, "kl": 2.359375, "learning_rate": 7.428491087329219e-06, "loss": 0.0017, "step": 2666 }, { "clip_ratio": 0.08143321424722672, "epoch": 0.810145808019441, "grad_norm": 1.545611337737932, "kl": 2.359375, "learning_rate": 7.426173320261318e-06, "loss": -0.0095, "step": 2667 }, { "clip_ratio": 0.16769330203533173, "epoch": 0.81044957472661, "grad_norm": 1.1928405706938354, "kl": 2.375, "learning_rate": 7.423854871098446e-06, "loss": -0.0178, "step": 2668 }, { "clip_ratio": 0.01312968973070383, "completions/clipped_ratio": 0.0, "completions/max_length": 207.0, "completions/max_terminated_length": 207.0, "completions/mean_length": 159.7857208251953, "completions/mean_terminated_length": 159.7857208251953, "completions/min_length": 107.0, "completions/min_terminated_length": 107.0, "epoch": 0.8107533414337789, "grad_norm": 1.944541063170926, "kl": 2.171875, "learning_rate": 7.421535740492408e-06, "loss": 0.0076, "num_tokens": 12616310.0, "reward": 2.9741876125335693, "reward_std": 0.05285712331533432, "rewards/check_gptzero_func/mean": 0.9821428656578064, "rewards/check_gptzero_func/std": 0.13363061845302582, "rewards/check_originality_func/mean": 1.0, "rewards/check_originality_func/std": 0.0, "rewards/check_winston_local_func/mean": 0.9920445680618286, "rewards/check_winston_local_func/std": 0.0032343289349228144, "step": 2669 }, { "clip_ratio": 0.03057098388671875, "epoch": 0.8110571081409478, "grad_norm": 1.5265936549773365, "kl": 2.1875, "learning_rate": 7.419215929095211e-06, "loss": 0.0034, "step": 2670 }, { "clip_ratio": 0.06645475327968597, "epoch": 0.8113608748481167, "grad_norm": 1.4659522185543978, "kl": 2.1875, "learning_rate": 7.416895437559043e-06, "loss": -0.0043, "step": 2671 }, { "clip_ratio": 0.12457464635372162, "epoch": 0.8116646415552855, "grad_norm": 1.221260112872923, "kl": 2.1875, "learning_rate": 7.4145742665362885e-06, "loss": -0.0131, "step": 2672 }, { "clip_ratio": 0.013413218781352043, "completions/clipped_ratio": 0.0, "completions/max_length": 221.0, "completions/max_terminated_length": 221.0, "completions/mean_length": 155.75, "completions/mean_terminated_length": 155.75, "completions/min_length": 91.0, "completions/min_terminated_length": 91.0, "epoch": 0.8119684082624544, "grad_norm": 1.8502619699671803, "kl": 2.203125, "learning_rate": 7.4122524166795215e-06, "loss": 0.0129, "num_tokens": 12636648.0, "reward": 2.95578670501709, "reward_std": 0.10381079465150833, "rewards/check_gptzero_func/mean": 1.0, "rewards/check_gptzero_func/std": 0.0, "rewards/check_originality_func/mean": 0.9642857313156128, "rewards/check_originality_func/std": 0.187256321310997, "rewards/check_winston_local_func/mean": 0.991500735282898, "rewards/check_winston_local_func/std": 0.00422172574326396, "step": 2673 }, { "clip_ratio": 0.026915481314063072, "epoch": 0.8122721749696233, "grad_norm": 1.5271289902714773, "kl": 2.203125, "learning_rate": 7.40992988864151e-06, "loss": 0.0058, "step": 2674 }, { "clip_ratio": 0.05514051392674446, "epoch": 0.8125759416767923, "grad_norm": 0.9941145621920704, "kl": 2.203125, "learning_rate": 7.407606683075207e-06, "loss": 0.001, "step": 2675 }, { "clip_ratio": 0.10731584578752518, "epoch": 0.8128797083839611, "grad_norm": 0.8180767213906096, "kl": 2.203125, "learning_rate": 7.4052828006337606e-06, "loss": -0.0041, "step": 2676 }, { "clip_ratio": 0.014581811614334583, "completions/clipped_ratio": 0.0, "completions/max_length": 198.0, "completions/max_terminated_length": 198.0, "completions/mean_length": 164.58929443359375, "completions/mean_terminated_length": 164.58929443359375, "completions/min_length": 113.0, "completions/min_terminated_length": 113.0, "epoch": 0.81318347509113, "grad_norm": 1.9744577481914412, "kl": 2.0625, "learning_rate": 7.402958241970508e-06, "loss": 0.0178, "num_tokens": 12658153.0, "reward": 2.975322961807251, "reward_std": 0.05250334367156029, "rewards/check_gptzero_func/mean": 1.0, "rewards/check_gptzero_func/std": 0.0, "rewards/check_originality_func/mean": 0.9821428656578064, "rewards/check_originality_func/std": 0.13363061845302582, "rewards/check_winston_local_func/mean": 0.993179976940155, "rewards/check_winston_local_func/std": 0.0023808879777789116, "step": 2677 }, { "clip_ratio": 0.02337494120001793, "epoch": 0.8134872417982989, "grad_norm": 1.6125668906170132, "kl": 2.0625, "learning_rate": 7.400633007738977e-06, "loss": 0.0109, "step": 2678 }, { "clip_ratio": 0.04729249328374863, "epoch": 0.8137910085054678, "grad_norm": 1.188573060496788, "kl": 2.0625, "learning_rate": 7.398307098592882e-06, "loss": 0.0057, "step": 2679 }, { "clip_ratio": 0.08646649867296219, "epoch": 0.8140947752126367, "grad_norm": 1.0395255835721484, "kl": 2.0625, "learning_rate": 7.395980515186132e-06, "loss": -0.0021, "step": 2680 }, { "clip_ratio": 0.010984860360622406, "completions/clipped_ratio": 0.0, "completions/max_length": 245.0, "completions/max_terminated_length": 245.0, "completions/mean_length": 141.92857360839844, "completions/mean_terminated_length": 141.92857360839844, "completions/min_length": 102.0, "completions/min_terminated_length": 102.0, "epoch": 0.8143985419198055, "grad_norm": 2.5909386374119983, "kl": 2.40625, "learning_rate": 7.393653258172819e-06, "loss": 0.0236, "num_tokens": 12676445.0, "reward": 2.9737865924835205, "reward_std": 0.052601054310798645, "rewards/check_gptzero_func/mean": 0.9821428656578064, "rewards/check_gptzero_func/std": 0.13363061845302582, "rewards/check_originality_func/mean": 1.0, "rewards/check_originality_func/std": 0.0, "rewards/check_winston_local_func/mean": 0.991643488407135, "rewards/check_winston_local_func/std": 0.0025186887942254543, "step": 2681 }, { "clip_ratio": 0.041347574442625046, "epoch": 0.8147023086269745, "grad_norm": 2.2427648087505307, "kl": 2.390625, "learning_rate": 7.391325328207236e-06, "loss": 0.0122, "step": 2682 }, { "clip_ratio": 0.09364684671163559, "epoch": 0.8150060753341434, "grad_norm": 1.612009847235431, "kl": 2.390625, "learning_rate": 7.388996725943854e-06, "loss": -0.0011, "step": 2683 }, { "clip_ratio": 0.17095865309238434, "epoch": 0.8153098420413123, "grad_norm": 1.0676305858642283, "kl": 2.40625, "learning_rate": 7.386667452037334e-06, "loss": -0.0082, "step": 2684 }, { "clip_ratio": 0.014808038249611855, "completions/clipped_ratio": 0.017857142857142905, "completions/max_length": 256.0, "completions/max_terminated_length": 194.0, "completions/mean_length": 152.67857360839844, "completions/mean_terminated_length": 150.79998779296875, "completions/min_length": 108.0, "completions/min_terminated_length": 108.0, "epoch": 0.8156136087484812, "grad_norm": 2.730819781406187, "kl": 2.125, "learning_rate": 7.3843375071425315e-06, "loss": 0.0377, "num_tokens": 12696371.0, "reward": 2.973726749420166, "reward_std": 0.05435556545853615, "rewards/check_gptzero_func/mean": 1.0, "rewards/check_gptzero_func/std": 0.0, "rewards/check_originality_func/mean": 0.9821428656578064, "rewards/check_originality_func/std": 0.13363061845302582, "rewards/check_winston_local_func/mean": 0.991583526134491, "rewards/check_winston_local_func/std": 0.006890955846756697, "step": 2685 }, { "clip_ratio": 0.018916338682174683, "epoch": 0.81591737545565, "grad_norm": 2.8008181938440324, "kl": 2.140625, "learning_rate": 7.382006891914487e-06, "loss": 0.0318, "step": 2686 }, { "clip_ratio": 0.05377443507313728, "epoch": 0.8162211421628189, "grad_norm": 1.2975184846343466, "kl": 2.140625, "learning_rate": 7.379675607008432e-06, "loss": 0.0235, "step": 2687 }, { "clip_ratio": 0.10184673964977264, "epoch": 0.8165249088699879, "grad_norm": 0.8651333782422708, "kl": 2.140625, "learning_rate": 7.377343653079782e-06, "loss": 0.0183, "step": 2688 }, { "clip_ratio": 0.011874116957187653, "completions/clipped_ratio": 0.0357142857142857, "completions/max_length": 256.0, "completions/max_terminated_length": 198.0, "completions/mean_length": 162.46429443359375, "completions/mean_terminated_length": 159.0, "completions/min_length": 112.0, "completions/min_terminated_length": 112.0, "epoch": 0.8168286755771568, "grad_norm": 3.1004273569416165, "kl": 2.28125, "learning_rate": 7.375011030784145e-06, "loss": 0.0075, "num_tokens": 12716893.0, "reward": 2.9910943508148193, "reward_std": 0.0032692616805434227, "rewards/check_gptzero_func/mean": 1.0, "rewards/check_gptzero_func/std": 0.0, "rewards/check_originality_func/mean": 1.0, "rewards/check_originality_func/std": 0.0, "rewards/check_winston_local_func/mean": 0.9910942912101746, "rewards/check_winston_local_func/std": 0.004491506610065699, "step": 2689 }, { "clip_ratio": 0.025855766609311104, "epoch": 0.8171324422843257, "grad_norm": 2.654424427255758, "kl": 2.265625, "learning_rate": 7.372677740777314e-06, "loss": 0.0014, "step": 2690 }, { "clip_ratio": 0.08886414766311646, "epoch": 0.8174362089914945, "grad_norm": 2.0864714471725145, "kl": 2.265625, "learning_rate": 7.370343783715272e-06, "loss": -0.0096, "step": 2691 }, { "clip_ratio": 0.15320107340812683, "epoch": 0.8177399756986634, "grad_norm": 1.5213406313447966, "kl": 2.25, "learning_rate": 7.368009160254187e-06, "loss": -0.0192, "step": 2692 }, { "clip_ratio": 0.014447594992816448, "completions/clipped_ratio": 0.017857142857142905, "completions/max_length": 256.0, "completions/max_terminated_length": 253.0, "completions/mean_length": 156.98214721679688, "completions/mean_terminated_length": 155.1818084716797, "completions/min_length": 88.0, "completions/min_terminated_length": 88.0, "epoch": 0.8180437424058323, "grad_norm": 2.7160440929273504, "kl": 2.203125, "learning_rate": 7.365673871050415e-06, "loss": 0.0219, "num_tokens": 12737276.0, "reward": 2.9751055240631104, "reward_std": 0.05178527161478996, "rewards/check_gptzero_func/mean": 0.9821428656578064, "rewards/check_gptzero_func/std": 0.13363061845302582, "rewards/check_originality_func/mean": 1.0, "rewards/check_originality_func/std": 0.0, "rewards/check_winston_local_func/mean": 0.9929622411727905, "rewards/check_winston_local_func/std": 0.001996038481593132, "step": 2693 }, { "clip_ratio": 0.02380150370299816, "epoch": 0.8183475091130012, "grad_norm": 2.623088973007948, "kl": 2.203125, "learning_rate": 7.363337916760502e-06, "loss": 0.0158, "step": 2694 }, { "clip_ratio": 0.05850830301642418, "epoch": 0.81865127582017, "grad_norm": 1.7680393664959164, "kl": 2.1875, "learning_rate": 7.361001298041178e-06, "loss": 0.0069, "step": 2695 }, { "clip_ratio": 0.12006913870573044, "epoch": 0.818955042527339, "grad_norm": 1.1662561129318214, "kl": 2.171875, "learning_rate": 7.358664015549359e-06, "loss": -0.0012, "step": 2696 }, { "clip_ratio": 0.010622655041515827, "completions/clipped_ratio": 0.0, "completions/max_length": 231.0, "completions/max_terminated_length": 231.0, "completions/mean_length": 160.7857208251953, "completions/mean_terminated_length": 160.7857208251953, "completions/min_length": 96.0, "completions/min_terminated_length": 96.0, "epoch": 0.8192588092345079, "grad_norm": 2.5160352358778635, "kl": 2.015625, "learning_rate": 7.35632606994215e-06, "loss": 0.0067, "num_tokens": 12757776.0, "reward": 2.9556214809417725, "reward_std": 0.103349469602108, "rewards/check_gptzero_func/mean": 0.9642857313156128, "rewards/check_gptzero_func/std": 0.187256321310997, "rewards/check_originality_func/mean": 1.0, "rewards/check_originality_func/std": 0.0, "rewards/check_winston_local_func/mean": 0.9913355708122253, "rewards/check_winston_local_func/std": 0.0032310395035892725, "step": 2697 }, { "clip_ratio": 0.03060622327029705, "epoch": 0.8195625759416768, "grad_norm": 1.883761375593053, "kl": 2.015625, "learning_rate": 7.353987461876841e-06, "loss": -0.0011, "step": 2698 }, { "clip_ratio": 0.07246198505163193, "epoch": 0.8198663426488457, "grad_norm": 1.1718092212803182, "kl": 2.03125, "learning_rate": 7.351648192010912e-06, "loss": -0.0064, "step": 2699 }, { "clip_ratio": 0.1151808649301529, "epoch": 0.8201701093560145, "grad_norm": 1.1967040972934906, "kl": 2.046875, "learning_rate": 7.349308261002023e-06, "loss": -0.0106, "step": 2700 }, { "clip_ratio": 0.007246758323162794, "completions/clipped_ratio": 0.017857142857142905, "completions/max_length": 256.0, "completions/max_terminated_length": 220.0, "completions/mean_length": 167.17857360839844, "completions/mean_terminated_length": 165.56362915039062, "completions/min_length": 106.0, "completions/min_terminated_length": 106.0, "epoch": 0.8204738760631834, "grad_norm": 2.641404495312225, "kl": 1.9609375, "learning_rate": 7.346967669508023e-06, "loss": 0.0213, "num_tokens": 12778962.0, "reward": 2.9196882247924805, "reward_std": 0.16883906722068787, "rewards/check_gptzero_func/mean": 0.9821428656578064, "rewards/check_gptzero_func/std": 0.13363061845302582, "rewards/check_originality_func/mean": 0.9464285969734192, "rewards/check_originality_func/std": 0.22720777988433838, "rewards/check_winston_local_func/mean": 0.9911166429519653, "rewards/check_winston_local_func/std": 0.0074690780602395535, "step": 2701 }, { "clip_ratio": 0.014006230048835278, "epoch": 0.8207776427703524, "grad_norm": 2.3560181732175614, "kl": 1.9765625, "learning_rate": 7.344626418186948e-06, "loss": 0.0155, "step": 2702 }, { "clip_ratio": 0.05524373799562454, "epoch": 0.8210814094775213, "grad_norm": 2.040594033487662, "kl": 1.984375, "learning_rate": 7.342284507697015e-06, "loss": 0.0038, "step": 2703 }, { "clip_ratio": 0.11648815125226974, "epoch": 0.8213851761846902, "grad_norm": 1.4485089513904696, "kl": 1.9921875, "learning_rate": 7.339941938696632e-06, "loss": -0.0051, "step": 2704 }, { "clip_ratio": 0.013805108144879341, "completions/clipped_ratio": 0.017857142857142905, "completions/max_length": 256.0, "completions/max_terminated_length": 244.0, "completions/mean_length": 171.37501525878906, "completions/mean_terminated_length": 169.83636474609375, "completions/min_length": 98.0, "completions/min_terminated_length": 98.0, "epoch": 0.821688942891859, "grad_norm": 2.327560189293615, "kl": 1.921875, "learning_rate": 7.337598711844389e-06, "loss": 0.0061, "num_tokens": 12800551.0, "reward": 2.993105173110962, "reward_std": 0.002124724444001913, "rewards/check_gptzero_func/mean": 1.0, "rewards/check_gptzero_func/std": 0.0, "rewards/check_originality_func/mean": 1.0, "rewards/check_originality_func/std": 0.0, "rewards/check_winston_local_func/mean": 0.9931048154830933, "rewards/check_winston_local_func/std": 0.0024254578165709972, "step": 2705 }, { "clip_ratio": 0.023627761751413345, "epoch": 0.8219927095990279, "grad_norm": 2.0753126780706013, "kl": 1.9296875, "learning_rate": 7.335254827799062e-06, "loss": 0.0019, "step": 2706 }, { "clip_ratio": 0.06572484225034714, "epoch": 0.8222964763061968, "grad_norm": 1.7272468783754031, "kl": 1.9296875, "learning_rate": 7.332910287219609e-06, "loss": -0.0067, "step": 2707 }, { "clip_ratio": 0.1189858540892601, "epoch": 0.8226002430133658, "grad_norm": 1.7049280695081448, "kl": 1.9375, "learning_rate": 7.330565090765176e-06, "loss": -0.0131, "step": 2708 }, { "clip_ratio": 0.018034275621175766, "completions/clipped_ratio": 0.0, "completions/max_length": 191.0, "completions/max_terminated_length": 191.0, "completions/mean_length": 139.98214721679688, "completions/mean_terminated_length": 139.98214721679688, "completions/min_length": 102.0, "completions/min_terminated_length": 102.0, "epoch": 0.8229040097205347, "grad_norm": 2.480685624915909, "kl": 2.296875, "learning_rate": 7.328219239095093e-06, "loss": 0.0139, "num_tokens": 12819526.0, "reward": 2.990990400314331, "reward_std": 0.004983409307897091, "rewards/check_gptzero_func/mean": 1.0, "rewards/check_gptzero_func/std": 0.0, "rewards/check_originality_func/mean": 1.0, "rewards/check_originality_func/std": 0.0, "rewards/check_winston_local_func/mean": 0.990990161895752, "rewards/check_winston_local_func/std": 0.006944353226572275, "step": 2709 }, { "clip_ratio": 0.039013300091028214, "epoch": 0.8232077764277035, "grad_norm": 1.5129399111912416, "kl": 2.296875, "learning_rate": 7.32587273286887e-06, "loss": 0.0084, "step": 2710 }, { "clip_ratio": 0.08132874965667725, "epoch": 0.8235115431348724, "grad_norm": 1.3036645130803997, "kl": 2.328125, "learning_rate": 7.3235255727462075e-06, "loss": 0.0025, "step": 2711 }, { "clip_ratio": 0.12197408825159073, "epoch": 0.8238153098420413, "grad_norm": 0.9968017435283731, "kl": 2.328125, "learning_rate": 7.321177759386986e-06, "loss": -0.0009, "step": 2712 }, { "clip_ratio": 0.018085496500134468, "completions/clipped_ratio": 0.0, "completions/max_length": 216.0, "completions/max_terminated_length": 216.0, "completions/mean_length": 166.46429443359375, "completions/mean_terminated_length": 166.46429443359375, "completions/min_length": 120.0, "completions/min_terminated_length": 120.0, "epoch": 0.8241190765492102, "grad_norm": 2.297282188034795, "kl": 2.265625, "learning_rate": 7.318829293451271e-06, "loss": 0.0125, "num_tokens": 12840776.0, "reward": 2.9034223556518555, "reward_std": 0.18351761996746063, "rewards/check_gptzero_func/mean": 0.9285714030265808, "rewards/check_gptzero_func/std": 0.25987008213996887, "rewards/check_originality_func/mean": 0.9821428656578064, "rewards/check_originality_func/std": 0.13363061845302582, "rewards/check_winston_local_func/mean": 0.9927079081535339, "rewards/check_winston_local_func/std": 0.0019396807765588164, "step": 2713 }, { "clip_ratio": 0.032032597810029984, "epoch": 0.824422843256379, "grad_norm": 2.019743898582842, "kl": 2.25, "learning_rate": 7.31648017559931e-06, "loss": 0.004, "step": 2714 }, { "clip_ratio": 0.09137852489948273, "epoch": 0.824726609963548, "grad_norm": 1.4839618942715183, "kl": 2.25, "learning_rate": 7.314130406491533e-06, "loss": -0.0052, "step": 2715 }, { "clip_ratio": 0.15938957035541534, "epoch": 0.8250303766707169, "grad_norm": 1.3161731911761425, "kl": 2.25, "learning_rate": 7.311779986788556e-06, "loss": -0.013, "step": 2716 }, { "clip_ratio": 0.011101010255515575, "completions/clipped_ratio": 0.0, "completions/max_length": 245.0, "completions/max_terminated_length": 245.0, "completions/mean_length": 149.98214721679688, "completions/mean_terminated_length": 149.98214721679688, "completions/min_length": 104.0, "completions/min_terminated_length": 104.0, "epoch": 0.8253341433778858, "grad_norm": 2.929706633002188, "kl": 2.296875, "learning_rate": 7.309428917151179e-06, "loss": 0.0098, "num_tokens": 12860511.0, "reward": 2.971097707748413, "reward_std": 0.05574784800410271, "rewards/check_gptzero_func/mean": 0.9821428656578064, "rewards/check_gptzero_func/std": 0.13363061845302582, "rewards/check_originality_func/mean": 1.0, "rewards/check_originality_func/std": 0.0, "rewards/check_winston_local_func/mean": 0.9889549612998962, "rewards/check_winston_local_func/std": 0.011932704597711563, "step": 2717 }, { "clip_ratio": 0.04554647579789162, "epoch": 0.8256379100850547, "grad_norm": 2.5159705086376536, "kl": 2.296875, "learning_rate": 7.307077198240381e-06, "loss": -0.002, "step": 2718 }, { "clip_ratio": 0.12801583111286163, "epoch": 0.8259416767922235, "grad_norm": 1.7470746899052352, "kl": 2.296875, "learning_rate": 7.304724830717323e-06, "loss": -0.0117, "step": 2719 }, { "clip_ratio": 0.20289812982082367, "epoch": 0.8262454434993924, "grad_norm": 1.2332478862233887, "kl": 2.34375, "learning_rate": 7.30237181524335e-06, "loss": -0.0198, "step": 2720 }, { "clip_ratio": 0.016323432326316833, "completions/clipped_ratio": 0.0, "completions/max_length": 197.0, "completions/max_terminated_length": 197.0, "completions/mean_length": 151.6428680419922, "completions/mean_terminated_length": 151.6428680419922, "completions/min_length": 99.0, "completions/min_terminated_length": 99.0, "epoch": 0.8265492102065614, "grad_norm": 2.4837254637310044, "kl": 2.328125, "learning_rate": 7.300018152479989e-06, "loss": 0.0248, "num_tokens": 12880419.0, "reward": 2.9928901195526123, "reward_std": 0.0019122313242405653, "rewards/check_gptzero_func/mean": 1.0, "rewards/check_gptzero_func/std": 0.0, "rewards/check_originality_func/mean": 1.0, "rewards/check_originality_func/std": 0.0, "rewards/check_winston_local_func/mean": 0.992889940738678, "rewards/check_winston_local_func/std": 0.002156487200409174, "step": 2721 }, { "clip_ratio": 0.03422228619456291, "epoch": 0.8268529769137303, "grad_norm": 1.963042969196983, "kl": 2.34375, "learning_rate": 7.2976638430889535e-06, "loss": 0.0176, "step": 2722 }, { "clip_ratio": 0.08593891561031342, "epoch": 0.8271567436208992, "grad_norm": 1.4803315357517068, "kl": 2.34375, "learning_rate": 7.295308887732131e-06, "loss": 0.0107, "step": 2723 }, { "clip_ratio": 0.15485438704490662, "epoch": 0.827460510328068, "grad_norm": 1.2063827248140624, "kl": 2.359375, "learning_rate": 7.292953287071595e-06, "loss": 0.004, "step": 2724 }, { "clip_ratio": 0.01870381459593773, "completions/clipped_ratio": 0.0, "completions/max_length": 199.0, "completions/max_terminated_length": 199.0, "completions/mean_length": 150.07144165039062, "completions/mean_terminated_length": 150.07144165039062, "completions/min_length": 115.0, "completions/min_terminated_length": 115.0, "epoch": 0.8277642770352369, "grad_norm": 4.486269101734899, "kl": 2.328125, "learning_rate": 7.2905970417696005e-06, "loss": 0.0143, "num_tokens": 12899855.0, "reward": 2.9569807052612305, "reward_std": 0.10285773128271103, "rewards/check_gptzero_func/mean": 0.9642857313156128, "rewards/check_gptzero_func/std": 0.187256321310997, "rewards/check_originality_func/mean": 1.0, "rewards/check_originality_func/std": 0.0, "rewards/check_winston_local_func/mean": 0.9926947951316833, "rewards/check_winston_local_func/std": 0.003458039602264762, "step": 2725 }, { "clip_ratio": 0.04812387749552727, "epoch": 0.8280680437424058, "grad_norm": 3.1103747963732737, "kl": 2.328125, "learning_rate": 7.288240152488583e-06, "loss": 0.0067, "step": 2726 }, { "clip_ratio": 0.10121645033359528, "epoch": 0.8283718104495748, "grad_norm": 1.4584144184518701, "kl": 2.34375, "learning_rate": 7.285882619891157e-06, "loss": -0.0021, "step": 2727 }, { "clip_ratio": 0.14659127593040466, "epoch": 0.8286755771567437, "grad_norm": 1.534427666786109, "kl": 2.359375, "learning_rate": 7.28352444464012e-06, "loss": -0.0067, "step": 2728 }, { "clip_ratio": 0.021694021299481392, "completions/clipped_ratio": 0.0, "completions/max_length": 212.0, "completions/max_terminated_length": 212.0, "completions/mean_length": 159.25, "completions/mean_terminated_length": 159.25, "completions/min_length": 115.0, "completions/min_terminated_length": 115.0, "epoch": 0.8289793438639125, "grad_norm": 3.9637583505440466, "kl": 2.296875, "learning_rate": 7.281165627398453e-06, "loss": 0.0133, "num_tokens": 12920397.0, "reward": 2.956955909729004, "reward_std": 0.10266878455877304, "rewards/check_gptzero_func/mean": 1.0, "rewards/check_gptzero_func/std": 0.0, "rewards/check_originality_func/mean": 0.9642857313156128, "rewards/check_originality_func/std": 0.187256321310997, "rewards/check_winston_local_func/mean": 0.9926700592041016, "rewards/check_winston_local_func/std": 0.0025070738047361374, "step": 2729 }, { "clip_ratio": 0.0529043972492218, "epoch": 0.8292831105710814, "grad_norm": 1.8833977760823972, "kl": 2.296875, "learning_rate": 7.278806168829313e-06, "loss": 0.0038, "step": 2730 }, { "clip_ratio": 0.10568126291036606, "epoch": 0.8295868772782503, "grad_norm": 1.2974463906878226, "kl": 2.34375, "learning_rate": 7.276446069596037e-06, "loss": -0.0015, "step": 2731 }, { "clip_ratio": 0.14318087697029114, "epoch": 0.8298906439854192, "grad_norm": 1.098177360774605, "kl": 2.359375, "learning_rate": 7.274085330362145e-06, "loss": -0.0045, "step": 2732 }, { "clip_ratio": 0.015063631348311901, "completions/clipped_ratio": 0.0, "completions/max_length": 197.0, "completions/max_terminated_length": 197.0, "completions/mean_length": 155.7857208251953, "completions/mean_terminated_length": 155.7857208251953, "completions/min_length": 88.0, "completions/min_terminated_length": 88.0, "epoch": 0.8301944106925881, "grad_norm": 2.1107942622142994, "kl": 2.15625, "learning_rate": 7.271723951791335e-06, "loss": 0.0174, "num_tokens": 12940265.0, "reward": 2.938586711883545, "reward_std": 0.11909795552492142, "rewards/check_gptzero_func/mean": 0.9464285969734192, "rewards/check_gptzero_func/std": 0.22720777988433838, "rewards/check_originality_func/mean": 1.0, "rewards/check_originality_func/std": 0.0, "rewards/check_winston_local_func/mean": 0.9921582341194153, "rewards/check_winston_local_func/std": 0.004506477154791355, "step": 2733 }, { "clip_ratio": 0.03242545202374458, "epoch": 0.830498177399757, "grad_norm": 1.766594728270342, "kl": 2.140625, "learning_rate": 7.269361934547488e-06, "loss": 0.0105, "step": 2734 }, { "clip_ratio": 0.0854564756155014, "epoch": 0.8308019441069259, "grad_norm": 1.2191169051018307, "kl": 2.140625, "learning_rate": 7.2669992792946595e-06, "loss": 0.0027, "step": 2735 }, { "clip_ratio": 0.12426991760730743, "epoch": 0.8311057108140948, "grad_norm": 0.9854571426874257, "kl": 2.15625, "learning_rate": 7.264635986697089e-06, "loss": -0.0008, "step": 2736 }, { "clip_ratio": 0.018627973273396492, "completions/clipped_ratio": 0.0535714285714286, "completions/max_length": 256.0, "completions/max_terminated_length": 190.0, "completions/mean_length": 158.94644165039062, "completions/mean_terminated_length": 153.4528350830078, "completions/min_length": 122.0, "completions/min_terminated_length": 122.0, "epoch": 0.8314094775212637, "grad_norm": 2.742483231111253, "kl": 2.265625, "learning_rate": 7.262272057419193e-06, "loss": 0.0478, "num_tokens": 12960382.0, "reward": 2.9198577404022217, "reward_std": 0.1268029659986496, "rewards/check_gptzero_func/mean": 0.9821428656578064, "rewards/check_gptzero_func/std": 0.13363061845302582, "rewards/check_originality_func/mean": 0.9464285969734192, "rewards/check_originality_func/std": 0.22720777988433838, "rewards/check_winston_local_func/mean": 0.9912859797477722, "rewards/check_winston_local_func/std": 0.005602674093097448, "step": 2737 }, { "clip_ratio": 0.037137072533369064, "epoch": 0.8317132442284325, "grad_norm": 2.1458174142566704, "kl": 2.25, "learning_rate": 7.259907492125565e-06, "loss": 0.0428, "step": 2738 }, { "clip_ratio": 0.07567007094621658, "epoch": 0.8320170109356014, "grad_norm": 1.631006989031098, "kl": 2.265625, "learning_rate": 7.2575422914809815e-06, "loss": 0.0354, "step": 2739 }, { "clip_ratio": 0.1368006467819214, "epoch": 0.8323207776427703, "grad_norm": 1.3063096355019386, "kl": 2.265625, "learning_rate": 7.255176456150395e-06, "loss": 0.0285, "step": 2740 }, { "clip_ratio": 0.013657049275934696, "completions/clipped_ratio": 0.0, "completions/max_length": 213.0, "completions/max_terminated_length": 213.0, "completions/mean_length": 171.8928680419922, "completions/mean_terminated_length": 171.8928680419922, "completions/min_length": 118.0, "completions/min_terminated_length": 118.0, "epoch": 0.8326245443499393, "grad_norm": 4.46216944706139, "kl": 2.09375, "learning_rate": 7.2528099867989366e-06, "loss": 0.0218, "num_tokens": 12981920.0, "reward": 2.9753479957580566, "reward_std": 0.05206143110990524, "rewards/check_gptzero_func/mean": 0.9821428656578064, "rewards/check_gptzero_func/std": 0.13363061845302582, "rewards/check_originality_func/mean": 1.0, "rewards/check_originality_func/std": 0.0, "rewards/check_winston_local_func/mean": 0.9932050704956055, "rewards/check_winston_local_func/std": 0.002219654619693756, "step": 2741 }, { "clip_ratio": 0.031639326363801956, "epoch": 0.8329283110571082, "grad_norm": 3.0451692960186736, "kl": 2.09375, "learning_rate": 7.250442884091917e-06, "loss": 0.0112, "step": 2742 }, { "clip_ratio": 0.10218121856451035, "epoch": 0.833232077764277, "grad_norm": 2.296585211110131, "kl": 2.109375, "learning_rate": 7.248075148694824e-06, "loss": 0.0025, "step": 2743 }, { "clip_ratio": 0.1526549756526947, "epoch": 0.8335358444714459, "grad_norm": 1.3538511189992735, "kl": 2.140625, "learning_rate": 7.245706781273323e-06, "loss": -0.0038, "step": 2744 }, { "clip_ratio": 0.014281312935054302, "completions/clipped_ratio": 0.0, "completions/max_length": 237.0, "completions/max_terminated_length": 237.0, "completions/mean_length": 167.94644165039062, "completions/mean_terminated_length": 167.94644165039062, "completions/min_length": 105.0, "completions/min_terminated_length": 105.0, "epoch": 0.8338396111786148, "grad_norm": 3.593130146067608, "kl": 2.21875, "learning_rate": 7.243337782493255e-06, "loss": 0.0184, "num_tokens": 13002861.0, "reward": 2.8794755935668945, "reward_std": 0.21964305639266968, "rewards/check_gptzero_func/mean": 0.9285714030265808, "rewards/check_gptzero_func/std": 0.25987008213996887, "rewards/check_originality_func/mean": 0.9642857313156128, "rewards/check_originality_func/std": 0.187256321310997, "rewards/check_winston_local_func/mean": 0.9866183400154114, "rewards/check_winston_local_func/std": 0.028185535222291946, "step": 2745 }, { "clip_ratio": 0.049641720950603485, "epoch": 0.8341433778857837, "grad_norm": 2.7219535850142784, "kl": 2.21875, "learning_rate": 7.240968153020645e-06, "loss": 0.0049, "step": 2746 }, { "clip_ratio": 0.12346158921718597, "epoch": 0.8344471445929527, "grad_norm": 1.8503404143766133, "kl": 2.234375, "learning_rate": 7.23859789352169e-06, "loss": -0.0063, "step": 2747 }, { "clip_ratio": 0.20639833807945251, "epoch": 0.8347509113001215, "grad_norm": 1.449056714327881, "kl": 2.265625, "learning_rate": 7.236227004662764e-06, "loss": -0.0141, "step": 2748 }, { "clip_ratio": 0.021974246948957443, "completions/clipped_ratio": 0.0, "completions/max_length": 235.0, "completions/max_terminated_length": 235.0, "completions/mean_length": 166.67857360839844, "completions/mean_terminated_length": 166.67857360839844, "completions/min_length": 120.0, "completions/min_terminated_length": 120.0, "epoch": 0.8350546780072904, "grad_norm": 3.90247437388308, "kl": 2.21875, "learning_rate": 7.2338554871104195e-06, "loss": 0.0042, "num_tokens": 13023579.0, "reward": 2.974055528640747, "reward_std": 0.05231037363409996, "rewards/check_gptzero_func/mean": 0.9821428656578064, "rewards/check_gptzero_func/std": 0.13363061845302582, "rewards/check_originality_func/mean": 1.0, "rewards/check_originality_func/std": 0.0, "rewards/check_winston_local_func/mean": 0.9919127225875854, "rewards/check_winston_local_func/std": 0.0021135960705578327, "step": 2749 }, { "clip_ratio": 0.045119185000658035, "epoch": 0.8353584447144593, "grad_norm": 2.7897070509672566, "kl": 2.203125, "learning_rate": 7.231483341531387e-06, "loss": 0.0027, "step": 2750 }, { "clip_ratio": 0.10599268972873688, "epoch": 0.8356622114216282, "grad_norm": 1.678756320258366, "kl": 2.203125, "learning_rate": 7.229110568592571e-06, "loss": -0.0101, "step": 2751 }, { "clip_ratio": 0.15338288247585297, "epoch": 0.8359659781287971, "grad_norm": 1.5642083960226463, "kl": 2.203125, "learning_rate": 7.226737168961054e-06, "loss": -0.0175, "step": 2752 }, { "clip_ratio": 0.019785627722740173, "completions/clipped_ratio": 0.0, "completions/max_length": 236.0, "completions/max_terminated_length": 236.0, "completions/mean_length": 167.2857208251953, "completions/mean_terminated_length": 167.2857208251953, "completions/min_length": 120.0, "completions/min_terminated_length": 120.0, "epoch": 0.836269744835966, "grad_norm": 5.975632472003448, "kl": 2.078125, "learning_rate": 7.224363143304093e-06, "loss": 0.016, "num_tokens": 13044691.0, "reward": 2.975367307662964, "reward_std": 0.05228177458047867, "rewards/check_gptzero_func/mean": 1.0, "rewards/check_gptzero_func/std": 0.0, "rewards/check_originality_func/mean": 0.9821428656578064, "rewards/check_originality_func/std": 0.13363061845302582, "rewards/check_winston_local_func/mean": 0.993224561214447, "rewards/check_winston_local_func/std": 0.0022579869255423546, "step": 2753 }, { "clip_ratio": 0.053123414516448975, "epoch": 0.8365735115431349, "grad_norm": 3.736664699459627, "kl": 2.078125, "learning_rate": 7.221988492289125e-06, "loss": 0.0008, "step": 2754 }, { "clip_ratio": 0.09571663290262222, "epoch": 0.8368772782503038, "grad_norm": 2.6547900121438484, "kl": 2.140625, "learning_rate": 7.2196132165837564e-06, "loss": -0.0076, "step": 2755 }, { "clip_ratio": 0.14699959754943848, "epoch": 0.8371810449574727, "grad_norm": 1.6750900451904918, "kl": 2.203125, "learning_rate": 7.217237316855774e-06, "loss": -0.0155, "step": 2756 }, { "clip_ratio": 0.020366033539175987, "completions/clipped_ratio": 0.0357142857142857, "completions/max_length": 256.0, "completions/max_terminated_length": 245.0, "completions/mean_length": 158.71429443359375, "completions/mean_terminated_length": 155.11111450195312, "completions/min_length": 95.0, "completions/min_terminated_length": 95.0, "epoch": 0.8374848116646415, "grad_norm": 3.000767244432168, "kl": 2.546875, "learning_rate": 7.214860793773138e-06, "loss": 0.0048, "num_tokens": 13064763.0, "reward": 2.99228835105896, "reward_std": 0.0026335695292800665, "rewards/check_gptzero_func/mean": 1.0, "rewards/check_gptzero_func/std": 0.0, "rewards/check_originality_func/mean": 1.0, "rewards/check_originality_func/std": 0.0, "rewards/check_winston_local_func/mean": 0.9922882914543152, "rewards/check_winston_local_func/std": 0.0035117596853524446, "step": 2757 }, { "clip_ratio": 0.036700665950775146, "epoch": 0.8377885783718104, "grad_norm": 2.646059190485799, "kl": 2.578125, "learning_rate": 7.212483648003989e-06, "loss": -0.0034, "step": 2758 }, { "clip_ratio": 0.06434240192174911, "epoch": 0.8380923450789793, "grad_norm": 2.301358072847025, "kl": 2.578125, "learning_rate": 7.210105880216634e-06, "loss": -0.0101, "step": 2759 }, { "clip_ratio": 0.11278843879699707, "epoch": 0.8383961117861483, "grad_norm": 1.6885219945681729, "kl": 2.53125, "learning_rate": 7.2077274910795605e-06, "loss": -0.0186, "step": 2760 }, { "clip_ratio": 0.021010566502809525, "completions/clipped_ratio": 0.0535714285714286, "completions/max_length": 256.0, "completions/max_terminated_length": 250.0, "completions/mean_length": 186.83929443359375, "completions/mean_terminated_length": 182.92453002929688, "completions/min_length": 122.0, "completions/min_terminated_length": 122.0, "epoch": 0.8386998784933172, "grad_norm": 5.76618990764804, "kl": 2.015625, "learning_rate": 7.205348481261429e-06, "loss": 0.004, "num_tokens": 13087202.0, "reward": 2.9201748371124268, "reward_std": 0.07738035917282104, "rewards/check_gptzero_func/mean": 1.0, "rewards/check_gptzero_func/std": 0.0, "rewards/check_originality_func/mean": 0.9285714030265808, "rewards/check_originality_func/std": 0.25987008213996887, "rewards/check_winston_local_func/mean": 0.9916031956672668, "rewards/check_winston_local_func/std": 0.005367598030716181, "step": 2761 }, { "clip_ratio": 0.03990182653069496, "epoch": 0.839003645200486, "grad_norm": 5.034224347431938, "kl": 2.015625, "learning_rate": 7.2029688514310755e-06, "loss": -0.011, "step": 2762 }, { "clip_ratio": 0.12375663965940475, "epoch": 0.8393074119076549, "grad_norm": 2.845606704499491, "kl": 1.984375, "learning_rate": 7.20058860225751e-06, "loss": -0.0289, "step": 2763 }, { "clip_ratio": 0.22961261868476868, "epoch": 0.8396111786148238, "grad_norm": 2.0989723449981756, "kl": 1.9609375, "learning_rate": 7.198207734409914e-06, "loss": -0.0392, "step": 2764 }, { "clip_ratio": 0.023185184225440025, "completions/clipped_ratio": 0.0, "completions/max_length": 235.0, "completions/max_terminated_length": 235.0, "completions/mean_length": 172.17857360839844, "completions/mean_terminated_length": 172.17857360839844, "completions/min_length": 133.0, "completions/min_terminated_length": 133.0, "epoch": 0.8399149453219927, "grad_norm": 3.605469296550882, "kl": 1.9921875, "learning_rate": 7.195826248557649e-06, "loss": 0.0334, "num_tokens": 13108740.0, "reward": 2.992142915725708, "reward_std": 0.0021136365830898285, "rewards/check_gptzero_func/mean": 1.0, "rewards/check_gptzero_func/std": 0.0, "rewards/check_originality_func/mean": 1.0, "rewards/check_originality_func/std": 0.0, "rewards/check_winston_local_func/mean": 0.9921427965164185, "rewards/check_winston_local_func/std": 0.0025610378943383694, "step": 2765 }, { "clip_ratio": 0.030870964750647545, "epoch": 0.8402187120291617, "grad_norm": 2.822349608393294, "kl": 2.0, "learning_rate": 7.1934441453702465e-06, "loss": 0.0251, "step": 2766 }, { "clip_ratio": 0.08346960693597794, "epoch": 0.8405224787363305, "grad_norm": 2.312746410023578, "kl": 2.015625, "learning_rate": 7.191061425517407e-06, "loss": 0.0164, "step": 2767 }, { "clip_ratio": 0.17587684094905853, "epoch": 0.8408262454434994, "grad_norm": 1.4377098065042293, "kl": 2.03125, "learning_rate": 7.188678089669014e-06, "loss": 0.0068, "step": 2768 }, { "clip_ratio": 0.01364436186850071, "completions/clipped_ratio": 0.0, "completions/max_length": 227.0, "completions/max_terminated_length": 227.0, "completions/mean_length": 162.80357360839844, "completions/mean_terminated_length": 162.80357360839844, "completions/min_length": 93.0, "completions/min_terminated_length": 93.0, "epoch": 0.8411300121506683, "grad_norm": 5.2053324946910005, "kl": 2.015625, "learning_rate": 7.1862941384951135e-06, "loss": 0.0192, "num_tokens": 13129601.0, "reward": 2.973881959915161, "reward_std": 0.05415954440832138, "rewards/check_gptzero_func/mean": 1.0, "rewards/check_gptzero_func/std": 0.0, "rewards/check_originality_func/mean": 0.9821428656578064, "rewards/check_originality_func/std": 0.13363061845302582, "rewards/check_winston_local_func/mean": 0.9917391538619995, "rewards/check_winston_local_func/std": 0.00470338761806488, "step": 2769 }, { "clip_ratio": 0.04759344831109047, "epoch": 0.8414337788578372, "grad_norm": 3.2189089524211045, "kl": 2.03125, "learning_rate": 7.183909572665936e-06, "loss": 0.0069, "step": 2770 }, { "clip_ratio": 0.09992735832929611, "epoch": 0.8417375455650061, "grad_norm": 1.5961645445013117, "kl": 2.046875, "learning_rate": 7.1815243928518764e-06, "loss": -0.0018, "step": 2771 }, { "clip_ratio": 0.16776929795742035, "epoch": 0.8420413122721749, "grad_norm": 1.1693390186279695, "kl": 2.09375, "learning_rate": 7.179138599723504e-06, "loss": -0.0054, "step": 2772 }, { "clip_ratio": 0.022855352610349655, "completions/clipped_ratio": 0.0, "completions/max_length": 232.0, "completions/max_terminated_length": 232.0, "completions/mean_length": 177.58929443359375, "completions/mean_terminated_length": 177.58929443359375, "completions/min_length": 115.0, "completions/min_terminated_length": 115.0, "epoch": 0.8423450789793439, "grad_norm": 4.847216692133516, "kl": 2.171875, "learning_rate": 7.176752193951563e-06, "loss": 0.0116, "num_tokens": 13151514.0, "reward": 2.992648124694824, "reward_std": 0.002182670868933201, "rewards/check_gptzero_func/mean": 1.0, "rewards/check_gptzero_func/std": 0.0, "rewards/check_originality_func/mean": 1.0, "rewards/check_originality_func/std": 0.0, "rewards/check_winston_local_func/mean": 0.9926480054855347, "rewards/check_winston_local_func/std": 0.002722113626077771, "step": 2773 }, { "clip_ratio": 0.060592930763959885, "epoch": 0.8426488456865128, "grad_norm": 3.1801300957039107, "kl": 2.125, "learning_rate": 7.174365176206966e-06, "loss": 0.0048, "step": 2774 }, { "clip_ratio": 0.11336313188076019, "epoch": 0.8429526123936817, "grad_norm": 2.1408298219951725, "kl": 2.109375, "learning_rate": 7.1719775471608025e-06, "loss": -0.0084, "step": 2775 }, { "clip_ratio": 0.18350611627101898, "epoch": 0.8432563791008505, "grad_norm": 1.4843625023568, "kl": 2.109375, "learning_rate": 7.169589307484327e-06, "loss": -0.0165, "step": 2776 }, { "clip_ratio": 0.02453210949897766, "completions/clipped_ratio": 0.0535714285714286, "completions/max_length": 256.0, "completions/max_terminated_length": 248.0, "completions/mean_length": 173.19644165039062, "completions/mean_terminated_length": 168.50942993164062, "completions/min_length": 121.0, "completions/min_terminated_length": 121.0, "epoch": 0.8435601458080194, "grad_norm": 4.749196099580069, "kl": 2.21875, "learning_rate": 7.167200457848974e-06, "loss": 0.0402, "num_tokens": 13172565.0, "reward": 2.9371578693389893, "reward_std": 0.12331020832061768, "rewards/check_gptzero_func/mean": 1.0, "rewards/check_gptzero_func/std": 0.0, "rewards/check_originality_func/mean": 0.9464285969734192, "rewards/check_originality_func/std": 0.22720777988433838, "rewards/check_winston_local_func/mean": 0.9907292723655701, "rewards/check_winston_local_func/std": 0.014996908605098724, "step": 2777 }, { "clip_ratio": 0.038593802601099014, "epoch": 0.8438639125151883, "grad_norm": 3.6750084892356796, "kl": 2.234375, "learning_rate": 7.164810998926345e-06, "loss": 0.029, "step": 2778 }, { "clip_ratio": 0.1076098307967186, "epoch": 0.8441676792223572, "grad_norm": 3.3237604365790805, "kl": 2.265625, "learning_rate": 7.16242093138821e-06, "loss": 0.0147, "step": 2779 }, { "clip_ratio": 0.17800545692443848, "epoch": 0.8444714459295262, "grad_norm": 3.252514657404133, "kl": 2.265625, "learning_rate": 7.160030255906516e-06, "loss": 0.0049, "step": 2780 }, { "clip_ratio": 0.013404933735728264, "completions/clipped_ratio": 0.0, "completions/max_length": 213.0, "completions/max_terminated_length": 213.0, "completions/mean_length": 162.07144165039062, "completions/mean_terminated_length": 162.07144165039062, "completions/min_length": 122.0, "completions/min_terminated_length": 122.0, "epoch": 0.844775212636695, "grad_norm": 4.400415681284922, "kl": 2.09375, "learning_rate": 7.157638973153375e-06, "loss": 0.0154, "num_tokens": 13193353.0, "reward": 2.9934420585632324, "reward_std": 0.0019292098004370928, "rewards/check_gptzero_func/mean": 1.0, "rewards/check_gptzero_func/std": 0.0, "rewards/check_originality_func/mean": 1.0, "rewards/check_originality_func/std": 0.0, "rewards/check_winston_local_func/mean": 0.9934418797492981, "rewards/check_winston_local_func/std": 0.0025751707144081593, "step": 2781 }, { "clip_ratio": 0.034056805074214935, "epoch": 0.8450789793438639, "grad_norm": 3.815002763843414, "kl": 2.09375, "learning_rate": 7.155247083801076e-06, "loss": 0.007, "step": 2782 }, { "clip_ratio": 0.08322127908468246, "epoch": 0.8453827460510328, "grad_norm": 2.452093769202848, "kl": 2.140625, "learning_rate": 7.1528545885220744e-06, "loss": -0.0039, "step": 2783 }, { "clip_ratio": 0.13489791750907898, "epoch": 0.8456865127582017, "grad_norm": 1.793933226213445, "kl": 2.140625, "learning_rate": 7.1504614879889975e-06, "loss": -0.0105, "step": 2784 }, { "clip_ratio": 0.023507023230195045, "completions/clipped_ratio": 0.0535714285714286, "completions/max_length": 256.0, "completions/max_terminated_length": 243.0, "completions/mean_length": 178.7857208251953, "completions/mean_terminated_length": 174.41510009765625, "completions/min_length": 125.0, "completions/min_terminated_length": 125.0, "epoch": 0.8459902794653706, "grad_norm": 5.2877499130590015, "kl": 2.21875, "learning_rate": 7.14806778287464e-06, "loss": 0.0002, "num_tokens": 13215621.0, "reward": 2.9752092361450195, "reward_std": 0.051915477961301804, "rewards/check_gptzero_func/mean": 0.9821428656578064, "rewards/check_gptzero_func/std": 0.13363061845302582, "rewards/check_originality_func/mean": 1.0, "rewards/check_originality_func/std": 0.0, "rewards/check_winston_local_func/mean": 0.9930662512779236, "rewards/check_winston_local_func/std": 0.0018200349295511842, "step": 2785 }, { "clip_ratio": 0.05655089393258095, "epoch": 0.8462940461725394, "grad_norm": 3.394374437921403, "kl": 2.265625, "learning_rate": 7.145673473851972e-06, "loss": -0.0142, "step": 2786 }, { "clip_ratio": 0.09768977016210556, "epoch": 0.8465978128797084, "grad_norm": 3.236716338799739, "kl": 2.328125, "learning_rate": 7.143278561594127e-06, "loss": -0.0202, "step": 2787 }, { "clip_ratio": 0.15738539397716522, "epoch": 0.8469015795868773, "grad_norm": 1.6593418075825792, "kl": 2.390625, "learning_rate": 7.140883046774412e-06, "loss": -0.0292, "step": 2788 }, { "clip_ratio": 0.019903942942619324, "completions/clipped_ratio": 0.0, "completions/max_length": 233.0, "completions/max_terminated_length": 233.0, "completions/mean_length": 170.25, "completions/mean_terminated_length": 170.25, "completions/min_length": 132.0, "completions/min_terminated_length": 132.0, "epoch": 0.8472053462940462, "grad_norm": 4.162833931930915, "kl": 2.265625, "learning_rate": 7.1384869300663025e-06, "loss": 0.019, "num_tokens": 13237003.0, "reward": 2.993757486343384, "reward_std": 0.0014085003640502691, "rewards/check_gptzero_func/mean": 1.0, "rewards/check_gptzero_func/std": 0.0, "rewards/check_originality_func/mean": 1.0, "rewards/check_originality_func/std": 0.0, "rewards/check_winston_local_func/mean": 0.9937573075294495, "rewards/check_winston_local_func/std": 0.001838683383539319, "step": 2789 }, { "clip_ratio": 0.07970471680164337, "epoch": 0.8475091130012151, "grad_norm": 2.720250467941742, "kl": 2.234375, "learning_rate": 7.136090212143447e-06, "loss": 0.0059, "step": 2790 }, { "clip_ratio": 0.1408817619085312, "epoch": 0.8478128797083839, "grad_norm": 1.599836434588104, "kl": 2.203125, "learning_rate": 7.133692893679655e-06, "loss": -0.0017, "step": 2791 }, { "clip_ratio": 0.20767594873905182, "epoch": 0.8481166464155528, "grad_norm": 2.8264226026826957, "kl": 2.171875, "learning_rate": 7.1312949753489106e-06, "loss": -0.0071, "step": 2792 }, { "clip_ratio": 0.020969578996300697, "completions/clipped_ratio": 0.0, "completions/max_length": 231.0, "completions/max_terminated_length": 231.0, "completions/mean_length": 156.23214721679688, "completions/mean_terminated_length": 156.23214721679688, "completions/min_length": 99.0, "completions/min_terminated_length": 99.0, "epoch": 0.8484204131227218, "grad_norm": 5.101102672538129, "kl": 2.1875, "learning_rate": 7.128896457825364e-06, "loss": 0.0136, "num_tokens": 13257480.0, "reward": 2.938149929046631, "reward_std": 0.15286122262477875, "rewards/check_gptzero_func/mean": 0.9464285969734192, "rewards/check_gptzero_func/std": 0.22720777988433838, "rewards/check_originality_func/mean": 1.0, "rewards/check_originality_func/std": 0.0, "rewards/check_winston_local_func/mean": 0.9917212724685669, "rewards/check_winston_local_func/std": 0.002812299644574523, "step": 2793 }, { "clip_ratio": 0.03964471444487572, "epoch": 0.8487241798298907, "grad_norm": 3.0470038409121796, "kl": 2.21875, "learning_rate": 7.126497341783337e-06, "loss": 0.0014, "step": 2794 }, { "clip_ratio": 0.09984607994556427, "epoch": 0.8490279465370595, "grad_norm": 1.9367938373304616, "kl": 2.234375, "learning_rate": 7.124097627897317e-06, "loss": -0.0093, "step": 2795 }, { "clip_ratio": 0.18605436384677887, "epoch": 0.8493317132442284, "grad_norm": 1.2854113971855914, "kl": 2.28125, "learning_rate": 7.12169731684196e-06, "loss": -0.0147, "step": 2796 }, { "clip_ratio": 0.023348616436123848, "completions/clipped_ratio": 0.017857142857142905, "completions/max_length": 256.0, "completions/max_terminated_length": 209.0, "completions/mean_length": 150.58929443359375, "completions/mean_terminated_length": 148.6727294921875, "completions/min_length": 93.0, "completions/min_terminated_length": 93.0, "epoch": 0.8496354799513973, "grad_norm": 5.139714513276113, "kl": 2.53125, "learning_rate": 7.119296409292091e-06, "loss": 0.0177, "num_tokens": 13276937.0, "reward": 2.9754159450531006, "reward_std": 0.05179673805832863, "rewards/check_gptzero_func/mean": 1.0, "rewards/check_gptzero_func/std": 0.0, "rewards/check_originality_func/mean": 0.9821428656578064, "rewards/check_originality_func/std": 0.13363061845302582, "rewards/check_winston_local_func/mean": 0.9932729601860046, "rewards/check_winston_local_func/std": 0.0021204554941505194, "step": 2797 }, { "clip_ratio": 0.06165309622883797, "epoch": 0.8499392466585662, "grad_norm": 3.2009187755916617, "kl": 2.53125, "learning_rate": 7.1168949059226984e-06, "loss": 0.0042, "step": 2798 }, { "clip_ratio": 0.11951414495706558, "epoch": 0.8502430133657352, "grad_norm": 2.285808865838868, "kl": 2.59375, "learning_rate": 7.114492807408945e-06, "loss": -0.0037, "step": 2799 }, { "clip_ratio": 0.17253825068473816, "epoch": 0.850546780072904, "grad_norm": 1.2133893137872498, "kl": 2.609375, "learning_rate": 7.112090114426157e-06, "loss": -0.0099, "step": 2800 }, { "clip_ratio": 0.022641530260443687, "completions/clipped_ratio": 0.0, "completions/max_length": 186.0, "completions/max_terminated_length": 186.0, "completions/mean_length": 141.98214721679688, "completions/mean_terminated_length": 141.98214721679688, "completions/min_length": 75.0, "completions/min_terminated_length": 75.0, "epoch": 0.8508505467800729, "grad_norm": 5.094749231372898, "kl": 2.59375, "learning_rate": 7.109686827649826e-06, "loss": 0.0102, "num_tokens": 13295960.0, "reward": 2.99283504486084, "reward_std": 0.003003594232723117, "rewards/check_gptzero_func/mean": 1.0, "rewards/check_gptzero_func/std": 0.0, "rewards/check_originality_func/mean": 1.0, "rewards/check_originality_func/std": 0.0, "rewards/check_winston_local_func/mean": 0.9928350448608398, "rewards/check_winston_local_func/std": 0.003532466245815158, "step": 2801 }, { "clip_ratio": 0.07318922132253647, "epoch": 0.8511543134872418, "grad_norm": 2.135688745226668, "kl": 2.59375, "learning_rate": 7.107282947755617e-06, "loss": -0.0036, "step": 2802 }, { "clip_ratio": 0.10825807601213455, "epoch": 0.8514580801944107, "grad_norm": 1.550813624487324, "kl": 2.609375, "learning_rate": 7.104878475419352e-06, "loss": -0.008, "step": 2803 }, { "clip_ratio": 0.15434782207012177, "epoch": 0.8517618469015796, "grad_norm": 1.2999896107447064, "kl": 2.59375, "learning_rate": 7.102473411317028e-06, "loss": -0.0127, "step": 2804 }, { "clip_ratio": 0.017656611278653145, "completions/clipped_ratio": 0.017857142857142905, "completions/max_length": 256.0, "completions/max_terminated_length": 221.0, "completions/mean_length": 162.2857208251953, "completions/mean_terminated_length": 160.58181762695312, "completions/min_length": 40.0, "completions/min_terminated_length": 40.0, "epoch": 0.8520656136087484, "grad_norm": 3.116746489134023, "kl": 2.03125, "learning_rate": 7.100067756124803e-06, "loss": 0.0073, "num_tokens": 13317264.0, "reward": 2.95637583732605, "reward_std": 0.10281423479318619, "rewards/check_gptzero_func/mean": 0.9642857313156128, "rewards/check_gptzero_func/std": 0.187256321310997, "rewards/check_originality_func/mean": 1.0, "rewards/check_originality_func/std": 0.0, "rewards/check_winston_local_func/mean": 0.9920900464057922, "rewards/check_winston_local_func/std": 0.0029151965864002705, "step": 2805 }, { "clip_ratio": 0.03527214005589485, "epoch": 0.8523693803159174, "grad_norm": 3.2022260309386303, "kl": 2.03125, "learning_rate": 7.097661510519007e-06, "loss": -0.001, "step": 2806 }, { "clip_ratio": 0.09407271444797516, "epoch": 0.8526731470230863, "grad_norm": 1.298938708123232, "kl": 2.046875, "learning_rate": 7.095254675176131e-06, "loss": -0.0105, "step": 2807 }, { "clip_ratio": 0.13329751789569855, "epoch": 0.8529769137302552, "grad_norm": 1.1499126947460572, "kl": 2.078125, "learning_rate": 7.092847250772832e-06, "loss": -0.0159, "step": 2808 }, { "clip_ratio": 0.019582360982894897, "completions/clipped_ratio": 0.0, "completions/max_length": 224.0, "completions/max_terminated_length": 224.0, "completions/mean_length": 156.46429443359375, "completions/mean_terminated_length": 156.46429443359375, "completions/min_length": 104.0, "completions/min_terminated_length": 104.0, "epoch": 0.8532806804374241, "grad_norm": 3.534761893348813, "kl": 2.296875, "learning_rate": 7.090439237985934e-06, "loss": 0.0085, "num_tokens": 13337594.0, "reward": 2.9754951000213623, "reward_std": 0.052140410989522934, "rewards/check_gptzero_func/mean": 0.9821428656578064, "rewards/check_gptzero_func/std": 0.13363061845302582, "rewards/check_originality_func/mean": 1.0, "rewards/check_originality_func/std": 0.0, "rewards/check_winston_local_func/mean": 0.9933520555496216, "rewards/check_winston_local_func/std": 0.0018740460509434342, "step": 2809 }, { "clip_ratio": 0.028271440416574478, "epoch": 0.8535844471445929, "grad_norm": 3.3838394737541813, "kl": 2.296875, "learning_rate": 7.088030637492429e-06, "loss": 0.0048, "step": 2810 }, { "clip_ratio": 0.10545729100704193, "epoch": 0.8538882138517618, "grad_norm": 1.3327239551430803, "kl": 2.34375, "learning_rate": 7.085621449969468e-06, "loss": -0.0059, "step": 2811 }, { "clip_ratio": 0.15105196833610535, "epoch": 0.8541919805589308, "grad_norm": 1.7463047114263848, "kl": 2.359375, "learning_rate": 7.083211676094373e-06, "loss": -0.0115, "step": 2812 }, { "clip_ratio": 0.016402455046772957, "completions/clipped_ratio": 0.0, "completions/max_length": 225.0, "completions/max_terminated_length": 225.0, "completions/mean_length": 156.4107208251953, "completions/mean_terminated_length": 156.4107208251953, "completions/min_length": 99.0, "completions/min_terminated_length": 99.0, "epoch": 0.8544957472660997, "grad_norm": 4.577249535386419, "kl": 2.234375, "learning_rate": 7.080801316544628e-06, "loss": 0.0277, "num_tokens": 13357793.0, "reward": 2.9936864376068115, "reward_std": 0.001865399768576026, "rewards/check_gptzero_func/mean": 1.0, "rewards/check_gptzero_func/std": 0.0, "rewards/check_originality_func/mean": 1.0, "rewards/check_originality_func/std": 0.0, "rewards/check_winston_local_func/mean": 0.993686318397522, "rewards/check_winston_local_func/std": 0.002051419345661998, "step": 2813 }, { "clip_ratio": 0.09374070167541504, "epoch": 0.8547995139732685, "grad_norm": 2.678171779152002, "kl": 2.265625, "learning_rate": 7.078390371997882e-06, "loss": 0.0154, "step": 2814 }, { "clip_ratio": 0.16923286020755768, "epoch": 0.8551032806804374, "grad_norm": 2.745929784930188, "kl": 2.28125, "learning_rate": 7.0759788431319475e-06, "loss": 0.0077, "step": 2815 }, { "clip_ratio": 0.2123192697763443, "epoch": 0.8554070473876063, "grad_norm": 1.8351790091042435, "kl": 2.265625, "learning_rate": 7.073566730624804e-06, "loss": -0.0003, "step": 2816 }, { "clip_ratio": 0.024562295526266098, "completions/clipped_ratio": 0.0, "completions/max_length": 223.0, "completions/max_terminated_length": 223.0, "completions/mean_length": 162.19644165039062, "completions/mean_terminated_length": 162.19644165039062, "completions/min_length": 110.0, "completions/min_terminated_length": 110.0, "epoch": 0.8557108140947752, "grad_norm": 3.9218467533734085, "kl": 2.25, "learning_rate": 7.0711540351545915e-06, "loss": 0.0069, "num_tokens": 13378388.0, "reward": 2.9583382606506348, "reward_std": 0.1018020436167717, "rewards/check_gptzero_func/mean": 0.9821428656578064, "rewards/check_gptzero_func/std": 0.13363061845302582, "rewards/check_originality_func/mean": 0.9821428656578064, "rewards/check_originality_func/std": 0.13363061845302582, "rewards/check_winston_local_func/mean": 0.9940523505210876, "rewards/check_winston_local_func/std": 0.001741789048537612, "step": 2817 }, { "clip_ratio": 0.050208527594804764, "epoch": 0.8560145808019441, "grad_norm": 3.3732050879831283, "kl": 2.25, "learning_rate": 7.068740757399618e-06, "loss": -0.0026, "step": 2818 }, { "clip_ratio": 0.12475419789552689, "epoch": 0.856318347509113, "grad_norm": 1.5320206997707868, "kl": 2.234375, "learning_rate": 7.0663268980383515e-06, "loss": -0.0089, "step": 2819 }, { "clip_ratio": 0.18905937671661377, "epoch": 0.8566221142162819, "grad_norm": 1.5567007926729919, "kl": 2.25, "learning_rate": 7.063912457749426e-06, "loss": -0.016, "step": 2820 }, { "clip_ratio": 0.023044168949127197, "completions/clipped_ratio": 0.0, "completions/max_length": 219.0, "completions/max_terminated_length": 219.0, "completions/mean_length": 147.0178680419922, "completions/mean_terminated_length": 147.0178680419922, "completions/min_length": 100.0, "completions/min_terminated_length": 100.0, "epoch": 0.8569258809234508, "grad_norm": 4.781806422647491, "kl": 2.484375, "learning_rate": 7.061497437211636e-06, "loss": 0.0318, "num_tokens": 13397509.0, "reward": 2.992464542388916, "reward_std": 0.0019653458148241043, "rewards/check_gptzero_func/mean": 1.0, "rewards/check_gptzero_func/std": 0.0, "rewards/check_originality_func/mean": 1.0, "rewards/check_originality_func/std": 0.0, "rewards/check_winston_local_func/mean": 0.9924640655517578, "rewards/check_winston_local_func/std": 0.0020775224547833204, "step": 2821 }, { "clip_ratio": 0.07941297441720963, "epoch": 0.8572296476306197, "grad_norm": 3.273846742399244, "kl": 2.484375, "learning_rate": 7.059081837103943e-06, "loss": 0.0133, "step": 2822 }, { "clip_ratio": 0.20035460591316223, "epoch": 0.8575334143377886, "grad_norm": 1.3895313343438869, "kl": 2.53125, "learning_rate": 7.056665658105469e-06, "loss": 0.0074, "step": 2823 }, { "clip_ratio": 0.24145933985710144, "epoch": 0.8578371810449574, "grad_norm": 1.4637301249382777, "kl": 2.59375, "learning_rate": 7.054248900895498e-06, "loss": 0.0018, "step": 2824 }, { "clip_ratio": 0.021087434142827988, "completions/clipped_ratio": 0.0357142857142857, "completions/max_length": 256.0, "completions/max_terminated_length": 239.0, "completions/mean_length": 167.7678680419922, "completions/mean_terminated_length": 164.5, "completions/min_length": 132.0, "completions/min_terminated_length": 132.0, "epoch": 0.8581409477521263, "grad_norm": 8.047518893309048, "kl": 2.40625, "learning_rate": 7.0518315661534785e-06, "loss": 0.042, "num_tokens": 13418760.0, "reward": 2.9729509353637695, "reward_std": 0.052836377173662186, "rewards/check_gptzero_func/mean": 0.9821428656578064, "rewards/check_gptzero_func/std": 0.13363061845302582, "rewards/check_originality_func/mean": 1.0, "rewards/check_originality_func/std": 0.0, "rewards/check_winston_local_func/mean": 0.9908079504966736, "rewards/check_winston_local_func/std": 0.010717153549194336, "step": 2825 }, { "clip_ratio": 0.037351105362176895, "epoch": 0.8584447144592953, "grad_norm": 5.201143265327784, "kl": 2.421875, "learning_rate": 7.049413654559021e-06, "loss": 0.0345, "step": 2826 }, { "clip_ratio": 0.08650074899196625, "epoch": 0.8587484811664642, "grad_norm": 3.4206088184472465, "kl": 2.484375, "learning_rate": 7.046995166791899e-06, "loss": 0.0145, "step": 2827 }, { "clip_ratio": 0.1468183696269989, "epoch": 0.8590522478736331, "grad_norm": 2.493958434700366, "kl": 2.5, "learning_rate": 7.044576103532042e-06, "loss": 0.0051, "step": 2828 }, { "clip_ratio": 0.04128111153841019, "completions/clipped_ratio": 0.0357142857142857, "completions/max_length": 256.0, "completions/max_terminated_length": 252.0, "completions/mean_length": 169.10714721679688, "completions/mean_terminated_length": 165.88888549804688, "completions/min_length": 97.0, "completions/min_terminated_length": 97.0, "epoch": 0.8593560145808019, "grad_norm": 6.723686870242643, "kl": 2.53125, "learning_rate": 7.042156465459551e-06, "loss": 0.0367, "num_tokens": 13439766.0, "reward": 2.939464807510376, "reward_std": 0.11764097958803177, "rewards/check_gptzero_func/mean": 0.9821428656578064, "rewards/check_gptzero_func/std": 0.13363061845302582, "rewards/check_originality_func/mean": 0.9642857313156128, "rewards/check_originality_func/std": 0.187256321310997, "rewards/check_winston_local_func/mean": 0.9930359721183777, "rewards/check_winston_local_func/std": 0.0020185892935842276, "step": 2829 }, { "clip_ratio": 0.06677287817001343, "epoch": 0.8596597812879708, "grad_norm": 5.475389012365805, "kl": 2.59375, "learning_rate": 7.039736253254682e-06, "loss": 0.0262, "step": 2830 }, { "clip_ratio": 0.17793098092079163, "epoch": 0.8599635479951397, "grad_norm": 3.3406481340217073, "kl": 2.640625, "learning_rate": 7.0373154675978535e-06, "loss": 0.0064, "step": 2831 }, { "clip_ratio": 0.23910948634147644, "epoch": 0.8602673147023087, "grad_norm": 3.201405199639088, "kl": 2.671875, "learning_rate": 7.034894109169648e-06, "loss": -0.0016, "step": 2832 }, { "clip_ratio": 0.030367927625775337, "completions/clipped_ratio": 0.017857142857142905, "completions/max_length": 256.0, "completions/max_terminated_length": 251.0, "completions/mean_length": 153.58929443359375, "completions/mean_terminated_length": 151.72726440429688, "completions/min_length": 105.0, "completions/min_terminated_length": 105.0, "epoch": 0.8605710814094775, "grad_norm": 4.323742774722242, "kl": 2.71875, "learning_rate": 7.032472178650802e-06, "loss": 0.0272, "num_tokens": 13459391.0, "reward": 2.9752018451690674, "reward_std": 0.051466215401887894, "rewards/check_gptzero_func/mean": 0.9821428656578064, "rewards/check_gptzero_func/std": 0.13363061845302582, "rewards/check_originality_func/mean": 1.0, "rewards/check_originality_func/std": 0.0, "rewards/check_winston_local_func/mean": 0.993058979511261, "rewards/check_winston_local_func/std": 0.0015972736291587353, "step": 2833 }, { "clip_ratio": 0.0862133726477623, "epoch": 0.8608748481166464, "grad_norm": 3.2497253592074533, "kl": 2.671875, "learning_rate": 7.0300496767222225e-06, "loss": 0.0182, "step": 2834 }, { "clip_ratio": 0.14163173735141754, "epoch": 0.8611786148238153, "grad_norm": 2.392761490880362, "kl": 2.640625, "learning_rate": 7.02762660406497e-06, "loss": 0.0077, "step": 2835 }, { "clip_ratio": 0.21167179942131042, "epoch": 0.8614823815309842, "grad_norm": 1.8911332830096985, "kl": 2.671875, "learning_rate": 7.025202961360268e-06, "loss": 0.0008, "step": 2836 }, { "clip_ratio": 0.0296663586050272, "completions/clipped_ratio": 0.0, "completions/max_length": 228.0, "completions/max_terminated_length": 228.0, "completions/mean_length": 166.98214721679688, "completions/mean_terminated_length": 166.98214721679688, "completions/min_length": 128.0, "completions/min_terminated_length": 128.0, "epoch": 0.8617861482381531, "grad_norm": 4.48124575274872, "kl": 2.234375, "learning_rate": 7.0227787492895e-06, "loss": 0.0265, "num_tokens": 13480750.0, "reward": 2.993285894393921, "reward_std": 0.0015849367482587695, "rewards/check_gptzero_func/mean": 1.0, "rewards/check_gptzero_func/std": 0.0, "rewards/check_originality_func/mean": 1.0, "rewards/check_originality_func/std": 0.0, "rewards/check_winston_local_func/mean": 0.9932857751846313, "rewards/check_winston_local_func/std": 0.0017332083079963923, "step": 2837 }, { "clip_ratio": 0.06145917624235153, "epoch": 0.862089914945322, "grad_norm": 3.203373667944012, "kl": 2.25, "learning_rate": 7.0203539685342114e-06, "loss": 0.0168, "step": 2838 }, { "clip_ratio": 0.1305791735649109, "epoch": 0.8623936816524909, "grad_norm": 2.1872602346914873, "kl": 2.265625, "learning_rate": 7.017928619776104e-06, "loss": 0.0049, "step": 2839 }, { "clip_ratio": 0.21719186007976532, "epoch": 0.8626974483596598, "grad_norm": 1.5970574852005655, "kl": 2.296875, "learning_rate": 7.015502703697039e-06, "loss": -0.0039, "step": 2840 }, { "clip_ratio": 0.02971232496201992, "completions/clipped_ratio": 0.017857142857142905, "completions/max_length": 256.0, "completions/max_terminated_length": 216.0, "completions/mean_length": 164.35714721679688, "completions/mean_terminated_length": 162.69090270996094, "completions/min_length": 77.0, "completions/min_terminated_length": 77.0, "epoch": 0.8630012150668287, "grad_norm": 4.056418651882453, "kl": 2.71875, "learning_rate": 7.0130762209790426e-06, "loss": 0.0306, "num_tokens": 13501138.0, "reward": 2.9165453910827637, "reward_std": 0.21860751509666443, "rewards/check_gptzero_func/mean": 0.9464285969734192, "rewards/check_gptzero_func/std": 0.22720777988433838, "rewards/check_originality_func/mean": 0.9821428656578064, "rewards/check_originality_func/std": 0.13363061845302582, "rewards/check_winston_local_func/mean": 0.9879739880561829, "rewards/check_winston_local_func/std": 0.03786463290452957, "step": 2841 }, { "clip_ratio": 0.026182910427451134, "epoch": 0.8633049817739976, "grad_norm": 3.3212911917561767, "kl": 2.71875, "learning_rate": 7.010649172304294e-06, "loss": 0.0241, "step": 2842 }, { "clip_ratio": 0.09027334302663803, "epoch": 0.8636087484811664, "grad_norm": 1.9602219157341634, "kl": 2.71875, "learning_rate": 7.008221558355137e-06, "loss": 0.0095, "step": 2843 }, { "clip_ratio": 0.16066792607307434, "epoch": 0.8639125151883353, "grad_norm": 1.6202647687685936, "kl": 2.734375, "learning_rate": 7.0057933798140695e-06, "loss": -0.001, "step": 2844 }, { "clip_ratio": 0.035528820008039474, "completions/clipped_ratio": 0.0, "completions/max_length": 252.0, "completions/max_terminated_length": 252.0, "completions/mean_length": 169.83929443359375, "completions/mean_terminated_length": 169.83929443359375, "completions/min_length": 117.0, "completions/min_terminated_length": 117.0, "epoch": 0.8642162818955043, "grad_norm": 3.9439028272566414, "kl": 2.203125, "learning_rate": 7.0033646373637525e-06, "loss": 0.0129, "num_tokens": 13522473.0, "reward": 2.9178075790405273, "reward_std": 0.08530645072460175, "rewards/check_gptzero_func/mean": 0.9464285969734192, "rewards/check_gptzero_func/std": 0.22720777988433838, "rewards/check_originality_func/mean": 0.9821428656578064, "rewards/check_originality_func/std": 0.13363061845302582, "rewards/check_winston_local_func/mean": 0.9892361760139465, "rewards/check_winston_local_func/std": 0.017297428101301193, "step": 2845 }, { "clip_ratio": 0.06303291767835617, "epoch": 0.8645200486026732, "grad_norm": 3.309652803532394, "kl": 2.171875, "learning_rate": 7.000935331687e-06, "loss": 0.0039, "step": 2846 }, { "clip_ratio": 0.10385645925998688, "epoch": 0.8648238153098421, "grad_norm": 3.0449950602874116, "kl": 2.1875, "learning_rate": 6.998505463466787e-06, "loss": -0.0081, "step": 2847 }, { "clip_ratio": 0.20074240863323212, "epoch": 0.8651275820170109, "grad_norm": 1.7051252484300021, "kl": 2.21875, "learning_rate": 6.996075033386252e-06, "loss": -0.0171, "step": 2848 }, { "clip_ratio": 0.02611926570534706, "completions/clipped_ratio": 0.017857142857142905, "completions/max_length": 256.0, "completions/max_terminated_length": 252.0, "completions/mean_length": 153.1428680419922, "completions/mean_terminated_length": 151.27272033691406, "completions/min_length": 83.0, "completions/min_terminated_length": 83.0, "epoch": 0.8654313487241798, "grad_norm": 6.1011440098337, "kl": 2.703125, "learning_rate": 6.993644042128684e-06, "loss": 0.0056, "num_tokens": 13541969.0, "reward": 2.974029779434204, "reward_std": 0.053436413407325745, "rewards/check_gptzero_func/mean": 0.9821428656578064, "rewards/check_gptzero_func/std": 0.13363061845302582, "rewards/check_originality_func/mean": 1.0, "rewards/check_originality_func/std": 0.0, "rewards/check_winston_local_func/mean": 0.9918864965438843, "rewards/check_winston_local_func/std": 0.003370941150933504, "step": 2849 }, { "clip_ratio": 0.057842645794153214, "epoch": 0.8657351154313487, "grad_norm": 3.765126491754108, "kl": 2.703125, "learning_rate": 6.991212490377532e-06, "loss": -0.0086, "step": 2850 }, { "clip_ratio": 0.13646134734153748, "epoch": 0.8660388821385177, "grad_norm": 3.6282716774347867, "kl": 2.71875, "learning_rate": 6.9887803788164064e-06, "loss": -0.0194, "step": 2851 }, { "clip_ratio": 0.2620219886302948, "epoch": 0.8663426488456865, "grad_norm": 2.673932519075127, "kl": 2.671875, "learning_rate": 6.986347708129065e-06, "loss": -0.0244, "step": 2852 }, { "clip_ratio": 0.02145541086792946, "completions/clipped_ratio": 0.0, "completions/max_length": 243.0, "completions/max_terminated_length": 243.0, "completions/mean_length": 162.69644165039062, "completions/mean_terminated_length": 162.69644165039062, "completions/min_length": 136.0, "completions/min_terminated_length": 136.0, "epoch": 0.8666464155528554, "grad_norm": 8.149662882805192, "kl": 2.421875, "learning_rate": 6.983914478999437e-06, "loss": 0.0085, "num_tokens": 13562592.0, "reward": 2.9389748573303223, "reward_std": 0.15383729338645935, "rewards/check_gptzero_func/mean": 0.9642857313156128, "rewards/check_gptzero_func/std": 0.187256321310997, "rewards/check_originality_func/mean": 0.9821428656578064, "rewards/check_originality_func/std": 0.13363061845302582, "rewards/check_winston_local_func/mean": 0.9925457835197449, "rewards/check_winston_local_func/std": 0.0039812601171433926, "step": 2853 }, { "clip_ratio": 0.04954156652092934, "epoch": 0.8669501822600243, "grad_norm": 3.7113227045144197, "kl": 2.421875, "learning_rate": 6.9814806921115956e-06, "loss": -0.005, "step": 2854 }, { "clip_ratio": 0.13832725584506989, "epoch": 0.8672539489671932, "grad_norm": 2.228255972127135, "kl": 2.40625, "learning_rate": 6.979046348149777e-06, "loss": -0.0154, "step": 2855 }, { "clip_ratio": 0.2386307418346405, "epoch": 0.8675577156743621, "grad_norm": 1.5681690459299507, "kl": 2.375, "learning_rate": 6.976611447798376e-06, "loss": -0.0216, "step": 2856 }, { "clip_ratio": 0.02137024886906147, "completions/clipped_ratio": 0.0714285714285714, "completions/max_length": 256.0, "completions/max_terminated_length": 253.0, "completions/mean_length": 173.94644165039062, "completions/mean_terminated_length": 167.63462829589844, "completions/min_length": 124.0, "completions/min_terminated_length": 124.0, "epoch": 0.8678614823815309, "grad_norm": 3.779614931756381, "kl": 2.390625, "learning_rate": 6.97417599174194e-06, "loss": 0.0194, "num_tokens": 13583765.0, "reward": 2.9563441276550293, "reward_std": 0.06938161700963974, "rewards/check_gptzero_func/mean": 0.9821428656578064, "rewards/check_gptzero_func/std": 0.13363061845302582, "rewards/check_originality_func/mean": 0.9821428656578064, "rewards/check_originality_func/std": 0.13363061845302582, "rewards/check_winston_local_func/mean": 0.9920584559440613, "rewards/check_winston_local_func/std": 0.004301462322473526, "step": 2857 }, { "clip_ratio": 0.04808719828724861, "epoch": 0.8681652490886999, "grad_norm": 3.4938656970823447, "kl": 2.34375, "learning_rate": 6.971739980665172e-06, "loss": 0.0093, "step": 2858 }, { "clip_ratio": 0.15967358648777008, "epoch": 0.8684690157958688, "grad_norm": 3.1283787897366815, "kl": 2.28125, "learning_rate": 6.969303415252932e-06, "loss": -0.0036, "step": 2859 }, { "clip_ratio": 0.2061479091644287, "epoch": 0.8687727825030377, "grad_norm": 2.072112532183692, "kl": 2.28125, "learning_rate": 6.966866296190243e-06, "loss": -0.0104, "step": 2860 }, { "clip_ratio": 0.02217222936451435, "completions/clipped_ratio": 0.0, "completions/max_length": 208.0, "completions/max_terminated_length": 208.0, "completions/mean_length": 162.83929443359375, "completions/mean_terminated_length": 162.83929443359375, "completions/min_length": 116.0, "completions/min_terminated_length": 116.0, "epoch": 0.8690765492102066, "grad_norm": 4.988807610203645, "kl": 2.203125, "learning_rate": 6.96442862416227e-06, "loss": 0.0157, "num_tokens": 13604796.0, "reward": 2.9200587272644043, "reward_std": 0.16734328866004944, "rewards/check_gptzero_func/mean": 0.9285714030265808, "rewards/check_gptzero_func/std": 0.25987008213996887, "rewards/check_originality_func/mean": 1.0, "rewards/check_originality_func/std": 0.0, "rewards/check_winston_local_func/mean": 0.991487443447113, "rewards/check_winston_local_func/std": 0.0051938993856310844, "step": 2861 }, { "clip_ratio": 0.04882458224892616, "epoch": 0.8693803159173754, "grad_norm": 3.4838030766899246, "kl": 2.21875, "learning_rate": 6.961990399854344e-06, "loss": 0.0023, "step": 2862 }, { "clip_ratio": 0.13234283030033112, "epoch": 0.8696840826245443, "grad_norm": 3.1578112843365567, "kl": 2.25, "learning_rate": 6.959551623951949e-06, "loss": -0.0079, "step": 2863 }, { "clip_ratio": 0.2013016939163208, "epoch": 0.8699878493317132, "grad_norm": 2.542105748795695, "kl": 2.234375, "learning_rate": 6.95711229714072e-06, "loss": -0.0153, "step": 2864 }, { "clip_ratio": 0.02197141945362091, "completions/clipped_ratio": 0.017857142857142905, "completions/max_length": 256.0, "completions/max_terminated_length": 243.0, "completions/mean_length": 181.58929443359375, "completions/mean_terminated_length": 180.23635864257812, "completions/min_length": 137.0, "completions/min_terminated_length": 137.0, "epoch": 0.8702916160388822, "grad_norm": 4.807870155098541, "kl": 2.078125, "learning_rate": 6.954672420106452e-06, "loss": 0.0112, "num_tokens": 13627053.0, "reward": 2.9571714401245117, "reward_std": 0.10240652412176132, "rewards/check_gptzero_func/mean": 0.9642857313156128, "rewards/check_gptzero_func/std": 0.187256321310997, "rewards/check_originality_func/mean": 1.0, "rewards/check_originality_func/std": 0.0, "rewards/check_winston_local_func/mean": 0.9928854703903198, "rewards/check_winston_local_func/std": 0.0022124559618532658, "step": 2865 }, { "clip_ratio": 0.05465969070792198, "epoch": 0.8705953827460511, "grad_norm": 3.597549414871811, "kl": 2.046875, "learning_rate": 6.952231993535092e-06, "loss": 0.0009, "step": 2866 }, { "clip_ratio": 0.1300048679113388, "epoch": 0.8708991494532199, "grad_norm": 2.053536181078877, "kl": 2.046875, "learning_rate": 6.949791018112743e-06, "loss": -0.0128, "step": 2867 }, { "clip_ratio": 0.22450745105743408, "epoch": 0.8712029161603888, "grad_norm": 1.5058962314562947, "kl": 2.03125, "learning_rate": 6.947349494525661e-06, "loss": -0.0198, "step": 2868 }, { "clip_ratio": 0.016854455694556236, "completions/clipped_ratio": 0.017857142857142905, "completions/max_length": 256.0, "completions/max_terminated_length": 210.0, "completions/mean_length": 166.85714721679688, "completions/mean_terminated_length": 165.23635864257812, "completions/min_length": 96.0, "completions/min_terminated_length": 96.0, "epoch": 0.8715066828675577, "grad_norm": 3.467360188953214, "kl": 2.265625, "learning_rate": 6.9449074234602545e-06, "loss": 0.0074, "num_tokens": 13647797.0, "reward": 2.902714729309082, "reward_std": 0.1739153116941452, "rewards/check_gptzero_func/mean": 0.9642857313156128, "rewards/check_gptzero_func/std": 0.187256321310997, "rewards/check_originality_func/mean": 0.9464285969734192, "rewards/check_originality_func/std": 0.22720777988433838, "rewards/check_winston_local_func/mean": 0.9920002222061157, "rewards/check_winston_local_func/std": 0.003777120728045702, "step": 2869 }, { "clip_ratio": 0.03495778888463974, "epoch": 0.8718104495747266, "grad_norm": 2.8518429589942635, "kl": 2.265625, "learning_rate": 6.942464805603091e-06, "loss": -0.001, "step": 2870 }, { "clip_ratio": 0.09774564206600189, "epoch": 0.8721142162818954, "grad_norm": 2.1280562241074255, "kl": 2.265625, "learning_rate": 6.940021641640887e-06, "loss": -0.0128, "step": 2871 }, { "clip_ratio": 0.1661946326494217, "epoch": 0.8724179829890644, "grad_norm": 1.6722265874324223, "kl": 2.265625, "learning_rate": 6.9375779322605154e-06, "loss": -0.0178, "step": 2872 }, { "clip_ratio": 0.010465257801115513, "completions/clipped_ratio": 0.0, "completions/max_length": 205.0, "completions/max_terminated_length": 205.0, "completions/mean_length": 167.82144165039062, "completions/mean_terminated_length": 167.82144165039062, "completions/min_length": 126.0, "completions/min_terminated_length": 126.0, "epoch": 0.8727217496962333, "grad_norm": 3.1454680599656286, "kl": 1.8671875, "learning_rate": 6.935133678148999e-06, "loss": 0.0185, "num_tokens": 13669299.0, "reward": 2.7948992252349854, "reward_std": 0.31742534041404724, "rewards/check_gptzero_func/mean": 0.875, "rewards/check_gptzero_func/std": 0.3337118923664093, "rewards/check_originality_func/mean": 0.9285714030265808, "rewards/check_originality_func/std": 0.25987008213996887, "rewards/check_winston_local_func/mean": 0.991327702999115, "rewards/check_winston_local_func/std": 0.007904176600277424, "step": 2873 }, { "clip_ratio": 0.02446736767888069, "epoch": 0.8730255164034022, "grad_norm": 2.503258504368229, "kl": 1.875, "learning_rate": 6.93268887999352e-06, "loss": 0.0104, "step": 2874 }, { "clip_ratio": 0.07050265371799469, "epoch": 0.8733292831105711, "grad_norm": 1.6262196032720733, "kl": 1.890625, "learning_rate": 6.930243538481407e-06, "loss": 0.0021, "step": 2875 }, { "clip_ratio": 0.10785811394453049, "epoch": 0.8736330498177399, "grad_norm": 1.459679619731626, "kl": 1.890625, "learning_rate": 6.927797654300142e-06, "loss": -0.0035, "step": 2876 }, { "clip_ratio": 0.011160453781485558, "completions/clipped_ratio": 0.0535714285714286, "completions/max_length": 256.0, "completions/max_terminated_length": 234.0, "completions/mean_length": 172.85714721679688, "completions/mean_terminated_length": 168.15093994140625, "completions/min_length": 69.0, "completions/min_terminated_length": 69.0, "epoch": 0.8739368165249088, "grad_norm": 5.620192338303827, "kl": 2.03125, "learning_rate": 6.925351228137366e-06, "loss": 0.036, "num_tokens": 13690731.0, "reward": 2.79644513130188, "reward_std": 0.24483749270439148, "rewards/check_gptzero_func/mean": 0.8571428656578064, "rewards/check_gptzero_func/std": 0.3530939221382141, "rewards/check_originality_func/mean": 0.9464285969734192, "rewards/check_originality_func/std": 0.22720777988433838, "rewards/check_winston_local_func/mean": 0.9928733110427856, "rewards/check_winston_local_func/std": 0.0028057375457137823, "step": 2877 }, { "clip_ratio": 0.029277827590703964, "epoch": 0.8742405832320778, "grad_norm": 3.9221785042494552, "kl": 2.046875, "learning_rate": 6.922904260680864e-06, "loss": 0.0238, "step": 2878 }, { "clip_ratio": 0.07969755679368973, "epoch": 0.8745443499392467, "grad_norm": 2.2083755907641227, "kl": 2.078125, "learning_rate": 6.920456752618579e-06, "loss": 0.0131, "step": 2879 }, { "clip_ratio": 0.1457197666168213, "epoch": 0.8748481166464156, "grad_norm": 1.7752972245237932, "kl": 2.109375, "learning_rate": 6.918008704638603e-06, "loss": 0.0027, "step": 2880 }, { "clip_ratio": 0.020703190937638283, "completions/clipped_ratio": 0.0, "completions/max_length": 243.0, "completions/max_terminated_length": 243.0, "completions/mean_length": 175.58929443359375, "completions/mean_terminated_length": 175.58929443359375, "completions/min_length": 114.0, "completions/min_terminated_length": 114.0, "epoch": 0.8751518833535844, "grad_norm": 3.5663375094323975, "kl": 1.8671875, "learning_rate": 6.915560117429181e-06, "loss": 0.0072, "num_tokens": 13712156.0, "reward": 2.938998222351074, "reward_std": 0.119165800511837, "rewards/check_gptzero_func/mean": 1.0, "rewards/check_gptzero_func/std": 0.0, "rewards/check_originality_func/mean": 0.9464285969734192, "rewards/check_originality_func/std": 0.22720777988433838, "rewards/check_winston_local_func/mean": 0.9925697445869446, "rewards/check_winston_local_func/std": 0.003398485481739044, "step": 2881 }, { "clip_ratio": 0.04273146018385887, "epoch": 0.8754556500607533, "grad_norm": 2.9947711669713906, "kl": 1.8671875, "learning_rate": 6.913110991678711e-06, "loss": -0.0034, "step": 2882 }, { "clip_ratio": 0.11384344100952148, "epoch": 0.8757594167679222, "grad_norm": 2.3359105482096276, "kl": 1.8671875, "learning_rate": 6.9106613280757375e-06, "loss": -0.0126, "step": 2883 }, { "clip_ratio": 0.17083308100700378, "epoch": 0.8760631834750912, "grad_norm": 1.5298207473541998, "kl": 1.875, "learning_rate": 6.9082111273089625e-06, "loss": -0.0184, "step": 2884 }, { "clip_ratio": 0.012130905874073505, "completions/clipped_ratio": 0.0357142857142857, "completions/max_length": 256.0, "completions/max_terminated_length": 239.0, "completions/mean_length": 172.19644165039062, "completions/mean_terminated_length": 169.09259033203125, "completions/min_length": 128.0, "completions/min_terminated_length": 128.0, "epoch": 0.8763669501822601, "grad_norm": 4.343774944646072, "kl": 2.046875, "learning_rate": 6.9057603900672355e-06, "loss": 0.0006, "num_tokens": 13733103.0, "reward": 2.7947208881378174, "reward_std": 0.3309199810028076, "rewards/check_gptzero_func/mean": 0.8571428656578064, "rewards/check_gptzero_func/std": 0.3530939221382141, "rewards/check_originality_func/mean": 0.9464285969734192, "rewards/check_originality_func/std": 0.22720777988433838, "rewards/check_winston_local_func/mean": 0.9911492466926575, "rewards/check_winston_local_func/std": 0.007067815400660038, "step": 2885 }, { "clip_ratio": 0.02905506081879139, "epoch": 0.8766707168894289, "grad_norm": 3.8964069629638827, "kl": 2.03125, "learning_rate": 6.903309117039557e-06, "loss": -0.0048, "step": 2886 }, { "clip_ratio": 0.09491239488124847, "epoch": 0.8769744835965978, "grad_norm": 2.699112599938771, "kl": 2.03125, "learning_rate": 6.90085730891508e-06, "loss": -0.0211, "step": 2887 }, { "clip_ratio": 0.19392180442810059, "epoch": 0.8772782503037667, "grad_norm": 1.8653062497908004, "kl": 2.03125, "learning_rate": 6.898404966383103e-06, "loss": -0.0327, "step": 2888 }, { "clip_ratio": 0.016465703025460243, "completions/clipped_ratio": 0.0, "completions/max_length": 226.0, "completions/max_terminated_length": 226.0, "completions/mean_length": 179.37501525878906, "completions/mean_terminated_length": 179.37501525878906, "completions/min_length": 120.0, "completions/min_terminated_length": 120.0, "epoch": 0.8775820170109356, "grad_norm": 3.2698805389240557, "kl": 1.828125, "learning_rate": 6.89595209013308e-06, "loss": 0.0156, "num_tokens": 13755076.0, "reward": 2.8320658206939697, "reward_std": 0.193765789270401, "rewards/check_gptzero_func/mean": 0.9821428656578064, "rewards/check_gptzero_func/std": 0.13363061845302582, "rewards/check_originality_func/mean": 0.8571428656578064, "rewards/check_originality_func/std": 0.3530939221382141, "rewards/check_winston_local_func/mean": 0.9927798509597778, "rewards/check_winston_local_func/std": 0.0021187213715165854, "step": 2889 }, { "clip_ratio": 0.0610177107155323, "epoch": 0.8778857837181046, "grad_norm": 2.581914766627428, "kl": 1.8125, "learning_rate": 6.893498680854614e-06, "loss": 0.003, "step": 2890 }, { "clip_ratio": 0.09829668700695038, "epoch": 0.8781895504252734, "grad_norm": 1.853043643285347, "kl": 1.8203125, "learning_rate": 6.891044739237458e-06, "loss": -0.0067, "step": 2891 }, { "clip_ratio": 0.17180313169956207, "epoch": 0.8784933171324423, "grad_norm": 1.1221451136230145, "kl": 1.828125, "learning_rate": 6.8885902659715105e-06, "loss": -0.0184, "step": 2892 }, { "clip_ratio": 0.02279156632721424, "completions/clipped_ratio": 0.0, "completions/max_length": 250.0, "completions/max_terminated_length": 250.0, "completions/mean_length": 163.21429443359375, "completions/mean_terminated_length": 163.21429443359375, "completions/min_length": 107.0, "completions/min_terminated_length": 107.0, "epoch": 0.8787970838396112, "grad_norm": 3.488739455143939, "kl": 2.234375, "learning_rate": 6.886135261746825e-06, "loss": 0.0216, "num_tokens": 13775352.0, "reward": 2.7248947620391846, "reward_std": 0.3113386332988739, "rewards/check_gptzero_func/mean": 0.8571428656578064, "rewards/check_gptzero_func/std": 0.3530939221382141, "rewards/check_originality_func/mean": 0.875, "rewards/check_originality_func/std": 0.3337118923664093, "rewards/check_winston_local_func/mean": 0.9927516579627991, "rewards/check_winston_local_func/std": 0.0036854648496955633, "step": 2893 }, { "clip_ratio": 0.044680364429950714, "epoch": 0.8791008505467801, "grad_norm": 3.0984973395550464, "kl": 2.21875, "learning_rate": 6.8836797272536005e-06, "loss": 0.0114, "step": 2894 }, { "clip_ratio": 0.08012192696332932, "epoch": 0.8794046172539489, "grad_norm": 2.24622322997758, "kl": 2.234375, "learning_rate": 6.8812236631821886e-06, "loss": 0.0013, "step": 2895 }, { "clip_ratio": 0.1512739211320877, "epoch": 0.8797083839611178, "grad_norm": 1.7085154374052123, "kl": 2.265625, "learning_rate": 6.878767070223087e-06, "loss": -0.0052, "step": 2896 }, { "clip_ratio": 0.04997389763593674, "completions/clipped_ratio": 0.0535714285714286, "completions/max_length": 256.0, "completions/max_terminated_length": 253.0, "completions/mean_length": 185.6607208251953, "completions/mean_terminated_length": 181.6792449951172, "completions/min_length": 21.0, "completions/min_terminated_length": 21.0, "epoch": 0.8800121506682868, "grad_norm": 5.443393486382868, "kl": 2.21875, "learning_rate": 6.876309949066942e-06, "loss": 0.0219, "num_tokens": 13796949.0, "reward": 2.88531231880188, "reward_std": 0.26865914463996887, "rewards/check_gptzero_func/mean": 0.9642857313156128, "rewards/check_gptzero_func/std": 0.187256321310997, "rewards/check_originality_func/mean": 0.9285714030265808, "rewards/check_originality_func/std": 0.25987008213996887, "rewards/check_winston_local_func/mean": 0.9924551844596863, "rewards/check_winston_local_func/std": 0.0020706329960376024, "step": 2897 }, { "clip_ratio": 0.055484216660261154, "epoch": 0.8803159173754557, "grad_norm": 3.668118636989168, "kl": 2.234375, "learning_rate": 6.873852300404551e-06, "loss": 0.0031, "step": 2898 }, { "clip_ratio": 0.13295622169971466, "epoch": 0.8806196840826246, "grad_norm": 3.027456165683402, "kl": 2.25, "learning_rate": 6.871394124926857e-06, "loss": -0.0132, "step": 2899 }, { "clip_ratio": 0.22350285947322845, "epoch": 0.8809234507897934, "grad_norm": 2.4485983759286, "kl": 2.25, "learning_rate": 6.86893542332495e-06, "loss": -0.0202, "step": 2900 }, { "clip_ratio": 0.008508026599884033, "completions/clipped_ratio": 0.017857142857142905, "completions/max_length": 256.0, "completions/max_terminated_length": 235.0, "completions/mean_length": 174.6428680419922, "completions/mean_terminated_length": 173.16363525390625, "completions/min_length": 105.0, "completions/min_terminated_length": 105.0, "epoch": 1.0150454863540939, "grad_norm": 2.212723097364759, "kl": 1.859375, "learning_rate": 6.866476196290071e-06, "loss": 0.003, "num_tokens": 13818393.0, "reward": 2.8496923446655273, "reward_std": 0.002045696135610342, "rewards/check_gptzero_func/mean": 1.0, "rewards/check_gptzero_func/std": 0.0, "rewards/check_originality_func/mean": 1.0, "rewards/check_originality_func/std": 0.0, "rewards/check_winston_local_func/mean": 0.8496923446655273, "rewards/check_winston_local_func/std": 0.35006988048553467, "step": 2901 }, { "clip_ratio": 0.02657606266438961, "epoch": 1.0153953813855843, "grad_norm": 1.816804400277233, "kl": 1.859375, "learning_rate": 5.73931550841847e-06, "loss": -0.0031, "step": 2902 }, { "clip_ratio": 0.07014858722686768, "epoch": 1.015745276417075, "grad_norm": 1.0766908361470018, "kl": 1.859375, "learning_rate": 5.736295289314593e-06, "loss": -0.0084, "step": 2903 }, { "clip_ratio": 0.10761847347021103, "epoch": 1.0160951714485655, "grad_norm": 0.7917307988065578, "kl": 1.859375, "learning_rate": 5.733274795579541e-06, "loss": -0.0117, "step": 2904 }, { "clip_ratio": 0.010465504601597786, "completions/clipped_ratio": 0.0357142857142857, "completions/max_length": 256.0, "completions/max_terminated_length": 246.0, "completions/mean_length": 164.33929443359375, "completions/mean_terminated_length": 160.94444274902344, "completions/min_length": 83.0, "completions/min_terminated_length": 83.0, "epoch": 1.016445066480056, "grad_norm": 4.012565128607679, "kl": 2.078125, "learning_rate": 5.730254028339932e-06, "loss": 0.0028, "num_tokens": 13838292.0, "reward": 2.955983877182007, "reward_std": 0.06830395013093948, "rewards/check_gptzero_func/mean": 0.9642857313156128, "rewards/check_gptzero_func/std": 0.187256321310997, "rewards/check_originality_func/mean": 1.0, "rewards/check_originality_func/std": 0.0, "rewards/check_winston_local_func/mean": 0.9916982650756836, "rewards/check_winston_local_func/std": 0.0035846212413161993, "step": 2905 }, { "clip_ratio": 0.029811495915055275, "epoch": 1.0167949615115466, "grad_norm": 3.692544447549636, "kl": 2.078125, "learning_rate": 5.727232988722481e-06, "loss": -0.0102, "step": 2906 }, { "clip_ratio": 0.07566817849874496, "epoch": 1.017144856543037, "grad_norm": 2.72110618804022, "kl": 2.078125, "learning_rate": 5.724211677854007e-06, "loss": -0.0208, "step": 2907 }, { "clip_ratio": 0.14082790911197662, "epoch": 1.0174947515745276, "grad_norm": 1.8254083572844468, "kl": 2.09375, "learning_rate": 5.721190096861431e-06, "loss": -0.0304, "step": 2908 }, { "clip_ratio": 0.014130047522485256, "completions/clipped_ratio": 0.017857142857142905, "completions/max_length": 256.0, "completions/max_terminated_length": 256.0, "completions/mean_length": 181.80357360839844, "completions/mean_terminated_length": 180.4545440673828, "completions/min_length": 97.0, "completions/min_terminated_length": 97.0, "epoch": 1.0178446466060183, "grad_norm": 2.488295585296881, "kl": 1.953125, "learning_rate": 5.718168246871775e-06, "loss": 0.0134, "num_tokens": 13860081.0, "reward": 2.794058322906494, "reward_std": 0.16097743809223175, "rewards/check_gptzero_func/mean": 0.8571428656578064, "rewards/check_gptzero_func/std": 0.3530939221382141, "rewards/check_originality_func/mean": 0.9464285969734192, "rewards/check_originality_func/std": 0.22720777988433838, "rewards/check_winston_local_func/mean": 0.9904866814613342, "rewards/check_winston_local_func/std": 0.006276190280914307, "step": 2909 }, { "clip_ratio": 0.03665953874588013, "epoch": 1.0181945416375087, "grad_norm": 1.8191265971495967, "kl": 1.9609375, "learning_rate": 5.715146129012161e-06, "loss": 0.007, "step": 2910 }, { "clip_ratio": 0.06970115005970001, "epoch": 1.0185444366689993, "grad_norm": 1.298494072796272, "kl": 1.9609375, "learning_rate": 5.712123744409809e-06, "loss": 0.0022, "step": 2911 }, { "clip_ratio": 0.0984921008348465, "epoch": 1.01889433170049, "grad_norm": 1.014662040647944, "kl": 1.9765625, "learning_rate": 5.7091010941920385e-06, "loss": -0.0012, "step": 2912 }, { "clip_ratio": 0.02974449284374714, "completions/clipped_ratio": 0.1964285714285714, "completions/max_length": 256.0, "completions/max_terminated_length": 243.0, "completions/mean_length": 176.57144165039062, "completions/mean_terminated_length": 157.1555633544922, "completions/min_length": 109.0, "completions/min_terminated_length": 109.0, "epoch": 1.0192442267319803, "grad_norm": 4.004915023681568, "kl": 1.984375, "learning_rate": 5.706078179486273e-06, "loss": 0.0163, "num_tokens": 13881689.0, "reward": 2.775686025619507, "reward_std": 0.24514003098011017, "rewards/check_gptzero_func/mean": 0.9464285969734192, "rewards/check_gptzero_func/std": 0.22720777988433838, "rewards/check_originality_func/mean": 0.8392857313156128, "rewards/check_originality_func/std": 0.3705909848213196, "rewards/check_winston_local_func/mean": 0.9899716973304749, "rewards/check_winston_local_func/std": 0.0073460424318909645, "step": 2913 }, { "clip_ratio": 0.045735739171504974, "epoch": 1.019594121763471, "grad_norm": 3.4878236779185574, "kl": 1.9765625, "learning_rate": 5.703055001420028e-06, "loss": 0.0096, "step": 2914 }, { "clip_ratio": 0.11200960725545883, "epoch": 1.0199440167949616, "grad_norm": 1.8416226683243249, "kl": 1.9765625, "learning_rate": 5.7000315611209225e-06, "loss": -0.0031, "step": 2915 }, { "clip_ratio": 0.1464008241891861, "epoch": 1.020293911826452, "grad_norm": 1.8502126728527803, "kl": 1.984375, "learning_rate": 5.697007859716669e-06, "loss": -0.0061, "step": 2916 }, { "clip_ratio": 0.01757216267287731, "completions/clipped_ratio": 0.017857142857142905, "completions/max_length": 256.0, "completions/max_terminated_length": 240.0, "completions/mean_length": 171.55357360839844, "completions/mean_terminated_length": 170.01817321777344, "completions/min_length": 125.0, "completions/min_terminated_length": 125.0, "epoch": 1.0206438068579426, "grad_norm": 6.768417920258454, "kl": 2.078125, "learning_rate": 5.69398389833508e-06, "loss": 0.009, "num_tokens": 13902240.0, "reward": 2.991164207458496, "reward_std": 0.002686533145606518, "rewards/check_gptzero_func/mean": 1.0, "rewards/check_gptzero_func/std": 0.0, "rewards/check_originality_func/mean": 1.0, "rewards/check_originality_func/std": 0.0, "rewards/check_winston_local_func/mean": 0.9911640882492065, "rewards/check_winston_local_func/std": 0.003402040107175708, "step": 2917 }, { "clip_ratio": 0.05726001411676407, "epoch": 1.020993701889433, "grad_norm": 4.781125720125103, "kl": 2.078125, "learning_rate": 5.6909596781040655e-06, "loss": -0.0051, "step": 2918 }, { "clip_ratio": 0.11836200952529907, "epoch": 1.0213435969209237, "grad_norm": 3.0388492919447625, "kl": 2.09375, "learning_rate": 5.687935200151632e-06, "loss": -0.0179, "step": 2919 }, { "clip_ratio": 0.19128486514091492, "epoch": 1.0216934919524143, "grad_norm": 1.8377454942098341, "kl": 2.140625, "learning_rate": 5.684910465605878e-06, "loss": -0.0251, "step": 2920 }, { "clip_ratio": 0.020535385236144066, "completions/clipped_ratio": 0.0714285714285714, "completions/max_length": 256.0, "completions/max_terminated_length": 241.0, "completions/mean_length": 177.46429443359375, "completions/mean_terminated_length": 171.42308044433594, "completions/min_length": 83.0, "completions/min_terminated_length": 83.0, "epoch": 1.0220433869839047, "grad_norm": 3.5016261708614382, "kl": 1.984375, "learning_rate": 5.681885475595002e-06, "loss": 0.0137, "num_tokens": 13923706.0, "reward": 2.877340078353882, "reward_std": 0.07893340289592743, "rewards/check_gptzero_func/mean": 1.0, "rewards/check_gptzero_func/std": 0.0, "rewards/check_originality_func/mean": 0.9821428656578064, "rewards/check_originality_func/std": 0.13363061845302582, "rewards/check_winston_local_func/mean": 0.8951972126960754, "rewards/check_winston_local_func/std": 0.25047919154167175, "step": 2921 }, { "clip_ratio": 0.04295936971902847, "epoch": 1.0223932820153954, "grad_norm": 2.4911489390533292, "kl": 1.9609375, "learning_rate": 5.6788602312473e-06, "loss": 0.0103, "step": 2922 }, { "clip_ratio": 0.0768522173166275, "epoch": 1.022743177046886, "grad_norm": 1.9446638489824808, "kl": 1.9609375, "learning_rate": 5.675834733691155e-06, "loss": 0.0028, "step": 2923 }, { "clip_ratio": 0.1281847208738327, "epoch": 1.0230930720783764, "grad_norm": 1.6016668705363313, "kl": 1.9609375, "learning_rate": 5.67280898405505e-06, "loss": -0.0029, "step": 2924 }, { "clip_ratio": 0.01954737678170204, "completions/clipped_ratio": 0.1428571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 241.0, "completions/mean_length": 178.30357360839844, "completions/mean_terminated_length": 165.3541717529297, "completions/min_length": 90.0, "completions/min_terminated_length": 90.0, "epoch": 1.023442967109867, "grad_norm": 6.041051103017317, "kl": 1.9765625, "learning_rate": 5.669782983467562e-06, "loss": 0.0005, "num_tokens": 13945467.0, "reward": 2.740748167037964, "reward_std": 0.16755010187625885, "rewards/check_gptzero_func/mean": 0.8928571343421936, "rewards/check_gptzero_func/std": 0.3120938837528229, "rewards/check_originality_func/mean": 0.8571428656578064, "rewards/check_originality_func/std": 0.3530939221382141, "rewards/check_winston_local_func/mean": 0.9907479882240295, "rewards/check_winston_local_func/std": 0.00644368352368474, "step": 2925 }, { "clip_ratio": 0.03327177092432976, "epoch": 1.0237928621413577, "grad_norm": 5.474162755695637, "kl": 1.9765625, "learning_rate": 5.666756733057361e-06, "loss": -0.0094, "step": 2926 }, { "clip_ratio": 0.08305276930332184, "epoch": 1.024142757172848, "grad_norm": 2.346876823955614, "kl": 1.953125, "learning_rate": 5.663730233953209e-06, "loss": -0.0233, "step": 2927 }, { "clip_ratio": 0.13135696947574615, "epoch": 1.0244926522043387, "grad_norm": 1.7650275121072694, "kl": 1.953125, "learning_rate": 5.6607034872839605e-06, "loss": -0.0273, "step": 2928 }, { "clip_ratio": 0.019614199176430702, "completions/clipped_ratio": 0.0892857142857143, "completions/max_length": 256.0, "completions/max_terminated_length": 251.0, "completions/mean_length": 145.10714721679688, "completions/mean_terminated_length": 134.2353057861328, "completions/min_length": 67.0, "completions/min_terminated_length": 67.0, "epoch": 1.0248425472358293, "grad_norm": 3.101210449400742, "kl": 2.421875, "learning_rate": 5.657676494178565e-06, "loss": 0.0117, "num_tokens": 13963241.0, "reward": 2.773376703262329, "reward_std": 0.23453480005264282, "rewards/check_gptzero_func/mean": 0.875, "rewards/check_gptzero_func/std": 0.3337118923664093, "rewards/check_originality_func/mean": 0.9107142686843872, "rewards/check_originality_func/std": 0.28773635625839233, "rewards/check_winston_local_func/mean": 0.9876623153686523, "rewards/check_winston_local_func/std": 0.017314953729510307, "step": 2929 }, { "clip_ratio": 0.03431819751858711, "epoch": 1.0251924422673198, "grad_norm": 2.320007468464847, "kl": 2.40625, "learning_rate": 5.654649255766062e-06, "loss": 0.0056, "step": 2930 }, { "clip_ratio": 0.06356845796108246, "epoch": 1.0255423372988104, "grad_norm": 1.6600477094152224, "kl": 2.421875, "learning_rate": 5.651621773175582e-06, "loss": -0.0002, "step": 2931 }, { "clip_ratio": 0.10407207906246185, "epoch": 1.0258922323303008, "grad_norm": 1.6219168951409877, "kl": 2.421875, "learning_rate": 5.648594047536349e-06, "loss": -0.0064, "step": 2932 }, { "clip_ratio": 0.014092189259827137, "completions/clipped_ratio": 0.017857142857142905, "completions/max_length": 256.0, "completions/max_terminated_length": 255.0, "completions/mean_length": 180.80357360839844, "completions/mean_terminated_length": 179.4363555908203, "completions/min_length": 75.0, "completions/min_terminated_length": 75.0, "epoch": 1.0262421273617914, "grad_norm": 4.431367654296805, "kl": 1.921875, "learning_rate": 5.645566079977673e-06, "loss": 0.0185, "num_tokens": 13984854.0, "reward": 2.988321542739868, "reward_std": 0.006182943936437368, "rewards/check_gptzero_func/mean": 1.0, "rewards/check_gptzero_func/std": 0.0, "rewards/check_originality_func/mean": 1.0, "rewards/check_originality_func/std": 0.0, "rewards/check_winston_local_func/mean": 0.9883213639259338, "rewards/check_winston_local_func/std": 0.009641297161579132, "step": 2933 }, { "clip_ratio": 0.03753158077597618, "epoch": 1.026592022393282, "grad_norm": 2.563551588603373, "kl": 1.9296875, "learning_rate": 5.6425378716289605e-06, "loss": 0.0084, "step": 2934 }, { "clip_ratio": 0.07139673084020615, "epoch": 1.0269419174247725, "grad_norm": 2.008662503925312, "kl": 1.953125, "learning_rate": 5.639509423619701e-06, "loss": 0.0066, "step": 2935 }, { "clip_ratio": 0.10418520867824554, "epoch": 1.027291812456263, "grad_norm": 2.1339521724726818, "kl": 1.9921875, "learning_rate": 5.63648073707948e-06, "loss": 0.0016, "step": 2936 }, { "clip_ratio": 0.02532629854977131, "completions/clipped_ratio": 0.1428571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 251.0, "completions/mean_length": 162.35714721679688, "completions/mean_terminated_length": 146.75, "completions/min_length": 83.0, "completions/min_terminated_length": 83.0, "epoch": 1.0276417074877537, "grad_norm": 4.965403199126494, "kl": 2.25, "learning_rate": 5.633451813137969e-06, "loss": 0.0106, "num_tokens": 14004706.0, "reward": 2.9009106159210205, "reward_std": 0.07746421545743942, "rewards/check_gptzero_func/mean": 1.0, "rewards/check_gptzero_func/std": 0.0, "rewards/check_originality_func/mean": 0.9107142686843872, "rewards/check_originality_func/std": 0.28773635625839233, "rewards/check_winston_local_func/mean": 0.990196168422699, "rewards/check_winston_local_func/std": 0.006542951334267855, "step": 2937 }, { "clip_ratio": 0.04214828088879585, "epoch": 1.0279916025192442, "grad_norm": 3.409104064749198, "kl": 2.25, "learning_rate": 5.630422652924928e-06, "loss": -0.0032, "step": 2938 }, { "clip_ratio": 0.08159526437520981, "epoch": 1.0283414975507348, "grad_norm": 1.956039340010915, "kl": 2.25, "learning_rate": 5.627393257570201e-06, "loss": -0.012, "step": 2939 }, { "clip_ratio": 0.16733790934085846, "epoch": 1.0286913925822254, "grad_norm": 1.4421493170475055, "kl": 2.234375, "learning_rate": 5.624363628203727e-06, "loss": -0.0187, "step": 2940 }, { "clip_ratio": 0.017061052843928337, "completions/clipped_ratio": 0.0357142857142857, "completions/max_length": 256.0, "completions/max_terminated_length": 236.0, "completions/mean_length": 178.85714721679688, "completions/mean_terminated_length": 176.0, "completions/min_length": 125.0, "completions/min_terminated_length": 125.0, "epoch": 1.0290412876137158, "grad_norm": 4.94496294008011, "kl": 1.9296875, "learning_rate": 5.621333765955529e-06, "loss": 0.0397, "num_tokens": 14026346.0, "reward": 2.938385248184204, "reward_std": 0.15367268025875092, "rewards/check_gptzero_func/mean": 0.9821428656578064, "rewards/check_gptzero_func/std": 0.13363061845302582, "rewards/check_originality_func/mean": 0.9642857313156128, "rewards/check_originality_func/std": 0.187256321310997, "rewards/check_winston_local_func/mean": 0.9919564127922058, "rewards/check_winston_local_func/std": 0.003083372488617897, "step": 2941 }, { "clip_ratio": 0.036609433591365814, "epoch": 1.0293911826452065, "grad_norm": 4.2150231701548755, "kl": 1.921875, "learning_rate": 5.618303671955719e-06, "loss": 0.0289, "step": 2942 }, { "clip_ratio": 0.07795790582895279, "epoch": 1.029741077676697, "grad_norm": 2.374765950153936, "kl": 1.921875, "learning_rate": 5.61527334733449e-06, "loss": 0.0189, "step": 2943 }, { "clip_ratio": 0.1183653250336647, "epoch": 1.0300909727081875, "grad_norm": 1.934282726887374, "kl": 1.953125, "learning_rate": 5.612242793222125e-06, "loss": 0.0131, "step": 2944 }, { "clip_ratio": 0.03069748356938362, "completions/clipped_ratio": 0.0, "completions/max_length": 229.0, "completions/max_terminated_length": 229.0, "completions/mean_length": 155.07144165039062, "completions/mean_terminated_length": 155.07144165039062, "completions/min_length": 83.0, "completions/min_terminated_length": 83.0, "epoch": 1.0304408677396781, "grad_norm": 2.8933885158042227, "kl": 2.21875, "learning_rate": 5.609212010748992e-06, "loss": 0.01, "num_tokens": 14045310.0, "reward": 2.9206056594848633, "reward_std": 0.0773301050066948, "rewards/check_gptzero_func/mean": 0.9285714030265808, "rewards/check_gptzero_func/std": 0.25987008213996887, "rewards/check_originality_func/mean": 1.0, "rewards/check_originality_func/std": 0.0, "rewards/check_winston_local_func/mean": 0.9920340776443481, "rewards/check_winston_local_func/std": 0.0023288154043257236, "step": 2945 }, { "clip_ratio": 0.03771844878792763, "epoch": 1.0307907627711685, "grad_norm": 2.1041664272128178, "kl": 2.21875, "learning_rate": 5.6061810010455465e-06, "loss": 0.0055, "step": 2946 }, { "clip_ratio": 0.08765089511871338, "epoch": 1.0311406578026592, "grad_norm": 1.643385272286021, "kl": 2.234375, "learning_rate": 5.603149765242324e-06, "loss": -0.001, "step": 2947 }, { "clip_ratio": 0.1366853266954422, "epoch": 1.0314905528341498, "grad_norm": 1.103404594711221, "kl": 2.234375, "learning_rate": 5.600118304469948e-06, "loss": -0.0054, "step": 2948 }, { "clip_ratio": 0.010616423562169075, "completions/clipped_ratio": 0.017857142857142905, "completions/max_length": 256.0, "completions/max_terminated_length": 211.0, "completions/mean_length": 128.55357360839844, "completions/mean_terminated_length": 126.23635864257812, "completions/min_length": 65.0, "completions/min_terminated_length": 65.0, "epoch": 1.0318404478656402, "grad_norm": 3.9518894915217047, "kl": 2.625, "learning_rate": 5.5970866198591235e-06, "loss": 0.0138, "num_tokens": 14061069.0, "reward": 2.935309410095215, "reward_std": 0.12488730251789093, "rewards/check_gptzero_func/mean": 1.0, "rewards/check_gptzero_func/std": 0.0, "rewards/check_originality_func/mean": 0.9464285969734192, "rewards/check_originality_func/std": 0.22720777988433838, "rewards/check_winston_local_func/mean": 0.9888805150985718, "rewards/check_winston_local_func/std": 0.02376384660601616, "step": 2949 }, { "clip_ratio": 0.04781137779355049, "epoch": 1.0321903428971309, "grad_norm": 2.9738154423322913, "kl": 2.625, "learning_rate": 5.594054712540641e-06, "loss": 0.0053, "step": 2950 }, { "clip_ratio": 0.09614534676074982, "epoch": 1.0325402379286215, "grad_norm": 2.167647236718368, "kl": 2.625, "learning_rate": 5.591022583645373e-06, "loss": -0.0012, "step": 2951 }, { "clip_ratio": 0.15482184290885925, "epoch": 1.032890132960112, "grad_norm": 1.5955657888491102, "kl": 2.65625, "learning_rate": 5.587990234304275e-06, "loss": -0.0071, "step": 2952 }, { "clip_ratio": 0.020596850663423538, "completions/clipped_ratio": 0.0535714285714286, "completions/max_length": 256.0, "completions/max_terminated_length": 250.0, "completions/mean_length": 164.0, "completions/mean_terminated_length": 158.79244995117188, "completions/min_length": 65.0, "completions/min_terminated_length": 65.0, "epoch": 1.0332400279916025, "grad_norm": 4.436757845749753, "kl": 2.28125, "learning_rate": 5.584957665648385e-06, "loss": 0.0161, "num_tokens": 14081229.0, "reward": 2.7419466972351074, "reward_std": 0.30322715640068054, "rewards/check_gptzero_func/mean": 0.8571428656578064, "rewards/check_gptzero_func/std": 0.3530939221382141, "rewards/check_originality_func/mean": 0.8928571343421936, "rewards/check_originality_func/std": 0.3120938837528229, "rewards/check_winston_local_func/mean": 0.9919466972351074, "rewards/check_winston_local_func/std": 0.002527419477701187, "step": 2953 }, { "clip_ratio": 0.039179135113954544, "epoch": 1.0335899230230932, "grad_norm": 3.3426832607094417, "kl": 2.28125, "learning_rate": 5.581924878808822e-06, "loss": 0.0072, "step": 2954 }, { "clip_ratio": 0.09542650729417801, "epoch": 1.0339398180545836, "grad_norm": 2.424692993653941, "kl": 2.328125, "learning_rate": 5.578891874916785e-06, "loss": -0.0038, "step": 2955 }, { "clip_ratio": 0.1497231423854828, "epoch": 1.0342897130860742, "grad_norm": 2.1056194083389026, "kl": 2.328125, "learning_rate": 5.575858655103559e-06, "loss": -0.013, "step": 2956 }, { "clip_ratio": 0.019419420510530472, "completions/clipped_ratio": 0.017857142857142905, "completions/max_length": 256.0, "completions/max_terminated_length": 249.0, "completions/mean_length": 167.125, "completions/mean_terminated_length": 165.5090789794922, "completions/min_length": 108.0, "completions/min_terminated_length": 108.0, "epoch": 1.0346396081175648, "grad_norm": 3.0503365065850963, "kl": 2.125, "learning_rate": 5.572825220500505e-06, "loss": -0.0005, "num_tokens": 14102004.0, "reward": 2.974355459213257, "reward_std": 0.05183061212301254, "rewards/check_gptzero_func/mean": 0.9821428656578064, "rewards/check_gptzero_func/std": 0.13363061845302582, "rewards/check_originality_func/mean": 1.0, "rewards/check_originality_func/std": 0.0, "rewards/check_winston_local_func/mean": 0.99221271276474, "rewards/check_winston_local_func/std": 0.0022048589307814837, "step": 2957 }, { "clip_ratio": 0.05264102667570114, "epoch": 1.0349895031490552, "grad_norm": 2.563010820168722, "kl": 2.125, "learning_rate": 5.569791572239065e-06, "loss": -0.0074, "step": 2958 }, { "clip_ratio": 0.0991988554596901, "epoch": 1.0353393981805459, "grad_norm": 2.3446641610759507, "kl": 2.140625, "learning_rate": 5.566757711450761e-06, "loss": -0.0155, "step": 2959 }, { "clip_ratio": 0.1569141000509262, "epoch": 1.0356892932120363, "grad_norm": 1.5510851058089283, "kl": 2.140625, "learning_rate": 5.5637236392671945e-06, "loss": -0.0252, "step": 2960 }, { "clip_ratio": 0.024682700634002686, "completions/clipped_ratio": 0.125, "completions/max_length": 256.0, "completions/max_terminated_length": 255.0, "completions/mean_length": 162.2678680419922, "completions/mean_terminated_length": 148.87754821777344, "completions/min_length": 65.0, "completions/min_terminated_length": 65.0, "epoch": 1.036039188243527, "grad_norm": 4.608884906149297, "kl": 2.28125, "learning_rate": 5.560689356820045e-06, "loss": 0.0156, "num_tokens": 14121891.0, "reward": 2.916081666946411, "reward_std": 0.1785403937101364, "rewards/check_gptzero_func/mean": 1.0, "rewards/check_gptzero_func/std": 0.0, "rewards/check_originality_func/mean": 0.9285714030265808, "rewards/check_originality_func/std": 0.25987008213996887, "rewards/check_winston_local_func/mean": 0.9875100255012512, "rewards/check_winston_local_func/std": 0.03963566571474075, "step": 2961 }, { "clip_ratio": 0.04515165835618973, "epoch": 1.0363890832750176, "grad_norm": 3.937226815654861, "kl": 2.28125, "learning_rate": 5.5576548652410734e-06, "loss": 0.0047, "step": 2962 }, { "clip_ratio": 0.09310547262430191, "epoch": 1.036738978306508, "grad_norm": 3.0002407709864243, "kl": 2.28125, "learning_rate": 5.554620165662115e-06, "loss": -0.0069, "step": 2963 }, { "clip_ratio": 0.17749540507793427, "epoch": 1.0370888733379986, "grad_norm": 2.230848149881132, "kl": 2.28125, "learning_rate": 5.551585259215083e-06, "loss": -0.016, "step": 2964 }, { "clip_ratio": 0.01038341410458088, "completions/clipped_ratio": 0.1607142857142857, "completions/max_length": 256.0, "completions/max_terminated_length": 234.0, "completions/mean_length": 158.7857208251953, "completions/mean_terminated_length": 140.1702117919922, "completions/min_length": 88.0, "completions/min_terminated_length": 88.0, "epoch": 1.0374387683694892, "grad_norm": 2.70065811062113, "kl": 1.921875, "learning_rate": 5.548550147031971e-06, "loss": 0.0031, "num_tokens": 14141839.0, "reward": 2.6915249824523926, "reward_std": 0.25406894087791443, "rewards/check_gptzero_func/mean": 0.9821428656578064, "rewards/check_gptzero_func/std": 0.13363061845302582, "rewards/check_originality_func/mean": 0.7321428656578064, "rewards/check_originality_func/std": 0.44685041904449463, "rewards/check_winston_local_func/mean": 0.9772391319274902, "rewards/check_winston_local_func/std": 0.060543205589056015, "step": 2965 }, { "clip_ratio": 0.03633688762784004, "epoch": 1.0377886634009796, "grad_norm": 2.0068998465352808, "kl": 1.9140625, "learning_rate": 5.545514830244844e-06, "loss": -0.0005, "step": 2966 }, { "clip_ratio": 0.07568030059337616, "epoch": 1.0381385584324703, "grad_norm": 1.4271288972556637, "kl": 1.8984375, "learning_rate": 5.54247930998585e-06, "loss": -0.0052, "step": 2967 }, { "clip_ratio": 0.1327541321516037, "epoch": 1.038488453463961, "grad_norm": 1.416821954802062, "kl": 1.890625, "learning_rate": 5.5394435873872055e-06, "loss": -0.0093, "step": 2968 }, { "clip_ratio": 0.012021416798233986, "completions/clipped_ratio": 0.1428571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 253.0, "completions/mean_length": 161.42857360839844, "completions/mean_terminated_length": 145.6666717529297, "completions/min_length": 88.0, "completions/min_terminated_length": 88.0, "epoch": 1.0388383484954513, "grad_norm": 2.082988726949345, "kl": 2.046875, "learning_rate": 5.53640766358121e-06, "loss": 0.0178, "num_tokens": 14162223.0, "reward": 2.8298325538635254, "reward_std": 0.11862089484930038, "rewards/check_gptzero_func/mean": 1.0, "rewards/check_gptzero_func/std": 0.0, "rewards/check_originality_func/mean": 0.8392857313156128, "rewards/check_originality_func/std": 0.3705909848213196, "rewards/check_winston_local_func/mean": 0.9905465245246887, "rewards/check_winston_local_func/std": 0.0032736589200794697, "step": 2969 }, { "clip_ratio": 0.024231670424342155, "epoch": 1.039188243526942, "grad_norm": 1.7496863904336266, "kl": 2.046875, "learning_rate": 5.533371539700232e-06, "loss": 0.0134, "step": 2970 }, { "clip_ratio": 0.05300416424870491, "epoch": 1.0395381385584326, "grad_norm": 1.3552094896793092, "kl": 2.046875, "learning_rate": 5.530335216876717e-06, "loss": 0.0071, "step": 2971 }, { "clip_ratio": 0.09776788204908371, "epoch": 1.039888033589923, "grad_norm": 0.9858576510989038, "kl": 2.046875, "learning_rate": 5.527298696243188e-06, "loss": 0.0035, "step": 2972 }, { "clip_ratio": 0.013266326859593391, "completions/clipped_ratio": 0.0, "completions/max_length": 249.0, "completions/max_terminated_length": 249.0, "completions/mean_length": 153.33929443359375, "completions/mean_terminated_length": 153.33929443359375, "completions/min_length": 80.0, "completions/min_terminated_length": 80.0, "epoch": 1.0402379286214136, "grad_norm": 2.827115845094396, "kl": 2.21875, "learning_rate": 5.524261978932234e-06, "loss": 0.0142, "num_tokens": 14180970.0, "reward": 2.901212215423584, "reward_std": 0.1287986934185028, "rewards/check_gptzero_func/mean": 0.9821428656578064, "rewards/check_gptzero_func/std": 0.13363061845302582, "rewards/check_originality_func/mean": 0.9285714030265808, "rewards/check_originality_func/std": 0.25987008213996887, "rewards/check_winston_local_func/mean": 0.990497887134552, "rewards/check_winston_local_func/std": 0.005601636599749327, "step": 2973 }, { "clip_ratio": 0.019679052755236626, "epoch": 1.040587823652904, "grad_norm": 2.6797062791262, "kl": 2.21875, "learning_rate": 5.521225066076524e-06, "loss": 0.0103, "step": 2974 }, { "clip_ratio": 0.049189623445272446, "epoch": 1.0409377186843947, "grad_norm": 1.7213092170019009, "kl": 2.21875, "learning_rate": 5.5181879588087976e-06, "loss": 0.0021, "step": 2975 }, { "clip_ratio": 0.09843909740447998, "epoch": 1.0412876137158853, "grad_norm": 1.5859561801474922, "kl": 2.21875, "learning_rate": 5.5151506582618675e-06, "loss": -0.0034, "step": 2976 }, { "clip_ratio": 0.010038331151008606, "completions/clipped_ratio": 0.2678571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 244.0, "completions/mean_length": 184.4107208251953, "completions/mean_terminated_length": 158.21951293945312, "completions/min_length": 122.0, "completions/min_terminated_length": 122.0, "epoch": 1.0416375087473757, "grad_norm": 2.7723599457130046, "kl": 1.7890625, "learning_rate": 5.512113165568615e-06, "loss": 0.0106, "num_tokens": 14203625.0, "reward": 2.9026682376861572, "reward_std": 0.18420985341072083, "rewards/check_gptzero_func/mean": 1.0, "rewards/check_gptzero_func/std": 0.0, "rewards/check_originality_func/mean": 0.9107142686843872, "rewards/check_originality_func/std": 0.28773635625839233, "rewards/check_winston_local_func/mean": 0.9919538497924805, "rewards/check_winston_local_func/std": 0.003582259640097618, "step": 2977 }, { "clip_ratio": 0.016789065673947334, "epoch": 1.0419874037788663, "grad_norm": 2.7321758931345057, "kl": 1.7890625, "learning_rate": 5.509075481862001e-06, "loss": 0.0022, "step": 2978 }, { "clip_ratio": 0.0427837148308754, "epoch": 1.042337298810357, "grad_norm": 1.6118160520472393, "kl": 1.7890625, "learning_rate": 5.50603760827505e-06, "loss": -0.0063, "step": 2979 }, { "clip_ratio": 0.08164671808481216, "epoch": 1.0426871938418474, "grad_norm": 1.2131928124345304, "kl": 1.796875, "learning_rate": 5.502999545940859e-06, "loss": -0.0122, "step": 2980 }, { "clip_ratio": 0.018740952014923096, "completions/clipped_ratio": 0.0, "completions/max_length": 245.0, "completions/max_terminated_length": 245.0, "completions/mean_length": 171.85714721679688, "completions/mean_terminated_length": 171.85714721679688, "completions/min_length": 106.0, "completions/min_terminated_length": 106.0, "epoch": 1.043037088873338, "grad_norm": 2.5417707493711896, "kl": 1.8828125, "learning_rate": 5.4999612959925995e-06, "loss": 0.0069, "num_tokens": 14224777.0, "reward": 2.9917290210723877, "reward_std": 0.0031145017128437757, "rewards/check_gptzero_func/mean": 1.0, "rewards/check_gptzero_func/std": 0.0, "rewards/check_originality_func/mean": 1.0, "rewards/check_originality_func/std": 0.0, "rewards/check_winston_local_func/mean": 0.9917290806770325, "rewards/check_winston_local_func/std": 0.004113874398171902, "step": 2981 }, { "clip_ratio": 0.026085203513503075, "epoch": 1.0433869839048286, "grad_norm": 2.360911102326627, "kl": 1.8828125, "learning_rate": 5.496922859563507e-06, "loss": 0.0018, "step": 2982 }, { "clip_ratio": 0.060908060520887375, "epoch": 1.043736878936319, "grad_norm": 2.0080965246447384, "kl": 1.8828125, "learning_rate": 5.493884237786891e-06, "loss": -0.0066, "step": 2983 }, { "clip_ratio": 0.11911710351705551, "epoch": 1.0440867739678097, "grad_norm": 1.6626473261863808, "kl": 1.8828125, "learning_rate": 5.49084543179613e-06, "loss": -0.0134, "step": 2984 }, { "clip_ratio": 0.011752516962587833, "completions/clipped_ratio": 0.0535714285714286, "completions/max_length": 256.0, "completions/max_terminated_length": 254.0, "completions/mean_length": 155.71429443359375, "completions/mean_terminated_length": 150.03773498535156, "completions/min_length": 84.0, "completions/min_terminated_length": 84.0, "epoch": 1.0444366689993, "grad_norm": 2.052542201612757, "kl": 1.984375, "learning_rate": 5.487806442724668e-06, "loss": 0.0108, "num_tokens": 14244593.0, "reward": 2.916327714920044, "reward_std": 0.17040468752384186, "rewards/check_gptzero_func/mean": 0.9642857313156128, "rewards/check_gptzero_func/std": 0.187256321310997, "rewards/check_originality_func/mean": 0.9642857313156128, "rewards/check_originality_func/std": 0.187256321310997, "rewards/check_winston_local_func/mean": 0.9877563118934631, "rewards/check_winston_local_func/std": 0.011776238679885864, "step": 2985 }, { "clip_ratio": 0.016773881390690804, "epoch": 1.0447865640307907, "grad_norm": 1.670789494856157, "kl": 1.984375, "learning_rate": 5.48476727170602e-06, "loss": 0.0048, "step": 2986 }, { "clip_ratio": 0.03988916054368019, "epoch": 1.0451364590622814, "grad_norm": 1.339750604541236, "kl": 2.0, "learning_rate": 5.4817279198737676e-06, "loss": -0.0001, "step": 2987 }, { "clip_ratio": 0.0841217041015625, "epoch": 1.0454863540937718, "grad_norm": 1.08073956808543, "kl": 2.015625, "learning_rate": 5.478688388361558e-06, "loss": -0.0051, "step": 2988 }, { "clip_ratio": 0.020270301029086113, "completions/clipped_ratio": 0.0, "completions/max_length": 253.0, "completions/max_terminated_length": 253.0, "completions/mean_length": 144.85714721679688, "completions/mean_terminated_length": 144.85714721679688, "completions/min_length": 90.0, "completions/min_terminated_length": 90.0, "epoch": 1.0458362491252624, "grad_norm": 3.718158839830306, "kl": 2.234375, "learning_rate": 5.475648678303112e-06, "loss": 0.0048, "num_tokens": 14262585.0, "reward": 2.9911015033721924, "reward_std": 0.002847041003406048, "rewards/check_gptzero_func/mean": 1.0, "rewards/check_gptzero_func/std": 0.0, "rewards/check_originality_func/mean": 1.0, "rewards/check_originality_func/std": 0.0, "rewards/check_winston_local_func/mean": 0.9911016225814819, "rewards/check_winston_local_func/std": 0.003094530664384365, "step": 2989 }, { "clip_ratio": 0.034513749182224274, "epoch": 1.046186144156753, "grad_norm": 3.351238632895407, "kl": 2.234375, "learning_rate": 5.472608790832211e-06, "loss": 0.0001, "step": 2990 }, { "clip_ratio": 0.0863841101527214, "epoch": 1.0465360391882434, "grad_norm": 2.2155285307197476, "kl": 2.234375, "learning_rate": 5.469568727082703e-06, "loss": -0.0068, "step": 2991 }, { "clip_ratio": 0.13088111579418182, "epoch": 1.046885934219734, "grad_norm": 1.6667103063772846, "kl": 2.234375, "learning_rate": 5.466528488188505e-06, "loss": -0.0113, "step": 2992 }, { "clip_ratio": 0.010350950993597507, "completions/clipped_ratio": 0.0, "completions/max_length": 220.0, "completions/max_terminated_length": 220.0, "completions/mean_length": 157.60714721679688, "completions/mean_terminated_length": 157.60714721679688, "completions/min_length": 82.0, "completions/min_terminated_length": 82.0, "epoch": 1.0472358292512247, "grad_norm": 3.0917831579253097, "kl": 2.15625, "learning_rate": 5.4634880752835954e-06, "loss": 0.0076, "num_tokens": 14281531.0, "reward": 2.707174777984619, "reward_std": 0.24934178590774536, "rewards/check_gptzero_func/mean": 0.7857142686843872, "rewards/check_gptzero_func/std": 0.4140393137931824, "rewards/check_originality_func/mean": 0.9464285969734192, "rewards/check_originality_func/std": 0.22720777988433838, "rewards/check_winston_local_func/mean": 0.9750315546989441, "rewards/check_winston_local_func/std": 0.07101523876190186, "step": 2993 }, { "clip_ratio": 0.020427923649549484, "epoch": 1.0475857242827151, "grad_norm": 2.8780575852643797, "kl": 2.15625, "learning_rate": 5.4604474895020195e-06, "loss": 0.0033, "step": 2994 }, { "clip_ratio": 0.06706845015287399, "epoch": 1.0479356193142058, "grad_norm": 1.4958514346346228, "kl": 2.15625, "learning_rate": 5.45740673197789e-06, "loss": -0.0024, "step": 2995 }, { "clip_ratio": 0.09721199423074722, "epoch": 1.0482855143456964, "grad_norm": 1.4294753416713992, "kl": 2.171875, "learning_rate": 5.454365803845378e-06, "loss": -0.0064, "step": 2996 }, { "clip_ratio": 0.01165732927620411, "completions/clipped_ratio": 0.0535714285714286, "completions/max_length": 256.0, "completions/max_terminated_length": 254.0, "completions/mean_length": 153.25, "completions/mean_terminated_length": 147.4339599609375, "completions/min_length": 71.0, "completions/min_terminated_length": 71.0, "epoch": 1.0486354093771868, "grad_norm": 2.150785170554603, "kl": 2.140625, "learning_rate": 5.451324706238721e-06, "loss": 0.0093, "num_tokens": 14300241.0, "reward": 2.9194576740264893, "reward_std": 0.13405311107635498, "rewards/check_gptzero_func/mean": 1.0, "rewards/check_gptzero_func/std": 0.0, "rewards/check_originality_func/mean": 0.9285714030265808, "rewards/check_originality_func/std": 0.25987008213996887, "rewards/check_winston_local_func/mean": 0.9908863306045532, "rewards/check_winston_local_func/std": 0.004539735149592161, "step": 2997 }, { "clip_ratio": 0.012909970246255398, "epoch": 1.0489853044086774, "grad_norm": 2.0825334200700105, "kl": 2.15625, "learning_rate": 5.448283440292219e-06, "loss": 0.0067, "step": 2998 }, { "clip_ratio": 0.03564988449215889, "epoch": 1.0493351994401678, "grad_norm": 1.4828958468836708, "kl": 2.15625, "learning_rate": 5.445242007140238e-06, "loss": -0.0016, "step": 2999 }, { "clip_ratio": 0.08077813684940338, "epoch": 1.0496850944716585, "grad_norm": 1.2723850848079281, "kl": 2.140625, "learning_rate": 5.442200407917202e-06, "loss": -0.0068, "step": 3000 }, { "clip_ratio": 0.015133687295019627, "completions/clipped_ratio": 0.0, "completions/max_length": 240.0, "completions/max_terminated_length": 240.0, "completions/mean_length": 153.0178680419922, "completions/mean_terminated_length": 153.0178680419922, "completions/min_length": 79.0, "completions/min_terminated_length": 79.0, "epoch": 1.050034989503149, "grad_norm": 2.810241853483439, "kl": 2.171875, "learning_rate": 5.439158643757599e-06, "loss": 0.0151, "num_tokens": 14319066.0, "reward": 3.9690141677856445, "reward_std": 0.03835548833012581, "rewards/check_gptzero_func/mean": 1.0, "rewards/check_gptzero_func/std": 0.0, "rewards/check_originality_func/mean": 1.0, "rewards/check_originality_func/std": 0.0, "rewards/check_winston_local_func/mean": 0.9850853681564331, "rewards/check_winston_local_func/std": 0.020299192517995834, "rewards/sentence_count_match_reward_logic/mean": 0.9839285612106323, "rewards/sentence_count_match_reward_logic/std": 0.04964157193899155, "step": 3001 }, { "clip_ratio": 0.02070591412484646, "epoch": 1.0503848845346395, "grad_norm": 2.672493200089234, "kl": 2.1875, "learning_rate": 5.436116715795978e-06, "loss": 0.0087, "step": 3002 }, { "clip_ratio": 0.039430346339941025, "epoch": 1.0507347795661302, "grad_norm": 2.4525788133705047, "kl": 2.171875, "learning_rate": 5.433074625166949e-06, "loss": 0.0029, "step": 3003 }, { "clip_ratio": 0.0822756290435791, "epoch": 1.0510846745976208, "grad_norm": 1.6201695117453312, "kl": 2.1875, "learning_rate": 5.430032373005184e-06, "loss": -0.0051, "step": 3004 }, { "clip_ratio": 0.015114284120500088, "completions/clipped_ratio": 0.0714285714285714, "completions/max_length": 255.0, "completions/max_terminated_length": 255.0, "completions/mean_length": 155.2678680419922, "completions/mean_terminated_length": 167.2115478515625, "completions/min_length": 0.0, "completions/min_terminated_length": 101.0, "epoch": 1.0514345696291112, "grad_norm": 2.9037708237137827, "kl": 2.015625, "learning_rate": 5.426989960445415e-06, "loss": -0.0636, "num_tokens": 14339065.0, "reward": 3.7653284072875977, "reward_std": 0.33294153213500977, "rewards/check_gptzero_func/mean": 0.9107142686843872, "rewards/check_gptzero_func/std": 0.28773635625839233, "rewards/check_originality_func/mean": 0.875, "rewards/check_originality_func/std": 0.3337118923664093, "rewards/check_winston_local_func/mean": 0.9923621416091919, "rewards/check_winston_local_func/std": 0.002128463238477707, "rewards/sentence_count_match_reward_logic/mean": 0.9872519373893738, "rewards/sentence_count_match_reward_logic/std": 0.0424250066280365, "step": 3005 }, { "clip_ratio": 0.02035653404891491, "epoch": 1.0517844646606018, "grad_norm": 2.7475870324195806, "kl": 2.03125, "learning_rate": 5.423947388622431e-06, "loss": -0.0689, "step": 3006 }, { "clip_ratio": 0.04024800285696983, "epoch": 1.0521343596920925, "grad_norm": 2.1988586686916913, "kl": 2.03125, "learning_rate": 5.420904658671082e-06, "loss": -0.0751, "step": 3007 }, { "clip_ratio": 0.10450894385576248, "epoch": 1.0524842547235829, "grad_norm": 1.259061098758635, "kl": 2.03125, "learning_rate": 5.4178617717262805e-06, "loss": -0.0849, "step": 3008 }, { "clip_ratio": 0.013682009652256966, "completions/clipped_ratio": 0.0714285714285714, "completions/max_length": 235.0, "completions/max_terminated_length": 235.0, "completions/mean_length": 165.46429443359375, "completions/mean_terminated_length": 178.19232177734375, "completions/min_length": 0.0, "completions/min_terminated_length": 119.0, "epoch": 1.0528341497550735, "grad_norm": 3.1577255575397176, "kl": 1.6953125, "learning_rate": 5.414818728922993e-06, "loss": -0.0164, "num_tokens": 14360147.0, "reward": 3.927868127822876, "reward_std": 0.12019401788711548, "rewards/check_gptzero_func/mean": 1.0, "rewards/check_gptzero_func/std": 0.0, "rewards/check_originality_func/mean": 0.9642857313156128, "rewards/check_originality_func/std": 0.187256321310997, "rewards/check_winston_local_func/mean": 0.9906586408615112, "rewards/check_winston_local_func/std": 0.0036381427198648453, "rewards/sentence_count_match_reward_logic/mean": 0.972923755645752, "rewards/sentence_count_match_reward_logic/std": 0.0704304575920105, "step": 3009 }, { "clip_ratio": 0.030437257140874863, "epoch": 1.0531840447865641, "grad_norm": 2.6757700543676255, "kl": 1.703125, "learning_rate": 5.411775531396245e-06, "loss": -0.0225, "step": 3010 }, { "clip_ratio": 0.06756015121936798, "epoch": 1.0535339398180545, "grad_norm": 1.955092197981652, "kl": 1.703125, "learning_rate": 5.4087321802811224e-06, "loss": -0.029, "step": 3011 }, { "clip_ratio": 0.11205606162548065, "epoch": 1.0538838348495452, "grad_norm": 1.2893037716244078, "kl": 1.7265625, "learning_rate": 5.405688676712764e-06, "loss": -0.0344, "step": 3012 }, { "clip_ratio": 0.01014427375048399, "completions/clipped_ratio": 0.125, "completions/max_length": 255.0, "completions/max_terminated_length": 255.0, "completions/mean_length": 131.625, "completions/mean_terminated_length": 150.42857360839844, "completions/min_length": 0.0, "completions/min_terminated_length": 68.0, "epoch": 1.0542337298810356, "grad_norm": 2.6059447779981286, "kl": 2.09375, "learning_rate": 5.402645021826367e-06, "loss": -0.0187, "num_tokens": 14378590.0, "reward": 3.5257062911987305, "reward_std": 0.34437692165374756, "rewards/check_gptzero_func/mean": 0.7321428656578064, "rewards/check_gptzero_func/std": 0.446850448846817, "rewards/check_originality_func/mean": 0.875, "rewards/check_originality_func/std": 0.3337118923664093, "rewards/check_winston_local_func/mean": 0.971709668636322, "rewards/check_winston_local_func/std": 0.07038688659667969, "rewards/sentence_count_match_reward_logic/mean": 0.9468536972999573, "rewards/sentence_count_match_reward_logic/std": 0.07822348922491074, "step": 3013 }, { "clip_ratio": 0.022118082270026207, "epoch": 1.0545836249125262, "grad_norm": 1.9684795095458392, "kl": 2.109375, "learning_rate": 5.399601216757189e-06, "loss": -0.0256, "step": 3014 }, { "clip_ratio": 0.060954976826906204, "epoch": 1.0549335199440169, "grad_norm": 1.3707197345562063, "kl": 2.109375, "learning_rate": 5.39655726264054e-06, "loss": -0.03, "step": 3015 }, { "clip_ratio": 0.08947420865297318, "epoch": 1.0552834149755073, "grad_norm": 1.2351777420201293, "kl": 2.109375, "learning_rate": 5.393513160611785e-06, "loss": -0.0326, "step": 3016 }, { "clip_ratio": 0.037743981927633286, "completions/clipped_ratio": 0.0357142857142857, "completions/max_length": 241.0, "completions/max_terminated_length": 241.0, "completions/mean_length": 128.375, "completions/mean_terminated_length": 133.12962341308594, "completions/min_length": 0.0, "completions/min_terminated_length": 68.0, "epoch": 1.055633310006998, "grad_norm": 3.8421938037288945, "kl": 2.609375, "learning_rate": 5.390468911806345e-06, "loss": -0.02, "num_tokens": 14394859.0, "reward": 3.8859145641326904, "reward_std": 0.21676309406757355, "rewards/check_gptzero_func/mean": 1.0, "rewards/check_gptzero_func/std": 0.0, "rewards/check_originality_func/mean": 0.9107142686843872, "rewards/check_originality_func/std": 0.28773635625839233, "rewards/check_winston_local_func/mean": 0.9841285347938538, "rewards/check_winston_local_func/std": 0.029874393716454506, "rewards/sentence_count_match_reward_logic/mean": 0.9910714030265808, "rewards/sentence_count_match_reward_logic/std": 0.05253323167562485, "step": 3017 }, { "clip_ratio": 0.057489972561597824, "epoch": 1.0559832050384885, "grad_norm": 2.2546488129747955, "kl": 2.625, "learning_rate": 5.387424517359696e-06, "loss": -0.0249, "step": 3018 }, { "clip_ratio": 0.07525009661912918, "epoch": 1.056333100069979, "grad_norm": 1.9551112535626913, "kl": 2.640625, "learning_rate": 5.384379978407369e-06, "loss": -0.0278, "step": 3019 }, { "clip_ratio": 0.10372042655944824, "epoch": 1.0566829951014696, "grad_norm": 2.358380815118254, "kl": 2.625, "learning_rate": 5.381335296084949e-06, "loss": -0.0342, "step": 3020 }, { "clip_ratio": 0.021829649806022644, "completions/clipped_ratio": 0.1071428571428571, "completions/max_length": 235.0, "completions/max_terminated_length": 235.0, "completions/mean_length": 132.55357360839844, "completions/mean_terminated_length": 148.45999145507812, "completions/min_length": 0.0, "completions/min_terminated_length": 57.0, "epoch": 1.0570328901329602, "grad_norm": 3.7507284761955573, "kl": 2.25, "learning_rate": 5.3782904715280705e-06, "loss": -0.0277, "num_tokens": 14412074.0, "reward": 3.8183329105377197, "reward_std": 0.25274747610092163, "rewards/check_gptzero_func/mean": 1.0, "rewards/check_gptzero_func/std": 0.0, "rewards/check_originality_func/mean": 0.875, "rewards/check_originality_func/std": 0.3337118923664093, "rewards/check_winston_local_func/mean": 0.9845176339149475, "rewards/check_winston_local_func/std": 0.025009101256728172, "rewards/sentence_count_match_reward_logic/mean": 0.9588151574134827, "rewards/sentence_count_match_reward_logic/std": 0.08373341709375381, "step": 3021 }, { "clip_ratio": 0.04046761244535446, "epoch": 1.0573827851644506, "grad_norm": 2.9161905217808672, "kl": 2.25, "learning_rate": 5.375245505872429e-06, "loss": -0.036, "step": 3022 }, { "clip_ratio": 0.0826931968331337, "epoch": 1.0577326801959412, "grad_norm": 1.572094919569261, "kl": 2.25, "learning_rate": 5.372200400253763e-06, "loss": -0.0417, "step": 3023 }, { "clip_ratio": 0.13703478872776031, "epoch": 1.0580825752274319, "grad_norm": 1.1083598821887146, "kl": 2.265625, "learning_rate": 5.3691551558078715e-06, "loss": -0.0462, "step": 3024 }, { "clip_ratio": 0.015232743695378304, "completions/clipped_ratio": 0.0, "completions/max_length": 224.0, "completions/max_terminated_length": 224.0, "completions/mean_length": 140.0357208251953, "completions/mean_terminated_length": 140.0357208251953, "completions/min_length": 74.0, "completions/min_terminated_length": 74.0, "epoch": 1.0584324702589223, "grad_norm": 2.27453225271881, "kl": 2.71875, "learning_rate": 5.3661097736706e-06, "loss": 0.0019, "num_tokens": 14429532.0, "reward": 3.8669376373291016, "reward_std": 0.24537970125675201, "rewards/check_gptzero_func/mean": 1.0, "rewards/check_gptzero_func/std": 0.0, "rewards/check_originality_func/mean": 0.8928571343421936, "rewards/check_originality_func/std": 0.31209391355514526, "rewards/check_winston_local_func/mean": 0.9915122389793396, "rewards/check_winston_local_func/std": 0.003711689030751586, "rewards/sentence_count_match_reward_logic/mean": 0.982568085193634, "rewards/sentence_count_match_reward_logic/std": 0.05086169391870499, "step": 3025 }, { "clip_ratio": 0.02040010690689087, "epoch": 1.058782365290413, "grad_norm": 1.637917755891273, "kl": 2.71875, "learning_rate": 5.3630642549778475e-06, "loss": -0.0028, "step": 3026 }, { "clip_ratio": 0.04121536761522293, "epoch": 1.0591322603219033, "grad_norm": 0.9662813343396732, "kl": 2.75, "learning_rate": 5.360018600865566e-06, "loss": -0.0057, "step": 3027 }, { "clip_ratio": 0.06136297062039375, "epoch": 1.059482155353394, "grad_norm": 1.2534660005865246, "kl": 2.796875, "learning_rate": 5.356972812469753e-06, "loss": -0.0083, "step": 3028 }, { "clip_ratio": 0.01716042496263981, "completions/clipped_ratio": 0.017857142857142905, "completions/max_length": 232.0, "completions/max_terminated_length": 232.0, "completions/mean_length": 167.57144165039062, "completions/mean_terminated_length": 170.61817932128906, "completions/min_length": 0.0, "completions/min_terminated_length": 103.0, "epoch": 1.0598320503848846, "grad_norm": 3.939583202779671, "kl": 1.953125, "learning_rate": 5.35392689092646e-06, "loss": 0.0264, "num_tokens": 14449996.0, "reward": 3.5200600624084473, "reward_std": 0.5086892247200012, "rewards/check_gptzero_func/mean": 0.6428571343421936, "rewards/check_gptzero_func/std": 0.48349377512931824, "rewards/check_originality_func/mean": 0.8928571343421936, "rewards/check_originality_func/std": 0.3120938837528229, "rewards/check_winston_local_func/mean": 0.9863296151161194, "rewards/check_winston_local_func/std": 0.02019653469324112, "rewards/sentence_count_match_reward_logic/mean": 0.9980158805847168, "rewards/sentence_count_match_reward_logic/std": 0.014847845770418644, "step": 3029 }, { "clip_ratio": 0.024108240380883217, "epoch": 1.060181945416375, "grad_norm": 2.761984882858165, "kl": 1.9609375, "learning_rate": 5.3508808373717845e-06, "loss": 0.0201, "step": 3030 }, { "clip_ratio": 0.04707685485482216, "epoch": 1.0605318404478656, "grad_norm": 1.8718737887733867, "kl": 1.984375, "learning_rate": 5.3478346529418775e-06, "loss": 0.0155, "step": 3031 }, { "clip_ratio": 0.08211620897054672, "epoch": 1.0608817354793563, "grad_norm": 1.3555819083282903, "kl": 2.0, "learning_rate": 5.3447883387729395e-06, "loss": 0.0099, "step": 3032 }, { "clip_ratio": 0.01625978760421276, "completions/clipped_ratio": 0.125, "completions/max_length": 245.0, "completions/max_terminated_length": 245.0, "completions/mean_length": 156.9107208251953, "completions/mean_terminated_length": 179.32652282714844, "completions/min_length": 0.0, "completions/min_terminated_length": 119.0, "epoch": 1.0612316305108467, "grad_norm": 2.104225459761388, "kl": 1.9609375, "learning_rate": 5.341741896001212e-06, "loss": 0.0282, "num_tokens": 14470671.0, "reward": 3.790379285812378, "reward_std": 0.16514043509960175, "rewards/check_gptzero_func/mean": 0.9107142686843872, "rewards/check_gptzero_func/std": 0.28773635625839233, "rewards/check_originality_func/mean": 0.9642857313156128, "rewards/check_originality_func/std": 0.187256321310997, "rewards/check_winston_local_func/mean": 0.9882956743240356, "rewards/check_winston_local_func/std": 0.030340921133756638, "rewards/sentence_count_match_reward_logic/mean": 0.9270833134651184, "rewards/sentence_count_match_reward_logic/std": 0.09391896426677704, "step": 3033 }, { "clip_ratio": 0.028834255412220955, "epoch": 1.0615815255423373, "grad_norm": 1.888039053286925, "kl": 1.9609375, "learning_rate": 5.338695325762992e-06, "loss": 0.022, "step": 3034 }, { "clip_ratio": 0.06434811651706696, "epoch": 1.061931420573828, "grad_norm": 1.368331720860613, "kl": 1.953125, "learning_rate": 5.335648629194622e-06, "loss": 0.0161, "step": 3035 }, { "clip_ratio": 0.1336527168750763, "epoch": 1.0622813156053184, "grad_norm": 1.2546588382623582, "kl": 1.953125, "learning_rate": 5.332601807432489e-06, "loss": 0.0103, "step": 3036 }, { "clip_ratio": 0.02611778862774372, "completions/clipped_ratio": 0.0357142857142857, "completions/max_length": 233.0, "completions/max_terminated_length": 233.0, "completions/mean_length": 164.21429443359375, "completions/mean_terminated_length": 170.29629516601562, "completions/min_length": 0.0, "completions/min_terminated_length": 88.0, "epoch": 1.062631210636809, "grad_norm": 3.2795460414473774, "kl": 1.9921875, "learning_rate": 5.329554861613031e-06, "loss": -0.0131, "num_tokens": 14491147.0, "reward": 3.86728572845459, "reward_std": 0.2580018639564514, "rewards/check_gptzero_func/mean": 0.9285714030265808, "rewards/check_gptzero_func/std": 0.25987008213996887, "rewards/check_originality_func/mean": 0.9642857313156128, "rewards/check_originality_func/std": 0.187256321310997, "rewards/check_winston_local_func/mean": 0.9860356450080872, "rewards/check_winston_local_func/std": 0.014714566059410572, "rewards/sentence_count_match_reward_logic/mean": 0.9883928298950195, "rewards/sentence_count_match_reward_logic/std": 0.049535173922777176, "step": 3037 }, { "clip_ratio": 0.03000415302813053, "epoch": 1.0629811056682996, "grad_norm": 3.10828998167661, "kl": 1.9921875, "learning_rate": 5.326507792872729e-06, "loss": -0.0173, "step": 3038 }, { "clip_ratio": 0.09936472028493881, "epoch": 1.06333100069979, "grad_norm": 2.143817432620962, "kl": 2.0, "learning_rate": 5.3234606023481085e-06, "loss": -0.0249, "step": 3039 }, { "clip_ratio": 0.17262080311775208, "epoch": 1.0636808957312807, "grad_norm": 1.7835027248261897, "kl": 2.03125, "learning_rate": 5.320413291175744e-06, "loss": -0.0297, "step": 3040 }, { "clip_ratio": 0.021200301125645638, "completions/clipped_ratio": 0.1964285714285714, "completions/max_length": 256.0, "completions/max_terminated_length": 256.0, "completions/mean_length": 146.3928680419922, "completions/mean_terminated_length": 182.1777801513672, "completions/min_length": 0.0, "completions/min_terminated_length": 104.0, "epoch": 1.064030790762771, "grad_norm": 3.0926454132876993, "kl": 1.84375, "learning_rate": 5.317365860492256e-06, "loss": -0.0306, "num_tokens": 14511953.0, "reward": 3.8474090099334717, "reward_std": 0.17085593938827515, "rewards/check_gptzero_func/mean": 0.9464285969734192, "rewards/check_gptzero_func/std": 0.22720777988433838, "rewards/check_originality_func/mean": 0.9642857313156128, "rewards/check_originality_func/std": 0.187256321310997, "rewards/check_winston_local_func/mean": 0.9921870231628418, "rewards/check_winston_local_func/std": 0.0024084008764475584, "rewards/sentence_count_match_reward_logic/mean": 0.9445075392723083, "rewards/sentence_count_match_reward_logic/std": 0.10145265609025955, "step": 3041 }, { "clip_ratio": 0.04407704249024391, "epoch": 1.0643806857942617, "grad_norm": 2.4788565533138383, "kl": 1.8359375, "learning_rate": 5.314318311434307e-06, "loss": -0.037, "step": 3042 }, { "clip_ratio": 0.09315555542707443, "epoch": 1.0647305808257523, "grad_norm": 1.7873593939877481, "kl": 1.84375, "learning_rate": 5.311270645138601e-06, "loss": -0.0464, "step": 3043 }, { "clip_ratio": 0.13374534249305725, "epoch": 1.0650804758572427, "grad_norm": 1.8359568170613974, "kl": 1.8359375, "learning_rate": 5.3082228627418895e-06, "loss": -0.0506, "step": 3044 }, { "clip_ratio": 0.031687792390584946, "completions/clipped_ratio": 0.0892857142857143, "completions/max_length": 254.0, "completions/max_terminated_length": 254.0, "completions/mean_length": 155.4107208251953, "completions/mean_terminated_length": 170.64706420898438, "completions/min_length": 0.0, "completions/min_terminated_length": 112.0, "epoch": 1.0654303708887334, "grad_norm": 2.911920383674559, "kl": 1.8828125, "learning_rate": 5.3051749653809685e-06, "loss": -0.0856, "num_tokens": 14532840.0, "reward": 3.7575645446777344, "reward_std": 0.33128032088279724, "rewards/check_gptzero_func/mean": 0.9821428656578064, "rewards/check_gptzero_func/std": 0.13363061845302582, "rewards/check_originality_func/mean": 0.8392857313156128, "rewards/check_originality_func/std": 0.3705909848213196, "rewards/check_winston_local_func/mean": 0.9753433465957642, "rewards/check_winston_local_func/std": 0.057148367166519165, "rewards/sentence_count_match_reward_logic/mean": 0.9607922434806824, "rewards/sentence_count_match_reward_logic/std": 0.07401727885007858, "step": 3045 }, { "clip_ratio": 0.02914193458855152, "epoch": 1.065780265920224, "grad_norm": 2.7886894596680745, "kl": 1.8671875, "learning_rate": 5.3021269541926735e-06, "loss": -0.0876, "step": 3046 }, { "clip_ratio": 0.07068236917257309, "epoch": 1.0661301609517144, "grad_norm": 2.2046920313827933, "kl": 1.8671875, "learning_rate": 5.299078830313883e-06, "loss": -0.0921, "step": 3047 }, { "clip_ratio": 0.10260064154863358, "epoch": 1.066480055983205, "grad_norm": 1.724132785282033, "kl": 1.8671875, "learning_rate": 5.296030594881523e-06, "loss": -0.0977, "step": 3048 }, { "clip_ratio": 0.025066768750548363, "completions/clipped_ratio": 0.2678571428571429, "completions/max_length": 248.0, "completions/max_terminated_length": 248.0, "completions/mean_length": 121.83929443359375, "completions/mean_terminated_length": 166.4146270751953, "completions/min_length": 0.0, "completions/min_terminated_length": 82.0, "epoch": 1.0668299510146957, "grad_norm": 2.8190007991280597, "kl": 1.9453125, "learning_rate": 5.292982249032549e-06, "loss": 0.009, "num_tokens": 14552879.0, "reward": 3.777986526489258, "reward_std": 0.2613278329372406, "rewards/check_gptzero_func/mean": 0.9642857313156128, "rewards/check_gptzero_func/std": 0.187256321310997, "rewards/check_originality_func/mean": 0.8571428656578064, "rewards/check_originality_func/std": 0.3530939221382141, "rewards/check_winston_local_func/mean": 0.9920451045036316, "rewards/check_winston_local_func/std": 0.00309037696570158, "rewards/sentence_count_match_reward_logic/mean": 0.9645124673843384, "rewards/sentence_count_match_reward_logic/std": 0.0690879076719284, "step": 3049 }, { "clip_ratio": 0.0419003963470459, "epoch": 1.067179846046186, "grad_norm": 1.854990747503435, "kl": 1.9453125, "learning_rate": 5.28993379390397e-06, "loss": 0.0022, "step": 3050 }, { "clip_ratio": 0.0942889079451561, "epoch": 1.0675297410776767, "grad_norm": 1.0183874070338663, "kl": 1.9765625, "learning_rate": 5.28688523063283e-06, "loss": -0.0019, "step": 3051 }, { "clip_ratio": 0.1287861466407776, "epoch": 1.0678796361091671, "grad_norm": 0.9518490901120868, "kl": 2.0, "learning_rate": 5.2838365603562135e-06, "loss": -0.0046, "step": 3052 }, { "clip_ratio": 0.025831827893853188, "completions/clipped_ratio": 0.1428571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 256.0, "completions/mean_length": 120.9464340209961, "completions/mean_terminated_length": 141.1041717529297, "completions/min_length": 0.0, "completions/min_terminated_length": 60.0, "epoch": 1.0682295311406578, "grad_norm": 2.6374637712344486, "kl": 2.46875, "learning_rate": 5.2807877842112475e-06, "loss": -0.0337, "num_tokens": 14570012.0, "reward": 3.6401312351226807, "reward_std": 0.3021279275417328, "rewards/check_gptzero_func/mean": 0.875, "rewards/check_gptzero_func/std": 0.3337118923664093, "rewards/check_originality_func/mean": 0.8214285969734192, "rewards/check_originality_func/std": 0.3864591419696808, "rewards/check_winston_local_func/mean": 0.969099223613739, "rewards/check_winston_local_func/std": 0.08699356764554977, "rewards/sentence_count_match_reward_logic/mean": 0.9746031761169434, "rewards/sentence_count_match_reward_logic/std": 0.0656304731965065, "step": 3053 }, { "clip_ratio": 0.03396891430020332, "epoch": 1.0685794261721484, "grad_norm": 2.4392444541413516, "kl": 2.46875, "learning_rate": 5.277738903335097e-06, "loss": -0.0378, "step": 3054 }, { "clip_ratio": 0.08153598755598068, "epoch": 1.0689293212036388, "grad_norm": 1.253953989632105, "kl": 2.46875, "learning_rate": 5.274689918864962e-06, "loss": -0.0449, "step": 3055 }, { "clip_ratio": 0.1156974509358406, "epoch": 1.0692792162351294, "grad_norm": 1.1774123888662753, "kl": 2.515625, "learning_rate": 5.2716408319380865e-06, "loss": -0.0457, "step": 3056 }, { "clip_ratio": 0.017473479732871056, "completions/clipped_ratio": 0.0535714285714286, "completions/max_length": 235.0, "completions/max_terminated_length": 235.0, "completions/mean_length": 122.14286041259766, "completions/mean_terminated_length": 129.05661010742188, "completions/min_length": 0.0, "completions/min_terminated_length": 59.0, "epoch": 1.06962911126662, "grad_norm": 3.5542608159552076, "kl": 2.734375, "learning_rate": 5.2685916436917524e-06, "loss": -0.0202, "num_tokens": 14585684.0, "reward": 3.775099039077759, "reward_std": 0.3447791635990143, "rewards/check_gptzero_func/mean": 0.9464285969734192, "rewards/check_gptzero_func/std": 0.22720777988433838, "rewards/check_originality_func/mean": 0.8928571343421936, "rewards/check_originality_func/std": 0.3120938837528229, "rewards/check_winston_local_func/mean": 0.9865784049034119, "rewards/check_winston_local_func/std": 0.01675347425043583, "rewards/sentence_count_match_reward_logic/mean": 0.949234664440155, "rewards/sentence_count_match_reward_logic/std": 0.11981397867202759, "step": 3057 }, { "clip_ratio": 0.02990180253982544, "epoch": 1.0699790062981105, "grad_norm": 2.390651400411448, "kl": 2.734375, "learning_rate": 5.265542355263277e-06, "loss": -0.0259, "step": 3058 }, { "clip_ratio": 0.06116872653365135, "epoch": 1.0703289013296011, "grad_norm": 1.3708031107738141, "kl": 2.75, "learning_rate": 5.262492967790017e-06, "loss": -0.031, "step": 3059 }, { "clip_ratio": 0.08653330057859421, "epoch": 1.0706787963610918, "grad_norm": 1.238192291135986, "kl": 2.75, "learning_rate": 5.259443482409364e-06, "loss": -0.0331, "step": 3060 }, { "clip_ratio": 0.021247465163469315, "completions/clipped_ratio": 0.0535714285714286, "completions/max_length": 237.0, "completions/max_terminated_length": 237.0, "completions/mean_length": 153.48214721679688, "completions/mean_terminated_length": 162.16981506347656, "completions/min_length": 0.0, "completions/min_terminated_length": 95.0, "epoch": 1.0710286913925822, "grad_norm": 3.483404888115707, "kl": 2.1875, "learning_rate": 5.256393900258747e-06, "loss": -0.0204, "num_tokens": 14604775.0, "reward": 3.8779964447021484, "reward_std": 0.18169023096561432, "rewards/check_gptzero_func/mean": 0.9642857313156128, "rewards/check_gptzero_func/std": 0.187256321310997, "rewards/check_originality_func/mean": 0.9642857313156128, "rewards/check_originality_func/std": 0.187256321310997, "rewards/check_winston_local_func/mean": 0.9909850358963013, "rewards/check_winston_local_func/std": 0.004928562790155411, "rewards/sentence_count_match_reward_logic/mean": 0.9584396481513977, "rewards/sentence_count_match_reward_logic/std": 0.0921124815940857, "step": 3061 }, { "clip_ratio": 0.02753777988255024, "epoch": 1.0713785864240728, "grad_norm": 1.900532539547796, "kl": 2.171875, "learning_rate": 5.253344222475633e-06, "loss": -0.0289, "step": 3062 }, { "clip_ratio": 0.053332071751356125, "epoch": 1.0717284814555634, "grad_norm": 1.336804400137702, "kl": 2.1875, "learning_rate": 5.250294450197521e-06, "loss": -0.0346, "step": 3063 }, { "clip_ratio": 0.0746031329035759, "epoch": 1.0720783764870538, "grad_norm": 1.1251992612257538, "kl": 2.171875, "learning_rate": 5.247244584561948e-06, "loss": -0.0376, "step": 3064 }, { "clip_ratio": 0.0179140567779541, "completions/clipped_ratio": 0.0714285714285714, "completions/max_length": 234.0, "completions/max_terminated_length": 234.0, "completions/mean_length": 157.25, "completions/mean_terminated_length": 169.34616088867188, "completions/min_length": 0.0, "completions/min_terminated_length": 84.0, "epoch": 1.0724282715185445, "grad_norm": 2.994290112262331, "kl": 2.0, "learning_rate": 5.244194626706485e-06, "loss": -0.0278, "num_tokens": 14624501.0, "reward": 3.8751392364501953, "reward_std": 0.24938277900218964, "rewards/check_gptzero_func/mean": 0.9464285969734192, "rewards/check_gptzero_func/std": 0.22720777988433838, "rewards/check_originality_func/mean": 0.9464285969734192, "rewards/check_originality_func/std": 0.22720777988433838, "rewards/check_winston_local_func/mean": 0.9904661178588867, "rewards/check_winston_local_func/std": 0.004363304935395718, "rewards/sentence_count_match_reward_logic/mean": 0.991815447807312, "rewards/sentence_count_match_reward_logic/std": 0.03501410782337189, "step": 3065 }, { "clip_ratio": 0.050835080444812775, "epoch": 1.072778166550035, "grad_norm": 1.9915998705051603, "kl": 2.03125, "learning_rate": 5.2411445777687365e-06, "loss": -0.0297, "step": 3066 }, { "clip_ratio": 0.084258072078228, "epoch": 1.0731280615815255, "grad_norm": 1.5617861928676942, "kl": 2.046875, "learning_rate": 5.238094438886344e-06, "loss": -0.0345, "step": 3067 }, { "clip_ratio": 0.11279736459255219, "epoch": 1.0734779566130161, "grad_norm": 0.8832595287242584, "kl": 2.046875, "learning_rate": 5.235044211196978e-06, "loss": -0.0392, "step": 3068 }, { "clip_ratio": 0.022810855880379677, "completions/clipped_ratio": 0.1785714285714286, "completions/max_length": 255.0, "completions/max_terminated_length": 255.0, "completions/mean_length": 144.08929443359375, "completions/mean_terminated_length": 175.4130401611328, "completions/min_length": 0.0, "completions/min_terminated_length": 99.0, "epoch": 1.0738278516445066, "grad_norm": 3.437395528796828, "kl": 1.8671875, "learning_rate": 5.231993895838348e-06, "loss": 0.0051, "num_tokens": 14644690.0, "reward": 3.84627103805542, "reward_std": 0.17598949372768402, "rewards/check_gptzero_func/mean": 1.0, "rewards/check_gptzero_func/std": 0.0, "rewards/check_originality_func/mean": 0.8928571343421936, "rewards/check_originality_func/std": 0.31209391355514526, "rewards/check_winston_local_func/mean": 0.9926994442939758, "rewards/check_winston_local_func/std": 0.002306822221726179, "rewards/sentence_count_match_reward_logic/mean": 0.9607142806053162, "rewards/sentence_count_match_reward_logic/std": 0.0724673941731453, "step": 3069 }, { "clip_ratio": 0.04358586296439171, "epoch": 1.0741777466759972, "grad_norm": 2.614346616735897, "kl": 1.8671875, "learning_rate": 5.2289434939481875e-06, "loss": -0.002, "step": 3070 }, { "clip_ratio": 0.06564038246870041, "epoch": 1.0745276417074878, "grad_norm": 1.9973148398567917, "kl": 1.875, "learning_rate": 5.2258930066642725e-06, "loss": -0.0072, "step": 3071 }, { "clip_ratio": 0.11591995507478714, "epoch": 1.0748775367389782, "grad_norm": 1.1883748516402453, "kl": 1.8828125, "learning_rate": 5.222842435124403e-06, "loss": -0.0146, "step": 3072 }, { "clip_ratio": 0.017754625529050827, "completions/clipped_ratio": 0.1607142857142857, "completions/max_length": 250.0, "completions/max_terminated_length": 250.0, "completions/mean_length": 138.98214721679688, "completions/mean_terminated_length": 165.59573364257812, "completions/min_length": 0.0, "completions/min_terminated_length": 98.0, "epoch": 1.0752274317704689, "grad_norm": 5.94094178550758, "kl": 2.0, "learning_rate": 5.219791780466416e-06, "loss": -0.0731, "num_tokens": 14663329.0, "reward": 3.775031089782715, "reward_std": 0.22978712618350983, "rewards/check_gptzero_func/mean": 1.0, "rewards/check_gptzero_func/std": 0.0, "rewards/check_originality_func/mean": 0.8571428656578064, "rewards/check_originality_func/std": 0.3530939221382141, "rewards/check_winston_local_func/mean": 0.9900110363960266, "rewards/check_winston_local_func/std": 0.0048414571210742, "rewards/sentence_count_match_reward_logic/mean": 0.9278770089149475, "rewards/sentence_count_match_reward_logic/std": 0.1066766157746315, "step": 3073 }, { "clip_ratio": 0.04754570126533508, "epoch": 1.0755773268019595, "grad_norm": 3.9538705187331247, "kl": 2.015625, "learning_rate": 5.216741043828174e-06, "loss": -0.0833, "step": 3074 }, { "clip_ratio": 0.10027360171079636, "epoch": 1.07592722183345, "grad_norm": 1.6255901838094495, "kl": 2.015625, "learning_rate": 5.213690226347573e-06, "loss": -0.0906, "step": 3075 }, { "clip_ratio": 0.15732677280902863, "epoch": 1.0762771168649405, "grad_norm": 1.6447357089167625, "kl": 2.03125, "learning_rate": 5.210639329162541e-06, "loss": -0.0916, "step": 3076 }, { "clip_ratio": 0.027001017704606056, "completions/clipped_ratio": 0.0535714285714286, "completions/max_length": 254.0, "completions/max_terminated_length": 254.0, "completions/mean_length": 148.8928680419922, "completions/mean_terminated_length": 157.3207550048828, "completions/min_length": 0.0, "completions/min_terminated_length": 94.0, "epoch": 1.0766270118964312, "grad_norm": 2.3445154583423045, "kl": 1.8828125, "learning_rate": 5.207588353411032e-06, "loss": -0.0278, "num_tokens": 14682267.0, "reward": 3.8933298587799072, "reward_std": 0.20423085987567902, "rewards/check_gptzero_func/mean": 0.9642857313156128, "rewards/check_gptzero_func/std": 0.187256321310997, "rewards/check_originality_func/mean": 0.9642857313156128, "rewards/check_originality_func/std": 0.187256321310997, "rewards/check_winston_local_func/mean": 0.9756708145141602, "rewards/check_winston_local_func/std": 0.05464845895767212, "rewards/sentence_count_match_reward_logic/mean": 0.9890872836112976, "rewards/sentence_count_match_reward_logic/std": 0.058189716190099716, "step": 3077 }, { "clip_ratio": 0.04345674067735672, "epoch": 1.0769769069279216, "grad_norm": 2.161764057390981, "kl": 1.8828125, "learning_rate": 5.2045373002310326e-06, "loss": -0.0318, "step": 3078 }, { "clip_ratio": 0.07131576538085938, "epoch": 1.0773268019594122, "grad_norm": 1.6475710735075102, "kl": 1.890625, "learning_rate": 5.2014861707605565e-06, "loss": -0.0384, "step": 3079 }, { "clip_ratio": 0.08691740036010742, "epoch": 1.0776766969909026, "grad_norm": 1.249155354773994, "kl": 1.9140625, "learning_rate": 5.198434966137645e-06, "loss": -0.043, "step": 3080 }, { "clip_ratio": 0.020749662071466446, "completions/clipped_ratio": 0.2857142857142857, "completions/max_length": 248.0, "completions/max_terminated_length": 248.0, "completions/mean_length": 97.66072082519531, "completions/mean_terminated_length": 136.72500610351562, "completions/min_length": 0.0, "completions/min_terminated_length": 79.0, "epoch": 1.0780265920223933, "grad_norm": 2.9792814082959724, "kl": 2.53125, "learning_rate": 5.1953836875003695e-06, "loss": 0.0156, "num_tokens": 14699632.0, "reward": 3.664335250854492, "reward_std": 0.16603320837020874, "rewards/check_gptzero_func/mean": 0.9821428656578064, "rewards/check_gptzero_func/std": 0.13363061845302582, "rewards/check_originality_func/mean": 0.75, "rewards/check_originality_func/std": 0.43693143129348755, "rewards/check_winston_local_func/mean": 0.9917021989822388, "rewards/check_winston_local_func/std": 0.0038833143189549446, "rewards/sentence_count_match_reward_logic/mean": 0.9404897093772888, "rewards/sentence_count_match_reward_logic/std": 0.1064353808760643, "step": 3081 }, { "clip_ratio": 0.04261306673288345, "epoch": 1.078376487053884, "grad_norm": 2.4412495875560016, "kl": 2.546875, "learning_rate": 5.192332335986828e-06, "loss": 0.0101, "step": 3082 }, { "clip_ratio": 0.08827245980501175, "epoch": 1.0787263820853743, "grad_norm": 1.274380489059626, "kl": 2.5625, "learning_rate": 5.189280912735144e-06, "loss": 0.0045, "step": 3083 }, { "clip_ratio": 0.1514931619167328, "epoch": 1.079076277116865, "grad_norm": 0.7687977144530836, "kl": 2.59375, "learning_rate": 5.186229418883472e-06, "loss": 0.0011, "step": 3084 }, { "clip_ratio": 0.022271141409873962, "completions/clipped_ratio": 0.1607142857142857, "completions/max_length": 254.0, "completions/max_terminated_length": 254.0, "completions/mean_length": 146.6428680419922, "completions/mean_terminated_length": 174.72340393066406, "completions/min_length": 0.0, "completions/min_terminated_length": 106.0, "epoch": 1.0794261721483556, "grad_norm": 2.528147202271601, "kl": 1.9765625, "learning_rate": 5.183177855569989e-06, "loss": -0.0292, "num_tokens": 14720612.0, "reward": 3.8086442947387695, "reward_std": 0.1742449849843979, "rewards/check_gptzero_func/mean": 0.9821428656578064, "rewards/check_gptzero_func/std": 0.13363061845302582, "rewards/check_originality_func/mean": 0.875, "rewards/check_originality_func/std": 0.3337118923664093, "rewards/check_winston_local_func/mean": 0.991113007068634, "rewards/check_winston_local_func/std": 0.0028463054914027452, "rewards/sentence_count_match_reward_logic/mean": 0.9603883028030396, "rewards/sentence_count_match_reward_logic/std": 0.06332269310951233, "step": 3085 }, { "clip_ratio": 0.02664274163544178, "epoch": 1.079776067179846, "grad_norm": 2.2865653720858714, "kl": 1.9921875, "learning_rate": 5.180126223932901e-06, "loss": -0.0324, "step": 3086 }, { "clip_ratio": 0.06672801077365875, "epoch": 1.0801259622113366, "grad_norm": 1.3374557094674897, "kl": 2.0, "learning_rate": 5.177074525110433e-06, "loss": -0.039, "step": 3087 }, { "clip_ratio": 0.09816527366638184, "epoch": 1.0804758572428272, "grad_norm": 1.076451689629475, "kl": 2.015625, "learning_rate": 5.174022760240846e-06, "loss": -0.0413, "step": 3088 }, { "clip_ratio": 0.021191056817770004, "completions/clipped_ratio": 0.017857142857142905, "completions/max_length": 234.0, "completions/max_terminated_length": 234.0, "completions/mean_length": 154.71429443359375, "completions/mean_terminated_length": 157.5272674560547, "completions/min_length": 0.0, "completions/min_terminated_length": 106.0, "epoch": 1.0808257522743177, "grad_norm": 7.1470288530068045, "kl": 2.21875, "learning_rate": 5.170970930462414e-06, "loss": 0.004, "num_tokens": 14739380.0, "reward": 3.949662446975708, "reward_std": 0.09346218407154083, "rewards/check_gptzero_func/mean": 1.0, "rewards/check_gptzero_func/std": 0.0, "rewards/check_originality_func/mean": 0.9821428656578064, "rewards/check_originality_func/std": 0.13363061845302582, "rewards/check_winston_local_func/mean": 0.9921368956565857, "rewards/check_winston_local_func/std": 0.003108204109594226, "rewards/sentence_count_match_reward_logic/mean": 0.9753826856613159, "rewards/sentence_count_match_reward_logic/std": 0.06806249916553497, "step": 3089 }, { "clip_ratio": 0.061547886580228806, "epoch": 1.0811756473058083, "grad_norm": 2.781199562995365, "kl": 2.296875, "learning_rate": 5.1679190369134455e-06, "loss": -0.0059, "step": 3090 }, { "clip_ratio": 0.07558566331863403, "epoch": 1.0815255423372987, "grad_norm": 2.3954812565177463, "kl": 2.34375, "learning_rate": 5.164867080732265e-06, "loss": -0.0076, "step": 3091 }, { "clip_ratio": 0.09055197983980179, "epoch": 1.0818754373687893, "grad_norm": 1.649392608890169, "kl": 2.375, "learning_rate": 5.161815063057224e-06, "loss": -0.0103, "step": 3092 }, { "clip_ratio": 0.033323295414447784, "completions/clipped_ratio": 0.25, "completions/max_length": 256.0, "completions/max_terminated_length": 256.0, "completions/mean_length": 104.12500762939453, "completions/mean_terminated_length": 138.83334350585938, "completions/min_length": 0.0, "completions/min_terminated_length": 80.0, "epoch": 1.08222533240028, "grad_norm": 3.2810723111988844, "kl": 2.515625, "learning_rate": 5.158762985026694e-06, "loss": -0.0334, "num_tokens": 14756203.0, "reward": 3.6951546669006348, "reward_std": 0.2638990879058838, "rewards/check_gptzero_func/mean": 0.875, "rewards/check_gptzero_func/std": 0.3337118923664093, "rewards/check_originality_func/mean": 0.875, "rewards/check_originality_func/std": 0.3337118923664093, "rewards/check_winston_local_func/mean": 0.9863249063491821, "rewards/check_winston_local_func/std": 0.013378576375544071, "rewards/sentence_count_match_reward_logic/mean": 0.9588293433189392, "rewards/sentence_count_match_reward_logic/std": 0.08015681058168411, "step": 3093 }, { "clip_ratio": 0.07513118535280228, "epoch": 1.0825752274317704, "grad_norm": 2.1509050289729235, "kl": 2.546875, "learning_rate": 5.155710847779074e-06, "loss": -0.038, "step": 3094 }, { "clip_ratio": 0.12116104364395142, "epoch": 1.082925122463261, "grad_norm": 1.1099539666957852, "kl": 2.609375, "learning_rate": 5.152658652452783e-06, "loss": -0.0435, "step": 3095 }, { "clip_ratio": 0.1505127251148224, "epoch": 1.0832750174947516, "grad_norm": 1.0291371331122168, "kl": 2.609375, "learning_rate": 5.149606400186259e-06, "loss": -0.0442, "step": 3096 }, { "clip_ratio": 0.01753237284719944, "completions/clipped_ratio": 0.0714285714285714, "completions/max_length": 255.0, "completions/max_terminated_length": 255.0, "completions/mean_length": 176.62501525878906, "completions/mean_terminated_length": 190.2115478515625, "completions/min_length": 0.0, "completions/min_terminated_length": 133.0, "epoch": 1.083624912526242, "grad_norm": 5.5330724938502005, "kl": 1.6796875, "learning_rate": 5.146554092117963e-06, "loss": 0.0009, "num_tokens": 14778750.0, "reward": 3.798550605773926, "reward_std": 0.17538565397262573, "rewards/check_gptzero_func/mean": 0.8571428656578064, "rewards/check_gptzero_func/std": 0.3530939221382141, "rewards/check_originality_func/mean": 0.9642857313156128, "rewards/check_originality_func/std": 0.187256321310997, "rewards/check_winston_local_func/mean": 0.9911524057388306, "rewards/check_winston_local_func/std": 0.0030503750313073397, "rewards/sentence_count_match_reward_logic/mean": 0.9859694242477417, "rewards/sentence_count_match_reward_logic/std": 0.040963903069496155, "step": 3097 }, { "clip_ratio": 0.056437768042087555, "epoch": 1.0839748075577327, "grad_norm": 2.343167242006129, "kl": 1.671875, "learning_rate": 5.143501729386376e-06, "loss": -0.0098, "step": 3098 }, { "clip_ratio": 0.0778920128941536, "epoch": 1.0843247025892233, "grad_norm": 1.9712063894036016, "kl": 1.671875, "learning_rate": 5.140449313130004e-06, "loss": -0.0141, "step": 3099 }, { "clip_ratio": 0.10083992779254913, "epoch": 1.0846745976207137, "grad_norm": 1.8116697570386109, "kl": 1.6796875, "learning_rate": 5.137396844487368e-06, "loss": -0.0202, "step": 3100 }, { "clip_ratio": 0.02512526884675026, "completions/clipped_ratio": 0.017857142857142905, "completions/max_length": 253.0, "completions/max_terminated_length": 253.0, "completions/mean_length": 154.1607208251953, "completions/mean_terminated_length": 156.96363830566406, "completions/min_length": 0.0, "completions/min_terminated_length": 70.0, "epoch": 1.0850244926522044, "grad_norm": 5.297105746979393, "kl": 2.296875, "learning_rate": 5.1343443245970095e-06, "loss": 0.023, "num_tokens": 14797631.0, "reward": 3.8732073307037354, "reward_std": 0.20622695982456207, "rewards/check_gptzero_func/mean": 1.0, "rewards/check_gptzero_func/std": 0.0, "rewards/check_originality_func/mean": 0.9285714030265808, "rewards/check_originality_func/std": 0.25987008213996887, "rewards/check_winston_local_func/mean": 0.9905114769935608, "rewards/check_winston_local_func/std": 0.007341587916016579, "rewards/sentence_count_match_reward_logic/mean": 0.9541241526603699, "rewards/sentence_count_match_reward_logic/std": 0.09779626131057739, "step": 3101 }, { "clip_ratio": 0.0459706112742424, "epoch": 1.085374387683695, "grad_norm": 3.1914544677210137, "kl": 2.328125, "learning_rate": 5.131291754597488e-06, "loss": 0.0175, "step": 3102 }, { "clip_ratio": 0.0736338272690773, "epoch": 1.0857242827151854, "grad_norm": 2.603822895762989, "kl": 2.328125, "learning_rate": 5.128239135627384e-06, "loss": 0.0124, "step": 3103 }, { "clip_ratio": 0.11249543726444244, "epoch": 1.086074177746676, "grad_norm": 2.031367807252684, "kl": 2.328125, "learning_rate": 5.125186468825297e-06, "loss": 0.0071, "step": 3104 }, { "clip_ratio": 0.02977713569998741, "completions/clipped_ratio": 0.017857142857142905, "completions/max_length": 234.0, "completions/max_terminated_length": 234.0, "completions/mean_length": 172.0178680419922, "completions/mean_terminated_length": 175.14544677734375, "completions/min_length": 0.0, "completions/min_terminated_length": 141.0, "epoch": 1.0864240727781667, "grad_norm": 1.4796339925036708, "kl": 1.9375, "learning_rate": 5.122133755329843e-06, "loss": -0.0175, "num_tokens": 14818768.0, "reward": 3.871187925338745, "reward_std": 0.17026151716709137, "rewards/check_gptzero_func/mean": 1.0, "rewards/check_gptzero_func/std": 0.0, "rewards/check_originality_func/mean": 0.9107142686843872, "rewards/check_originality_func/std": 0.28773635625839233, "rewards/check_winston_local_func/mean": 0.9916810393333435, "rewards/check_winston_local_func/std": 0.002780524082481861, "rewards/sentence_count_match_reward_logic/mean": 0.9687924981117249, "rewards/sentence_count_match_reward_logic/std": 0.07833518087863922, "step": 3105 }, { "clip_ratio": 0.028906848281621933, "epoch": 1.086773967809657, "grad_norm": 1.501570348657306, "kl": 1.9375, "learning_rate": 5.119080996279654e-06, "loss": -0.0206, "step": 3106 }, { "clip_ratio": 0.04435069113969803, "epoch": 1.0871238628411477, "grad_norm": 1.2027769402192894, "kl": 1.9375, "learning_rate": 5.116028192813379e-06, "loss": -0.0256, "step": 3107 }, { "clip_ratio": 0.06678379327058792, "epoch": 1.0874737578726381, "grad_norm": 0.8934033944041544, "kl": 1.9375, "learning_rate": 5.112975346069687e-06, "loss": -0.0296, "step": 3108 }, { "clip_ratio": 0.02446397766470909, "completions/clipped_ratio": 0.1071428571428571, "completions/max_length": 256.0, "completions/max_terminated_length": 256.0, "completions/mean_length": 169.125, "completions/mean_terminated_length": 189.4199981689453, "completions/min_length": 0.0, "completions/min_terminated_length": 131.0, "epoch": 1.0878236529041287, "grad_norm": 4.239014804130677, "kl": 1.8203125, "learning_rate": 5.10992245718726e-06, "loss": -0.0645, "num_tokens": 14840583.0, "reward": 3.8890628814697266, "reward_std": 0.17564508318901062, "rewards/check_gptzero_func/mean": 1.0, "rewards/check_gptzero_func/std": 0.0, "rewards/check_originality_func/mean": 0.9464285969734192, "rewards/check_originality_func/std": 0.22720777988433838, "rewards/check_winston_local_func/mean": 0.9902886152267456, "rewards/check_winston_local_func/std": 0.005193950608372688, "rewards/sentence_count_match_reward_logic/mean": 0.9523455500602722, "rewards/sentence_count_match_reward_logic/std": 0.0835207849740982, "step": 3109 }, { "clip_ratio": 0.03317072615027428, "epoch": 1.0881735479356194, "grad_norm": 4.005762537635607, "kl": 1.8203125, "learning_rate": 5.1068695273047975e-06, "loss": -0.0665, "step": 3110 }, { "clip_ratio": 0.08604307472705841, "epoch": 1.0885234429671098, "grad_norm": 3.2262886798653754, "kl": 1.8203125, "learning_rate": 5.103816557561014e-06, "loss": -0.0772, "step": 3111 }, { "clip_ratio": 0.14102938771247864, "epoch": 1.0888733379986004, "grad_norm": 2.6187382775120263, "kl": 1.828125, "learning_rate": 5.100763549094634e-06, "loss": -0.0866, "step": 3112 }, { "clip_ratio": 0.014534756541252136, "completions/clipped_ratio": 0.0, "completions/max_length": 228.0, "completions/max_terminated_length": 228.0, "completions/mean_length": 148.23214721679688, "completions/mean_terminated_length": 148.23214721679688, "completions/min_length": 63.0, "completions/min_terminated_length": 63.0, "epoch": 1.089223233030091, "grad_norm": 4.934217886096276, "kl": 2.296875, "learning_rate": 5.097710503044404e-06, "loss": 0.0241, "num_tokens": 14858940.0, "reward": 3.913100481033325, "reward_std": 0.11732801795005798, "rewards/check_gptzero_func/mean": 1.0, "rewards/check_gptzero_func/std": 0.0, "rewards/check_originality_func/mean": 0.9821428656578064, "rewards/check_originality_func/std": 0.13363061845302582, "rewards/check_winston_local_func/mean": 0.9914382100105286, "rewards/check_winston_local_func/std": 0.003665786935016513, "rewards/sentence_count_match_reward_logic/mean": 0.9395195245742798, "rewards/sentence_count_match_reward_logic/std": 0.1152297779917717, "step": 3113 }, { "clip_ratio": 0.04004940018057823, "epoch": 1.0895731280615815, "grad_norm": 3.683994053952703, "kl": 2.296875, "learning_rate": 5.094657420549083e-06, "loss": 0.0121, "step": 3114 }, { "clip_ratio": 0.08874119818210602, "epoch": 1.089923023093072, "grad_norm": 2.31714963705785, "kl": 2.328125, "learning_rate": 5.091604302747439e-06, "loss": 0.0033, "step": 3115 }, { "clip_ratio": 0.1373739242553711, "epoch": 1.0902729181245627, "grad_norm": 1.6345119291244394, "kl": 2.34375, "learning_rate": 5.088551150778257e-06, "loss": -0.0043, "step": 3116 }, { "clip_ratio": 0.03173951059579849, "completions/clipped_ratio": 0.0, "completions/max_length": 238.0, "completions/max_terminated_length": 238.0, "completions/mean_length": 152.32144165039062, "completions/mean_terminated_length": 152.32144165039062, "completions/min_length": 64.0, "completions/min_terminated_length": 64.0, "epoch": 1.0906228131560531, "grad_norm": 6.334998412481995, "kl": 2.390625, "learning_rate": 5.085497965780335e-06, "loss": 0.0169, "num_tokens": 14877518.0, "reward": 3.9387400150299072, "reward_std": 0.10391148924827576, "rewards/check_gptzero_func/mean": 0.9821428656578064, "rewards/check_gptzero_func/std": 0.13363061845302582, "rewards/check_originality_func/mean": 0.9821428656578064, "rewards/check_originality_func/std": 0.13363061845302582, "rewards/check_winston_local_func/mean": 0.9889096617698669, "rewards/check_winston_local_func/std": 0.013797370716929436, "rewards/sentence_count_match_reward_logic/mean": 0.9855442047119141, "rewards/sentence_count_match_reward_logic/std": 0.04667430743575096, "step": 3117 }, { "clip_ratio": 0.038999345153570175, "epoch": 1.0909727081875438, "grad_norm": 3.979320634220717, "kl": 2.40625, "learning_rate": 5.08244474889248e-06, "loss": 0.0091, "step": 3118 }, { "clip_ratio": 0.06977181881666183, "epoch": 1.0913226032190342, "grad_norm": 3.0010743555515496, "kl": 2.421875, "learning_rate": 5.079391501253515e-06, "loss": 0.0003, "step": 3119 }, { "clip_ratio": 0.13117964565753937, "epoch": 1.0916724982505248, "grad_norm": 2.3502143592478695, "kl": 2.46875, "learning_rate": 5.07633822400227e-06, "loss": -0.0057, "step": 3120 }, { "clip_ratio": 0.027280433103442192, "completions/clipped_ratio": 0.017857142857142905, "completions/max_length": 254.0, "completions/max_terminated_length": 254.0, "completions/mean_length": 136.85714721679688, "completions/mean_terminated_length": 139.34544372558594, "completions/min_length": 0.0, "completions/min_terminated_length": 87.0, "epoch": 1.0920223932820154, "grad_norm": 4.025987239409655, "kl": 2.71875, "learning_rate": 5.07328491827759e-06, "loss": 0.0134, "num_tokens": 14894478.0, "reward": 3.8910815715789795, "reward_std": 0.1753634661436081, "rewards/check_gptzero_func/mean": 0.9821428656578064, "rewards/check_gptzero_func/std": 0.13363061845302582, "rewards/check_originality_func/mean": 0.9642857313156128, "rewards/check_originality_func/std": 0.187256321310997, "rewards/check_winston_local_func/mean": 0.9870848655700684, "rewards/check_winston_local_func/std": 0.022914394736289978, "rewards/sentence_count_match_reward_logic/mean": 0.9575680494308472, "rewards/sentence_count_match_reward_logic/std": 0.09229110181331635, "step": 3121 }, { "clip_ratio": 0.04685910791158676, "epoch": 1.0923722883135059, "grad_norm": 4.035862659069525, "kl": 2.703125, "learning_rate": 5.070231585218329e-06, "loss": 0.0048, "step": 3122 }, { "clip_ratio": 0.11399779468774796, "epoch": 1.0927221833449965, "grad_norm": 2.4556977049598934, "kl": 2.671875, "learning_rate": 5.067178225963351e-06, "loss": -0.0029, "step": 3123 }, { "clip_ratio": 0.19117386639118195, "epoch": 1.0930720783764871, "grad_norm": 1.6764748479703695, "kl": 2.65625, "learning_rate": 5.064124841651533e-06, "loss": -0.0071, "step": 3124 }, { "clip_ratio": 0.03174065798521042, "completions/clipped_ratio": 0.125, "completions/max_length": 252.0, "completions/max_terminated_length": 252.0, "completions/mean_length": 125.66072082519531, "completions/mean_terminated_length": 143.61224365234375, "completions/min_length": 0.0, "completions/min_terminated_length": 61.0, "epoch": 1.0934219734079775, "grad_norm": 3.673111759129635, "kl": 2.171875, "learning_rate": 5.061071433421754e-06, "loss": -0.0083, "num_tokens": 14912307.0, "reward": 3.7476840019226074, "reward_std": 0.3494087755680084, "rewards/check_gptzero_func/mean": 0.9107142686843872, "rewards/check_gptzero_func/std": 0.28773635625839233, "rewards/check_originality_func/mean": 0.875, "rewards/check_originality_func/std": 0.3337118923664093, "rewards/check_winston_local_func/mean": 0.9813149571418762, "rewards/check_winston_local_func/std": 0.03241117671132088, "rewards/sentence_count_match_reward_logic/mean": 0.980654776096344, "rewards/sentence_count_match_reward_logic/std": 0.061012234538793564, "step": 3125 }, { "clip_ratio": 0.05329544097185135, "epoch": 1.0937718684394682, "grad_norm": 2.827246530115788, "kl": 2.15625, "learning_rate": 5.058018002412909e-06, "loss": -0.0085, "step": 3126 }, { "clip_ratio": 0.12107634544372559, "epoch": 1.0941217634709588, "grad_norm": 1.9548068799820113, "kl": 2.15625, "learning_rate": 5.0549645497638974e-06, "loss": -0.0176, "step": 3127 }, { "clip_ratio": 0.13721852004528046, "epoch": 1.0944716585024492, "grad_norm": 1.5825847757832532, "kl": 2.15625, "learning_rate": 5.05191107661363e-06, "loss": -0.0216, "step": 3128 }, { "clip_ratio": 0.01439643744379282, "completions/clipped_ratio": 0.0, "completions/max_length": 220.0, "completions/max_terminated_length": 220.0, "completions/mean_length": 160.6428680419922, "completions/mean_terminated_length": 160.6428680419922, "completions/min_length": 107.0, "completions/min_terminated_length": 107.0, "epoch": 1.0948215535339398, "grad_norm": 5.449680184110023, "kl": 1.9453125, "learning_rate": 5.048857584101023e-06, "loss": 0.0203, "num_tokens": 14932527.0, "reward": 3.9115421772003174, "reward_std": 0.08593987673521042, "rewards/check_gptzero_func/mean": 1.0, "rewards/check_gptzero_func/std": 0.0, "rewards/check_originality_func/mean": 0.9821428656578064, "rewards/check_originality_func/std": 0.13363061845302582, "rewards/check_winston_local_func/mean": 0.9914317727088928, "rewards/check_winston_local_func/std": 0.00556298578158021, "rewards/sentence_count_match_reward_logic/mean": 0.9379677176475525, "rewards/sentence_count_match_reward_logic/std": 0.0880214124917984, "step": 3129 }, { "clip_ratio": 0.04200921580195427, "epoch": 1.0951714485654305, "grad_norm": 2.3152722009101536, "kl": 1.9453125, "learning_rate": 5.045804073364997e-06, "loss": 0.0156, "step": 3130 }, { "clip_ratio": 0.05706975981593132, "epoch": 1.0955213435969209, "grad_norm": 1.5434532720051586, "kl": 1.9765625, "learning_rate": 5.042750545544486e-06, "loss": 0.0103, "step": 3131 }, { "clip_ratio": 0.07965034991502762, "epoch": 1.0958712386284115, "grad_norm": 1.2108765060560487, "kl": 2.0, "learning_rate": 5.039697001778427e-06, "loss": 0.0067, "step": 3132 }, { "clip_ratio": 0.0188708808273077, "completions/clipped_ratio": 0.0, "completions/max_length": 214.0, "completions/max_terminated_length": 214.0, "completions/mean_length": 149.96429443359375, "completions/mean_terminated_length": 149.96429443359375, "completions/min_length": 80.0, "completions/min_terminated_length": 80.0, "epoch": 1.0962211336599021, "grad_norm": 1.5102116607821332, "kl": 2.234375, "learning_rate": 5.0366434432057624e-06, "loss": 0.0121, "num_tokens": 14950925.0, "reward": 3.8299219608306885, "reward_std": 0.27958327531814575, "rewards/check_gptzero_func/mean": 0.9285714030265808, "rewards/check_gptzero_func/std": 0.25987008213996887, "rewards/check_originality_func/mean": 0.9285714030265808, "rewards/check_originality_func/std": 0.25987008213996887, "rewards/check_winston_local_func/mean": 0.9904869794845581, "rewards/check_winston_local_func/std": 0.005486452952027321, "rewards/sentence_count_match_reward_logic/mean": 0.9822916388511658, "rewards/sentence_count_match_reward_logic/std": 0.06879302859306335, "step": 3133 }, { "clip_ratio": 0.03310994431376457, "epoch": 1.0965710286913926, "grad_norm": 1.318968283525745, "kl": 2.234375, "learning_rate": 5.03358987096544e-06, "loss": 0.008, "step": 3134 }, { "clip_ratio": 0.0501098670065403, "epoch": 1.0969209237228832, "grad_norm": 1.0933071882681193, "kl": 2.234375, "learning_rate": 5.030536286196413e-06, "loss": 0.0034, "step": 3135 }, { "clip_ratio": 0.07575742155313492, "epoch": 1.0972708187543736, "grad_norm": 0.9063369939373775, "kl": 2.234375, "learning_rate": 5.02748269003764e-06, "loss": 0.0007, "step": 3136 }, { "clip_ratio": 0.011882086284458637, "completions/clipped_ratio": 0.2142857142857143, "completions/max_length": 253.0, "completions/max_terminated_length": 253.0, "completions/mean_length": 154.3928680419922, "completions/mean_terminated_length": 196.5, "completions/min_length": 0.0, "completions/min_terminated_length": 95.0, "epoch": 1.0976207137858642, "grad_norm": 3.6783236969892514, "kl": 1.7265625, "learning_rate": 5.024429083628084e-06, "loss": -0.0174, "num_tokens": 14973179.0, "reward": 3.7670366764068604, "reward_std": 0.23012174665927887, "rewards/check_gptzero_func/mean": 1.0, "rewards/check_gptzero_func/std": 0.0, "rewards/check_originality_func/mean": 0.8392857313156128, "rewards/check_originality_func/std": 0.3705909848213196, "rewards/check_winston_local_func/mean": 0.980535626411438, "rewards/check_winston_local_func/std": 0.027725983411073685, "rewards/sentence_count_match_reward_logic/mean": 0.9472151398658752, "rewards/sentence_count_match_reward_logic/std": 0.09483879059553146, "step": 3137 }, { "clip_ratio": 0.01992013305425644, "epoch": 1.0979706088173549, "grad_norm": 2.951341845916398, "kl": 1.7265625, "learning_rate": 5.021375468106712e-06, "loss": -0.0264, "step": 3138 }, { "clip_ratio": 0.046126384288072586, "epoch": 1.0983205038488453, "grad_norm": 2.7118312759979237, "kl": 1.7421875, "learning_rate": 5.018321844612491e-06, "loss": -0.0318, "step": 3139 }, { "clip_ratio": 0.11835992336273193, "epoch": 1.098670398880336, "grad_norm": 1.3545684867282521, "kl": 1.75, "learning_rate": 5.0152682142843966e-06, "loss": -0.0397, "step": 3140 }, { "clip_ratio": 0.01821846142411232, "completions/clipped_ratio": 0.0892857142857143, "completions/max_length": 252.0, "completions/max_terminated_length": 252.0, "completions/mean_length": 149.17857360839844, "completions/mean_terminated_length": 163.80392456054688, "completions/min_length": 0.0, "completions/min_terminated_length": 108.0, "epoch": 1.0990202939118265, "grad_norm": 0.9838772061602118, "kl": 1.875, "learning_rate": 5.012214578261402e-06, "loss": -0.0, "num_tokens": 14993301.0, "reward": 3.825024366378784, "reward_std": 0.20688089728355408, "rewards/check_gptzero_func/mean": 0.9642857313156128, "rewards/check_gptzero_func/std": 0.187256321310997, "rewards/check_originality_func/mean": 0.9464285969734192, "rewards/check_originality_func/std": 0.22720777988433838, "rewards/check_winston_local_func/mean": 0.9395009875297546, "rewards/check_winston_local_func/std": 0.1295425146818161, "rewards/sentence_count_match_reward_logic/mean": 0.9748086929321289, "rewards/sentence_count_match_reward_logic/std": 0.07856643199920654, "step": 3141 }, { "clip_ratio": 0.016457531601190567, "epoch": 1.099370188943317, "grad_norm": 0.9900214101965079, "kl": 1.875, "learning_rate": 5.009160937682485e-06, "loss": -0.0013, "step": 3142 }, { "clip_ratio": 0.025954198092222214, "epoch": 1.0997200839748076, "grad_norm": 0.9262841509945201, "kl": 1.8671875, "learning_rate": 5.006107293686627e-06, "loss": -0.0034, "step": 3143 }, { "clip_ratio": 0.052364230155944824, "epoch": 1.1000699790062982, "grad_norm": 0.7561221914555339, "kl": 1.859375, "learning_rate": 5.0030536474128046e-06, "loss": -0.007, "step": 3144 }, { "clip_ratio": 0.02202058583498001, "completions/clipped_ratio": 0.2142857142857143, "completions/max_length": 256.0, "completions/max_terminated_length": 256.0, "completions/mean_length": 130.48214721679688, "completions/mean_terminated_length": 166.0681915283203, "completions/min_length": 0.0, "completions/min_terminated_length": 107.0, "epoch": 1.1004198740377886, "grad_norm": 3.3337703201093016, "kl": 2.015625, "learning_rate": 5e-06, "loss": -0.0253, "num_tokens": 15012488.0, "reward": 3.876190662384033, "reward_std": 0.17003652453422546, "rewards/check_gptzero_func/mean": 1.0, "rewards/check_gptzero_func/std": 0.0, "rewards/check_originality_func/mean": 0.9285714030265808, "rewards/check_originality_func/std": 0.25987008213996887, "rewards/check_winston_local_func/mean": 0.9909743070602417, "rewards/check_winston_local_func/std": 0.0035049656871706247, "rewards/sentence_count_match_reward_logic/mean": 0.9566448926925659, "rewards/sentence_count_match_reward_logic/std": 0.08360406756401062, "step": 3145 }, { "clip_ratio": 0.06357920169830322, "epoch": 1.1007697690692793, "grad_norm": 3.3742685636522625, "kl": 2.015625, "learning_rate": 4.996946352587197e-06, "loss": -0.0273, "step": 3146 }, { "clip_ratio": 0.07757798582315445, "epoch": 1.1011196641007697, "grad_norm": 2.0695824295290306, "kl": 2.015625, "learning_rate": 4.993892706313374e-06, "loss": -0.0314, "step": 3147 }, { "clip_ratio": 0.1134178638458252, "epoch": 1.1014695591322603, "grad_norm": 2.262466364194477, "kl": 2.015625, "learning_rate": 4.9908390623175156e-06, "loss": -0.0351, "step": 3148 }, { "clip_ratio": 0.021729031577706337, "completions/clipped_ratio": 0.1428571428571429, "completions/max_length": 251.0, "completions/max_terminated_length": 251.0, "completions/mean_length": 129.69644165039062, "completions/mean_terminated_length": 151.3125, "completions/min_length": 0.0, "completions/min_terminated_length": 75.0, "epoch": 1.101819454163751, "grad_norm": 3.541711444073933, "kl": 2.234375, "learning_rate": 4.987785421738599e-06, "loss": -0.0229, "num_tokens": 15030615.0, "reward": 3.877412796020508, "reward_std": 0.1821896731853485, "rewards/check_gptzero_func/mean": 1.0, "rewards/check_gptzero_func/std": 0.0, "rewards/check_originality_func/mean": 0.9285714030265808, "rewards/check_originality_func/std": 0.25987008213996887, "rewards/check_winston_local_func/mean": 0.9890835881233215, "rewards/check_winston_local_func/std": 0.007228672970086336, "rewards/sentence_count_match_reward_logic/mean": 0.9597576856613159, "rewards/sentence_count_match_reward_logic/std": 0.08503349870443344, "step": 3149 }, { "clip_ratio": 0.034026943147182465, "epoch": 1.1021693491952413, "grad_norm": 3.207128254464276, "kl": 2.203125, "learning_rate": 4.984731785715605e-06, "loss": -0.0288, "step": 3150 }, { "clip_ratio": 0.0855565220117569, "epoch": 1.102519244226732, "grad_norm": 1.7509334150890339, "kl": 2.203125, "learning_rate": 4.981678155387511e-06, "loss": -0.0318, "step": 3151 }, { "clip_ratio": 0.14078067243099213, "epoch": 1.1028691392582226, "grad_norm": 1.0439418101469218, "kl": 2.171875, "learning_rate": 4.97862453189329e-06, "loss": -0.0359, "step": 3152 }, { "clip_ratio": 0.013767249882221222, "completions/clipped_ratio": 0.1964285714285714, "completions/max_length": 255.0, "completions/max_terminated_length": 255.0, "completions/mean_length": 127.80357360839844, "completions/mean_terminated_length": 159.04444885253906, "completions/min_length": 0.0, "completions/min_terminated_length": 107.0, "epoch": 1.103219034289713, "grad_norm": 1.2720860000704126, "kl": 2.078125, "learning_rate": 4.975570916371917e-06, "loss": -0.0203, "num_tokens": 15049740.0, "reward": 3.852163553237915, "reward_std": 0.17676442861557007, "rewards/check_gptzero_func/mean": 1.0, "rewards/check_gptzero_func/std": 0.0, "rewards/check_originality_func/mean": 0.9464285969734192, "rewards/check_originality_func/std": 0.22720777988433838, "rewards/check_winston_local_func/mean": 0.983548641204834, "rewards/check_winston_local_func/std": 0.03220970556139946, "rewards/sentence_count_match_reward_logic/mean": 0.9221860766410828, "rewards/sentence_count_match_reward_logic/std": 0.1026434674859047, "step": 3153 }, { "clip_ratio": 0.02140546776354313, "epoch": 1.1035689293212037, "grad_norm": 0.8838460479924126, "kl": 2.078125, "learning_rate": 4.972517309962361e-06, "loss": -0.022, "step": 3154 }, { "clip_ratio": 0.031505949795246124, "epoch": 1.1039188243526943, "grad_norm": 1.0376388605001954, "kl": 2.140625, "learning_rate": 4.969463713803589e-06, "loss": -0.0226, "step": 3155 }, { "clip_ratio": 0.04656979441642761, "epoch": 1.1042687193841847, "grad_norm": 1.541016304247783, "kl": 2.234375, "learning_rate": 4.9664101290345625e-06, "loss": -0.0236, "step": 3156 }, { "clip_ratio": 0.014764835126698017, "completions/clipped_ratio": 0.0, "completions/max_length": 255.0, "completions/max_terminated_length": 255.0, "completions/mean_length": 171.5357208251953, "completions/mean_terminated_length": 171.5357208251953, "completions/min_length": 105.0, "completions/min_terminated_length": 105.0, "epoch": 1.1046186144156753, "grad_norm": 2.663555005050117, "kl": 1.75, "learning_rate": 4.963356556794238e-06, "loss": 0.01, "num_tokens": 15071098.0, "reward": 3.774381160736084, "reward_std": 0.2507038712501526, "rewards/check_gptzero_func/mean": 0.9285714030265808, "rewards/check_gptzero_func/std": 0.25987008213996887, "rewards/check_originality_func/mean": 0.8928571343421936, "rewards/check_originality_func/std": 0.3120938837528229, "rewards/check_winston_local_func/mean": 0.9876747131347656, "rewards/check_winston_local_func/std": 0.020088564604520798, "rewards/sentence_count_match_reward_logic/mean": 0.9652777910232544, "rewards/sentence_count_match_reward_logic/std": 0.07266182452440262, "step": 3157 }, { "clip_ratio": 0.022094637155532837, "epoch": 1.1049685094471657, "grad_norm": 1.9012226494108453, "kl": 1.75, "learning_rate": 4.9603029982215735e-06, "loss": 0.0065, "step": 3158 }, { "clip_ratio": 0.04768182709813118, "epoch": 1.1053184044786564, "grad_norm": 0.8421960115211824, "kl": 1.7578125, "learning_rate": 4.957249454455514e-06, "loss": 0.0034, "step": 3159 }, { "clip_ratio": 0.0597313828766346, "epoch": 1.105668299510147, "grad_norm": 0.798990407005675, "kl": 1.7734375, "learning_rate": 4.954195926635004e-06, "loss": 0.0015, "step": 3160 }, { "clip_ratio": 0.019016887992620468, "completions/clipped_ratio": 0.0, "completions/max_length": 226.0, "completions/max_terminated_length": 226.0, "completions/mean_length": 144.10714721679688, "completions/mean_terminated_length": 144.10714721679688, "completions/min_length": 73.0, "completions/min_terminated_length": 73.0, "epoch": 1.1060181945416374, "grad_norm": 1.2525803278521037, "kl": 2.375, "learning_rate": 4.95114241589898e-06, "loss": 0.013, "num_tokens": 15089312.0, "reward": 3.8775665760040283, "reward_std": 0.2398373782634735, "rewards/check_gptzero_func/mean": 0.9642857313156128, "rewards/check_gptzero_func/std": 0.187256321310997, "rewards/check_originality_func/mean": 0.9464285969734192, "rewards/check_originality_func/std": 0.22720777988433838, "rewards/check_winston_local_func/mean": 0.9850282669067383, "rewards/check_winston_local_func/std": 0.01884986273944378, "rewards/sentence_count_match_reward_logic/mean": 0.9818239808082581, "rewards/sentence_count_match_reward_logic/std": 0.061056241393089294, "step": 3161 }, { "clip_ratio": 0.02597772888839245, "epoch": 1.106368089573128, "grad_norm": 1.1427493769208428, "kl": 2.375, "learning_rate": 4.9480889233863705e-06, "loss": 0.0082, "step": 3162 }, { "clip_ratio": 0.0442386195063591, "epoch": 1.1067179846046187, "grad_norm": 0.7731289858534003, "kl": 2.375, "learning_rate": 4.945035450236103e-06, "loss": 0.0077, "step": 3163 }, { "clip_ratio": 0.06534040719270706, "epoch": 1.107067879636109, "grad_norm": 0.6812383589060572, "kl": 2.375, "learning_rate": 4.941981997587092e-06, "loss": 0.0043, "step": 3164 }, { "clip_ratio": 0.01446485985070467, "completions/clipped_ratio": 0.0, "completions/max_length": 211.0, "completions/max_terminated_length": 211.0, "completions/mean_length": 162.96429443359375, "completions/mean_terminated_length": 162.96429443359375, "completions/min_length": 113.0, "completions/min_terminated_length": 113.0, "epoch": 1.1074177746675997, "grad_norm": 1.8921877382974492, "kl": 1.984375, "learning_rate": 4.938928566578247e-06, "loss": 0.0129, "num_tokens": 15109662.0, "reward": 3.9567737579345703, "reward_std": 0.06984603404998779, "rewards/check_gptzero_func/mean": 1.0, "rewards/check_gptzero_func/std": 0.0, "rewards/check_originality_func/mean": 0.9821428656578064, "rewards/check_originality_func/std": 0.13363061845302582, "rewards/check_winston_local_func/mean": 0.9896178841590881, "rewards/check_winston_local_func/std": 0.006004807539284229, "rewards/sentence_count_match_reward_logic/mean": 0.9850127100944519, "rewards/sentence_count_match_reward_logic/std": 0.04371128976345062, "step": 3165 }, { "clip_ratio": 0.027706775814294815, "epoch": 1.1077676696990904, "grad_norm": 1.3156628615298087, "kl": 1.9765625, "learning_rate": 4.93587515834847e-06, "loss": 0.0079, "step": 3166 }, { "clip_ratio": 0.04417593032121658, "epoch": 1.1081175647305808, "grad_norm": 1.0630565434006722, "kl": 1.984375, "learning_rate": 4.932821774036648e-06, "loss": 0.0056, "step": 3167 }, { "clip_ratio": 0.059623222798109055, "epoch": 1.1084674597620714, "grad_norm": 0.9442360468944013, "kl": 1.9921875, "learning_rate": 4.9297684147816714e-06, "loss": 0.0028, "step": 3168 }, { "clip_ratio": 0.020709766075015068, "completions/clipped_ratio": 0.0892857142857143, "completions/max_length": 256.0, "completions/max_terminated_length": 256.0, "completions/mean_length": 179.05357360839844, "completions/mean_terminated_length": 196.60784912109375, "completions/min_length": 0.0, "completions/min_terminated_length": 128.0, "epoch": 1.108817354793562, "grad_norm": 1.5270362892772091, "kl": 1.609375, "learning_rate": 4.926715081722412e-06, "loss": -0.0877, "num_tokens": 15133377.0, "reward": 3.896367311477661, "reward_std": 0.12917308509349823, "rewards/check_gptzero_func/mean": 0.9821428656578064, "rewards/check_gptzero_func/std": 0.13363061845302582, "rewards/check_originality_func/mean": 0.9285714030265808, "rewards/check_originality_func/std": 0.25987008213996887, "rewards/check_winston_local_func/mean": 0.9876369833946228, "rewards/check_winston_local_func/std": 0.01578989438712597, "rewards/sentence_count_match_reward_logic/mean": 0.9980158805847168, "rewards/sentence_count_match_reward_logic/std": 0.014847845770418644, "step": 3169 }, { "clip_ratio": 0.01858372986316681, "epoch": 1.1091672498250524, "grad_norm": 1.5358817625154766, "kl": 1.609375, "learning_rate": 4.923661775997732e-06, "loss": -0.0905, "step": 3170 }, { "clip_ratio": 0.045428067445755005, "epoch": 1.109517144856543, "grad_norm": 0.9097853665859539, "kl": 1.609375, "learning_rate": 4.920608498746488e-06, "loss": -0.0923, "step": 3171 }, { "clip_ratio": 0.07316745817661285, "epoch": 1.1098670398880337, "grad_norm": 0.9015352830996584, "kl": 1.609375, "learning_rate": 4.917555251107521e-06, "loss": -0.0961, "step": 3172 }, { "clip_ratio": 0.015864375978708267, "completions/clipped_ratio": 0.2321428571428571, "completions/max_length": 256.0, "completions/max_terminated_length": 256.0, "completions/mean_length": 132.23214721679688, "completions/mean_terminated_length": 172.2093048095703, "completions/min_length": 0.0, "completions/min_terminated_length": 110.0, "epoch": 1.1102169349195241, "grad_norm": 0.8077441501223647, "kl": 1.75, "learning_rate": 4.914502034219667e-06, "loss": -0.0926, "num_tokens": 15154094.0, "reward": 3.7005295753479004, "reward_std": 0.1575392484664917, "rewards/check_gptzero_func/mean": 0.8571428656578064, "rewards/check_gptzero_func/std": 0.3530939221382141, "rewards/check_originality_func/mean": 0.875, "rewards/check_originality_func/std": 0.3337118923664093, "rewards/check_winston_local_func/mean": 0.9913740158081055, "rewards/check_winston_local_func/std": 0.004366809502243996, "rewards/sentence_count_match_reward_logic/mean": 0.9770125150680542, "rewards/sentence_count_match_reward_logic/std": 0.055628471076488495, "step": 3173 }, { "clip_ratio": 0.023109273985028267, "epoch": 1.1105668299510147, "grad_norm": 0.8000695025994891, "kl": 1.75, "learning_rate": 4.9114488492217445e-06, "loss": -0.0943, "step": 3174 }, { "clip_ratio": 0.031605951488018036, "epoch": 1.1109167249825052, "grad_norm": 0.7148496800974534, "kl": 1.75, "learning_rate": 4.908395697252562e-06, "loss": -0.0958, "step": 3175 }, { "clip_ratio": 0.03908546641469002, "epoch": 1.1112666200139958, "grad_norm": 0.6542949418780444, "kl": 1.75, "learning_rate": 4.905342579450919e-06, "loss": -0.0973, "step": 3176 }, { "clip_ratio": 0.017132246866822243, "completions/clipped_ratio": 0.1428571428571429, "completions/max_length": 200.0, "completions/max_terminated_length": 200.0, "completions/mean_length": 113.71429443359375, "completions/mean_terminated_length": 132.6666717529297, "completions/min_length": 0.0, "completions/min_terminated_length": 62.0, "epoch": 1.1116165150454864, "grad_norm": 2.997478037596703, "kl": 2.390625, "learning_rate": 4.902289496955596e-06, "loss": 0.0187, "num_tokens": 15171342.0, "reward": 3.667043685913086, "reward_std": 0.27780041098594666, "rewards/check_gptzero_func/mean": 0.9285714030265808, "rewards/check_gptzero_func/std": 0.25987008213996887, "rewards/check_originality_func/mean": 0.8214285969734192, "rewards/check_originality_func/std": 0.3864591419696808, "rewards/check_winston_local_func/mean": 0.983795166015625, "rewards/check_winston_local_func/std": 0.020354807376861572, "rewards/sentence_count_match_reward_logic/mean": 0.9332483410835266, "rewards/sentence_count_match_reward_logic/std": 0.09691911190748215, "step": 3177 }, { "clip_ratio": 0.02127121388912201, "epoch": 1.1119664100769768, "grad_norm": 2.3529305657206128, "kl": 2.40625, "learning_rate": 4.899236450905368e-06, "loss": 0.0144, "step": 3178 }, { "clip_ratio": 0.07061126828193665, "epoch": 1.1123163051084675, "grad_norm": 1.5045437636560581, "kl": 2.40625, "learning_rate": 4.89618344243899e-06, "loss": 0.0082, "step": 3179 }, { "clip_ratio": 0.1045432761311531, "epoch": 1.112666200139958, "grad_norm": 1.1607719753016643, "kl": 2.40625, "learning_rate": 4.893130472695204e-06, "loss": 0.0069, "step": 3180 }, { "clip_ratio": 0.01736919954419136, "completions/clipped_ratio": 0.017857142857142905, "completions/max_length": 241.0, "completions/max_terminated_length": 241.0, "completions/mean_length": 152.17857360839844, "completions/mean_terminated_length": 154.94544982910156, "completions/min_length": 0.0, "completions/min_terminated_length": 109.0, "epoch": 1.1130160951714485, "grad_norm": 0.9517641971049154, "kl": 2.03125, "learning_rate": 4.890077542812742e-06, "loss": -0.0241, "num_tokens": 15190520.0, "reward": 3.9236721992492676, "reward_std": 0.14135228097438812, "rewards/check_gptzero_func/mean": 1.0, "rewards/check_gptzero_func/std": 0.0, "rewards/check_originality_func/mean": 0.9464285969734192, "rewards/check_originality_func/std": 0.22720777988433838, "rewards/check_winston_local_func/mean": 0.9920393824577332, "rewards/check_winston_local_func/std": 0.0018581906333565712, "rewards/sentence_count_match_reward_logic/mean": 0.9852041006088257, "rewards/sentence_count_match_reward_logic/std": 0.048423320055007935, "step": 3181 }, { "clip_ratio": 0.02078499086201191, "epoch": 1.1133659902029391, "grad_norm": 0.8216215591568813, "kl": 2.03125, "learning_rate": 4.887024653930314e-06, "loss": -0.0249, "step": 3182 }, { "clip_ratio": 0.03003700077533722, "epoch": 1.1137158852344298, "grad_norm": 0.6682984633363346, "kl": 2.03125, "learning_rate": 4.883971807186622e-06, "loss": -0.0273, "step": 3183 }, { "clip_ratio": 0.04267624765634537, "epoch": 1.1140657802659202, "grad_norm": 0.6118598255894387, "kl": 2.03125, "learning_rate": 4.880919003720349e-06, "loss": -0.0291, "step": 3184 }, { "clip_ratio": 0.027350153774023056, "completions/clipped_ratio": 0.0, "completions/max_length": 251.0, "completions/max_terminated_length": 251.0, "completions/mean_length": 169.44644165039062, "completions/mean_terminated_length": 169.44644165039062, "completions/min_length": 103.0, "completions/min_terminated_length": 103.0, "epoch": 1.1144156752974108, "grad_norm": 5.753085507725993, "kl": 1.9765625, "learning_rate": 4.877866244670158e-06, "loss": 0.0229, "num_tokens": 15211361.0, "reward": 3.75999116897583, "reward_std": 0.1644207239151001, "rewards/check_gptzero_func/mean": 0.8571428656578064, "rewards/check_gptzero_func/std": 0.3530939221382141, "rewards/check_originality_func/mean": 0.9464285969734192, "rewards/check_originality_func/std": 0.22720777988433838, "rewards/check_winston_local_func/mean": 0.991736888885498, "rewards/check_winston_local_func/std": 0.005033204797655344, "rewards/sentence_count_match_reward_logic/mean": 0.9646825790405273, "rewards/sentence_count_match_reward_logic/std": 0.07749174535274506, "step": 3185 }, { "clip_ratio": 0.029649674892425537, "epoch": 1.1147655703289012, "grad_norm": 2.823053484315431, "kl": 1.9765625, "learning_rate": 4.874813531174704e-06, "loss": 0.0201, "step": 3186 }, { "clip_ratio": 0.05378556251525879, "epoch": 1.1151154653603919, "grad_norm": 1.8094967438090956, "kl": 1.9765625, "learning_rate": 4.871760864372616e-06, "loss": 0.0153, "step": 3187 }, { "clip_ratio": 0.07163112610578537, "epoch": 1.1154653603918825, "grad_norm": 1.1256509069448868, "kl": 1.984375, "learning_rate": 4.868708245402513e-06, "loss": 0.0128, "step": 3188 }, { "clip_ratio": 0.012499557808041573, "completions/clipped_ratio": 0.0, "completions/max_length": 220.0, "completions/max_terminated_length": 220.0, "completions/mean_length": 125.62500762939453, "completions/mean_terminated_length": 125.62500762939453, "completions/min_length": 50.0, "completions/min_terminated_length": 50.0, "epoch": 1.115815255423373, "grad_norm": 1.2461049218610283, "kl": 2.28125, "learning_rate": 4.865655675402993e-06, "loss": 0.0097, "num_tokens": 15227188.0, "reward": 3.792328119277954, "reward_std": 0.2056473344564438, "rewards/check_gptzero_func/mean": 0.875, "rewards/check_gptzero_func/std": 0.3337118923664093, "rewards/check_originality_func/mean": 1.0, "rewards/check_originality_func/std": 0.0, "rewards/check_winston_local_func/mean": 0.9557632207870483, "rewards/check_winston_local_func/std": 0.11568307876586914, "rewards/sentence_count_match_reward_logic/mean": 0.9615646004676819, "rewards/sentence_count_match_reward_logic/std": 0.07082200795412064, "step": 3189 }, { "clip_ratio": 0.017216691747307777, "epoch": 1.1161651504548635, "grad_norm": 1.1715488122090196, "kl": 2.28125, "learning_rate": 4.862603155512633e-06, "loss": 0.0084, "step": 3190 }, { "clip_ratio": 0.03339911624789238, "epoch": 1.1165150454863542, "grad_norm": 1.0532380714395062, "kl": 2.28125, "learning_rate": 4.859550686869997e-06, "loss": 0.0056, "step": 3191 }, { "clip_ratio": 0.05637324973940849, "epoch": 1.1168649405178446, "grad_norm": 0.8233088480212729, "kl": 2.296875, "learning_rate": 4.8564982706136236e-06, "loss": 0.003, "step": 3192 }, { "clip_ratio": 0.019835038110613823, "completions/clipped_ratio": 0.0, "completions/max_length": 253.0, "completions/max_terminated_length": 253.0, "completions/mean_length": 163.19644165039062, "completions/mean_terminated_length": 163.19644165039062, "completions/min_length": 85.0, "completions/min_terminated_length": 85.0, "epoch": 1.1172148355493352, "grad_norm": 1.396828930489136, "kl": 2.078125, "learning_rate": 4.853445907882039e-06, "loss": 0.0164, "num_tokens": 15246943.0, "reward": 3.8648841381073, "reward_std": 0.10179373621940613, "rewards/check_gptzero_func/mean": 1.0, "rewards/check_gptzero_func/std": 0.0, "rewards/check_originality_func/mean": 0.9107142686843872, "rewards/check_originality_func/std": 0.28773635625839233, "rewards/check_winston_local_func/mean": 0.9809552431106567, "rewards/check_winston_local_func/std": 0.03396942466497421, "rewards/sentence_count_match_reward_logic/mean": 0.9732142686843872, "rewards/sentence_count_match_reward_logic/std": 0.05815526470541954, "step": 3193 }, { "clip_ratio": 0.02686157450079918, "epoch": 1.1175647305808258, "grad_norm": 1.5430510588099373, "kl": 2.078125, "learning_rate": 4.850393599813743e-06, "loss": 0.0132, "step": 3194 }, { "clip_ratio": 0.03768926113843918, "epoch": 1.1179146256123162, "grad_norm": 1.323919284750665, "kl": 2.078125, "learning_rate": 4.847341347547218e-06, "loss": 0.009, "step": 3195 }, { "clip_ratio": 0.0699266865849495, "epoch": 1.1182645206438069, "grad_norm": 0.9467010556432002, "kl": 2.078125, "learning_rate": 4.8442891522209265e-06, "loss": 0.0047, "step": 3196 }, { "clip_ratio": 0.015677573159337044, "completions/clipped_ratio": 0.1428571428571429, "completions/max_length": 229.0, "completions/max_terminated_length": 229.0, "completions/mean_length": 150.0178680419922, "completions/mean_terminated_length": 175.02084350585938, "completions/min_length": 0.0, "completions/min_terminated_length": 100.0, "epoch": 1.1186144156752975, "grad_norm": 1.9422187893332643, "kl": 1.8359375, "learning_rate": 4.841237014973305e-06, "loss": -0.0027, "num_tokens": 15268440.0, "reward": 3.937662124633789, "reward_std": 0.048007335513830185, "rewards/check_gptzero_func/mean": 1.0, "rewards/check_gptzero_func/std": 0.0, "rewards/check_originality_func/mean": 1.0, "rewards/check_originality_func/std": 0.0, "rewards/check_winston_local_func/mean": 0.9912334084510803, "rewards/check_winston_local_func/std": 0.0032169846817851067, "rewards/sentence_count_match_reward_logic/mean": 0.9464285969734192, "rewards/sentence_count_match_reward_logic/std": 0.08912643790245056, "step": 3197 }, { "clip_ratio": 0.017837075516581535, "epoch": 1.118964310706788, "grad_norm": 1.3702991774612996, "kl": 1.84375, "learning_rate": 4.838184936942778e-06, "loss": -0.0048, "step": 3198 }, { "clip_ratio": 0.045659516006708145, "epoch": 1.1193142057382786, "grad_norm": 0.9790510747167867, "kl": 1.8359375, "learning_rate": 4.835132919267736e-06, "loss": -0.0078, "step": 3199 }, { "clip_ratio": 0.07921002805233002, "epoch": 1.119664100769769, "grad_norm": 0.8761775994853555, "kl": 1.84375, "learning_rate": 4.832080963086555e-06, "loss": -0.0102, "step": 3200 }, { "clip_ratio": 0.017932189628481865, "completions/clipped_ratio": 0.1428571428571429, "completions/max_length": 245.0, "completions/max_terminated_length": 245.0, "completions/mean_length": 144.75, "completions/mean_terminated_length": 168.875, "completions/min_length": 0.0, "completions/min_terminated_length": 114.0, "epoch": 1.1200139958012596, "grad_norm": 1.0748071148825762, "kl": 2.078125, "learning_rate": 4.8290290695375866e-06, "loss": 0.0151, "num_tokens": 15288426.0, "reward": 3.8421175479888916, "reward_std": 0.11197474598884583, "rewards/check_gptzero_func/mean": 0.9821428656578064, "rewards/check_gptzero_func/std": 0.13363061845302582, "rewards/check_originality_func/mean": 0.875, "rewards/check_originality_func/std": 0.3337118923664093, "rewards/check_winston_local_func/mean": 0.9923087358474731, "rewards/check_winston_local_func/std": 0.0030361535027623177, "rewards/sentence_count_match_reward_logic/mean": 0.9926658272743225, "rewards/sentence_count_match_reward_logic/std": 0.0311678946018219, "step": 3201 }, { "clip_ratio": 0.019376181066036224, "epoch": 1.1203638908327502, "grad_norm": 1.0682007980340629, "kl": 2.078125, "learning_rate": 4.825977239759154e-06, "loss": 0.0134, "step": 3202 }, { "clip_ratio": 0.04693543538451195, "epoch": 1.1207137858642406, "grad_norm": 0.861366495652315, "kl": 2.078125, "learning_rate": 4.8229254748895674e-06, "loss": 0.0113, "step": 3203 }, { "clip_ratio": 0.06853976100683212, "epoch": 1.1210636808957313, "grad_norm": 0.9751343806529492, "kl": 2.078125, "learning_rate": 4.819873776067102e-06, "loss": 0.0093, "step": 3204 }, { "clip_ratio": 0.021452993154525757, "completions/clipped_ratio": 0.1785714285714286, "completions/max_length": 256.0, "completions/max_terminated_length": 256.0, "completions/mean_length": 161.08929443359375, "completions/mean_terminated_length": 196.10870361328125, "completions/min_length": 0.0, "completions/min_terminated_length": 147.0, "epoch": 1.121413575927222, "grad_norm": 1.3143708844127913, "kl": 1.7734375, "learning_rate": 4.8168221444300124e-06, "loss": -0.0108, "num_tokens": 15311431.0, "reward": 3.9105963706970215, "reward_std": 0.1636006087064743, "rewards/check_gptzero_func/mean": 1.0, "rewards/check_gptzero_func/std": 0.0, "rewards/check_originality_func/mean": 0.9642857313156128, "rewards/check_originality_func/std": 0.187256321310997, "rewards/check_winston_local_func/mean": 0.9839266538619995, "rewards/check_winston_local_func/std": 0.018756203353405, "rewards/sentence_count_match_reward_logic/mean": 0.962384045124054, "rewards/sentence_count_match_reward_logic/std": 0.107459157705307, "step": 3205 }, { "clip_ratio": 0.031274665147066116, "epoch": 1.1217634709587123, "grad_norm": 1.0430701648642757, "kl": 1.78125, "learning_rate": 4.81377058111653e-06, "loss": -0.0135, "step": 3206 }, { "clip_ratio": 0.04619443789124489, "epoch": 1.122113365990203, "grad_norm": 0.9536117899312397, "kl": 1.78125, "learning_rate": 4.8107190872648586e-06, "loss": -0.0156, "step": 3207 }, { "clip_ratio": 0.06398040056228638, "epoch": 1.1224632610216936, "grad_norm": 0.816237383954305, "kl": 1.8125, "learning_rate": 4.807667664013174e-06, "loss": -0.019, "step": 3208 }, { "clip_ratio": 0.017850762233138084, "completions/clipped_ratio": 0.0, "completions/max_length": 227.0, "completions/max_terminated_length": 227.0, "completions/mean_length": 141.6428680419922, "completions/mean_terminated_length": 141.6428680419922, "completions/min_length": 80.0, "completions/min_terminated_length": 80.0, "epoch": 1.122813156053184, "grad_norm": 1.6255731490112313, "kl": 2.53125, "learning_rate": 4.804616312499633e-06, "loss": 0.0097, "num_tokens": 15328955.0, "reward": 3.9190011024475098, "reward_std": 0.07846236228942871, "rewards/check_gptzero_func/mean": 0.9285714030265808, "rewards/check_gptzero_func/std": 0.25987008213996887, "rewards/check_originality_func/mean": 1.0, "rewards/check_originality_func/std": 0.0, "rewards/check_winston_local_func/mean": 0.9904293417930603, "rewards/check_winston_local_func/std": 0.0048119742423295975, "rewards/sentence_count_match_reward_logic/mean": 1.0, "rewards/sentence_count_match_reward_logic/std": 0.0, "step": 3209 }, { "clip_ratio": 0.021828915923833847, "epoch": 1.1231630510846746, "grad_norm": 1.241760244848994, "kl": 2.515625, "learning_rate": 4.801565033862355e-06, "loss": 0.0074, "step": 3210 }, { "clip_ratio": 0.03346629813313484, "epoch": 1.1235129461161653, "grad_norm": 0.9409913710465532, "kl": 2.53125, "learning_rate": 4.798513829239444e-06, "loss": 0.0044, "step": 3211 }, { "clip_ratio": 0.05055808275938034, "epoch": 1.1238628411476557, "grad_norm": 0.8714196165902948, "kl": 2.53125, "learning_rate": 4.79546269976897e-06, "loss": 0.0025, "step": 3212 }, { "clip_ratio": 0.017504693940281868, "completions/clipped_ratio": 0.1964285714285714, "completions/max_length": 245.0, "completions/max_terminated_length": 245.0, "completions/mean_length": 117.41072082519531, "completions/mean_terminated_length": 146.11111450195312, "completions/min_length": 0.0, "completions/min_terminated_length": 74.0, "epoch": 1.1242127361791463, "grad_norm": 1.5697565979102555, "kl": 2.234375, "learning_rate": 4.7924116465889684e-06, "loss": -0.048, "num_tokens": 15346482.0, "reward": 3.6365694999694824, "reward_std": 0.3570118844509125, "rewards/check_gptzero_func/mean": 0.7857142686843872, "rewards/check_gptzero_func/std": 0.4140393137931824, "rewards/check_originality_func/mean": 0.875, "rewards/check_originality_func/std": 0.3337118923664093, "rewards/check_winston_local_func/mean": 0.9894605278968811, "rewards/check_winston_local_func/std": 0.005088403820991516, "rewards/sentence_count_match_reward_logic/mean": 0.9863945841789246, "rewards/sentence_count_match_reward_logic/std": 0.04398636147379875, "step": 3213 }, { "clip_ratio": 0.021537482738494873, "epoch": 1.1245626312106367, "grad_norm": 1.5222519365557128, "kl": 2.234375, "learning_rate": 4.7893606708374616e-06, "loss": -0.0526, "step": 3214 }, { "clip_ratio": 0.04737157002091408, "epoch": 1.1249125262421273, "grad_norm": 0.9161547002939325, "kl": 2.234375, "learning_rate": 4.786309773652427e-06, "loss": -0.055, "step": 3215 }, { "clip_ratio": 0.07498234510421753, "epoch": 1.125262421273618, "grad_norm": 1.112891903205593, "kl": 2.234375, "learning_rate": 4.783258956171828e-06, "loss": -0.0566, "step": 3216 }, { "clip_ratio": 0.013300309889018536, "completions/clipped_ratio": 0.0, "completions/max_length": 241.0, "completions/max_terminated_length": 241.0, "completions/mean_length": 184.98214721679688, "completions/mean_terminated_length": 184.98214721679688, "completions/min_length": 120.0, "completions/min_terminated_length": 120.0, "epoch": 1.1256123163051084, "grad_norm": 1.597109358780222, "kl": 1.875, "learning_rate": 4.7802082195335866e-06, "loss": 0.012, "num_tokens": 15369017.0, "reward": 3.7943482398986816, "reward_std": 0.30097562074661255, "rewards/check_gptzero_func/mean": 0.875, "rewards/check_gptzero_func/std": 0.3337118923664093, "rewards/check_originality_func/mean": 0.9464285969734192, "rewards/check_originality_func/std": 0.22720777988433838, "rewards/check_winston_local_func/mean": 0.9815928339958191, "rewards/check_winston_local_func/std": 0.0410529300570488, "rewards/sentence_count_match_reward_logic/mean": 0.9913265109062195, "rewards/sentence_count_match_reward_logic/std": 0.037320107221603394, "step": 3217 }, { "clip_ratio": 0.016099650412797928, "epoch": 1.125962211336599, "grad_norm": 1.3841623473219, "kl": 1.875, "learning_rate": 4.777157564875598e-06, "loss": 0.0096, "step": 3218 }, { "clip_ratio": 0.03437552973628044, "epoch": 1.1263121063680896, "grad_norm": 0.8990708196418414, "kl": 1.875, "learning_rate": 4.774106993335729e-06, "loss": 0.0061, "step": 3219 }, { "clip_ratio": 0.05796518176794052, "epoch": 1.12666200139958, "grad_norm": 0.8262647113695366, "kl": 1.875, "learning_rate": 4.7710565060518125e-06, "loss": 0.0036, "step": 3220 }, { "clip_ratio": 0.021615251898765564, "completions/clipped_ratio": 0.0357142857142857, "completions/max_length": 256.0, "completions/max_terminated_length": 256.0, "completions/mean_length": 180.98214721679688, "completions/mean_terminated_length": 187.6851806640625, "completions/min_length": 0.0, "completions/min_terminated_length": 111.0, "epoch": 1.1270118964310707, "grad_norm": 1.327225065381924, "kl": 1.953125, "learning_rate": 4.768006104161655e-06, "loss": -0.07, "num_tokens": 15390896.0, "reward": 3.890097141265869, "reward_std": 0.21427203714847565, "rewards/check_gptzero_func/mean": 0.9821428656578064, "rewards/check_gptzero_func/std": 0.13363061845302582, "rewards/check_originality_func/mean": 0.9464285969734192, "rewards/check_originality_func/std": 0.22720777988433838, "rewards/check_winston_local_func/mean": 0.9920316934585571, "rewards/check_winston_local_func/std": 0.003620307659730315, "rewards/sentence_count_match_reward_logic/mean": 0.9694940447807312, "rewards/sentence_count_match_reward_logic/std": 0.06801146268844604, "step": 3221 }, { "clip_ratio": 0.022223638370633125, "epoch": 1.1273617914625613, "grad_norm": 1.229792358041951, "kl": 1.9609375, "learning_rate": 4.764955788803023e-06, "loss": -0.0721, "step": 3222 }, { "clip_ratio": 0.03404201939702034, "epoch": 1.1277116864940517, "grad_norm": 1.040423519813669, "kl": 1.9609375, "learning_rate": 4.761905561113658e-06, "loss": -0.0751, "step": 3223 }, { "clip_ratio": 0.05148839205503464, "epoch": 1.1280615815255424, "grad_norm": 1.097690535453888, "kl": 1.9609375, "learning_rate": 4.758855422231265e-06, "loss": -0.079, "step": 3224 }, { "clip_ratio": 0.021972492337226868, "completions/clipped_ratio": 0.0357142857142857, "completions/max_length": 235.0, "completions/max_terminated_length": 235.0, "completions/mean_length": 139.0178680419922, "completions/mean_terminated_length": 144.1666717529297, "completions/min_length": 0.0, "completions/min_terminated_length": 81.0, "epoch": 1.1284114765570328, "grad_norm": 1.9407483113541895, "kl": 2.640625, "learning_rate": 4.755805373293516e-06, "loss": -0.0724, "num_tokens": 15407825.0, "reward": 3.9191839694976807, "reward_std": 0.16286270320415497, "rewards/check_gptzero_func/mean": 1.0, "rewards/check_gptzero_func/std": 0.0, "rewards/check_originality_func/mean": 0.9642857313156128, "rewards/check_originality_func/std": 0.187256321310997, "rewards/check_winston_local_func/mean": 0.992313027381897, "rewards/check_winston_local_func/std": 0.002604829613119364, "rewards/sentence_count_match_reward_logic/mean": 0.9625850319862366, "rewards/sentence_count_match_reward_logic/std": 0.10611778497695923, "step": 3225 }, { "clip_ratio": 0.0477052703499794, "epoch": 1.1287613715885234, "grad_norm": 1.0126173549759139, "kl": 2.640625, "learning_rate": 4.7527554154380535e-06, "loss": -0.0747, "step": 3226 }, { "clip_ratio": 0.05401439219713211, "epoch": 1.129111266620014, "grad_norm": 0.7991504979736809, "kl": 2.640625, "learning_rate": 4.749705549802482e-06, "loss": -0.0761, "step": 3227 }, { "clip_ratio": 0.06817349046468735, "epoch": 1.1294611616515045, "grad_norm": 0.885056540255478, "kl": 2.65625, "learning_rate": 4.746655777524369e-06, "loss": -0.077, "step": 3228 }, { "clip_ratio": 0.019252808764576912, "completions/clipped_ratio": 0.4285714285714286, "completions/max_length": 228.0, "completions/max_terminated_length": 228.0, "completions/mean_length": 105.03572082519531, "completions/mean_terminated_length": 183.8125, "completions/min_length": 0.0, "completions/min_terminated_length": 130.0, "epoch": 1.129811056682995, "grad_norm": 1.8243506048196878, "kl": 1.9921875, "learning_rate": 4.743606099741255e-06, "loss": 0.0052, "num_tokens": 15430443.0, "reward": 3.5170984268188477, "reward_std": 0.27609720826148987, "rewards/check_gptzero_func/mean": 0.9107142686843872, "rewards/check_gptzero_func/std": 0.28773635625839233, "rewards/check_originality_func/mean": 0.7678571343421936, "rewards/check_originality_func/std": 0.4260205328464508, "rewards/check_winston_local_func/mean": 0.9418432116508484, "rewards/check_winston_local_func/std": 0.1228116974234581, "rewards/sentence_count_match_reward_logic/mean": 0.8966836929321289, "rewards/sentence_count_match_reward_logic/std": 0.13758710026741028, "step": 3229 }, { "clip_ratio": 0.02477196976542473, "epoch": 1.1301609517144857, "grad_norm": 1.1367795597125232, "kl": 1.9921875, "learning_rate": 4.740556517590637e-06, "loss": 0.0031, "step": 3230 }, { "clip_ratio": 0.04453916847705841, "epoch": 1.1305108467459761, "grad_norm": 0.8358306543444055, "kl": 2.0, "learning_rate": 4.737507032209985e-06, "loss": -0.0004, "step": 3231 }, { "clip_ratio": 0.05997177213430405, "epoch": 1.1308607417774668, "grad_norm": 0.6399937046650671, "kl": 2.0, "learning_rate": 4.734457644736724e-06, "loss": -0.0017, "step": 3232 }, { "clip_ratio": 0.027443161234259605, "completions/clipped_ratio": 0.2678571428571429, "completions/max_length": 254.0, "completions/max_terminated_length": 254.0, "completions/mean_length": 130.67857360839844, "completions/mean_terminated_length": 178.48779296875, "completions/min_length": 0.0, "completions/min_terminated_length": 108.0, "epoch": 1.1312106368089574, "grad_norm": 2.336302090018025, "kl": 2.0625, "learning_rate": 4.731408356308248e-06, "loss": 0.003, "num_tokens": 15452257.0, "reward": 3.6711032390594482, "reward_std": 0.2653835713863373, "rewards/check_gptzero_func/mean": 0.9107142686843872, "rewards/check_gptzero_func/std": 0.28773635625839233, "rewards/check_originality_func/mean": 0.8571428656578064, "rewards/check_originality_func/std": 0.3530939221382141, "rewards/check_winston_local_func/mean": 0.9904412031173706, "rewards/check_winston_local_func/std": 0.007068814244121313, "rewards/sentence_count_match_reward_logic/mean": 0.9128047227859497, "rewards/sentence_count_match_reward_logic/std": 0.16278523206710815, "step": 3233 }, { "clip_ratio": 0.01932336390018463, "epoch": 1.1315605318404478, "grad_norm": 1.9373444449444492, "kl": 2.09375, "learning_rate": 4.728359168061915e-06, "loss": 0.0012, "step": 3234 }, { "clip_ratio": 0.05663207173347473, "epoch": 1.1319104268719384, "grad_norm": 1.5964397116878632, "kl": 2.109375, "learning_rate": 4.725310081135039e-06, "loss": -0.0043, "step": 3235 }, { "clip_ratio": 0.07533516734838486, "epoch": 1.132260321903429, "grad_norm": 6.15902989904061, "kl": 2.640625, "learning_rate": 4.722261096664905e-06, "loss": -0.0028, "step": 3236 }, { "clip_ratio": 0.012963729910552502, "completions/clipped_ratio": 0.1428571428571429, "completions/max_length": 217.0, "completions/max_terminated_length": 217.0, "completions/mean_length": 143.4107208251953, "completions/mean_terminated_length": 167.3125, "completions/min_length": 0.0, "completions/min_terminated_length": 115.0, "epoch": 1.1326102169349195, "grad_norm": 1.339749549221018, "kl": 1.9921875, "learning_rate": 4.719212215788753e-06, "loss": 0.0149, "num_tokens": 15472512.0, "reward": 3.738840103149414, "reward_std": 0.16855376958847046, "rewards/check_gptzero_func/mean": 0.9285714030265808, "rewards/check_gptzero_func/std": 0.25987008213996887, "rewards/check_originality_func/mean": 0.8928571343421936, "rewards/check_originality_func/std": 0.3120938837528229, "rewards/check_winston_local_func/mean": 0.9900302886962891, "rewards/check_winston_local_func/std": 0.004237096291035414, "rewards/sentence_count_match_reward_logic/mean": 0.9273809790611267, "rewards/sentence_count_match_reward_logic/std": 0.1218365952372551, "step": 3237 }, { "clip_ratio": 0.024792427197098732, "epoch": 1.13296011196641, "grad_norm": 1.0383679065624727, "kl": 1.9921875, "learning_rate": 4.7161634396437864e-06, "loss": 0.0114, "step": 3238 }, { "clip_ratio": 0.0502261258661747, "epoch": 1.1333100069979007, "grad_norm": 1.1065751798430594, "kl": 1.9921875, "learning_rate": 4.713114769367171e-06, "loss": 0.0096, "step": 3239 }, { "clip_ratio": 0.07028321176767349, "epoch": 1.1336599020293912, "grad_norm": 0.8982068022956807, "kl": 1.9921875, "learning_rate": 4.71006620609603e-06, "loss": 0.0075, "step": 3240 }, { "clip_ratio": 0.02694416046142578, "completions/clipped_ratio": 0.017857142857142905, "completions/max_length": 235.0, "completions/max_terminated_length": 235.0, "completions/mean_length": 167.35714721679688, "completions/mean_terminated_length": 170.39999389648438, "completions/min_length": 0.0, "completions/min_terminated_length": 104.0, "epoch": 1.1340097970608818, "grad_norm": 2.440921434958879, "kl": 2.34375, "learning_rate": 4.707017750967452e-06, "loss": -0.0184, "num_tokens": 15492852.0, "reward": 3.932244300842285, "reward_std": 0.15037901699543, "rewards/check_gptzero_func/mean": 1.0, "rewards/check_gptzero_func/std": 0.0, "rewards/check_originality_func/mean": 0.9642857313156128, "rewards/check_originality_func/std": 0.187256321310997, "rewards/check_winston_local_func/mean": 0.9914703369140625, "rewards/check_winston_local_func/std": 0.011726070195436478, "rewards/sentence_count_match_reward_logic/mean": 0.9764881134033203, "rewards/sentence_count_match_reward_logic/std": 0.07544659823179245, "step": 3241 }, { "clip_ratio": 0.024419594556093216, "epoch": 1.1343596920923722, "grad_norm": 2.4577985325532397, "kl": 2.328125, "learning_rate": 4.7039694051184805e-06, "loss": -0.0195, "step": 3242 }, { "clip_ratio": 0.04036426171660423, "epoch": 1.1347095871238628, "grad_norm": 2.4774577731059035, "kl": 2.328125, "learning_rate": 4.700921169686116e-06, "loss": -0.0226, "step": 3243 }, { "clip_ratio": 0.06529339402914047, "epoch": 1.1350594821553535, "grad_norm": 1.2399989562086822, "kl": 2.28125, "learning_rate": 4.697873045807328e-06, "loss": -0.0274, "step": 3244 }, { "clip_ratio": 0.021150287240743637, "completions/clipped_ratio": 0.0357142857142857, "completions/max_length": 250.0, "completions/max_terminated_length": 250.0, "completions/mean_length": 145.69644165039062, "completions/mean_terminated_length": 151.09259033203125, "completions/min_length": 0.0, "completions/min_terminated_length": 106.0, "epoch": 1.1354093771868439, "grad_norm": 0.8824822414861718, "kl": 2.28125, "learning_rate": 4.6948250346190315e-06, "loss": -0.0475, "num_tokens": 15510635.0, "reward": 3.929931879043579, "reward_std": 0.1727026253938675, "rewards/check_gptzero_func/mean": 1.0, "rewards/check_gptzero_func/std": 0.0, "rewards/check_originality_func/mean": 0.9464285969734192, "rewards/check_originality_func/std": 0.22720777988433838, "rewards/check_winston_local_func/mean": 0.9926014542579651, "rewards/check_winston_local_func/std": 0.006732689682394266, "rewards/sentence_count_match_reward_logic/mean": 0.9909014105796814, "rewards/sentence_count_match_reward_logic/std": 0.038975510746240616, "step": 3245 }, { "clip_ratio": 0.026714548468589783, "epoch": 1.1357592722183345, "grad_norm": 0.9215455790629331, "kl": 2.265625, "learning_rate": 4.691777137258111e-06, "loss": -0.0493, "step": 3246 }, { "clip_ratio": 0.03758794441819191, "epoch": 1.1361091672498251, "grad_norm": 0.6785060134322298, "kl": 2.265625, "learning_rate": 4.688729354861401e-06, "loss": -0.0504, "step": 3247 }, { "clip_ratio": 0.06138088181614876, "epoch": 1.1364590622813155, "grad_norm": 0.5623864434695384, "kl": 2.265625, "learning_rate": 4.6856816885656956e-06, "loss": -0.0525, "step": 3248 }, { "clip_ratio": 0.016815312206745148, "completions/clipped_ratio": 0.1607142857142857, "completions/max_length": 253.0, "completions/max_terminated_length": 253.0, "completions/mean_length": 128.125, "completions/mean_terminated_length": 152.65957641601562, "completions/min_length": 0.0, "completions/min_terminated_length": 92.0, "epoch": 1.1368089573128062, "grad_norm": 3.306776522120281, "kl": 2.140625, "learning_rate": 4.6826341395077456e-06, "loss": -0.0492, "num_tokens": 15528402.0, "reward": 3.795137405395508, "reward_std": 0.2747722566127777, "rewards/check_gptzero_func/mean": 0.9642857313156128, "rewards/check_gptzero_func/std": 0.187256321310997, "rewards/check_originality_func/mean": 0.875, "rewards/check_originality_func/std": 0.3337118923664093, "rewards/check_winston_local_func/mean": 0.9915657043457031, "rewards/check_winston_local_func/std": 0.007636996451765299, "rewards/sentence_count_match_reward_logic/mean": 0.9642857313156128, "rewards/sentence_count_match_reward_logic/std": 0.08601970970630646, "step": 3249 }, { "clip_ratio": 0.04216674715280533, "epoch": 1.1371588523442968, "grad_norm": 2.813791745697, "kl": 2.140625, "learning_rate": 4.679586708824258e-06, "loss": -0.0546, "step": 3250 }, { "clip_ratio": 0.07898585498332977, "epoch": 1.1375087473757872, "grad_norm": 1.10006430399445, "kl": 2.15625, "learning_rate": 4.676539397651893e-06, "loss": -0.059, "step": 3251 }, { "clip_ratio": 0.10347455739974976, "epoch": 1.1378586424072779, "grad_norm": 0.9635008861602864, "kl": 2.1875, "learning_rate": 4.673492207127274e-06, "loss": -0.061, "step": 3252 }, { "clip_ratio": 0.016509681940078735, "completions/clipped_ratio": 0.2321428571428571, "completions/max_length": 241.0, "completions/max_terminated_length": 241.0, "completions/mean_length": 115.96429443359375, "completions/mean_terminated_length": 151.02325439453125, "completions/min_length": 0.0, "completions/min_terminated_length": 117.0, "epoch": 1.1382085374387683, "grad_norm": 1.1123910831912451, "kl": 2.03125, "learning_rate": 4.670445138386971e-06, "loss": 0.0675, "num_tokens": 15546416.0, "reward": 3.9002861976623535, "reward_std": 0.13771045207977295, "rewards/check_gptzero_func/mean": 1.0, "rewards/check_gptzero_func/std": 0.0, "rewards/check_originality_func/mean": 0.9285714030265808, "rewards/check_originality_func/std": 0.25987008213996887, "rewards/check_winston_local_func/mean": 0.9910528063774109, "rewards/check_winston_local_func/std": 0.004426054656505585, "rewards/sentence_count_match_reward_logic/mean": 0.9806618690490723, "rewards/sentence_count_match_reward_logic/std": 0.04888589307665825, "step": 3253 }, { "clip_ratio": 0.02195805311203003, "epoch": 1.138558432470259, "grad_norm": 0.9907506406601009, "kl": 2.03125, "learning_rate": 4.667398192567513e-06, "loss": 0.0658, "step": 3254 }, { "clip_ratio": 0.028944814577698708, "epoch": 1.1389083275017495, "grad_norm": 0.931182012865515, "kl": 2.03125, "learning_rate": 4.66435137080538e-06, "loss": 0.064, "step": 3255 }, { "clip_ratio": 0.06382358819246292, "epoch": 1.13925822253324, "grad_norm": 0.7661230481743011, "kl": 2.015625, "learning_rate": 4.661304674237009e-06, "loss": 0.0603, "step": 3256 }, { "clip_ratio": 0.021530015394091606, "completions/clipped_ratio": 0.0, "completions/max_length": 226.0, "completions/max_terminated_length": 226.0, "completions/mean_length": 150.4107208251953, "completions/mean_terminated_length": 150.4107208251953, "completions/min_length": 84.0, "completions/min_terminated_length": 84.0, "epoch": 1.1396081175647306, "grad_norm": 1.8919584570140169, "kl": 2.34375, "learning_rate": 4.658258103998789e-06, "loss": 0.013, "num_tokens": 15564615.0, "reward": 3.9794836044311523, "reward_std": 0.030710598453879356, "rewards/check_gptzero_func/mean": 1.0, "rewards/check_gptzero_func/std": 0.0, "rewards/check_originality_func/mean": 1.0, "rewards/check_originality_func/std": 0.0, "rewards/check_winston_local_func/mean": 0.9919837117195129, "rewards/check_winston_local_func/std": 0.006067381706088781, "rewards/sentence_count_match_reward_logic/mean": 0.987500011920929, "rewards/sentence_count_match_reward_logic/std": 0.05330017954111099, "step": 3257 }, { "clip_ratio": 0.02532907761633396, "epoch": 1.1399580125962212, "grad_norm": 1.4522554510140007, "kl": 2.359375, "learning_rate": 4.655211661227062e-06, "loss": 0.0104, "step": 3258 }, { "clip_ratio": 0.03736560791730881, "epoch": 1.1403079076277116, "grad_norm": 1.2564586966445208, "kl": 2.359375, "learning_rate": 4.652165347058123e-06, "loss": 0.0074, "step": 3259 }, { "clip_ratio": 0.05845613032579422, "epoch": 1.1406578026592022, "grad_norm": 0.8278576369154578, "kl": 2.34375, "learning_rate": 4.649119162628218e-06, "loss": 0.0043, "step": 3260 }, { "clip_ratio": 0.013287435285747051, "completions/clipped_ratio": 0.0, "completions/max_length": 231.0, "completions/max_terminated_length": 231.0, "completions/mean_length": 172.82144165039062, "completions/mean_terminated_length": 172.82144165039062, "completions/min_length": 84.0, "completions/min_terminated_length": 84.0, "epoch": 1.1410076976906929, "grad_norm": 1.8170055231945748, "kl": 2.015625, "learning_rate": 4.646073109073542e-06, "loss": 0.0148, "num_tokens": 15585573.0, "reward": 3.843973159790039, "reward_std": 0.1012692004442215, "rewards/check_gptzero_func/mean": 0.8571428656578064, "rewards/check_gptzero_func/std": 0.3530939221382141, "rewards/check_originality_func/mean": 1.0, "rewards/check_originality_func/std": 0.0, "rewards/check_winston_local_func/mean": 0.9923572540283203, "rewards/check_winston_local_func/std": 0.0024938490241765976, "rewards/sentence_count_match_reward_logic/mean": 0.9944728016853333, "rewards/sentence_count_match_reward_logic/std": 0.029068926349282265, "step": 3261 }, { "clip_ratio": 0.015272388234734535, "epoch": 1.1413575927221833, "grad_norm": 1.3254907723674636, "kl": 2.015625, "learning_rate": 4.643027187530249e-06, "loss": 0.0136, "step": 3262 }, { "clip_ratio": 0.027754629030823708, "epoch": 1.141707487753674, "grad_norm": 0.8825782797902552, "kl": 2.015625, "learning_rate": 4.639981399134435e-06, "loss": 0.0118, "step": 3263 }, { "clip_ratio": 0.032567478716373444, "epoch": 1.1420573827851643, "grad_norm": 0.8132600529698127, "kl": 2.03125, "learning_rate": 4.636935745022153e-06, "loss": 0.0111, "step": 3264 }, { "clip_ratio": 0.014301687479019165, "completions/clipped_ratio": 0.125, "completions/max_length": 243.0, "completions/max_terminated_length": 243.0, "completions/mean_length": 163.7857208251953, "completions/mean_terminated_length": 187.1836700439453, "completions/min_length": 0.0, "completions/min_terminated_length": 128.0, "epoch": 1.142407277816655, "grad_norm": 1.7390993509827632, "kl": 1.8125, "learning_rate": 4.633890226329402e-06, "loss": 0.0014, "num_tokens": 15607225.0, "reward": 3.8742847442626953, "reward_std": 0.21411600708961487, "rewards/check_gptzero_func/mean": 0.9285714030265808, "rewards/check_gptzero_func/std": 0.25987008213996887, "rewards/check_originality_func/mean": 0.9642857313156128, "rewards/check_originality_func/std": 0.187256321310997, "rewards/check_winston_local_func/mean": 0.9918438792228699, "rewards/check_winston_local_func/std": 0.0028620208613574505, "rewards/sentence_count_match_reward_logic/mean": 0.9895833134651184, "rewards/sentence_count_match_reward_logic/std": 0.04512188956141472, "step": 3265 }, { "clip_ratio": 0.020716430619359016, "epoch": 1.1427571728481456, "grad_norm": 1.5784385755933554, "kl": 1.8125, "learning_rate": 4.630844844192129e-06, "loss": -0.0009, "step": 3266 }, { "clip_ratio": 0.045372702181339264, "epoch": 1.1431070678796362, "grad_norm": 1.083811253423662, "kl": 1.8125, "learning_rate": 4.6277995997462385e-06, "loss": -0.005, "step": 3267 }, { "clip_ratio": 0.06978762894868851, "epoch": 1.1434569629111266, "grad_norm": 1.1007136451313222, "kl": 1.8125, "learning_rate": 4.624754494127573e-06, "loss": -0.0081, "step": 3268 }, { "clip_ratio": 0.02349892258644104, "completions/clipped_ratio": 0.0357142857142857, "completions/max_length": 254.0, "completions/max_terminated_length": 254.0, "completions/mean_length": 147.75, "completions/mean_terminated_length": 153.22222900390625, "completions/min_length": 0.0, "completions/min_terminated_length": 54.0, "epoch": 1.1438068579426173, "grad_norm": 1.9229920254626933, "kl": 2.390625, "learning_rate": 4.621709528471931e-06, "loss": 0.001, "num_tokens": 15625779.0, "reward": 3.825984477996826, "reward_std": 0.180109441280365, "rewards/check_gptzero_func/mean": 0.9285714030265808, "rewards/check_gptzero_func/std": 0.25987008213996887, "rewards/check_originality_func/mean": 0.9285714030265808, "rewards/check_originality_func/std": 0.25987008213996887, "rewards/check_winston_local_func/mean": 0.992651104927063, "rewards/check_winston_local_func/std": 0.002735472284257412, "rewards/sentence_count_match_reward_logic/mean": 0.976190447807312, "rewards/sentence_count_match_reward_logic/std": 0.06688276678323746, "step": 3269 }, { "clip_ratio": 0.05373205989599228, "epoch": 1.1441567529741077, "grad_norm": 1.8119667373664228, "kl": 2.375, "learning_rate": 4.618664703915055e-06, "loss": 0.0005, "step": 3270 }, { "clip_ratio": 0.04324989765882492, "epoch": 1.1445066480055983, "grad_norm": 1.5931219348887669, "kl": 2.390625, "learning_rate": 4.615620021592632e-06, "loss": -0.0057, "step": 3271 }, { "clip_ratio": 0.06898365914821625, "epoch": 1.144856543037089, "grad_norm": 1.2772418866087438, "kl": 2.375, "learning_rate": 4.6125754826403056e-06, "loss": -0.0053, "step": 3272 }, { "clip_ratio": 0.014100867323577404, "completions/clipped_ratio": 0.017857142857142905, "completions/max_length": 243.0, "completions/max_terminated_length": 243.0, "completions/mean_length": 172.17857360839844, "completions/mean_terminated_length": 175.30908203125, "completions/min_length": 0.0, "completions/min_terminated_length": 98.0, "epoch": 1.1452064380685794, "grad_norm": 1.2892085525342019, "kl": 1.9921875, "learning_rate": 4.609531088193655e-06, "loss": -0.0391, "num_tokens": 15646893.0, "reward": 3.8856966495513916, "reward_std": 0.1414491832256317, "rewards/check_gptzero_func/mean": 0.9642857313156128, "rewards/check_gptzero_func/std": 0.187256321310997, "rewards/check_originality_func/mean": 0.9821428656578064, "rewards/check_originality_func/std": 0.13363061845302582, "rewards/check_winston_local_func/mean": 0.9540001153945923, "rewards/check_winston_local_func/std": 0.09685391187667847, "rewards/sentence_count_match_reward_logic/mean": 0.9852678179740906, "rewards/sentence_count_match_reward_logic/std": 0.043894704431295395, "step": 3273 }, { "clip_ratio": 0.01738632284104824, "epoch": 1.14555633310007, "grad_norm": 1.0908823054261327, "kl": 1.9921875, "learning_rate": 4.6064868393882165e-06, "loss": -0.0413, "step": 3274 }, { "clip_ratio": 0.0321667455136776, "epoch": 1.1459062281315606, "grad_norm": 0.9291764865803548, "kl": 1.9921875, "learning_rate": 4.603442737359462e-06, "loss": -0.0446, "step": 3275 }, { "clip_ratio": 0.044463012367486954, "epoch": 1.146256123163051, "grad_norm": 0.8798640002012408, "kl": 2.0, "learning_rate": 4.600398783242811e-06, "loss": -0.0477, "step": 3276 }, { "clip_ratio": 0.012073405086994171, "completions/clipped_ratio": 0.1428571428571429, "completions/max_length": 256.0, "completions/max_terminated_length": 256.0, "completions/mean_length": 151.67857360839844, "completions/mean_terminated_length": 176.95834350585938, "completions/min_length": 0.0, "completions/min_terminated_length": 113.0, "epoch": 1.1466060181945417, "grad_norm": 2.746751836984044, "kl": 1.8515625, "learning_rate": 4.5973549781736335e-06, "loss": 0.0129, "num_tokens": 15667739.0, "reward": 3.803680896759033, "reward_std": 0.07643870264291763, "rewards/check_gptzero_func/mean": 0.9642857313156128, "rewards/check_gptzero_func/std": 0.187256321310997, "rewards/check_originality_func/mean": 0.8571428656578064, "rewards/check_originality_func/std": 0.3530939221382141, "rewards/check_winston_local_func/mean": 0.9887005686759949, "rewards/check_winston_local_func/std": 0.010030610486865044, "rewards/sentence_count_match_reward_logic/mean": 0.9935516119003296, "rewards/sentence_count_match_reward_logic/std": 0.027391798794269562, "step": 3277 }, { "clip_ratio": 0.02089749276638031, "epoch": 1.1469559132260323, "grad_norm": 1.4756629348525232, "kl": 1.8671875, "learning_rate": 4.594311323287237e-06, "loss": 0.0104, "step": 3278 }, { "clip_ratio": 0.03732641413807869, "epoch": 1.1473058082575227, "grad_norm": 0.9001307190025177, "kl": 1.8515625, "learning_rate": 4.591267819718879e-06, "loss": 0.0079, "step": 3279 }, { "clip_ratio": 0.054530877619981766, "epoch": 1.1476557032890133, "grad_norm": 0.7552099791989498, "kl": 1.8671875, "learning_rate": 4.5882244686037556e-06, "loss": 0.0061, "step": 3280 }, { "clip_ratio": 0.024393370375037193, "completions/clipped_ratio": 0.0357142857142857, "completions/max_length": 254.0, "completions/max_terminated_length": 254.0, "completions/mean_length": 169.82144165039062, "completions/mean_terminated_length": 176.11111450195312, "completions/min_length": 0.0, "completions/min_terminated_length": 107.0, "epoch": 1.1480055983205038, "grad_norm": 2.4093705170464985, "kl": 2.015625, "learning_rate": 4.585181271077008e-06, "loss": -0.0246, "num_tokens": 15688065.0, "reward": 3.9252712726593018, "reward_std": 0.17150701582431793, "rewards/check_gptzero_func/mean": 1.0, "rewards/check_gptzero_func/std": 0.0, "rewards/check_originality_func/mean": 0.9464285969734192, "rewards/check_originality_func/std": 0.22720777988433838, "rewards/check_winston_local_func/mean": 0.9918813109397888, "rewards/check_winston_local_func/std": 0.0032471988815814257, "rewards/sentence_count_match_reward_logic/mean": 0.9869614839553833, "rewards/sentence_count_match_reward_logic/std": 0.04246867448091507, "step": 3281 }, { "clip_ratio": 0.02524002455174923, "epoch": 1.1483554933519944, "grad_norm": 2.3504585009262895, "kl": 2.015625, "learning_rate": 4.582138228273721e-06, "loss": -0.0274, "step": 3282 }, { "clip_ratio": 0.04075231775641441, "epoch": 1.148705388383485, "grad_norm": 1.0506951704200962, "kl": 2.015625, "learning_rate": 4.579095341328917e-06, "loss": -0.0323, "step": 3283 }, { "clip_ratio": 0.05855483561754227, "epoch": 1.1490552834149754, "grad_norm": 0.9992439126195652, "kl": 2.015625, "learning_rate": 4.57605261137757e-06, "loss": -0.0351, "step": 3284 }, { "clip_ratio": 0.022249184548854828, "completions/clipped_ratio": 0.3392857142857143, "completions/max_length": 253.0, "completions/max_terminated_length": 253.0, "completions/mean_length": 128.98214721679688, "completions/mean_terminated_length": 195.21621704101562, "completions/min_length": 0.0, "completions/min_terminated_length": 121.0, "epoch": 1.149405178446466, "grad_norm": 3.1118937363795447, "kl": 1.875, "learning_rate": 4.573010039554587e-06, "loss": -0.076, "num_tokens": 15708280.0, "reward": 3.7393674850463867, "reward_std": 0.22918865084648132, "rewards/check_gptzero_func/mean": 1.0, "rewards/check_gptzero_func/std": 0.0, "rewards/check_originality_func/mean": 0.7857142686843872, "rewards/check_originality_func/std": 0.4140393137931824, "rewards/check_winston_local_func/mean": 0.990926206111908, "rewards/check_winston_local_func/std": 0.004202321171760559, "rewards/sentence_count_match_reward_logic/mean": 0.9627267122268677, "rewards/sentence_count_match_reward_logic/std": 0.07902679592370987, "step": 3285 }, { "clip_ratio": 0.05655815079808235, "epoch": 1.1497550734779567, "grad_norm": 1.9424681505783323, "kl": 1.8671875, "learning_rate": 4.569967626994817e-06, "loss": -0.0807, "step": 3286 }, { "clip_ratio": 0.10964051634073257, "epoch": 1.150104968509447, "grad_norm": 1.2321897072589685, "kl": 1.8515625, "learning_rate": 4.566925374833052e-06, "loss": -0.0846, "step": 3287 }, { "clip_ratio": 0.12355011701583862, "epoch": 1.1504548635409377, "grad_norm": 0.9940896499490268, "kl": 1.8671875, "learning_rate": 4.563883284204025e-06, "loss": -0.0851, "step": 3288 }, { "clip_ratio": 0.015827197581529617, "completions/clipped_ratio": 0.2857142857142857, "completions/max_length": 224.0, "completions/max_terminated_length": 224.0, "completions/mean_length": 113.6964340209961, "completions/mean_terminated_length": 159.1750030517578, "completions/min_length": 0.0, "completions/min_terminated_length": 109.0, "epoch": 1.1508047585724284, "grad_norm": 1.306994800012314, "kl": 2.171875, "learning_rate": 4.560841356242402e-06, "loss": 0.0067, "num_tokens": 15726839.0, "reward": 3.665827512741089, "reward_std": 0.27737918496131897, "rewards/check_gptzero_func/mean": 0.9285714030265808, "rewards/check_gptzero_func/std": 0.25987008213996887, "rewards/check_originality_func/mean": 0.8214285969734192, "rewards/check_originality_func/std": 0.3864591419696808, "rewards/check_winston_local_func/mean": 0.9724460244178772, "rewards/check_winston_local_func/std": 0.05343243107199669, "rewards/sentence_count_match_reward_logic/mean": 0.9433814883232117, "rewards/sentence_count_match_reward_logic/std": 0.09169309586286545, "step": 3289 }, { "clip_ratio": 0.017852410674095154, "epoch": 1.1511546536039188, "grad_norm": 1.2088776628077693, "kl": 2.171875, "learning_rate": 4.557799592082799e-06, "loss": 0.0044, "step": 3290 }, { "clip_ratio": 0.03168850392103195, "epoch": 1.1515045486354094, "grad_norm": 0.842981898283618, "kl": 2.171875, "learning_rate": 4.554757992859763e-06, "loss": 0.003, "step": 3291 }, { "clip_ratio": 0.04662296921014786, "epoch": 1.1518544436668998, "grad_norm": 1.2616789295891548, "kl": 2.234375, "learning_rate": 4.551716559707782e-06, "loss": 0.0014, "step": 3292 }, { "clip_ratio": 0.017836114391684532, "completions/clipped_ratio": 0.125, "completions/max_length": 233.0, "completions/max_terminated_length": 233.0, "completions/mean_length": 134.67857360839844, "completions/mean_terminated_length": 153.91836547851562, "completions/min_length": 0.0, "completions/min_terminated_length": 108.0, "epoch": 1.1522043386983905, "grad_norm": 4.892886722939747, "kl": 2.1875, "learning_rate": 4.548675293761281e-06, "loss": 0.0072, "num_tokens": 15745781.0, "reward": 3.7380192279815674, "reward_std": 0.2815675139427185, "rewards/check_gptzero_func/mean": 0.9107142686843872, "rewards/check_gptzero_func/std": 0.28773635625839233, "rewards/check_originality_func/mean": 0.875, "rewards/check_originality_func/std": 0.3337118923664093, "rewards/check_winston_local_func/mean": 0.9906164407730103, "rewards/check_winston_local_func/std": 0.0063043758273124695, "rewards/sentence_count_match_reward_logic/mean": 0.9616883397102356, "rewards/sentence_count_match_reward_logic/std": 0.08454970270395279, "step": 3293 }, { "clip_ratio": 0.026501089334487915, "epoch": 1.152554233729881, "grad_norm": 4.330807351988101, "kl": 2.203125, "learning_rate": 4.545634196154623e-06, "loss": 0.0001, "step": 3294 }, { "clip_ratio": 0.05715193226933479, "epoch": 1.1529041287613715, "grad_norm": 1.7258039364287128, "kl": 2.203125, "learning_rate": 4.542593268022111e-06, "loss": -0.0081, "step": 3295 }, { "clip_ratio": 0.08770734816789627, "epoch": 1.1532540237928621, "grad_norm": 1.0020710324902509, "kl": 2.25, "learning_rate": 4.5395525104979804e-06, "loss": -0.0098, "step": 3296 }, { "clip_ratio": 0.02085031010210514, "completions/clipped_ratio": 0.1428571428571429, "completions/max_length": 255.0, "completions/max_terminated_length": 255.0, "completions/mean_length": 141.32144165039062, "completions/mean_terminated_length": 164.875, "completions/min_length": 0.0, "completions/min_terminated_length": 75.0, "epoch": 1.1536039188243528, "grad_norm": 1.248235475479552, "kl": 1.9921875, "learning_rate": 4.536511924716406e-06, "loss": -0.0319, "num_tokens": 15765263.0, "reward": 3.7140989303588867, "reward_std": 0.2605968415737152, "rewards/check_gptzero_func/mean": 0.875, "rewards/check_gptzero_func/std": 0.3337118923664093, "rewards/check_originality_func/mean": 0.875, "rewards/check_originality_func/std": 0.3337118923664093, "rewards/check_winston_local_func/mean": 0.9896941184997559, "rewards/check_winston_local_func/std": 0.015251453965902328, "rewards/sentence_count_match_reward_logic/mean": 0.9744047522544861, "rewards/sentence_count_match_reward_logic/std": 0.07470858842134476, "step": 3297 }, { "clip_ratio": 0.02166704647243023, "epoch": 1.1539538138558432, "grad_norm": 1.6802243598657325, "kl": 1.984375, "learning_rate": 4.533471511811498e-06, "loss": -0.0336, "step": 3298 }, { "clip_ratio": 0.03460276499390602, "epoch": 1.1543037088873338, "grad_norm": 1.080882164909672, "kl": 2.0, "learning_rate": 4.5304312729172975e-06, "loss": -0.0352, "step": 3299 }, { "clip_ratio": 0.042924679815769196, "epoch": 1.1546536039188244, "grad_norm": 1.00895994875875, "kl": 2.0, "learning_rate": 4.527391209167791e-06, "loss": -0.0369, "step": 3300 }, { "clip_ratio": 0.02822568267583847, "completions/clipped_ratio": 0.1607142857142857, "completions/max_length": 255.0, "completions/max_terminated_length": 255.0, "completions/mean_length": 124.53572082519531, "completions/mean_terminated_length": 148.3829803466797, "completions/min_length": 0.0, "completions/min_terminated_length": 74.0, "epoch": 1.1550034989503148, "grad_norm": 1.989093098514918, "kl": 2.328125, "learning_rate": 4.524351321696889e-06, "loss": -0.0267, "num_tokens": 15782405.0, "reward": 3.6672396659851074, "reward_std": 0.4173179268836975, "rewards/check_gptzero_func/mean": 0.875, "rewards/check_gptzero_func/std": 0.3337118923664093, "rewards/check_originality_func/mean": 0.8392857313156128, "rewards/check_originality_func/std": 0.3705909848213196, "rewards/check_winston_local_func/mean": 0.9911030530929565, "rewards/check_winston_local_func/std": 0.005690563470125198, "rewards/sentence_count_match_reward_logic/mean": 0.961850643157959, "rewards/sentence_count_match_reward_logic/std": 0.0826697126030922, "step": 3301 }, { "clip_ratio": 0.04084967076778412, "epoch": 1.1553533939818055, "grad_norm": 1.857989772230827, "kl": 2.328125, "learning_rate": 4.521311611638443e-06, "loss": -0.0313, "step": 3302 }, { "clip_ratio": 0.07423019409179688, "epoch": 1.1557032890132959, "grad_norm": 1.0344196836674655, "kl": 2.328125, "learning_rate": 4.518272080126236e-06, "loss": -0.0343, "step": 3303 }, { "clip_ratio": 0.09055232256650925, "epoch": 1.1560531840447865, "grad_norm": 0.8710812092719828, "kl": 2.34375, "learning_rate": 4.515232728293981e-06, "loss": -0.036, "step": 3304 }, { "clip_ratio": 0.015688231214880943, "completions/clipped_ratio": 0.1785714285714286, "completions/max_length": 235.0, "completions/max_terminated_length": 235.0, "completions/mean_length": 134.5, "completions/mean_terminated_length": 163.7391357421875, "completions/min_length": 0.0, "completions/min_terminated_length": 130.0, "epoch": 1.1564030790762772, "grad_norm": 2.2489187967007367, "kl": 1.9296875, "learning_rate": 4.512193557275333e-06, "loss": 0.0242, "num_tokens": 15800945.0, "reward": 3.809183120727539, "reward_std": 0.23597517609596252, "rewards/check_gptzero_func/mean": 0.9642857313156128, "rewards/check_gptzero_func/std": 0.187256321310997, "rewards/check_originality_func/mean": 0.8571428656578064, "rewards/check_originality_func/std": 0.3530939221382141, "rewards/check_winston_local_func/mean": 0.9925375580787659, "rewards/check_winston_local_func/std": 0.0031660401728004217, "rewards/sentence_count_match_reward_logic/mean": 0.9952168464660645, "rewards/sentence_count_match_reward_logic/std": 0.02513670176267624, "step": 3305 }, { "clip_ratio": 0.028361869975924492, "epoch": 1.1567529741077678, "grad_norm": 1.8312660215116046, "kl": 1.9296875, "learning_rate": 4.509154568203871e-06, "loss": 0.0208, "step": 3306 }, { "clip_ratio": 0.04462623596191406, "epoch": 1.1571028691392582, "grad_norm": 1.1278768005834865, "kl": 1.9296875, "learning_rate": 4.506115762213109e-06, "loss": 0.0184, "step": 3307 }, { "clip_ratio": 0.0673418715596199, "epoch": 1.1574527641707488, "grad_norm": 1.0947887002703485, "kl": 1.9921875, "learning_rate": 4.5030771404364945e-06, "loss": 0.0168, "step": 3308 }, { "clip_ratio": 0.025679320096969604, "completions/clipped_ratio": 0.017857142857142905, "completions/max_length": 255.0, "completions/max_terminated_length": 255.0, "completions/mean_length": 175.6428680419922, "completions/mean_terminated_length": 178.83636474609375, "completions/min_length": 0.0, "completions/min_terminated_length": 85.0, "epoch": 1.1578026592022392, "grad_norm": 2.535199027892552, "kl": 2.078125, "learning_rate": 4.500038704007402e-06, "loss": -0.0249, "num_tokens": 15821997.0, "reward": 3.896914482116699, "reward_std": 0.12266357243061066, "rewards/check_gptzero_func/mean": 1.0, "rewards/check_gptzero_func/std": 0.0, "rewards/check_originality_func/mean": 0.9642857313156128, "rewards/check_originality_func/std": 0.187256321310997, "rewards/check_winston_local_func/mean": 0.9861999154090881, "rewards/check_winston_local_func/std": 0.02817191742360592, "rewards/sentence_count_match_reward_logic/mean": 0.9464285969734192, "rewards/sentence_count_match_reward_logic/std": 0.10399661213159561, "step": 3309 }, { "clip_ratio": 0.03748153895139694, "epoch": 1.1581525542337299, "grad_norm": 1.9262290187000133, "kl": 2.078125, "learning_rate": 4.497000454059142e-06, "loss": -0.0269, "step": 3310 }, { "clip_ratio": 0.07194121181964874, "epoch": 1.1585024492652205, "grad_norm": 0.9976995663359426, "kl": 2.078125, "learning_rate": 4.493962391724951e-06, "loss": -0.031, "step": 3311 }, { "clip_ratio": 0.09255070239305496, "epoch": 1.158852344296711, "grad_norm": 1.0292394319557288, "kl": 2.078125, "learning_rate": 4.490924518138001e-06, "loss": -0.0323, "step": 3312 }, { "clip_ratio": 0.019066430628299713, "completions/clipped_ratio": 0.1428571428571429, "completions/max_length": 218.0, "completions/max_terminated_length": 218.0, "completions/mean_length": 119.17857360839844, "completions/mean_terminated_length": 139.0416717529297, "completions/min_length": 0.0, "completions/min_terminated_length": 74.0, "epoch": 1.1592022393282015, "grad_norm": 2.8180114316655676, "kl": 2.40625, "learning_rate": 4.487886834431386e-06, "loss": 0.0204, "num_tokens": 15838535.0, "reward": 3.83679461479187, "reward_std": 0.2421140819787979, "rewards/check_gptzero_func/mean": 0.9821428656578064, "rewards/check_gptzero_func/std": 0.13363061845302582, "rewards/check_originality_func/mean": 0.8928571343421936, "rewards/check_originality_func/std": 0.31209391355514526, "rewards/check_winston_local_func/mean": 0.9885802865028381, "rewards/check_winston_local_func/std": 0.0235020499676466, "rewards/sentence_count_match_reward_logic/mean": 0.9732142686843872, "rewards/sentence_count_match_reward_logic/std": 0.07300744950771332, "step": 3313 }, { "clip_ratio": 0.037030041217803955, "epoch": 1.1595521343596922, "grad_norm": 1.6876937389975988, "kl": 2.40625, "learning_rate": 4.484849341738134e-06, "loss": 0.0155, "step": 3314 }, { "clip_ratio": 0.06478648632764816, "epoch": 1.1599020293911826, "grad_norm": 1.3036372546471475, "kl": 2.421875, "learning_rate": 4.481812041191203e-06, "loss": 0.0124, "step": 3315 }, { "clip_ratio": 0.09100189059972763, "epoch": 1.1602519244226732, "grad_norm": 1.1152025862564339, "kl": 2.421875, "learning_rate": 4.478774933923477e-06, "loss": 0.0095, "step": 3316 }, { "clip_ratio": 0.02353578247129917, "completions/clipped_ratio": 0.3035714285714286, "completions/max_length": 256.0, "completions/max_terminated_length": 256.0, "completions/mean_length": 139.85714721679688, "completions/mean_terminated_length": 200.8205108642578, "completions/min_length": 0.0, "completions/min_terminated_length": 155.0, "epoch": 1.1606018194541639, "grad_norm": 1.4831794612280007, "kl": 1.6640625, "learning_rate": 4.475738021067768e-06, "loss": -0.0759, "num_tokens": 15859879.0, "reward": 3.7895090579986572, "reward_std": 0.28474441170692444, "rewards/check_gptzero_func/mean": 0.9107142686843872, "rewards/check_gptzero_func/std": 0.28773635625839233, "rewards/check_originality_func/mean": 0.9107142686843872, "rewards/check_originality_func/std": 0.28773635625839233, "rewards/check_winston_local_func/mean": 0.9907631278038025, "rewards/check_winston_local_func/std": 0.009322858415544033, "rewards/sentence_count_match_reward_logic/mean": 0.977317214012146, "rewards/sentence_count_match_reward_logic/std": 0.053438950330019, "step": 3317 }, { "clip_ratio": 0.06081482395529747, "epoch": 1.1609517144856543, "grad_norm": 1.1001389286707155, "kl": 1.6640625, "learning_rate": 4.472701303756814e-06, "loss": -0.0781, "step": 3318 }, { "clip_ratio": 0.08665851503610611, "epoch": 1.161301609517145, "grad_norm": 0.9637303230468841, "kl": 1.6640625, "learning_rate": 4.469664783123282e-06, "loss": -0.0809, "step": 3319 }, { "clip_ratio": 0.11318671703338623, "epoch": 1.1616515045486353, "grad_norm": 0.8093837101536042, "kl": 1.6640625, "learning_rate": 4.466628460299769e-06, "loss": -0.0839, "step": 3320 }, { "clip_ratio": 0.02060736157000065, "completions/clipped_ratio": 0.0, "completions/max_length": 256.0, "completions/max_terminated_length": 256.0, "completions/mean_length": 189.1428680419922, "completions/mean_terminated_length": 189.1428680419922, "completions/min_length": 122.0, "completions/min_terminated_length": 122.0, "epoch": 1.162001399580126, "grad_norm": 4.669180415783208, "kl": 1.8984375, "learning_rate": 4.463592336418791e-06, "loss": 0.0112, "num_tokens": 15882039.0, "reward": 3.8883674144744873, "reward_std": 0.1178150475025177, "rewards/check_gptzero_func/mean": 0.9285714030265808, "rewards/check_gptzero_func/std": 0.25987008213996887, "rewards/check_originality_func/mean": 1.0, "rewards/check_originality_func/std": 0.0, "rewards/check_winston_local_func/mean": 0.9925339818000793, "rewards/check_winston_local_func/std": 0.0017342990031465888, "rewards/sentence_count_match_reward_logic/mean": 0.9672618508338928, "rewards/sentence_count_match_reward_logic/std": 0.06391790509223938, "step": 3321 }, { "clip_ratio": 0.04378582164645195, "epoch": 1.1623512946116166, "grad_norm": 2.800378111672058, "kl": 1.890625, "learning_rate": 4.460556412612796e-06, "loss": 0.0073, "step": 3322 }, { "clip_ratio": 0.06826358288526535, "epoch": 1.162701189643107, "grad_norm": 6.7266909583705985, "kl": 1.890625, "learning_rate": 4.4575206900141535e-06, "loss": 0.006, "step": 3323 }, { "clip_ratio": 0.07854443043470383, "epoch": 1.1630510846745976, "grad_norm": 1.9444840001459596, "kl": 1.9140625, "learning_rate": 4.454485169755156e-06, "loss": -0.0004, "step": 3324 }, { "clip_ratio": 0.01853208616375923, "completions/clipped_ratio": 0.017857142857142905, "completions/max_length": 248.0, "completions/max_terminated_length": 248.0, "completions/mean_length": 168.375, "completions/mean_terminated_length": 171.4363555908203, "completions/min_length": 0.0, "completions/min_terminated_length": 118.0, "epoch": 1.1634009797060882, "grad_norm": 4.087145661335501, "kl": 2.296875, "learning_rate": 4.451449852968031e-06, "loss": -0.0146, "num_tokens": 15902132.0, "reward": 3.8197906017303467, "reward_std": 0.2809430658817291, "rewards/check_gptzero_func/mean": 0.9107142686843872, "rewards/check_gptzero_func/std": 0.28773635625839233, "rewards/check_originality_func/mean": 0.9285714030265808, "rewards/check_originality_func/std": 0.25987008213996887, "rewards/check_winston_local_func/mean": 0.9926222562789917, "rewards/check_winston_local_func/std": 0.0021016127429902554, "rewards/sentence_count_match_reward_logic/mean": 0.9878826141357422, "rewards/sentence_count_match_reward_logic/std": 0.03913891315460205, "step": 3325 }, { "clip_ratio": 0.039294932037591934, "epoch": 1.1637508747375787, "grad_norm": 2.9938912495321652, "kl": 2.296875, "learning_rate": 4.4484147407849185e-06, "loss": -0.0203, "step": 3326 }, { "clip_ratio": 0.1030806303024292, "epoch": 1.1641007697690693, "grad_norm": 1.9312747232825447, "kl": 2.265625, "learning_rate": 4.445379834337887e-06, "loss": -0.0282, "step": 3327 }, { "clip_ratio": 0.1535034477710724, "epoch": 1.16445066480056, "grad_norm": 1.674102240380347, "kl": 2.21875, "learning_rate": 4.442345134758928e-06, "loss": -0.0325, "step": 3328 }, { "clip_ratio": 0.022944198921322823, "completions/clipped_ratio": 0.0, "completions/max_length": 243.0, "completions/max_terminated_length": 243.0, "completions/mean_length": 175.19644165039062, "completions/mean_terminated_length": 175.19644165039062, "completions/min_length": 100.0, "completions/min_terminated_length": 100.0, "epoch": 1.1648005598320503, "grad_norm": 5.579023065460552, "kl": 1.921875, "learning_rate": 4.4393106431799544e-06, "loss": 0.023, "num_tokens": 15922983.0, "reward": 3.9829559326171875, "reward_std": 0.02953561209142208, "rewards/check_gptzero_func/mean": 1.0, "rewards/check_gptzero_func/std": 0.0, "rewards/check_originality_func/mean": 1.0, "rewards/check_originality_func/std": 0.0, "rewards/check_winston_local_func/mean": 0.9930464625358582, "rewards/check_winston_local_func/std": 0.001832599169574678, "rewards/sentence_count_match_reward_logic/mean": 0.989909291267395, "rewards/sentence_count_match_reward_logic/std": 0.04350314661860466, "step": 3329 }, { "clip_ratio": 0.029624003916978836, "epoch": 1.165150454863541, "grad_norm": 3.4999459971303857, "kl": 1.9140625, "learning_rate": 4.436276360732807e-06, "loss": 0.0111, "step": 3330 }, { "clip_ratio": 0.048195820301771164, "epoch": 1.1655003498950314, "grad_norm": 1.7032617413946494, "kl": 1.921875, "learning_rate": 4.433242288549242e-06, "loss": 0.0106, "step": 3331 }, { "clip_ratio": 0.06658301502466202, "epoch": 1.165850244926522, "grad_norm": 1.2994073114712226, "kl": 1.921875, "learning_rate": 4.430208427760937e-06, "loss": 0.0036, "step": 3332 }, { "clip_ratio": 0.01524712797254324, "completions/clipped_ratio": 0.0, "completions/max_length": 256.0, "completions/max_terminated_length": 256.0, "completions/mean_length": 205.48214721679688, "completions/mean_terminated_length": 205.48214721679688, "completions/min_length": 116.0, "completions/min_terminated_length": 116.0, "epoch": 1.1662001399580126, "grad_norm": 2.3777253660546838, "kl": 1.671875, "learning_rate": 4.427174779499498e-06, "loss": 0.0121, "num_tokens": 15946834.0, "reward": 3.8750202655792236, "reward_std": 0.20164205133914948, "rewards/check_gptzero_func/mean": 0.9464285969734192, "rewards/check_gptzero_func/std": 0.22720777988433838, "rewards/check_originality_func/mean": 0.9464285969734192, "rewards/check_originality_func/std": 0.22720777988433838, "rewards/check_winston_local_func/mean": 0.9881154298782349, "rewards/check_winston_local_func/std": 0.0157761387526989, "rewards/sentence_count_match_reward_logic/mean": 0.9940475821495056, "rewards/sentence_count_match_reward_logic/std": 0.031209392473101616, "step": 3333 }, { "clip_ratio": 0.022958440706133842, "epoch": 1.1665500349895033, "grad_norm": 2.1468029689175037, "kl": 1.671875, "learning_rate": 4.424141344896442e-06, "loss": 0.0089, "step": 3334 }, { "clip_ratio": 0.04355308786034584, "epoch": 1.1668999300209937, "grad_norm": 1.4545342604303693, "kl": 1.671875, "learning_rate": 4.421108125083216e-06, "loss": 0.0044, "step": 3335 }, { "clip_ratio": 0.0726902112364769, "epoch": 1.1672498250524843, "grad_norm": 1.2144499252551455, "kl": 1.6796875, "learning_rate": 4.418075121191181e-06, "loss": 0.0009, "step": 3336 }, { "clip_ratio": 0.020250514149665833, "completions/clipped_ratio": 0.2321428571428571, "completions/max_length": 251.0, "completions/max_terminated_length": 251.0, "completions/mean_length": 147.10714721679688, "completions/mean_terminated_length": 191.58139038085938, "completions/min_length": 0.0, "completions/min_terminated_length": 137.0, "epoch": 1.1675997200839747, "grad_norm": 2.188301621657754, "kl": 1.7109375, "learning_rate": 4.415042334351616e-06, "loss": -0.0022, "num_tokens": 15967600.0, "reward": 3.8106119632720947, "reward_std": 0.2555363178253174, "rewards/check_gptzero_func/mean": 0.9464285969734192, "rewards/check_gptzero_func/std": 0.22720777988433838, "rewards/check_originality_func/mean": 0.8928571343421936, "rewards/check_originality_func/std": 0.3120938837528229, "rewards/check_winston_local_func/mean": 0.992159366607666, "rewards/check_winston_local_func/std": 0.005526290275156498, "rewards/sentence_count_match_reward_logic/mean": 0.9791666269302368, "rewards/sentence_count_match_reward_logic/std": 0.053394854068756104, "step": 3337 }, { "clip_ratio": 0.06257258355617523, "epoch": 1.1679496151154654, "grad_norm": 2.2617301935330825, "kl": 1.7265625, "learning_rate": 4.4120097656957275e-06, "loss": -0.0042, "step": 3338 }, { "clip_ratio": 0.047064147889614105, "epoch": 1.168299510146956, "grad_norm": 1.8448342103056155, "kl": 1.703125, "learning_rate": 4.408977416354628e-06, "loss": -0.0078, "step": 3339 }, { "clip_ratio": 0.11036419868469238, "epoch": 1.1686494051784464, "grad_norm": 1.4763120190847803, "kl": 1.734375, "learning_rate": 4.4059452874593605e-06, "loss": -0.0067, "step": 3340 }, { "clip_ratio": 0.01433208491653204, "completions/clipped_ratio": 0.2321428571428571, "completions/max_length": 255.0, "completions/max_terminated_length": 255.0, "completions/mean_length": 131.6607208251953, "completions/mean_terminated_length": 171.46511840820312, "completions/min_length": 0.0, "completions/min_terminated_length": 103.0, "epoch": 1.168999300209937, "grad_norm": 2.381767721158398, "kl": 2.015625, "learning_rate": 4.402913380140878e-06, "loss": -0.0021, "num_tokens": 15986749.0, "reward": 3.913942337036133, "reward_std": 0.17505456507205963, "rewards/check_gptzero_func/mean": 0.9821428656578064, "rewards/check_gptzero_func/std": 0.13363061845302582, "rewards/check_originality_func/mean": 0.9642857313156128, "rewards/check_originality_func/std": 0.187256321310997, "rewards/check_winston_local_func/mean": 0.990330696105957, "rewards/check_winston_local_func/std": 0.008623655885457993, "rewards/sentence_count_match_reward_logic/mean": 0.9771825075149536, "rewards/sentence_count_match_reward_logic/std": 0.068883016705513, "step": 3341 }, { "clip_ratio": 0.019113672897219658, "epoch": 1.1693491952414277, "grad_norm": 2.2663407836865974, "kl": 2.03125, "learning_rate": 4.399881695530053e-06, "loss": -0.005, "step": 3342 }, { "clip_ratio": 0.04263291880488396, "epoch": 1.169699090272918, "grad_norm": 1.0674244247052729, "kl": 2.046875, "learning_rate": 4.396850234757677e-06, "loss": -0.0075, "step": 3343 }, { "clip_ratio": 0.07720579952001572, "epoch": 1.1700489853044087, "grad_norm": 1.067854187047705, "kl": 2.0625, "learning_rate": 4.393818998954454e-06, "loss": -0.011, "step": 3344 }, { "clip_ratio": 0.018054861575365067, "completions/clipped_ratio": 0.0, "completions/max_length": 228.0, "completions/max_terminated_length": 228.0, "completions/mean_length": 161.1428680419922, "completions/mean_terminated_length": 161.1428680419922, "completions/min_length": 93.0, "completions/min_terminated_length": 93.0, "epoch": 1.1703988803358993, "grad_norm": 2.5453833253351643, "kl": 2.1875, "learning_rate": 4.390787989251009e-06, "loss": 0.0069, "num_tokens": 16006189.0, "reward": 3.8546342849731445, "reward_std": 0.22428424656391144, "rewards/check_gptzero_func/mean": 0.9107142686843872, "rewards/check_gptzero_func/std": 0.28773635625839233, "rewards/check_originality_func/mean": 0.9821428656578064, "rewards/check_originality_func/std": 0.13363061845302582, "rewards/check_winston_local_func/mean": 0.984098494052887, "rewards/check_winston_local_func/std": 0.03367244452238083, "rewards/sentence_count_match_reward_logic/mean": 0.9776785969734192, "rewards/sentence_count_match_reward_logic/std": 0.06531019508838654, "step": 3345 }, { "clip_ratio": 0.02501007542014122, "epoch": 1.1707487753673897, "grad_norm": 2.5682455037696617, "kl": 2.1875, "learning_rate": 4.3877572067778775e-06, "loss": 0.0064, "step": 3346 }, { "clip_ratio": 0.0358392558991909, "epoch": 1.1710986703988804, "grad_norm": 1.487377918352194, "kl": 2.171875, "learning_rate": 4.3847266526655115e-06, "loss": -0.0009, "step": 3347 }, { "clip_ratio": 0.061335377395153046, "epoch": 1.1714485654303708, "grad_norm": 1.11715960311185, "kl": 2.171875, "learning_rate": 4.381696328044282e-06, "loss": -0.0055, "step": 3348 }, { "clip_ratio": 0.02647395431995392, "completions/clipped_ratio": 0.0892857142857143, "completions/max_length": 256.0, "completions/max_terminated_length": 256.0, "completions/mean_length": 183.48214721679688, "completions/mean_terminated_length": 201.47059631347656, "completions/min_length": 0.0, "completions/min_terminated_length": 112.0, "epoch": 1.1717984604618614, "grad_norm": 3.4048288494251833, "kl": 1.7109375, "learning_rate": 4.378666234044471e-06, "loss": -0.0566, "num_tokens": 16028800.0, "reward": 3.8618226051330566, "reward_std": 0.2269068956375122, "rewards/check_gptzero_func/mean": 0.9464285969734192, "rewards/check_gptzero_func/std": 0.22720777988433838, "rewards/check_originality_func/mean": 0.9464285969734192, "rewards/check_originality_func/std": 0.22720777988433838, "rewards/check_winston_local_func/mean": 0.9923143982887268, "rewards/check_winston_local_func/std": 0.0034933844581246376, "rewards/sentence_count_match_reward_logic/mean": 0.9766510725021362, "rewards/sentence_count_match_reward_logic/std": 0.056055761873722076, "step": 3349 }, { "clip_ratio": 0.033353157341480255, "epoch": 1.172148355493352, "grad_norm": 3.5535485019942428, "kl": 1.7109375, "learning_rate": 4.3756363717962745e-06, "loss": -0.0567, "step": 3350 }, { "clip_ratio": 0.0639442428946495, "epoch": 1.1724982505248425, "grad_norm": 3.1627682989841794, "kl": 1.7109375, "learning_rate": 4.372606742429803e-06, "loss": -0.0641, "step": 3351 }, { "clip_ratio": 0.09060811251401901, "epoch": 1.172848145556333, "grad_norm": 3.2988866045636756, "kl": 1.703125, "learning_rate": 4.369577347075075e-06, "loss": -0.0649, "step": 3352 }, { "clip_ratio": 0.017008136957883835, "completions/clipped_ratio": 0.2321428571428571, "completions/max_length": 255.0, "completions/max_terminated_length": 255.0, "completions/mean_length": 140.96429443359375, "completions/mean_terminated_length": 183.58139038085938, "completions/min_length": 0.0, "completions/min_terminated_length": 130.0, "epoch": 1.1731980405878237, "grad_norm": 4.679056827523166, "kl": 1.8125, "learning_rate": 4.366548186862033e-06, "loss": -0.0293, "num_tokens": 16049158.0, "reward": 3.755760908126831, "reward_std": 0.2094200998544693, "rewards/check_gptzero_func/mean": 0.9107142686843872, "rewards/check_gptzero_func/std": 0.28773635625839233, "rewards/check_originality_func/mean": 0.8928571343421936, "rewards/check_originality_func/std": 0.3120938837528229, "rewards/check_winston_local_func/mean": 0.9868832230567932, "rewards/check_winston_local_func/std": 0.025918323546648026, "rewards/sentence_count_match_reward_logic/mean": 0.9653061628341675, "rewards/sentence_count_match_reward_logic/std": 0.08290223777294159, "step": 3353 }, { "clip_ratio": 0.03749106451869011, "epoch": 1.1735479356193141, "grad_norm": 2.3436159738637743, "kl": 1.8125, "learning_rate": 4.36351926292052e-06, "loss": -0.0342, "step": 3354 }, { "clip_ratio": 0.06181475520133972, "epoch": 1.1738978306508048, "grad_norm": 1.6740872285507296, "kl": 1.8359375, "learning_rate": 4.3604905763803006e-06, "loss": -0.0374, "step": 3355 }, { "clip_ratio": 0.08331236243247986, "epoch": 1.1742477256822954, "grad_norm": 1.381481098647763, "kl": 1.8515625, "learning_rate": 4.357462128371043e-06, "loss": -0.0391, "step": 3356 }, { "clip_ratio": 0.021746430546045303, "completions/clipped_ratio": 0.0535714285714286, "completions/max_length": 249.0, "completions/max_terminated_length": 249.0, "completions/mean_length": 152.375, "completions/mean_terminated_length": 161.0, "completions/min_length": 0.0, "completions/min_terminated_length": 107.0, "epoch": 1.1745976207137858, "grad_norm": 2.722725369023039, "kl": 2.25, "learning_rate": 4.354433920022328e-06, "loss": -0.0204, "num_tokens": 16067715.0, "reward": 3.7732455730438232, "reward_std": 0.282168984413147, "rewards/check_gptzero_func/mean": 0.8392857313156128, "rewards/check_gptzero_func/std": 0.3705909848213196, "rewards/check_originality_func/mean": 0.9642857313156128, "rewards/check_originality_func/std": 0.187256321310997, "rewards/check_winston_local_func/mean": 0.9887217283248901, "rewards/check_winston_local_func/std": 0.01624451018869877, "rewards/sentence_count_match_reward_logic/mean": 0.9809523820877075, "rewards/sentence_count_match_reward_logic/std": 0.055725570768117905, "step": 3357 }, { "clip_ratio": 0.045836690813302994, "epoch": 1.1749475157452764, "grad_norm": 2.373519987299352, "kl": 2.234375, "learning_rate": 4.351405952463653e-06, "loss": -0.0246, "step": 3358 }, { "clip_ratio": 0.07365278899669647, "epoch": 1.1752974107767669, "grad_norm": 1.7109390050837785, "kl": 2.21875, "learning_rate": 4.348378226824418e-06, "loss": -0.0305, "step": 3359 }, { "clip_ratio": 0.12397105991840363, "epoch": 1.1756473058082575, "grad_norm": 1.0724741236177417, "kl": 2.21875, "learning_rate": 4.3453507442339395e-06, "loss": -0.0354, "step": 3360 }, { "clip_ratio": 0.0368046760559082, "completions/clipped_ratio": 0.1071428571428571, "completions/max_length": 244.0, "completions/max_terminated_length": 244.0, "completions/mean_length": 127.21429443359375, "completions/mean_terminated_length": 142.47999572753906, "completions/min_length": 0.0, "completions/min_terminated_length": 87.0, "epoch": 1.1759972008397481, "grad_norm": 1.8268974107461258, "kl": 2.15625, "learning_rate": 4.342323505821436e-06, "loss": -0.0273, "num_tokens": 16084783.0, "reward": 3.8310859203338623, "reward_std": 0.2678700089454651, "rewards/check_gptzero_func/mean": 0.9821428656578064, "rewards/check_gptzero_func/std": 0.13363061845302582, "rewards/check_originality_func/mean": 0.875, "rewards/check_originality_func/std": 0.3337118923664093, "rewards/check_winston_local_func/mean": 0.9880797266960144, "rewards/check_winston_local_func/std": 0.017062105238437653, "rewards/sentence_count_match_reward_logic/mean": 0.9858631491661072, "rewards/sentence_count_match_reward_logic/std": 0.06419508159160614, "step": 3361 }, { "clip_ratio": 0.025789672508835793, "epoch": 1.1763470958712385, "grad_norm": 2.042708656313441, "kl": 2.15625, "learning_rate": 4.3392965127160395e-06, "loss": -0.0306, "step": 3362 }, { "clip_ratio": 0.0683421716094017, "epoch": 1.1766969909027292, "grad_norm": 1.3444422654269879, "kl": 2.140625, "learning_rate": 4.336269766046792e-06, "loss": -0.0351, "step": 3363 }, { "clip_ratio": 0.09913388639688492, "epoch": 1.1770468859342198, "grad_norm": 0.9087096570768457, "kl": 2.140625, "learning_rate": 4.3332432669426396e-06, "loss": -0.0397, "step": 3364 }, { "clip_ratio": 0.017063187435269356, "completions/clipped_ratio": 0.0, "completions/max_length": 230.0, "completions/max_terminated_length": 230.0, "completions/mean_length": 170.1428680419922, "completions/mean_terminated_length": 170.1428680419922, "completions/min_length": 102.0, "completions/min_terminated_length": 102.0, "epoch": 1.1773967809657102, "grad_norm": 3.7378406286121795, "kl": 1.8671875, "learning_rate": 4.3302170165324385e-06, "loss": 0.0157, "num_tokens": 16105031.0, "reward": 3.823139190673828, "reward_std": 0.18077629804611206, "rewards/check_gptzero_func/mean": 0.9642857313156128, "rewards/check_gptzero_func/std": 0.187256321310997, "rewards/check_originality_func/mean": 0.9464285969734192, "rewards/check_originality_func/std": 0.22720777988433838, "rewards/check_winston_local_func/mean": 0.9523056149482727, "rewards/check_winston_local_func/std": 0.1269901543855667, "rewards/sentence_count_match_reward_logic/mean": 0.9601190686225891, "rewards/sentence_count_match_reward_logic/std": 0.08608655631542206, "step": 3365 }, { "clip_ratio": 0.037517301738262177, "epoch": 1.1777466759972008, "grad_norm": 1.8336862876181494, "kl": 1.8671875, "learning_rate": 4.327191015944952e-06, "loss": 0.012, "step": 3366 }, { "clip_ratio": 0.05799098685383797, "epoch": 1.1780965710286915, "grad_norm": 1.8330868761086025, "kl": 1.875, "learning_rate": 4.324165266308846e-06, "loss": 0.0108, "step": 3367 }, { "clip_ratio": 0.08253384381532669, "epoch": 1.1784464660601819, "grad_norm": 1.9910090604888637, "kl": 1.8828125, "learning_rate": 4.321139768752702e-06, "loss": 0.0057, "step": 3368 }, { "clip_ratio": 0.028562163934111595, "completions/clipped_ratio": 0.1607142857142857, "completions/max_length": 256.0, "completions/max_terminated_length": 256.0, "completions/mean_length": 131.96429443359375, "completions/mean_terminated_length": 157.23403930664062, "completions/min_length": 0.0, "completions/min_terminated_length": 67.0, "epoch": 1.1787963610916725, "grad_norm": 2.2486523758462313, "kl": 2.234375, "learning_rate": 4.318114524404999e-06, "loss": 0.0139, "num_tokens": 16123797.0, "reward": 3.5349035263061523, "reward_std": 0.39210236072540283, "rewards/check_gptzero_func/mean": 0.8571428656578064, "rewards/check_gptzero_func/std": 0.3530939221382141, "rewards/check_originality_func/mean": 0.8035714030265808, "rewards/check_originality_func/std": 0.40089187026023865, "rewards/check_winston_local_func/mean": 0.8991609215736389, "rewards/check_winston_local_func/std": 0.23569458723068237, "rewards/sentence_count_match_reward_logic/mean": 0.975028395652771, "rewards/sentence_count_match_reward_logic/std": 0.06759369373321533, "step": 3369 }, { "clip_ratio": 0.0348798893392086, "epoch": 1.179146256123163, "grad_norm": 2.561169005702942, "kl": 2.234375, "learning_rate": 4.3150895343941235e-06, "loss": 0.0117, "step": 3370 }, { "clip_ratio": 0.0466521717607975, "epoch": 1.1794961511546536, "grad_norm": 1.2608566060385045, "kl": 2.234375, "learning_rate": 4.312064799848371e-06, "loss": 0.0116, "step": 3371 }, { "clip_ratio": 0.06508716195821762, "epoch": 1.1798460461861442, "grad_norm": 1.561245760844114, "kl": 2.234375, "learning_rate": 4.3090403218959345e-06, "loss": 0.0109, "step": 3372 }, { "clip_ratio": 0.01776144839823246, "completions/clipped_ratio": 0.017857142857142905, "completions/max_length": 251.0, "completions/max_terminated_length": 251.0, "completions/mean_length": 169.1428680419922, "completions/mean_terminated_length": 172.21817016601562, "completions/min_length": 0.0, "completions/min_terminated_length": 90.0, "epoch": 1.1801959412176348, "grad_norm": 2.760196363360211, "kl": 2.015625, "learning_rate": 4.306016101664921e-06, "loss": -0.024, "num_tokens": 16144005.0, "reward": 3.8657898902893066, "reward_std": 0.14082396030426025, "rewards/check_gptzero_func/mean": 1.0, "rewards/check_gptzero_func/std": 0.0, "rewards/check_originality_func/mean": 0.8928571343421936, "rewards/check_originality_func/std": 0.31209391355514526, "rewards/check_winston_local_func/mean": 0.9921358823776245, "rewards/check_winston_local_func/std": 0.003179979743435979, "rewards/sentence_count_match_reward_logic/mean": 0.9807964563369751, "rewards/sentence_count_match_reward_logic/std": 0.05329763516783714, "step": 3373 }, { "clip_ratio": 0.024847451597452164, "epoch": 1.1805458362491252, "grad_norm": 2.3024220566069507, "kl": 2.0, "learning_rate": 4.302992140283333e-06, "loss": -0.0296, "step": 3374 }, { "clip_ratio": 0.04278593137860298, "epoch": 1.1808957312806159, "grad_norm": 1.760524273303733, "kl": 2.0, "learning_rate": 4.29996843887908e-06, "loss": -0.0356, "step": 3375 }, { "clip_ratio": 0.07365797460079193, "epoch": 1.1812456263121063, "grad_norm": 1.3618293080249981, "kl": 2.015625, "learning_rate": 4.296944998579974e-06, "loss": -0.0403, "step": 3376 }, { "clip_ratio": 0.020594343543052673, "completions/clipped_ratio": 0.125, "completions/max_length": 253.0, "completions/max_terminated_length": 253.0, "completions/mean_length": 154.625, "completions/mean_terminated_length": 176.7142791748047, "completions/min_length": 0.0, "completions/min_terminated_length": 66.0, "epoch": 1.181595521343597, "grad_norm": 2.298653795525105, "kl": 1.859375, "learning_rate": 4.293921820513728e-06, "loss": -0.0003, "num_tokens": 16164360.0, "reward": 3.583589792251587, "reward_std": 0.3608640432357788, "rewards/check_gptzero_func/mean": 0.7857142686843872, "rewards/check_gptzero_func/std": 0.4140393137931824, "rewards/check_originality_func/mean": 0.8392857313156128, "rewards/check_originality_func/std": 0.3705909848213196, "rewards/check_winston_local_func/mean": 0.9853752255439758, "rewards/check_winston_local_func/std": 0.012382023967802525, "rewards/sentence_count_match_reward_logic/mean": 0.9732142686843872, "rewards/sentence_count_match_reward_logic/std": 0.05815526470541954, "step": 3377 }, { "clip_ratio": 0.019725626334547997, "epoch": 1.1819454163750875, "grad_norm": 2.111286829366573, "kl": 1.859375, "learning_rate": 4.290898905807962e-06, "loss": -0.0019, "step": 3378 }, { "clip_ratio": 0.042604710906744, "epoch": 1.182295311406578, "grad_norm": 1.3139940203474094, "kl": 1.859375, "learning_rate": 4.287876255590193e-06, "loss": -0.0092, "step": 3379 }, { "clip_ratio": 0.06541799008846283, "epoch": 1.1826452064380686, "grad_norm": 0.9684446682709748, "kl": 1.859375, "learning_rate": 4.28485387098784e-06, "loss": -0.012, "step": 3380 }, { "clip_ratio": 0.011767012998461723, "completions/clipped_ratio": 0.0714285714285714, "completions/max_length": 253.0, "completions/max_terminated_length": 253.0, "completions/mean_length": 158.75, "completions/mean_terminated_length": 170.9615478515625, "completions/min_length": 0.0, "completions/min_terminated_length": 112.0, "epoch": 1.1829951014695592, "grad_norm": 2.2825657232039567, "kl": 1.75, "learning_rate": 4.281831753128226e-06, "loss": -0.0321, "num_tokens": 16184562.0, "reward": 3.5788586139678955, "reward_std": 0.31301748752593994, "rewards/check_gptzero_func/mean": 0.875, "rewards/check_gptzero_func/std": 0.3337118923664093, "rewards/check_originality_func/mean": 0.8035714030265808, "rewards/check_originality_func/std": 0.40089184045791626, "rewards/check_winston_local_func/mean": 0.9189519882202148, "rewards/check_winston_local_func/std": 0.19412076473236084, "rewards/sentence_count_match_reward_logic/mean": 0.9813350439071655, "rewards/sentence_count_match_reward_logic/std": 0.05592087656259537, "step": 3381 }, { "clip_ratio": 0.02643590047955513, "epoch": 1.1833449965010496, "grad_norm": 1.9109179031911123, "kl": 1.7421875, "learning_rate": 4.278809903138569e-06, "loss": -0.0351, "step": 3382 }, { "clip_ratio": 0.0396493561565876, "epoch": 1.1836948915325403, "grad_norm": 1.5545054156254836, "kl": 1.7421875, "learning_rate": 4.275788322145994e-06, "loss": -0.0388, "step": 3383 }, { "clip_ratio": 0.07428060472011566, "epoch": 1.184044786564031, "grad_norm": 1.0624284414857115, "kl": 1.7421875, "learning_rate": 4.272767011277522e-06, "loss": -0.0445, "step": 3384 }, { "clip_ratio": 0.016370190307497978, "completions/clipped_ratio": 0.1071428571428571, "completions/max_length": 255.0, "completions/max_terminated_length": 255.0, "completions/mean_length": 157.5, "completions/mean_terminated_length": 176.39999389648438, "completions/min_length": 0.0, "completions/min_terminated_length": 85.0, "epoch": 1.1843946815955213, "grad_norm": 2.106721148754636, "kl": 1.890625, "learning_rate": 4.26974597166007e-06, "loss": -0.0461, "num_tokens": 16204934.0, "reward": 3.820631504058838, "reward_std": 0.36859217286109924, "rewards/check_gptzero_func/mean": 0.9107142686843872, "rewards/check_gptzero_func/std": 0.28773635625839233, "rewards/check_originality_func/mean": 0.9464285969734192, "rewards/check_originality_func/std": 0.22720777988433838, "rewards/check_winston_local_func/mean": 0.990422785282135, "rewards/check_winston_local_func/std": 0.008142215199768543, "rewards/sentence_count_match_reward_logic/mean": 0.9730654954910278, "rewards/sentence_count_match_reward_logic/std": 0.07781730592250824, "step": 3385 }, { "clip_ratio": 0.023730427026748657, "epoch": 1.184744576627012, "grad_norm": 1.561644830029672, "kl": 1.890625, "learning_rate": 4.26672520442046e-06, "loss": -0.0484, "step": 3386 }, { "clip_ratio": 0.03287848085165024, "epoch": 1.1850944716585023, "grad_norm": 1.1662425167865755, "kl": 1.890625, "learning_rate": 4.263704710685407e-06, "loss": -0.0494, "step": 3387 }, { "clip_ratio": 0.055775582790374756, "epoch": 1.185444366689993, "grad_norm": 1.3271788011608119, "kl": 1.890625, "learning_rate": 4.260684491581531e-06, "loss": -0.053, "step": 3388 }, { "clip_ratio": 0.01970471628010273, "completions/clipped_ratio": 0.0357142857142857, "completions/max_length": 255.0, "completions/max_terminated_length": 255.0, "completions/mean_length": 146.44644165039062, "completions/mean_terminated_length": 151.87037658691406, "completions/min_length": 0.0, "completions/min_terminated_length": 85.0, "epoch": 1.1857942617214836, "grad_norm": 3.035322427830808, "kl": 2.078125, "learning_rate": 4.257664548235341e-06, "loss": -0.0341, "num_tokens": 16223191.0, "reward": 3.8211491107940674, "reward_std": 0.23465804755687714, "rewards/check_gptzero_func/mean": 0.9285714030265808, "rewards/check_gptzero_func/std": 0.25987008213996887, "rewards/check_originality_func/mean": 0.9285714030265808, "rewards/check_originality_func/std": 0.25987008213996887, "rewards/check_winston_local_func/mean": 0.9915074110031128, "rewards/check_winston_local_func/std": 0.003116357373073697, "rewards/sentence_count_match_reward_logic/mean": 0.9724985957145691, "rewards/sentence_count_match_reward_logic/std": 0.06617746502161026, "step": 3389 }, { "clip_ratio": 0.032396700233221054, "epoch": 1.186144156752974, "grad_norm": 2.798919629351125, "kl": 2.078125, "learning_rate": 4.254644881773249e-06, "loss": -0.0386, "step": 3390 }, { "clip_ratio": 0.07988319545984268, "epoch": 1.1864940517844647, "grad_norm": 1.4167957201727825, "kl": 2.0625, "learning_rate": 4.251625493321563e-06, "loss": -0.0446, "step": 3391 }, { "clip_ratio": 0.11696698516607285, "epoch": 1.1868439468159553, "grad_norm": 1.3141968665096435, "kl": 2.046875, "learning_rate": 4.248606384006482e-06, "loss": -0.0479, "step": 3392 }, { "clip_ratio": 0.01855579949915409, "completions/clipped_ratio": 0.0, "completions/max_length": 249.0, "completions/max_terminated_length": 249.0, "completions/mean_length": 162.19644165039062, "completions/mean_terminated_length": 162.19644165039062, "completions/min_length": 122.0, "completions/min_terminated_length": 122.0, "epoch": 1.1871938418474457, "grad_norm": 3.107630632783513, "kl": 2.09375, "learning_rate": 4.245587554954111e-06, "loss": 0.0177, "num_tokens": 16242666.0, "reward": 3.9493207931518555, "reward_std": 0.11873391270637512, "rewards/check_gptzero_func/mean": 1.0, "rewards/check_gptzero_func/std": 0.0, "rewards/check_originality_func/mean": 0.9642857313156128, "rewards/check_originality_func/std": 0.187256321310997, "rewards/check_winston_local_func/mean": 0.9924755096435547, "rewards/check_winston_local_func/std": 0.0016869123792275786, "rewards/sentence_count_match_reward_logic/mean": 0.9925594925880432, "rewards/sentence_count_match_reward_logic/std": 0.03188832476735115, "step": 3393 }, { "clip_ratio": 0.03728238865733147, "epoch": 1.1875437368789363, "grad_norm": 1.5598665766709456, "kl": 2.078125, "learning_rate": 4.242569007290442e-06, "loss": 0.0108, "step": 3394 }, { "clip_ratio": 0.06523195654153824, "epoch": 1.187893631910427, "grad_norm": 1.1481118762926035, "kl": 2.078125, "learning_rate": 4.239550742141363e-06, "loss": 0.0079, "step": 3395 }, { "clip_ratio": 0.07199031859636307, "epoch": 1.1882435269419174, "grad_norm": 1.3987505318442313, "kl": 2.078125, "learning_rate": 4.236532760632662e-06, "loss": 0.0055, "step": 3396 }, { "clip_ratio": 0.03184247761964798, "completions/clipped_ratio": 0.0, "completions/max_length": 242.0, "completions/max_terminated_length": 242.0, "completions/mean_length": 161.1607208251953, "completions/mean_terminated_length": 161.1607208251953, "completions/min_length": 85.0, "completions/min_terminated_length": 85.0, "epoch": 1.188593421973408, "grad_norm": 2.5033576907974733, "kl": 2.046875, "learning_rate": 4.233515063890013e-06, "loss": 0.0032, "num_tokens": 16262003.0, "reward": 3.897427797317505, "reward_std": 0.07322023808956146, "rewards/check_gptzero_func/mean": 1.0, "rewards/check_gptzero_func/std": 0.0, "rewards/check_originality_func/mean": 0.9107142686843872, "rewards/check_originality_func/std": 0.28773635625839233, "rewards/check_winston_local_func/mean": 0.9867131114006042, "rewards/check_winston_local_func/std": 0.016592001542448997, "rewards/sentence_count_match_reward_logic/mean": 1.0, "rewards/sentence_count_match_reward_logic/std": 0.0, "step": 3397 }, { "clip_ratio": 0.03829902410507202, "epoch": 1.1889433170048984, "grad_norm": 1.8276386242250322, "kl": 2.03125, "learning_rate": 4.230497653038993e-06, "loss": -0.0014, "step": 3398 }, { "clip_ratio": 0.07196594774723053, "epoch": 1.189293212036389, "grad_norm": 1.1924056600296689, "kl": 2.03125, "learning_rate": 4.227480529205066e-06, "loss": -0.0044, "step": 3399 }, { "clip_ratio": 0.09068147093057632, "epoch": 1.1896431070678797, "grad_norm": 0.8307093062021844, "kl": 2.03125, "learning_rate": 4.224463693513588e-06, "loss": -0.0063, "step": 3400 } ], "logging_steps": 1, "max_steps": 5716, "num_input_tokens_seen": 16262003, "num_train_epochs": 2, "save_steps": 100, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": true, "should_training_stop": false }, "attributes": {} } }, "total_flos": 0.0, "train_batch_size": 8, "trial_name": null, "trial_params": null }