{"current_steps": 5, "total_steps": 196, "loss": 0.5035, "lr": 4e-08, "epoch": 0.025591810620601407, "percentage": 2.55, "elapsed_time": "0:01:18", "remaining_time": "0:49:51", "rewards/chosen": 0.0016484160564447705, "logps/chosen": -439.02909128289474, "logits/chosen": -59713280.0, "rewards/rejected": 0.0017084883792059763, "logps/rejected": -198.39232235863096, "logits/rejected": -50413135.23809524, "rewards/margins": -6.007232276120577e-05, "regularization/tv": 0.004178593400865793, "regularization/advantage_range": 0.41785937547683716, "kl": 0.007727213203907013}
{"current_steps": 10, "total_steps": 196, "loss": 0.506, "lr": 9e-08, "epoch": 0.05118362124120281, "percentage": 5.1, "elapsed_time": "0:02:39", "remaining_time": "0:49:17", "rewards/chosen": 0.0006063263621423152, "logps/chosen": -429.93242694805195, "logits/chosen": -59292100.15584416, "rewards/rejected": 0.0008949153574116258, "logps/rejected": -192.85899849397592, "logits/rejected": -50021388.3373494, "rewards/margins": -0.00028858899526931065, "regularization/tv": 0.006631854921579361, "regularization/advantage_range": 0.6631855368614197, "kl": 0.019967058673501015, "stability/response_length_mean": 46.125, "stability/response_length_std": 27.231481552124023, "stability/response_length_var": 741.5535888671875, "stability/token_entropy_mean": 3.4091796875, "stability/repetition_rate_mean": 0.3512222170829773}
{"current_steps": 15, "total_steps": 196, "loss": 0.5058, "lr": 1.4e-07, "epoch": 0.07677543186180422, "percentage": 7.65, "elapsed_time": "0:03:58", "remaining_time": "0:48:01", "rewards/chosen": -0.0012753700668161566, "logps/chosen": -404.1435606060606, "logits/chosen": -56228826.763636366, "rewards/rejected": 0.002305462860292004, "logps/rejected": -200.67100554435484, "logits/rejected": -51931185.548387095, "rewards/margins": -0.003580832927108161, "regularization/tv": 0.005917664151638746, "regularization/advantage_range": 0.5917664766311646, "kl": 0.01295405812561512, "stability/response_length_mean": 27.125, "stability/response_length_std": 30.79163932800293, "stability/response_length_var": 948.125, "stability/token_entropy_mean": 3.7978515625, "stability/repetition_rate_mean": 0.2476128488779068}
{"current_steps": 20, "total_steps": 196, "loss": 0.5065, "lr": 1.8999999999999998e-07, "epoch": 0.10236724248240563, "percentage": 10.2, "elapsed_time": "0:05:20", "remaining_time": "0:46:57", "rewards/chosen": 0.0012270104140043258, "logps/chosen": -343.6844482421875, "logits/chosen": -51667904.0, "rewards/rejected": 0.005719001591205597, "logps/rejected": -225.60615234375, "logits/rejected": -48060899.2, "rewards/margins": -0.004491991177201272, "regularization/tv": 0.0055425032041966915, "regularization/advantage_range": 0.5542503595352173, "kl": 0.021185090765357018, "stability/response_length_mean": 44.0, "stability/response_length_std": 28.0611572265625, "stability/response_length_var": 787.4285888671875, "stability/token_entropy_mean": 4.4296875, "stability/repetition_rate_mean": 0.2155257910490036}
{"current_steps": 25, "total_steps": 196, "loss": 0.5061, "lr": 2.4e-07, "epoch": 0.12795905310300704, "percentage": 12.76, "elapsed_time": "0:06:40", "remaining_time": "0:45:36", "rewards/chosen": 0.0012486030315530712, "logps/chosen": -378.47896012931034, "logits/chosen": -56471661.46206897, "rewards/rejected": 0.001101766654423305, "logps/rejected": -202.78183035714287, "logits/rejected": -52027626.057142854, "rewards/margins": 0.00014683637712976623, "regularization/tv": 0.005499088205397129, "regularization/advantage_range": 0.5499088168144226, "kl": 0.01974540390074253, "stability/response_length_mean": 43.375, "stability/response_length_std": 28.54538917541504, "stability/response_length_var": 814.8392944335938, "stability/token_entropy_mean": 4.806640625, "stability/repetition_rate_mean": 0.17801816761493683}
{"current_steps": 30, "total_steps": 196, "loss": 0.5063, "lr": 2.9e-07, "epoch": 0.15355086372360843, "percentage": 15.31, "elapsed_time": "0:08:00", "remaining_time": "0:44:16", "rewards/chosen": 0.003298740258952915, "logps/chosen": -468.54829068791946, "logits/chosen": -57726900.40268456, "rewards/rejected": 0.0027849639368336104, "logps/rejected": -198.2200292397661, "logits/rejected": -51525524.21052632, "rewards/margins": 0.0005137763221193048, "regularization/tv": 0.0065843164920806885, "regularization/advantage_range": 0.6584316492080688, "kl": 0.012587564997375011, "stability/response_length_mean": 43.0, "stability/response_length_std": 29.602123260498047, "stability/response_length_var": 876.2857055664062, "stability/token_entropy_mean": 4.56640625, "stability/repetition_rate_mean": 0.1486235111951828}
{"current_steps": 35, "total_steps": 196, "loss": 0.5061, "lr": 3.4000000000000003e-07, "epoch": 0.17914267434420986, "percentage": 17.86, "elapsed_time": "0:09:21", "remaining_time": "0:43:02", "rewards/chosen": 0.0006445885836323605, "logps/chosen": -385.93008306962025, "logits/chosen": -57639534.17721519, "rewards/rejected": 0.0016157176391578016, "logps/rejected": -195.11032744984567, "logits/rejected": -53902835.358024694, "rewards/margins": -0.0009711290555254411, "regularization/tv": 0.006383036263287067, "regularization/advantage_range": 0.6383035778999329, "kl": 0.018261268734931946, "stability/response_length_mean": 53.375, "stability/response_length_std": 21.14533042907715, "stability/response_length_var": 447.125, "stability/token_entropy_mean": 5.37890625, "stability/repetition_rate_mean": 0.3546627163887024}
{"current_steps": 40, "total_steps": 196, "loss": 0.5064, "lr": 3.8999999999999997e-07, "epoch": 0.20473448496481125, "percentage": 20.41, "elapsed_time": "0:10:42", "remaining_time": "0:41:43", "rewards/chosen": -0.0009344659745693206, "logps/chosen": -421.91201171875, "logits/chosen": -56638668.8, "rewards/rejected": 0.0034651543945074083, "logps/rejected": -219.274609375, "logits/rejected": -50249897.6, "rewards/margins": -0.004399620369076729, "regularization/tv": 0.006221619900316, "regularization/advantage_range": 0.6221620440483093, "kl": 0.024992099031805992, "stability/response_length_mean": 35.0, "stability/response_length_std": 31.518701553344727, "stability/response_length_var": 993.4285888671875, "stability/token_entropy_mean": 4.7978515625, "stability/repetition_rate_mean": 0.15522204339504242}
{"current_steps": 45, "total_steps": 196, "loss": 0.5057, "lr": 4.3999999999999997e-07, "epoch": 0.23032629558541268, "percentage": 22.96, "elapsed_time": "0:12:02", "remaining_time": "0:40:24", "rewards/chosen": 0.00447692487063824, "logps/chosen": -352.2836881291946, "logits/chosen": -52711602.684563756, "rewards/rejected": 0.0021861761982677973, "logps/rejected": -243.8523163377193, "logits/rejected": -50789016.701754384, "rewards/margins": 0.0022907486723704423, "regularization/tv": 0.006039530038833618, "regularization/advantage_range": 0.6039530634880066, "kl": 0.02745547890663147, "stability/response_length_mean": 28.875, "stability/response_length_std": 29.87324333190918, "stability/response_length_var": 892.4107055664062, "stability/token_entropy_mean": 4.744140625, "stability/repetition_rate_mean": 0.1033080443739891}
{"current_steps": 50, "total_steps": 196, "loss": 0.5059, "lr": 4.9e-07, "epoch": 0.2559181062060141, "percentage": 25.51, "elapsed_time": "0:13:22", "remaining_time": "0:39:03", "rewards/chosen": 0.003006344873633161, "logps/chosen": -375.16906984060404, "logits/chosen": -54046757.798657715, "rewards/rejected": 0.0034306809219003423, "logps/rejected": -180.20630025584796, "logits/rejected": -48303999.25146199, "rewards/margins": -0.00042433604826718147, "regularization/tv": 0.005211771931499243, "regularization/advantage_range": 0.5211772322654724, "kl": 0.03547530993819237, "stability/response_length_mean": 55.375, "stability/response_length_std": 16.361431121826172, "stability/response_length_var": 267.6964416503906, "stability/token_entropy_mean": 3.8271484375, "stability/repetition_rate_mean": 0.4206075370311737}
{"current_steps": 55, "total_steps": 196, "loss": 0.5062, "lr": 5.4e-07, "epoch": 0.28150991682661547, "percentage": 28.06, "elapsed_time": "0:14:42", "remaining_time": "0:37:41", "rewards/chosen": -0.0024640397255465112, "logps/chosen": -344.536636257764, "logits/chosen": -56948996.770186335, "rewards/rejected": 0.003600421941505288, "logps/rejected": -230.4120233883648, "logits/rejected": -49573849.358490564, "rewards/margins": -0.006064461667051799, "regularization/tv": 0.006143749691545963, "regularization/advantage_range": 0.6143749952316284, "kl": 0.04145348072052002, "stability/response_length_mean": 24.5, "stability/response_length_std": 32.710853576660156, "stability/response_length_var": 1070.0, "stability/token_entropy_mean": 4.419921875, "stability/repetition_rate_mean": 0.1015625}
{"current_steps": 60, "total_steps": 196, "loss": 0.5058, "lr": 5.9e-07, "epoch": 0.30710172744721687, "percentage": 30.61, "elapsed_time": "0:16:03", "remaining_time": "0:36:24", "rewards/chosen": 0.006462704051624645, "logps/chosen": -359.6562026515152, "logits/chosen": -52678984.92121212, "rewards/rejected": 0.0043683340472559775, "logps/rejected": -178.18324092741935, "logits/rejected": -47834769.341935486, "rewards/margins": 0.0020943700043686677, "regularization/tv": 0.005441432818770409, "regularization/advantage_range": 0.5441433191299438, "kl": 0.03181527182459831, "stability/response_length_mean": 52.625, "stability/response_length_std": 21.75800323486328, "stability/response_length_var": 473.41070556640625, "stability/token_entropy_mean": 5.291015625, "stability/repetition_rate_mean": 0.21373416483402252}
{"current_steps": 65, "total_steps": 196, "loss": 0.5057, "lr": 6.4e-07, "epoch": 0.3326935380678183, "percentage": 33.16, "elapsed_time": "0:17:24", "remaining_time": "0:35:04", "rewards/chosen": 0.008781992235491353, "logps/chosen": -381.85897177419355, "logits/chosen": -57569332.8516129, "rewards/rejected": 0.0033120903101834386, "logps/rejected": -183.5492305871212, "logits/rejected": -47285291.442424245, "rewards/margins": 0.005469901925307915, "regularization/tv": 0.0054890247993171215, "regularization/advantage_range": 0.5489024519920349, "kl": 0.03296849876642227, "stability/response_length_mean": 20.625, "stability/response_length_std": 27.176342010498047, "stability/response_length_var": 738.5535888671875, "stability/token_entropy_mean": 5.544921875, "stability/repetition_rate_mean": 0.1857219785451889}
{"current_steps": 70, "total_steps": 196, "loss": 0.5056, "lr": 6.9e-07, "epoch": 0.3582853486884197, "percentage": 35.71, "elapsed_time": "0:18:44", "remaining_time": "0:33:43", "rewards/chosen": 0.0039039734871156753, "logps/chosen": -379.08941532258063, "logits/chosen": -52787147.1483871, "rewards/rejected": 0.002243069807688395, "logps/rejected": -220.3997159090909, "logits/rejected": -49043046.4, "rewards/margins": 0.00166090367942728, "regularization/tv": 0.00558048952370882, "regularization/advantage_range": 0.5580489039421082, "kl": 0.03457256406545639, "stability/response_length_mean": 33.875, "stability/response_length_std": 32.36151885986328, "stability/response_length_var": 1047.267822265625, "stability/token_entropy_mean": 5.80078125, "stability/repetition_rate_mean": 0.166015625}
{"current_steps": 75, "total_steps": 196, "loss": 0.5059, "lr": 7.4e-07, "epoch": 0.3838771593090211, "percentage": 38.27, "elapsed_time": "0:20:05", "remaining_time": "0:32:24", "rewards/chosen": 0.00810670611020681, "logps/chosen": -314.1897434543919, "logits/chosen": -49525379.45945946, "rewards/rejected": 0.0031651778276576554, "logps/rejected": -228.8531976744186, "logits/rejected": -47913144.55813953, "rewards/margins": 0.004941528282549155, "regularization/tv": 0.006349931005388498, "regularization/advantage_range": 0.6349931359291077, "kl": 0.03155888244509697, "stability/response_length_mean": 22.75, "stability/response_length_std": 31.07479476928711, "stability/response_length_var": 965.6428833007812, "stability/token_entropy_mean": 2.1484375, "stability/repetition_rate_mean": 0.08308292925357819}
{"current_steps": 80, "total_steps": 196, "loss": 0.5058, "lr": 7.9e-07, "epoch": 0.4094689699296225, "percentage": 40.82, "elapsed_time": "0:21:26", "remaining_time": "0:31:04", "rewards/chosen": 0.006083738330184229, "logps/chosen": -403.96926738410593, "logits/chosen": -58659575.52317881, "rewards/rejected": 0.0026157531865249723, "logps/rejected": -201.02965514053255, "logits/rejected": -53490039.66863906, "rewards/margins": 0.0034679851436592566, "regularization/tv": 0.005138265434652567, "regularization/advantage_range": 0.5138265490531921, "kl": 0.049778860062360764, "stability/response_length_mean": 42.0, "stability/response_length_std": 30.481843948364258, "stability/response_length_var": 929.1428833007812, "stability/token_entropy_mean": 5.26171875, "stability/repetition_rate_mean": 0.19283854961395264}
{"current_steps": 85, "total_steps": 196, "loss": 0.5055, "lr": 8.399999999999999e-07, "epoch": 0.4350607805502239, "percentage": 43.37, "elapsed_time": "0:22:46", "remaining_time": "0:29:44", "rewards/chosen": 0.005946203970140026, "logps/chosen": -358.27303427419355, "logits/chosen": -51509360.30967742, "rewards/rejected": -5.5818214561000016e-05, "logps/rejected": -230.4253787878788, "logits/rejected": -47167947.24848485, "rewards/margins": 0.006002022184701026, "regularization/tv": 0.0056246244348585606, "regularization/advantage_range": 0.5624624490737915, "kl": 0.03699667379260063, "stability/response_length_mean": 26.0, "stability/response_length_std": 28.97782325744629, "stability/response_length_var": 839.7142944335938, "stability/token_entropy_mean": 4.982421875, "stability/repetition_rate_mean": 0.1396019458770752}
{"current_steps": 90, "total_steps": 196, "loss": 0.5055, "lr": 8.9e-07, "epoch": 0.46065259117082535, "percentage": 45.92, "elapsed_time": "0:24:07", "remaining_time": "0:28:24", "rewards/chosen": 0.00867634858840551, "logps/chosen": -380.70262419871796, "logits/chosen": -58464859.897435896, "rewards/rejected": 0.004627868896577417, "logps/rejected": -187.1805330602134, "logits/rejected": -50616251.317073174, "rewards/margins": 0.004048479691828094, "regularization/tv": 0.006174459587782621, "regularization/advantage_range": 0.6174459457397461, "kl": 0.04819156974554062, "stability/response_length_mean": 17.375, "stability/response_length_std": 20.542203903198242, "stability/response_length_var": 421.9821472167969, "stability/token_entropy_mean": 4.6875, "stability/repetition_rate_mean": 0.20654930174350739}
{"current_steps": 95, "total_steps": 196, "loss": 0.5057, "lr": 9.399999999999999e-07, "epoch": 0.48624440179142675, "percentage": 48.47, "elapsed_time": "0:25:27", "remaining_time": "0:27:03", "rewards/chosen": 0.0025773750075811074, "logps/chosen": -314.96761273734177, "logits/chosen": -50433743.39240506, "rewards/rejected": -0.0004016760690712634, "logps/rejected": -207.14773823302468, "logits/rejected": -49027005.62962963, "rewards/margins": 0.002979051076652371, "regularization/tv": 0.005851265043020248, "regularization/advantage_range": 0.585126519203186, "kl": 0.04422915354371071, "stability/response_length_mean": 27.0, "stability/response_length_std": 31.1264705657959, "stability/response_length_var": 968.8571166992188, "stability/token_entropy_mean": 3.669921875, "stability/repetition_rate_mean": 0.19569545984268188}
{"current_steps": 100, "total_steps": 196, "loss": 0.5055, "lr": 9.9e-07, "epoch": 0.5118362124120281, "percentage": 51.02, "elapsed_time": "0:26:47", "remaining_time": "0:25:43", "rewards/chosen": 0.004704180198746758, "logps/chosen": -359.8231102195946, "logits/chosen": -54773109.62162162, "rewards/rejected": 0.0017864440762719444, "logps/rejected": -216.5610465116279, "logits/rejected": -50978804.093023255, "rewards/margins": 0.002917736122474814, "regularization/tv": 0.006156648974865675, "regularization/advantage_range": 0.6156648993492126, "kl": 0.04536796733736992, "stability/response_length_mean": 38.0, "stability/response_length_std": 30.10457992553711, "stability/response_length_var": 906.2857055664062, "stability/token_entropy_mean": 4.140625, "stability/repetition_rate_mean": 0.23591694235801697}
{"current_steps": 105, "total_steps": 196, "loss": 0.5058, "lr": 9.999815554351224e-07, "epoch": 0.5374280230326296, "percentage": 53.57, "elapsed_time": "0:28:08", "remaining_time": "0:24:23", "rewards/chosen": 0.0068486849945711805, "logps/chosen": -399.8339608433735, "logits/chosen": -59440202.024096385, "rewards/rejected": 0.00588903876094075, "logps/rejected": -209.9449320211039, "logits/rejected": -53880659.116883114, "rewards/margins": 0.0009596462336304307, "regularization/tv": 0.008001342415809631, "regularization/advantage_range": 0.8001343011856079, "kl": 0.038910795003175735, "stability/response_length_mean": 28.0, "stability/response_length_std": 28.665559768676758, "stability/response_length_var": 821.7142944335938, "stability/token_entropy_mean": 3.41015625, "stability/repetition_rate_mean": 0.1702008843421936}
{"current_steps": 110, "total_steps": 196, "loss": 0.5056, "lr": 9.999066267225444e-07, "epoch": 0.5630198336532309, "percentage": 56.12, "elapsed_time": "0:29:29", "remaining_time": "0:23:03", "rewards/chosen": 0.002172079431005271, "logps/chosen": -378.54155685240966, "logits/chosen": -57453222.55421687, "rewards/rejected": 0.0018783260088462335, "logps/rejected": -206.06934862012986, "logits/rejected": -53678379.22077922, "rewards/margins": 0.00029375342215903744, "regularization/tv": 0.0063735246658325195, "regularization/advantage_range": 0.637352466583252, "kl": 0.04320025071501732, "stability/response_length_mean": 28.875, "stability/response_length_std": 29.911476135253906, "stability/response_length_var": 894.6964111328125, "stability/token_entropy_mean": 4.4052734375, "stability/repetition_rate_mean": 0.30853456258773804}
{"current_steps": 115, "total_steps": 196, "loss": 0.5057, "lr": 9.997740697079592e-07, "epoch": 0.5886116442738324, "percentage": 58.67, "elapsed_time": "0:30:51", "remaining_time": "0:21:43", "rewards/chosen": 0.00958028322533716, "logps/chosen": -364.49673655063293, "logits/chosen": -60699414.6835443, "rewards/rejected": 0.004186919809859476, "logps/rejected": -201.1546947337963, "logits/rejected": -53509745.777777776, "rewards/margins": 0.005393363415477684, "regularization/tv": 0.006492432206869125, "regularization/advantage_range": 0.6492432951927185, "kl": 0.04815962538123131, "stability/response_length_mean": 41.375, "stability/response_length_std": 31.25442886352539, "stability/response_length_var": 976.8392944335938, "stability/token_entropy_mean": 4.880859375, "stability/repetition_rate_mean": 0.2701823115348816}
{"current_steps": 120, "total_steps": 196, "loss": 0.5057, "lr": 9.995838996722914e-07, "epoch": 0.6142034548944337, "percentage": 61.22, "elapsed_time": "0:32:10", "remaining_time": "0:20:22", "rewards/chosen": 0.005139833182291268, "logps/chosen": -384.1935253267974, "logits/chosen": -57830406.69281046, "rewards/rejected": 0.0017575384257082455, "logps/rejected": -212.19879771706587, "logits/rejected": -54497776.670658685, "rewards/margins": 0.0033822947565830225, "regularization/tv": 0.006422989070415497, "regularization/advantage_range": 0.6422988772392273, "kl": 0.036807831376791, "stability/response_length_mean": 29.375, "stability/response_length_std": 29.548688888549805, "stability/response_length_var": 873.125, "stability/token_entropy_mean": 5.30859375, "stability/repetition_rate_mean": 0.17961646616458893}
{"current_steps": 125, "total_steps": 196, "loss": 0.5058, "lr": 9.993361385379874e-07, "epoch": 0.6397952655150352, "percentage": 63.78, "elapsed_time": "0:33:31", "remaining_time": "0:19:02", "rewards/chosen": 0.005247564348455978, "logps/chosen": -417.32681399828766, "logits/chosen": -60829906.4109589, "rewards/rejected": 0.004333444710435538, "logps/rejected": -216.3238595545977, "logits/rejected": -55816921.74712644, "rewards/margins": 0.00091411963802044, "regularization/tv": 0.007117211353033781, "regularization/advantage_range": 0.7117211222648621, "kl": 0.03840151056647301, "stability/response_length_mean": 15.25, "stability/response_length_std": 24.93276596069336, "stability/response_length_var": 621.6428833007812, "stability/token_entropy_mean": 3.7578125, "stability/repetition_rate_mean": 0.08598090708255768}
{"current_steps": 130, "total_steps": 196, "loss": 0.5057, "lr": 9.99030814866488e-07, "epoch": 0.6653870761356366, "percentage": 66.33, "elapsed_time": "0:34:51", "remaining_time": "0:17:41", "rewards/chosen": 0.0026500949936528358, "logps/chosen": -392.6314516129032, "logits/chosen": -57524488.258064516, "rewards/rejected": 0.0007959763209025065, "logps/rejected": -183.91096117424243, "logits/rejected": -53994719.41818182, "rewards/margins": 0.0018541186727503293, "regularization/tv": 0.006006088573485613, "regularization/advantage_range": 0.6006088852882385, "kl": 0.037142276763916016, "stability/response_length_mean": 38.0, "stability/response_length_std": 29.13515281677246, "stability/response_length_var": 848.8571166992188, "stability/token_entropy_mean": 3.921875, "stability/repetition_rate_mean": 0.18814630806446075}
{"current_steps": 135, "total_steps": 196, "loss": 0.5058, "lr": 9.986679638549358e-07, "epoch": 0.690978886756238, "percentage": 68.88, "elapsed_time": "0:36:11", "remaining_time": "0:16:21", "rewards/chosen": 0.007426799440676449, "logps/chosen": -401.7418999233129, "logits/chosen": -60006789.496932514, "rewards/rejected": 0.0023977617928936223, "logps/rejected": -247.84676054936307, "logits/rejected": -55216936.76433121, "rewards/margins": 0.005029037647782827, "regularization/tv": 0.00708754500374198, "regularization/advantage_range": 0.7087545394897461, "kl": 0.04310822859406471, "stability/response_length_mean": 29.0, "stability/response_length_std": 30.20406723022461, "stability/response_length_var": 912.2857055664062, "stability/token_entropy_mean": 3.4453125, "stability/repetition_rate_mean": 0.2250434011220932}
{"current_steps": 140, "total_steps": 196, "loss": 0.5052, "lr": 9.982476273321175e-07, "epoch": 0.7165706973768394, "percentage": 71.43, "elapsed_time": "0:37:31", "remaining_time": "0:15:00", "rewards/chosen": 0.009888766028664328, "logps/chosen": -367.0908380681818, "logits/chosen": -57250648.43636364, "rewards/rejected": -0.0006679048461298789, "logps/rejected": -190.04115423387097, "logits/rejected": -50280144.103225805, "rewards/margins": 0.010556670874794206, "regularization/tv": 0.007223478052765131, "regularization/advantage_range": 0.7223478555679321, "kl": 0.03307735547423363, "stability/response_length_mean": 56.25, "stability/response_length_std": 21.920310974121094, "stability/response_length_var": 480.5, "stability/token_entropy_mean": 4.919921875, "stability/repetition_rate_mean": 0.328125}
{"current_steps": 145, "total_steps": 196, "loss": 0.5056, "lr": 9.977698537536417e-07, "epoch": 0.7421625079974408, "percentage": 73.98, "elapsed_time": "0:38:52", "remaining_time": "0:13:40", "rewards/chosen": 0.005680616361549102, "logps/chosen": -330.21498493975906, "logits/chosen": -51311600.578313254, "rewards/rejected": -0.00267979773608121, "logps/rejected": -199.51471185064935, "logits/rejected": -51882898.28571428, "rewards/margins": 0.008360414097630312, "regularization/tv": 0.006130368448793888, "regularization/advantage_range": 0.6130368113517761, "kl": 0.04704681783914566, "stability/response_length_mean": 28.75, "stability/response_length_std": 29.86517333984375, "stability/response_length_var": 891.9285888671875, "stability/token_entropy_mean": 5.29296875, "stability/repetition_rate_mean": 0.25034263730049133}
{"current_steps": 150, "total_steps": 196, "loss": 0.5051, "lr": 9.972346981963546e-07, "epoch": 0.7677543186180422, "percentage": 76.53, "elapsed_time": "0:40:13", "remaining_time": "0:12:20", "rewards/chosen": 0.009661375775056727, "logps/chosen": -397.8378216911765, "logits/chosen": -55583340.42352941, "rewards/rejected": -0.0019295593102773032, "logps/rejected": -207.74522135416666, "logits/rejected": -50427801.6, "rewards/margins": 0.01159093508533403, "regularization/tv": 0.006044992245733738, "regularization/advantage_range": 0.6044992804527283, "kl": 0.04447220265865326, "stability/response_length_mean": 35.25, "stability/response_length_std": 31.01957893371582, "stability/response_length_var": 962.2142944335938, "stability/token_entropy_mean": 4.5859375, "stability/repetition_rate_mean": 0.2467447966337204}
{"current_steps": 155, "total_steps": 196, "loss": 0.5056, "lr": 9.966422223519885e-07, "epoch": 0.7933461292386437, "percentage": 79.08, "elapsed_time": "0:41:34", "remaining_time": "0:10:59", "rewards/chosen": 0.0040183291715734145, "logps/chosen": -360.94910386029414, "logits/chosen": -54226715.105882354, "rewards/rejected": 0.0015706966320673625, "logps/rejected": -191.28990885416667, "logits/rejected": -49573730.986666664, "rewards/margins": 0.0024476325395060517, "regularization/tv": 0.006386497523635626, "regularization/advantage_range": 0.6386498808860779, "kl": 0.03585250303149223, "stability/response_length_mean": 61.375, "stability/response_length_std": 7.424621105194092, "stability/response_length_var": 55.125, "stability/token_entropy_mean": 4.56640625, "stability/repetition_rate_mean": 0.3152253031730652}
{"current_steps": 160, "total_steps": 196, "loss": 0.5062, "lr": 9.959924945200523e-07, "epoch": 0.818937939859245, "percentage": 81.63, "elapsed_time": "0:42:55", "remaining_time": "0:09:39", "rewards/chosen": 0.006389315639223371, "logps/chosen": -416.8314267113095, "logits/chosen": -55488585.14285714, "rewards/rejected": -0.00017311762234098032, "logps/rejected": -186.89907997532896, "logits/rejected": -49255141.05263158, "rewards/margins": 0.006562433261564351, "regularization/tv": 0.007354104425758123, "regularization/advantage_range": 0.7354105114936829, "kl": 0.04733538627624512, "stability/response_length_mean": 26.25, "stability/response_length_std": 31.535694122314453, "stability/response_length_var": 994.5, "stability/token_entropy_mean": 3.8994140625, "stability/repetition_rate_mean": 0.205078125}
{"current_steps": 165, "total_steps": 196, "loss": 0.5055, "lr": 9.952855895999568e-07, "epoch": 0.8445297504798465, "percentage": 84.18, "elapsed_time": "0:44:15", "remaining_time": "0:08:18", "rewards/chosen": 0.008127493419866452, "logps/chosen": -370.83548850574715, "logits/chosen": -54670759.72413793, "rewards/rejected": -5.136907406865734e-05, "logps/rejected": -212.59262628424656, "logits/rejected": -50123060.60273973, "rewards/margins": 0.008178862493935108, "regularization/tv": 0.005651508923619986, "regularization/advantage_range": 0.5651509165763855, "kl": 0.039895690977573395, "stability/response_length_mean": 12.625, "stability/response_length_std": 21.460512161254883, "stability/response_length_var": 460.5535583496094, "stability/token_entropy_mean": 4.80078125, "stability/repetition_rate_mean": 0.20468750596046448}
{"current_steps": 170, "total_steps": 196, "loss": 0.5052, "lr": 9.9452158908238e-07, "epoch": 0.8701215611004478, "percentage": 86.73, "elapsed_time": "0:45:37", "remaining_time": "0:06:58", "rewards/chosen": 0.007143582112690401, "logps/chosen": -441.2704326923077, "logits/chosen": -59408686.95857988, "rewards/rejected": 0.0016791769210865956, "logps/rejected": -191.2410492549669, "logits/rejected": -52972838.993377484, "rewards/margins": 0.005464405191603806, "regularization/tv": 0.006181744392961264, "regularization/advantage_range": 0.6181744337081909, "kl": 0.06539209932088852, "stability/response_length_mean": 28.25, "stability/response_length_std": 30.844078063964844, "stability/response_length_var": 951.3571166992188, "stability/token_entropy_mean": 3.12353515625, "stability/repetition_rate_mean": 0.2809279263019562}
{"current_steps": 175, "total_steps": 196, "loss": 0.5054, "lr": 9.937005810398745e-07, "epoch": 0.8957133717210493, "percentage": 89.29, "elapsed_time": "0:46:57", "remaining_time": "0:05:38", "rewards/chosen": 0.009901675115148705, "logps/chosen": -356.33819653614455, "logits/chosen": -56948748.3373494, "rewards/rejected": 0.0023613097992810335, "logps/rejected": -194.78391335227272, "logits/rejected": -48448538.597402595, "rewards/margins": 0.007540365315867672, "regularization/tv": 0.007160508539527655, "regularization/advantage_range": 0.7160508632659912, "kl": 0.06210724264383316, "stability/response_length_mean": 32.25, "stability/response_length_std": 27.932315826416016, "stability/response_length_var": 780.2142944335938, "stability/token_entropy_mean": 4.6943359375, "stability/repetition_rate_mean": 0.2936941981315613}
{"current_steps": 180, "total_steps": 196, "loss": 0.5055, "lr": 9.928226601167138e-07, "epoch": 0.9213051823416507, "percentage": 91.84, "elapsed_time": "0:48:17", "remaining_time": "0:04:17", "rewards/chosen": 0.010499875238336669, "logps/chosen": -376.04083588957053, "logits/chosen": -52939769.71779141, "rewards/rejected": 0.004927185690327055, "logps/rejected": -211.24893013535032, "logits/rejected": -48362903.643312104, "rewards/margins": 0.005572689548009614, "regularization/tv": 0.006519151385873556, "regularization/advantage_range": 0.6519152522087097, "kl": 0.06741955131292343, "stability/response_length_mean": 40.375, "stability/response_length_std": 32.61874008178711, "stability/response_length_var": 1063.982177734375, "stability/token_entropy_mean": 3.35546875, "stability/repetition_rate_mean": 0.16796875}
{"current_steps": 185, "total_steps": 196, "loss": 0.5051, "lr": 9.918879275179817e-07, "epoch": 0.946896992962252, "percentage": 94.39, "elapsed_time": "0:49:37", "remaining_time": "0:02:57", "rewards/chosen": 0.007868100095678258, "logps/chosen": -311.92269483024694, "logits/chosen": -54950197.72839506, "rewards/rejected": 0.0036215963242929193, "logps/rejected": -197.00294204905063, "logits/rejected": -49892630.6835443, "rewards/margins": 0.004246503771385339, "regularization/tv": 0.0058816405944526196, "regularization/advantage_range": 0.5881640315055847, "kl": 0.0625443160533905, "stability/response_length_mean": 26.25, "stability/response_length_std": 31.45404815673828, "stability/response_length_var": 989.3571166992188, "stability/token_entropy_mean": 3.822265625, "stability/repetition_rate_mean": 0.20791903138160706}
{"current_steps": 190, "total_steps": 196, "loss": 0.5055, "lr": 9.90896490997906e-07, "epoch": 0.9724888035828535, "percentage": 96.94, "elapsed_time": "0:50:57", "remaining_time": "0:01:36", "rewards/chosen": 0.010125834070993755, "logps/chosen": -392.3040793413174, "logits/chosen": -53417732.598802395, "rewards/rejected": 0.0028987289254182305, "logps/rejected": -211.1696027369281, "logits/rejected": -50093725.28104575, "rewards/margins": 0.007227105145575524, "regularization/tv": 0.006623882334679365, "regularization/advantage_range": 0.6623881459236145, "kl": 0.06414353102445602, "stability/response_length_mean": 12.375, "stability/response_length_std": 22.057958602905273, "stability/response_length_var": 486.5535583496094, "stability/token_entropy_mean": 5.375, "stability/repetition_rate_mean": 0.1258285939693451}
{"current_steps": 195, "total_steps": 196, "loss": 0.5054, "lr": 9.898484648474362e-07, "epoch": 0.9980806142034548, "percentage": 99.49, "elapsed_time": "0:52:18", "remaining_time": "0:00:16", "rewards/chosen": 0.016048112992317447, "logps/chosen": -392.6244455645161, "logits/chosen": -60520414.96774194, "rewards/rejected": 0.0009160069805203062, "logps/rejected": -197.9101799242424, "logits/rejected": -54298518.4969697, "rewards/margins": 0.01513210601179714, "regularization/tv": 0.0068309674970805645, "regularization/advantage_range": 0.683096706867218, "kl": 0.06095733866095543, "stability/response_length_mean": 48.125, "stability/response_length_std": 29.39600372314453, "stability/response_length_var": 864.125, "stability/token_entropy_mean": 4.4453125, "stability/repetition_rate_mean": 0.412109375}
{"current_steps": 196, "total_steps": 196, "epoch": 1.0, "percentage": 100.0, "elapsed_time": "0:52:31", "remaining_time": "0:00:00"}