| { |
| "best_global_step": null, |
| "best_metric": null, |
| "best_model_checkpoint": null, |
| "epoch": 0.9998691270776077, |
| "eval_steps": 200, |
| "global_step": 955, |
| "is_hyper_param_search": false, |
| "is_local_process_zero": true, |
| "is_world_process_zero": true, |
| "log_history": [ |
| { |
| "epoch": 0.0010469833791388562, |
| "grad_norm": 46.5684700012207, |
| "kl": 0.04943779110908508, |
| "learning_rate": 0.0, |
| "logits/chosen": 442452480.0, |
| "logits/rejected": 465586784.0, |
| "logps/chosen": -192.3951231060606, |
| "logps/rejected": -244.5851814516129, |
| "loss": 4.0033, |
| "rewards/chosen": -0.002359318913835468, |
| "rewards/margins": -0.005345997827551815, |
| "rewards/rejected": 0.002986678913716347, |
| "step": 1 |
| }, |
| { |
| "epoch": 0.010469833791388562, |
| "grad_norm": 47.87677001953125, |
| "kl": 0.0966624915599823, |
| "learning_rate": 4.6875e-08, |
| "logits/chosen": 497818176.0, |
| "logits/rejected": 465912416.0, |
| "logps/chosen": -272.4681021973466, |
| "logps/rejected": -251.92019581056465, |
| "loss": 4.0037, |
| "rewards/chosen": -0.0018457298848166394, |
| "rewards/margins": -0.002352562184804766, |
| "rewards/rejected": 0.0005068322999881265, |
| "step": 10 |
| }, |
| { |
| "epoch": 0.020939667582777124, |
| "grad_norm": 52.892616271972656, |
| "kl": 0.07839385420084, |
| "learning_rate": 9.895833333333332e-08, |
| "logits/chosen": 495900416.0, |
| "logits/rejected": 479196576.0, |
| "logps/chosen": -270.1613023952096, |
| "logps/rejected": -259.60822610294116, |
| "loss": 3.9998, |
| "rewards/chosen": -0.0014621614517565972, |
| "rewards/margins": 0.001095909008419559, |
| "rewards/rejected": -0.0025580704601761563, |
| "step": 20 |
| }, |
| { |
| "epoch": 0.031409501374165684, |
| "grad_norm": 49.37065505981445, |
| "kl": 0.12432318925857544, |
| "learning_rate": 1.5104166666666664e-07, |
| "logits/chosen": 446004160.0, |
| "logits/rejected": 416264512.0, |
| "logps/chosen": -286.6748310810811, |
| "logps/rejected": -239.96737581433226, |
| "loss": 3.9988, |
| "rewards/chosen": 0.005302351516288322, |
| "rewards/margins": 0.001094049639710752, |
| "rewards/rejected": 0.00420830187657757, |
| "step": 30 |
| }, |
| { |
| "epoch": 0.04187933516555425, |
| "grad_norm": 56.20479965209961, |
| "kl": 0.1288590282201767, |
| "learning_rate": 2.03125e-07, |
| "logits/chosen": 469622016.0, |
| "logits/rejected": 471840608.0, |
| "logps/chosen": -299.05229591836735, |
| "logps/rejected": -275.7191874027994, |
| "loss": 4.0024, |
| "rewards/chosen": 0.011023993020529275, |
| "rewards/margins": -0.0016035611417646294, |
| "rewards/rejected": 0.012627554162293905, |
| "step": 40 |
| }, |
| { |
| "epoch": 0.05234916895694281, |
| "grad_norm": 53.468544006347656, |
| "kl": 0.15856146812438965, |
| "learning_rate": 2.552083333333333e-07, |
| "logits/chosen": 472065984.0, |
| "logits/rejected": 415794880.0, |
| "logps/chosen": -301.99, |
| "logps/rejected": -266.23648313492066, |
| "loss": 3.9892, |
| "rewards/chosen": 0.0272962394127479, |
| "rewards/margins": 0.012006511944408674, |
| "rewards/rejected": 0.015289727468339225, |
| "step": 50 |
| }, |
| { |
| "epoch": 0.06281900274833137, |
| "grad_norm": 55.099525451660156, |
| "kl": 0.1767362356185913, |
| "learning_rate": 3.0729166666666665e-07, |
| "logits/chosen": 458564768.0, |
| "logits/rejected": 465017408.0, |
| "logps/chosen": -291.2200561908517, |
| "logps/rejected": -270.51400445046437, |
| "loss": 3.9806, |
| "rewards/chosen": 0.03807779967972909, |
| "rewards/margins": 0.01917116797379836, |
| "rewards/rejected": 0.018906631705930727, |
| "step": 60 |
| }, |
| { |
| "epoch": 0.07328883653971993, |
| "grad_norm": 59.78743362426758, |
| "kl": 0.1566978394985199, |
| "learning_rate": 3.59375e-07, |
| "logits/chosen": 435970144.0, |
| "logits/rejected": 464205728.0, |
| "logps/chosen": -276.08103898713824, |
| "logps/rejected": -271.9755176671733, |
| "loss": 3.974, |
| "rewards/chosen": 0.05013132555308449, |
| "rewards/margins": 0.027480934417912214, |
| "rewards/rejected": 0.02265039113517228, |
| "step": 70 |
| }, |
| { |
| "epoch": 0.0837586703311085, |
| "grad_norm": 63.29967498779297, |
| "kl": 0.14287321269512177, |
| "learning_rate": 4.114583333333333e-07, |
| "logits/chosen": 465435904.0, |
| "logits/rejected": 445954368.0, |
| "logps/chosen": -301.7261437595712, |
| "logps/rejected": -253.4555173444976, |
| "loss": 3.9459, |
| "rewards/chosen": 0.07625812337738083, |
| "rewards/margins": 0.05526375141259004, |
| "rewards/rejected": 0.020994371964790794, |
| "step": 80 |
| }, |
| { |
| "epoch": 0.09422850412249706, |
| "grad_norm": 51.23120880126953, |
| "kl": 0.08589013665914536, |
| "learning_rate": 4.6354166666666664e-07, |
| "logits/chosen": 425856352.0, |
| "logits/rejected": 442634752.0, |
| "logps/chosen": -250.01865157480316, |
| "logps/rejected": -253.97698643410854, |
| "loss": 3.9365, |
| "rewards/chosen": 0.0922366164800689, |
| "rewards/margins": 0.06793148576547496, |
| "rewards/rejected": 0.02430513071459393, |
| "step": 90 |
| }, |
| { |
| "epoch": 0.10469833791388562, |
| "grad_norm": 53.67849349975586, |
| "kl": 0.06646991521120071, |
| "learning_rate": 4.999849525959245e-07, |
| "logits/chosen": 443815680.0, |
| "logits/rejected": 468530592.0, |
| "logps/chosen": -292.86275268151815, |
| "logps/rejected": -254.13021606083086, |
| "loss": 3.8786, |
| "rewards/chosen": 0.1379817798979605, |
| "rewards/margins": 0.13271600843962053, |
| "rewards/rejected": 0.005265771458339974, |
| "step": 100 |
| }, |
| { |
| "epoch": 0.11516817170527417, |
| "grad_norm": 55.30476379394531, |
| "kl": 0.008740690536797047, |
| "learning_rate": 4.997174935782199e-07, |
| "logits/chosen": 405325280.0, |
| "logits/rejected": 404096320.0, |
| "logps/chosen": -281.1175155520995, |
| "logps/rejected": -245.54272959183675, |
| "loss": 3.8465, |
| "rewards/chosen": 0.10175868104071491, |
| "rewards/margins": 0.15764128317543147, |
| "rewards/rejected": -0.05588260213471657, |
| "step": 110 |
| }, |
| { |
| "epoch": 0.12563800549666274, |
| "grad_norm": 53.23088073730469, |
| "kl": 0.0, |
| "learning_rate": 4.9911605954668e-07, |
| "logits/chosen": 413253440.0, |
| "logits/rejected": 466272960.0, |
| "logps/chosen": -265.9372119815668, |
| "logps/rejected": -285.0679401828299, |
| "loss": 3.7909, |
| "rewards/chosen": 0.09825083297518541, |
| "rewards/margins": 0.21899295158017598, |
| "rewards/rejected": -0.12074211860499057, |
| "step": 120 |
| }, |
| { |
| "epoch": 0.1361078392880513, |
| "grad_norm": 47.935428619384766, |
| "kl": 0.0031980276107788086, |
| "learning_rate": 4.981814548660135e-07, |
| "logits/chosen": 415410240.0, |
| "logits/rejected": 464793856.0, |
| "logps/chosen": -278.3420138888889, |
| "logps/rejected": -253.4058807237814, |
| "loss": 3.7401, |
| "rewards/chosen": 0.16747628080706495, |
| "rewards/margins": 0.27606504747137556, |
| "rewards/rejected": -0.10858876666431061, |
| "step": 130 |
| }, |
| { |
| "epoch": 0.14657767307943986, |
| "grad_norm": 45.16371154785156, |
| "kl": 0.0, |
| "learning_rate": 4.969149294871417e-07, |
| "logits/chosen": 419959424.0, |
| "logits/rejected": 448011200.0, |
| "logps/chosen": -259.45691892971246, |
| "logps/rejected": -274.75979548929666, |
| "loss": 3.6988, |
| "rewards/chosen": 0.15218203364850613, |
| "rewards/margins": 0.3255889087455651, |
| "rewards/rejected": -0.17340687509705896, |
| "step": 140 |
| }, |
| { |
| "epoch": 0.1570475068708284, |
| "grad_norm": 45.20751190185547, |
| "kl": 0.0, |
| "learning_rate": 4.953181772754997e-07, |
| "logits/chosen": 419500992.0, |
| "logits/rejected": 426865440.0, |
| "logps/chosen": -262.6239265267176, |
| "logps/rejected": -255.55855, |
| "loss": 3.5915, |
| "rewards/chosen": 0.18705672635376908, |
| "rewards/margins": 0.4586699343615816, |
| "rewards/rejected": -0.2716132080078125, |
| "step": 150 |
| }, |
| { |
| "epoch": 0.167517340662217, |
| "grad_norm": 43.46672058105469, |
| "kl": 0.0, |
| "learning_rate": 4.93393333745642e-07, |
| "logits/chosen": 402036128.0, |
| "logits/rejected": 407421024.0, |
| "logps/chosen": -256.07155539772725, |
| "logps/rejected": -253.9230515813253, |
| "loss": 3.5417, |
| "rewards/chosen": 0.35380408051726103, |
| "rewards/margins": 0.5338230570275502, |
| "rewards/rejected": -0.1800189765102892, |
| "step": 160 |
| }, |
| { |
| "epoch": 0.17798717445360554, |
| "grad_norm": 50.28922653198242, |
| "kl": 0.0, |
| "learning_rate": 4.9114297320518e-07, |
| "logits/chosen": 443569248.0, |
| "logits/rejected": 446337024.0, |
| "logps/chosen": -272.3035854231975, |
| "logps/rejected": -268.38152258566976, |
| "loss": 3.4368, |
| "rewards/chosen": 0.5461393254677703, |
| "rewards/margins": 0.6761910660725394, |
| "rewards/rejected": -0.13005174060476904, |
| "step": 170 |
| }, |
| { |
| "epoch": 0.18845700824499412, |
| "grad_norm": 50.34736633300781, |
| "kl": 0.0, |
| "learning_rate": 4.885701053118751e-07, |
| "logits/chosen": 420526400.0, |
| "logits/rejected": 440075456.0, |
| "logps/chosen": -267.0804433925811, |
| "logps/rejected": -268.7849772906793, |
| "loss": 3.3582, |
| "rewards/chosen": 0.5984241420739954, |
| "rewards/margins": 0.7952823357110659, |
| "rewards/rejected": -0.1968581936370705, |
| "step": 180 |
| }, |
| { |
| "epoch": 0.19892684203638267, |
| "grad_norm": 46.28329086303711, |
| "kl": 0.0, |
| "learning_rate": 4.856781710484872e-07, |
| "logits/chosen": 400002560.0, |
| "logits/rejected": 408255584.0, |
| "logps/chosen": -266.1100976874003, |
| "logps/rejected": -277.3754067764165, |
| "loss": 3.272, |
| "rewards/chosen": 0.658042360150643, |
| "rewards/margins": 0.9656070181593528, |
| "rewards/rejected": -0.3075646580087098, |
| "step": 190 |
| }, |
| { |
| "epoch": 0.20939667582777124, |
| "grad_norm": 37.88730239868164, |
| "kl": 0.0, |
| "learning_rate": 4.824710381207655e-07, |
| "logits/chosen": 394069760.0, |
| "logits/rejected": 439350080.0, |
| "logps/chosen": -280.22834606109325, |
| "logps/rejected": -273.49769661854106, |
| "loss": 3.2673, |
| "rewards/chosen": 0.34384584580203725, |
| "rewards/margins": 1.0138983249053806, |
| "rewards/rejected": -0.6700524791033434, |
| "step": 200 |
| }, |
| { |
| "epoch": 0.2198665096191598, |
| "grad_norm": 50.169498443603516, |
| "kl": 0.0, |
| "learning_rate": 4.789529957847353e-07, |
| "logits/chosen": 409849664.0, |
| "logits/rejected": 351628448.0, |
| "logps/chosen": -283.19119751908397, |
| "logps/rejected": -262.1562, |
| "loss": 3.1422, |
| "rewards/chosen": 0.7572274943344466, |
| "rewards/margins": 1.2206477580063215, |
| "rewards/rejected": -0.463420263671875, |
| "step": 210 |
| }, |
| { |
| "epoch": 0.23033634341054834, |
| "grad_norm": 54.83884048461914, |
| "kl": 0.0, |
| "learning_rate": 4.751287491101977e-07, |
| "logits/chosen": 400426208.0, |
| "logits/rejected": 369194944.0, |
| "logps/chosen": -270.1439732142857, |
| "logps/rejected": -257.53584414308176, |
| "loss": 3.2684, |
| "rewards/chosen": 0.4789316402458996, |
| "rewards/margins": 1.1337323431866424, |
| "rewards/rejected": -0.6548007029407429, |
| "step": 220 |
| }, |
| { |
| "epoch": 0.24080617720193692, |
| "grad_norm": 59.397918701171875, |
| "kl": 0.0, |
| "learning_rate": 4.710034126881159e-07, |
| "logits/chosen": 424935232.0, |
| "logits/rejected": 324887488.0, |
| "logps/chosen": -288.61760826055314, |
| "logps/rejected": -282.7670741989882, |
| "loss": 3.1502, |
| "rewards/chosen": 0.5312555082673762, |
| "rewards/margins": 1.3070566024126125, |
| "rewards/rejected": -0.7758010941452361, |
| "step": 230 |
| }, |
| { |
| "epoch": 0.25127601099332547, |
| "grad_norm": 54.53902053833008, |
| "kl": 0.0, |
| "learning_rate": 4.665825037903035e-07, |
| "logits/chosen": 411758592.0, |
| "logits/rejected": 355237504.0, |
| "logps/chosen": -266.5797621340524, |
| "logps/rejected": -262.10652733755944, |
| "loss": 3.1504, |
| "rewards/chosen": 0.7458241232003563, |
| "rewards/margins": 1.340257552513327, |
| "rewards/rejected": -0.5944334293129705, |
| "step": 240 |
| }, |
| { |
| "epoch": 0.261745844784714, |
| "grad_norm": 84.02921295166016, |
| "kl": 0.0, |
| "learning_rate": 4.618719349905619e-07, |
| "logits/chosen": 432440224.0, |
| "logits/rejected": 374901696.0, |
| "logps/chosen": -284.9515267175573, |
| "logps/rejected": -268.12985, |
| "loss": 3.0524, |
| "rewards/chosen": 0.4285235455927958, |
| "rewards/margins": 1.5875603619990457, |
| "rewards/rejected": -1.15903681640625, |
| "step": 250 |
| }, |
| { |
| "epoch": 0.2722156785761026, |
| "grad_norm": 44.28253173828125, |
| "kl": 0.0, |
| "learning_rate": 4.568780062571374e-07, |
| "logits/chosen": 350974560.0, |
| "logits/rejected": 379531712.0, |
| "logps/chosen": -270.08513621794873, |
| "logps/rejected": -280.8126429115854, |
| "loss": 3.0586, |
| "rewards/chosen": 0.14809322357177734, |
| "rewards/margins": 1.5623344560948813, |
| "rewards/rejected": -1.414241232523104, |
| "step": 260 |
| }, |
| { |
| "epoch": 0.2826855123674912, |
| "grad_norm": 56.65611267089844, |
| "kl": 0.0, |
| "learning_rate": 4.516073965270717e-07, |
| "logits/chosen": 353575936.0, |
| "logits/rejected": 341260640.0, |
| "logps/chosen": -261.7265013693271, |
| "logps/rejected": -284.40742004680186, |
| "loss": 3.0462, |
| "rewards/chosen": 0.5539226979716843, |
| "rewards/margins": 1.646275173808999, |
| "rewards/rejected": -1.0923524758373147, |
| "step": 270 |
| }, |
| { |
| "epoch": 0.2931553461588797, |
| "grad_norm": 117.23200988769531, |
| "kl": 0.0, |
| "learning_rate": 4.460671547737158e-07, |
| "logits/chosen": 336798048.0, |
| "logits/rejected": 346110912.0, |
| "logps/chosen": -299.6681069131833, |
| "logps/rejected": -268.43320193768994, |
| "loss": 3.0478, |
| "rewards/chosen": -0.01779557194357133, |
| "rewards/margins": 1.6294317994113867, |
| "rewards/rejected": -1.6472273713549581, |
| "step": 280 |
| }, |
| { |
| "epoch": 0.3036251799502683, |
| "grad_norm": 45.30254364013672, |
| "kl": 0.0, |
| "learning_rate": 4.40264690579353e-07, |
| "logits/chosen": 350954464.0, |
| "logits/rejected": 331866784.0, |
| "logps/chosen": -285.780487804878, |
| "logps/rejected": -266.4627904647436, |
| "loss": 2.9457, |
| "rewards/chosen": 0.3624885140395746, |
| "rewards/margins": 1.9047759982330341, |
| "rewards/rejected": -1.5422874841934595, |
| "step": 290 |
| }, |
| { |
| "epoch": 0.3140950137416568, |
| "grad_norm": 45.462738037109375, |
| "kl": 0.0, |
| "learning_rate": 4.3420776422553916e-07, |
| "logits/chosen": 356290880.0, |
| "logits/rejected": 351119168.0, |
| "logps/chosen": -276.1206756498471, |
| "logps/rejected": -268.070062899361, |
| "loss": 3.0097, |
| "rewards/chosen": 0.5417160360820432, |
| "rewards/margins": 1.7571349400544374, |
| "rewards/rejected": -1.2154189039723942, |
| "step": 300 |
| }, |
| { |
| "epoch": 0.32456484753304543, |
| "grad_norm": 47.83991622924805, |
| "kl": 0.0, |
| "learning_rate": 4.279044763144141e-07, |
| "logits/chosen": 334029376.0, |
| "logits/rejected": 379953376.0, |
| "logps/chosen": -260.419678514377, |
| "logps/rejected": -303.30186831039754, |
| "loss": 2.9571, |
| "rewards/chosen": 0.12563846819697858, |
| "rewards/margins": 1.8882139502067263, |
| "rewards/rejected": -1.7625754820097477, |
| "step": 310 |
| }, |
| { |
| "epoch": 0.335034681324434, |
| "grad_norm": 42.41203308105469, |
| "kl": 0.0, |
| "learning_rate": 4.213632569348639e-07, |
| "logits/chosen": 396048096.0, |
| "logits/rejected": 356900800.0, |
| "logps/chosen": -283.4178257686676, |
| "logps/rejected": -284.24552449748745, |
| "loss": 3.0977, |
| "rewards/chosen": -0.031586469703328034, |
| "rewards/margins": 1.8606463005934684, |
| "rewards/rejected": -1.8922327702967965, |
| "step": 320 |
| }, |
| { |
| "epoch": 0.34550451511582253, |
| "grad_norm": 38.14891815185547, |
| "kl": 0.0, |
| "learning_rate": 4.145928543880249e-07, |
| "logits/chosen": 355648192.0, |
| "logits/rejected": 368038208.0, |
| "logps/chosen": -278.34861275671403, |
| "logps/rejected": -277.72862731839257, |
| "loss": 2.8694, |
| "rewards/chosen": 0.5873398622630331, |
| "rewards/margins": 2.0932907428295904, |
| "rewards/rejected": -1.5059508805665571, |
| "step": 330 |
| }, |
| { |
| "epoch": 0.3559743489072111, |
| "grad_norm": 52.64259719848633, |
| "kl": 0.0, |
| "learning_rate": 4.076023234872057e-07, |
| "logits/chosen": 334901248.0, |
| "logits/rejected": 373347008.0, |
| "logps/chosen": -278.5432905896607, |
| "logps/rejected": -275.132209720121, |
| "loss": 2.9211, |
| "rewards/chosen": 0.4131505716598092, |
| "rewards/margins": 1.998353064331897, |
| "rewards/rejected": -1.5852024926720878, |
| "step": 340 |
| }, |
| { |
| "epoch": 0.3664441826985997, |
| "grad_norm": 47.56734085083008, |
| "kl": 0.0, |
| "learning_rate": 4.004010134478771e-07, |
| "logits/chosen": 377446912.0, |
| "logits/rejected": 370444192.0, |
| "logps/chosen": -268.446590470679, |
| "logps/rejected": -275.46061609968353, |
| "loss": 2.8817, |
| "rewards/chosen": 0.38401062105908806, |
| "rewards/margins": 2.1151071060521653, |
| "rewards/rejected": -1.7310964849930774, |
| "step": 350 |
| }, |
| { |
| "epoch": 0.37691401648998824, |
| "grad_norm": 44.92960739135742, |
| "kl": 0.0, |
| "learning_rate": 3.9299855538392534e-07, |
| "logits/chosen": 367343424.0, |
| "logits/rejected": 372150560.0, |
| "logps/chosen": -282.18478154388714, |
| "logps/rejected": -279.0036507009346, |
| "loss": 2.8888, |
| "rewards/chosen": 0.24498392421997453, |
| "rewards/margins": 2.0810528114254847, |
| "rewards/rejected": -1.8360688872055102, |
| "step": 360 |
| }, |
| { |
| "epoch": 0.3873838502813768, |
| "grad_norm": 70.89231872558594, |
| "kl": 0.0, |
| "learning_rate": 3.8540484942689075e-07, |
| "logits/chosen": 341587808.0, |
| "logits/rejected": 377246784.0, |
| "logps/chosen": -268.64557623407643, |
| "logps/rejected": -290.71287864263803, |
| "loss": 2.9263, |
| "rewards/chosen": 0.4795560897535579, |
| "rewards/margins": 2.0588525528598653, |
| "rewards/rejected": -1.5792964631063076, |
| "step": 370 |
| }, |
| { |
| "epoch": 0.39785368407276533, |
| "grad_norm": 48.053218841552734, |
| "kl": 0.0, |
| "learning_rate": 3.77630051485419e-07, |
| "logits/chosen": 373726304.0, |
| "logits/rejected": 332450912.0, |
| "logps/chosen": -287.8293694690266, |
| "logps/rejected": -280.6202242524917, |
| "loss": 2.9653, |
| "rewards/chosen": 0.13132064954369468, |
| "rewards/margins": 2.172828205736593, |
| "rewards/rejected": -2.0415075561928986, |
| "step": 380 |
| }, |
| { |
| "epoch": 0.4083235178641539, |
| "grad_norm": 73.88297271728516, |
| "kl": 0.0, |
| "learning_rate": 3.696845596626342e-07, |
| "logits/chosen": 357619776.0, |
| "logits/rejected": 357397952.0, |
| "logps/chosen": -258.85225694444443, |
| "logps/rejected": -277.76314903846156, |
| "loss": 2.9633, |
| "rewards/chosen": 0.49055417984250993, |
| "rewards/margins": 2.095406193063664, |
| "rewards/rejected": -1.604852013221154, |
| "step": 390 |
| }, |
| { |
| "epoch": 0.4187933516555425, |
| "grad_norm": 59.15121078491211, |
| "kl": 0.0, |
| "learning_rate": 3.61579000349597e-07, |
| "logits/chosen": 388324384.0, |
| "logits/rejected": 387673536.0, |
| "logps/chosen": -280.22964449541286, |
| "logps/rejected": -283.0034944089457, |
| "loss": 2.7056, |
| "rewards/chosen": 0.5564632590757598, |
| "rewards/margins": 2.5239340558009995, |
| "rewards/rejected": -1.9674707967252396, |
| "step": 400 |
| }, |
| { |
| "epoch": 0.42926318544693104, |
| "grad_norm": 72.18789672851562, |
| "kl": 0.0, |
| "learning_rate": 3.5332421401344837e-07, |
| "logits/chosen": 336218400.0, |
| "logits/rejected": 390127072.0, |
| "logps/chosen": -281.6167403198653, |
| "logps/rejected": -275.0251457725947, |
| "loss": 2.935, |
| "rewards/chosen": 0.14200486719407618, |
| "rewards/margins": 2.1233040867982127, |
| "rewards/rejected": -1.9812992196041364, |
| "step": 410 |
| }, |
| { |
| "epoch": 0.4397330192383196, |
| "grad_norm": 83.5944595336914, |
| "kl": 0.0, |
| "learning_rate": 3.4493124069924635e-07, |
| "logits/chosen": 369444512.0, |
| "logits/rejected": 383078592.0, |
| "logps/chosen": -284.5074, |
| "logps/rejected": -268.33742843511453, |
| "loss": 2.8723, |
| "rewards/chosen": 0.2378343994140625, |
| "rewards/margins": 2.372110997696505, |
| "rewards/rejected": -2.1342765982824425, |
| "step": 420 |
| }, |
| { |
| "epoch": 0.45020285302970814, |
| "grad_norm": 52.63675308227539, |
| "kl": 0.0, |
| "learning_rate": 3.3641130526488335e-07, |
| "logits/chosen": 376764608.0, |
| "logits/rejected": 398398816.0, |
| "logps/chosen": -252.68347723704866, |
| "logps/rejected": -299.20040338258167, |
| "loss": 3.0279, |
| "rewards/chosen": 0.16862698589426756, |
| "rewards/margins": 2.0355602834861224, |
| "rewards/rejected": -1.8669332975918547, |
| "step": 430 |
| }, |
| { |
| "epoch": 0.4606726868210967, |
| "grad_norm": 39.36049270629883, |
| "kl": 0.0, |
| "learning_rate": 3.2777580236883473e-07, |
| "logits/chosen": 365922400.0, |
| "logits/rejected": 392508864.0, |
| "logps/chosen": -252.91024361022363, |
| "logps/rejected": -279.8361525229358, |
| "loss": 2.8382, |
| "rewards/chosen": 0.5580993262342752, |
| "rewards/margins": 2.297070068182861, |
| "rewards/rejected": -1.7389707419485856, |
| "step": 440 |
| }, |
| { |
| "epoch": 0.4711425206124853, |
| "grad_norm": 57.87444305419922, |
| "kl": 0.0, |
| "learning_rate": 3.1903628123081196e-07, |
| "logits/chosen": 394673920.0, |
| "logits/rejected": 370509824.0, |
| "logps/chosen": -272.00308132763973, |
| "logps/rejected": -274.67553557389937, |
| "loss": 2.7844, |
| "rewards/chosen": 0.5175159051551582, |
| "rewards/margins": 2.5180846069412235, |
| "rewards/rejected": -2.000568701786065, |
| "step": 450 |
| }, |
| { |
| "epoch": 0.48161235440387384, |
| "grad_norm": 76.35614013671875, |
| "kl": 0.0, |
| "learning_rate": 3.1020443018570556e-07, |
| "logits/chosen": 327263424.0, |
| "logits/rejected": 376021536.0, |
| "logps/chosen": -267.0524807224026, |
| "logps/rejected": -267.99872929216866, |
| "loss": 2.9054, |
| "rewards/chosen": 0.571973280473189, |
| "rewards/margins": 2.1715556308708734, |
| "rewards/rejected": -1.5995823503976845, |
| "step": 460 |
| }, |
| { |
| "epoch": 0.4920821881952624, |
| "grad_norm": 65.82987976074219, |
| "kl": 0.0, |
| "learning_rate": 3.0129206105147343e-07, |
| "logits/chosen": 370748928.0, |
| "logits/rejected": 389477536.0, |
| "logps/chosen": -286.8205546492659, |
| "logps/rejected": -267.65315779610194, |
| "loss": 2.9417, |
| "rewards/chosen": 0.08930689825708286, |
| "rewards/margins": 2.134248538764954, |
| "rewards/rejected": -2.044941640507871, |
| "step": 470 |
| }, |
| { |
| "epoch": 0.5025520219866509, |
| "grad_norm": 69.5935287475586, |
| "kl": 0.0, |
| "learning_rate": 2.923110933318805e-07, |
| "logits/chosen": 372539360.0, |
| "logits/rejected": 354864128.0, |
| "logps/chosen": -271.98661380597014, |
| "logps/rejected": -262.8959016393443, |
| "loss": 2.9602, |
| "rewards/chosen": 0.0067851991795781835, |
| "rewards/margins": 2.1713433846944343, |
| "rewards/rejected": -2.1645581855148563, |
| "step": 480 |
| }, |
| { |
| "epoch": 0.5130218557780395, |
| "grad_norm": 75.75752258300781, |
| "kl": 0.0, |
| "learning_rate": 2.832735382752194e-07, |
| "logits/chosen": 403122240.0, |
| "logits/rejected": 371043840.0, |
| "logps/chosen": -270.78357101837673, |
| "logps/rejected": -284.7364433811802, |
| "loss": 2.9662, |
| "rewards/chosen": -0.02843327544184593, |
| "rewards/margins": 2.2282741669923647, |
| "rewards/rejected": -2.2567074424342106, |
| "step": 490 |
| }, |
| { |
| "epoch": 0.523491689569428, |
| "grad_norm": 67.58584594726562, |
| "kl": 0.0, |
| "learning_rate": 2.741914828103307e-07, |
| "logits/chosen": 372796864.0, |
| "logits/rejected": 383237824.0, |
| "logps/chosen": -266.3919005102041, |
| "logps/rejected": -270.33850116640747, |
| "loss": 2.8509, |
| "rewards/chosen": 0.3570691981532697, |
| "rewards/margins": 2.358759767566664, |
| "rewards/rejected": -2.001690569413394, |
| "step": 500 |
| }, |
| { |
| "epoch": 0.5339615233608166, |
| "grad_norm": 93.17213439941406, |
| "kl": 0.0, |
| "learning_rate": 2.650770733814065e-07, |
| "logits/chosen": 373166944.0, |
| "logits/rejected": 374013248.0, |
| "logps/chosen": -274.2689144736842, |
| "logps/rejected": -269.6120549387443, |
| "loss": 2.8906, |
| "rewards/chosen": 0.45857961554276316, |
| "rewards/margins": 2.376228702090428, |
| "rewards/rejected": -1.9176490865476645, |
| "step": 510 |
| }, |
| { |
| "epoch": 0.5444313571522053, |
| "grad_norm": 41.66130065917969, |
| "kl": 0.0, |
| "learning_rate": 2.55942499703198e-07, |
| "logits/chosen": 398498176.0, |
| "logits/rejected": 392743488.0, |
| "logps/chosen": -277.3498, |
| "logps/rejected": -268.6053673664122, |
| "loss": 2.9243, |
| "rewards/chosen": 0.3561779296875, |
| "rewards/margins": 2.193211826097328, |
| "rewards/rejected": -1.8370338964098283, |
| "step": 520 |
| }, |
| { |
| "epoch": 0.5549011909435938, |
| "grad_norm": 55.216609954833984, |
| "kl": 0.0, |
| "learning_rate": 2.467999784583527e-07, |
| "logits/chosen": 361071104.0, |
| "logits/rejected": 369749184.0, |
| "logps/chosen": -261.3549321086262, |
| "logps/rejected": -272.8801605504587, |
| "loss": 2.8446, |
| "rewards/chosen": 0.34026234294659796, |
| "rewards/margins": 2.427280370529262, |
| "rewards/rejected": -2.087018027582664, |
| "step": 530 |
| }, |
| { |
| "epoch": 0.5653710247349824, |
| "grad_norm": 63.85381317138672, |
| "kl": 0.0, |
| "learning_rate": 2.3766173695868388e-07, |
| "logits/chosen": 359954880.0, |
| "logits/rejected": 357861440.0, |
| "logps/chosen": -279.6579010336907, |
| "logps/rejected": -281.34379984051037, |
| "loss": 2.9617, |
| "rewards/chosen": 0.2960061603446832, |
| "rewards/margins": 2.205951756312586, |
| "rewards/rejected": -1.9099455959679028, |
| "step": 540 |
| }, |
| { |
| "epoch": 0.5758408585263709, |
| "grad_norm": 49.361106872558594, |
| "kl": 0.0, |
| "learning_rate": 2.285399967922253e-07, |
| "logits/chosen": 359154336.0, |
| "logits/rejected": 393155392.0, |
| "logps/chosen": -259.8390325479233, |
| "logps/rejected": -275.05624044342505, |
| "loss": 2.7826, |
| "rewards/chosen": 0.05572237983679238, |
| "rewards/margins": 2.533227845003077, |
| "rewards/rejected": -2.4775054651662844, |
| "step": 550 |
| }, |
| { |
| "epoch": 0.5863106923177595, |
| "grad_norm": 133.47369384765625, |
| "kl": 0.0, |
| "learning_rate": 2.194469574779397e-07, |
| "logits/chosen": 403979776.0, |
| "logits/rejected": 350524512.0, |
| "logps/chosen": -281.06496585735965, |
| "logps/rejected": -278.4915710547504, |
| "loss": 2.9209, |
| "rewards/chosen": -0.0682100171565286, |
| "rewards/margins": 2.379913441730347, |
| "rewards/rejected": -2.448123458886876, |
| "step": 560 |
| }, |
| { |
| "epoch": 0.596780526109148, |
| "grad_norm": 52.14252853393555, |
| "kl": 0.0, |
| "learning_rate": 2.1039478014994441e-07, |
| "logits/chosen": 348112192.0, |
| "logits/rejected": 400084896.0, |
| "logps/chosen": -262.22181181959564, |
| "logps/rejected": -282.3291306907378, |
| "loss": 2.8607, |
| "rewards/chosen": 0.08417754818455057, |
| "rewards/margins": 2.4870239519203237, |
| "rewards/rejected": -2.402846403735773, |
| "step": 570 |
| }, |
| { |
| "epoch": 0.6072503599005366, |
| "grad_norm": 102.21758270263672, |
| "kl": 0.0, |
| "learning_rate": 2.0139557129307149e-07, |
| "logits/chosen": 375497920.0, |
| "logits/rejected": 391752224.0, |
| "logps/chosen": -287.6094496417197, |
| "logps/rejected": -300.8546060199387, |
| "loss": 2.8128, |
| "rewards/chosen": 0.2170035246830837, |
| "rewards/margins": 2.57385417802137, |
| "rewards/rejected": -2.3568506533382863, |
| "step": 580 |
| }, |
| { |
| "epoch": 0.6177201936919251, |
| "grad_norm": 50.08977127075195, |
| "kl": 0.0, |
| "learning_rate": 1.9246136655151808e-07, |
| "logits/chosen": 391623968.0, |
| "logits/rejected": 371523136.0, |
| "logps/chosen": -284.78519864341087, |
| "logps/rejected": -299.40981791338584, |
| "loss": 2.7844, |
| "rewards/chosen": -0.06276658523914426, |
| "rewards/margins": 2.5907717236929426, |
| "rewards/rejected": -2.6535383089320868, |
| "step": 590 |
| }, |
| { |
| "epoch": 0.6281900274833137, |
| "grad_norm": 62.64336013793945, |
| "kl": 0.0, |
| "learning_rate": 1.8360411463223873e-07, |
| "logits/chosen": 372225920.0, |
| "logits/rejected": 384911008.0, |
| "logps/chosen": -274.0559916534181, |
| "logps/rejected": -288.66136232718895, |
| "loss": 2.8593, |
| "rewards/chosen": -0.07598057611947598, |
| "rewards/margins": 2.551059179424879, |
| "rewards/rejected": -2.627039755544355, |
| "step": 600 |
| }, |
| { |
| "epoch": 0.6386598612747023, |
| "grad_norm": 50.6363639831543, |
| "kl": 0.0, |
| "learning_rate": 1.7483566132460865e-07, |
| "logits/chosen": 373666784.0, |
| "logits/rejected": 393250304.0, |
| "logps/chosen": -285.63343057753167, |
| "logps/rejected": -270.8369984567901, |
| "loss": 2.8871, |
| "rewards/chosen": 0.03572451313839683, |
| "rewards/margins": 2.3185132018922547, |
| "rewards/rejected": -2.282788688753858, |
| "step": 610 |
| }, |
| { |
| "epoch": 0.6491296950660909, |
| "grad_norm": 70.63817596435547, |
| "kl": 0.0, |
| "learning_rate": 1.66167733657731e-07, |
| "logits/chosen": 388653408.0, |
| "logits/rejected": 387147584.0, |
| "logps/chosen": -286.9971341706539, |
| "logps/rejected": -280.6799866003063, |
| "loss": 2.9484, |
| "rewards/chosen": 0.25652475068063446, |
| "rewards/margins": 2.2528504903556916, |
| "rewards/rejected": -1.9963257396750573, |
| "step": 620 |
| }, |
| { |
| "epoch": 0.6595995288574794, |
| "grad_norm": 42.911075592041016, |
| "kl": 0.0, |
| "learning_rate": 1.5761192421657456e-07, |
| "logits/chosen": 380535040.0, |
| "logits/rejected": 384688384.0, |
| "logps/chosen": -277.87602336261983, |
| "logps/rejected": -287.9544629204893, |
| "loss": 2.7116, |
| "rewards/chosen": 0.6408637781112719, |
| "rewards/margins": 2.699830458515515, |
| "rewards/rejected": -2.058966680404243, |
| "step": 630 |
| }, |
| { |
| "epoch": 0.670069362648868, |
| "grad_norm": 33.01219177246094, |
| "kl": 0.0, |
| "learning_rate": 1.491796756379185e-07, |
| "logits/chosen": 426926528.0, |
| "logits/rejected": 395172800.0, |
| "logps/chosen": -296.3998833955224, |
| "logps/rejected": -276.0863729508197, |
| "loss": 2.8864, |
| "rewards/chosen": 0.3844701510756763, |
| "rewards/margins": 2.4492122425203484, |
| "rewards/rejected": -2.064742091444672, |
| "step": 640 |
| }, |
| { |
| "epoch": 0.6805391964402565, |
| "grad_norm": 37.57501220703125, |
| "kl": 0.0, |
| "learning_rate": 1.4088226530684071e-07, |
| "logits/chosen": 419599680.0, |
| "logits/rejected": 368290528.0, |
| "logps/chosen": -283.03209005376345, |
| "logps/rejected": -280.43193561208267, |
| "loss": 2.7923, |
| "rewards/chosen": 0.5528853690386185, |
| "rewards/margins": 2.5762015750079152, |
| "rewards/rejected": -2.0233162059692966, |
| "step": 650 |
| }, |
| { |
| "epoch": 0.6910090302316451, |
| "grad_norm": 73.64015197753906, |
| "kl": 0.0, |
| "learning_rate": 1.327307902742142e-07, |
| "logits/chosen": 411392192.0, |
| "logits/rejected": 383965600.0, |
| "logps/chosen": -270.74495192307694, |
| "logps/rejected": -289.49122023809525, |
| "loss": 2.7692, |
| "rewards/chosen": 0.3836714054987981, |
| "rewards/margins": 2.714577089712588, |
| "rewards/rejected": -2.3309056842137896, |
| "step": 660 |
| }, |
| { |
| "epoch": 0.7014788640230336, |
| "grad_norm": 80.35784912109375, |
| "kl": 0.0, |
| "learning_rate": 1.2473615241538523e-07, |
| "logits/chosen": 383171776.0, |
| "logits/rejected": 369880512.0, |
| "logps/chosen": -261.0426136363636, |
| "logps/rejected": -292.5834616174056, |
| "loss": 2.893, |
| "rewards/chosen": 0.42839020730486216, |
| "rewards/margins": 2.40669276047861, |
| "rewards/rejected": -1.978302553173748, |
| "step": 670 |
| }, |
| { |
| "epoch": 0.7119486978144222, |
| "grad_norm": 42.02663040161133, |
| "kl": 0.0, |
| "learning_rate": 1.169090438498816e-07, |
| "logits/chosen": 406482432.0, |
| "logits/rejected": 408727328.0, |
| "logps/chosen": -278.5440385367762, |
| "logps/rejected": -283.1681942277691, |
| "loss": 2.7501, |
| "rewards/chosen": 0.6435129467123925, |
| "rewards/margins": 2.670884500558921, |
| "rewards/rejected": -2.0273715538465287, |
| "step": 680 |
| }, |
| { |
| "epoch": 0.7224185316058107, |
| "grad_norm": 44.5306396484375, |
| "kl": 0.0, |
| "learning_rate": 1.0925993264165045e-07, |
| "logits/chosen": 379088128.0, |
| "logits/rejected": 389904960.0, |
| "logps/chosen": -274.40904552715654, |
| "logps/rejected": -286.7942708333333, |
| "loss": 2.8273, |
| "rewards/chosen": 0.4699281930161741, |
| "rewards/margins": 2.5310232645113, |
| "rewards/rejected": -2.061095071495126, |
| "step": 690 |
| }, |
| { |
| "epoch": 0.7328883653971994, |
| "grad_norm": 51.31837463378906, |
| "kl": 0.0, |
| "learning_rate": 1.0179904879894998e-07, |
| "logits/chosen": 404275392.0, |
| "logits/rejected": 392524608.0, |
| "logps/chosen": -271.8105407523511, |
| "logps/rejected": -288.45069119937693, |
| "loss": 2.8177, |
| "rewards/chosen": 0.28736071452078027, |
| "rewards/margins": 2.677813256929756, |
| "rewards/rejected": -2.3904525424089758, |
| "step": 700 |
| }, |
| { |
| "epoch": 0.7433581991885879, |
| "grad_norm": 65.66101837158203, |
| "kl": 0.0, |
| "learning_rate": 9.453637059262117e-08, |
| "logits/chosen": 381676384.0, |
| "logits/rejected": 391855936.0, |
| "logps/chosen": -266.3333492366412, |
| "logps/rejected": -267.5372, |
| "loss": 2.9462, |
| "rewards/chosen": 0.19959532759571805, |
| "rewards/margins": 2.291592788533218, |
| "rewards/rejected": -2.0919974609375, |
| "step": 710 |
| }, |
| { |
| "epoch": 0.7538280329799765, |
| "grad_norm": 87.61113739013672, |
| "kl": 0.0, |
| "learning_rate": 8.748161121103406e-08, |
| "logits/chosen": 391284128.0, |
| "logits/rejected": 399996864.0, |
| "logps/chosen": -276.3944143700787, |
| "logps/rejected": -298.0451792635659, |
| "loss": 2.737, |
| "rewards/chosen": 0.5421728900098425, |
| "rewards/margins": 2.7657016115456954, |
| "rewards/rejected": -2.223528721535853, |
| "step": 720 |
| }, |
| { |
| "epoch": 0.764297866771365, |
| "grad_norm": 83.69277954101562, |
| "kl": 0.0, |
| "learning_rate": 8.064420576955965e-08, |
| "logits/chosen": 411759936.0, |
| "logits/rejected": 421010976.0, |
| "logps/chosen": -281.4224365234375, |
| "logps/rejected": -293.0370361328125, |
| "loss": 2.8698, |
| "rewards/chosen": 0.0782856285572052, |
| "rewards/margins": 2.5774748146533963, |
| "rewards/rejected": -2.4991891860961912, |
| "step": 730 |
| }, |
| { |
| "epoch": 0.7747677005627536, |
| "grad_norm": 43.22333526611328, |
| "kl": 0.0, |
| "learning_rate": 7.403329869193922e-08, |
| "logits/chosen": 400044256.0, |
| "logits/rejected": 360019264.0, |
| "logps/chosen": -269.0556355606759, |
| "logps/rejected": -272.2297794117647, |
| "loss": 2.5935, |
| "rewards/chosen": 0.15169298960133448, |
| "rewards/margins": 3.1453841474411597, |
| "rewards/rejected": -2.9936911578398253, |
| "step": 740 |
| }, |
| { |
| "epoch": 0.7852375343541421, |
| "grad_norm": 55.45348358154297, |
| "kl": 0.0, |
| "learning_rate": 6.765773148042858e-08, |
| "logits/chosen": 402389952.0, |
| "logits/rejected": 381027904.0, |
| "logps/chosen": -278.2174539170507, |
| "logps/rejected": -275.59782392686805, |
| "loss": 2.9414, |
| "rewards/chosen": 0.10164003621231758, |
| "rewards/margins": 2.3010800265243208, |
| "rewards/rejected": -2.199439990312003, |
| "step": 750 |
| }, |
| { |
| "epoch": 0.7957073681455307, |
| "grad_norm": 88.17606353759766, |
| "kl": 0.0, |
| "learning_rate": 6.152603089107139e-08, |
| "logits/chosen": 405889216.0, |
| "logits/rejected": 379471808.0, |
| "logps/chosen": -268.80648306697105, |
| "logps/rejected": -274.56631219903693, |
| "loss": 2.9234, |
| "rewards/chosen": 0.23258001365255185, |
| "rewards/margins": 2.3599755700580496, |
| "rewards/rejected": -2.1273955564054976, |
| "step": 760 |
| }, |
| { |
| "epoch": 0.8061772019369192, |
| "grad_norm": 76.30126190185547, |
| "kl": 0.0, |
| "learning_rate": 5.5646397529920175e-08, |
| "logits/chosen": 395753920.0, |
| "logits/rejected": 400388992.0, |
| "logps/chosen": -295.6583118044515, |
| "logps/rejected": -280.01015264976957, |
| "loss": 2.6817, |
| "rewards/chosen": 0.3095470483049111, |
| "rewards/margins": 2.8186074406700223, |
| "rewards/rejected": -2.5090603923651114, |
| "step": 770 |
| }, |
| { |
| "epoch": 0.8166470357283078, |
| "grad_norm": 82.81681823730469, |
| "kl": 0.0, |
| "learning_rate": 5.002669488545111e-08, |
| "logits/chosen": 367751136.0, |
| "logits/rejected": 401200928.0, |
| "logps/chosen": -271.7482040229885, |
| "logps/rejected": -296.7671153129657, |
| "loss": 2.8095, |
| "rewards/chosen": 0.1301105543114673, |
| "rewards/margins": 2.4850469977028604, |
| "rewards/rejected": -2.3549364433913933, |
| "step": 780 |
| }, |
| { |
| "epoch": 0.8271168695196964, |
| "grad_norm": 62.69572448730469, |
| "kl": 0.0, |
| "learning_rate": 4.467443881184646e-08, |
| "logits/chosen": 372966144.0, |
| "logits/rejected": 386677760.0, |
| "logps/chosen": -278.3856132075472, |
| "logps/rejected": -270.80277076863354, |
| "loss": 2.9023, |
| "rewards/chosen": 0.03588581685000246, |
| "rewards/margins": 2.3716193608620366, |
| "rewards/rejected": -2.3357335440120344, |
| "step": 790 |
| }, |
| { |
| "epoch": 0.837586703311085, |
| "grad_norm": 43.35691833496094, |
| "kl": 0.0, |
| "learning_rate": 3.959678747720488e-08, |
| "logits/chosen": 410616064.0, |
| "logits/rejected": 374841760.0, |
| "logps/chosen": -270.33201388888887, |
| "logps/rejected": -281.9245867768595, |
| "loss": 2.8421, |
| "rewards/chosen": 0.23464515968605323, |
| "rewards/margins": 2.7391692267056813, |
| "rewards/rejected": -2.504524067019628, |
| "step": 800 |
| }, |
| { |
| "epoch": 0.8480565371024735, |
| "grad_norm": 44.70280838012695, |
| "kl": 0.0, |
| "learning_rate": 3.480053179012654e-08, |
| "logits/chosen": 351436864.0, |
| "logits/rejected": 401837312.0, |
| "logps/chosen": -260.5194256756757, |
| "logps/rejected": -283.58179723502303, |
| "loss": 3.0422, |
| "rewards/chosen": -0.020509217994580775, |
| "rewards/margins": 2.058081561815327, |
| "rewards/rejected": -2.078590779809908, |
| "step": 810 |
| }, |
| { |
| "epoch": 0.8585263708938621, |
| "grad_norm": 96.47203826904297, |
| "kl": 0.0, |
| "learning_rate": 3.029208631747446e-08, |
| "logits/chosen": 404733696.0, |
| "logits/rejected": 377852096.0, |
| "logps/chosen": -264.6643500766871, |
| "logps/rejected": -281.85546875, |
| "loss": 2.7119, |
| "rewards/chosen": 0.3689906582510544, |
| "rewards/margins": 2.8578583511376983, |
| "rewards/rejected": -2.488867692886644, |
| "step": 820 |
| }, |
| { |
| "epoch": 0.8689962046852506, |
| "grad_norm": 56.39802169799805, |
| "kl": 0.0, |
| "learning_rate": 2.607748070546037e-08, |
| "logits/chosen": 405348512.0, |
| "logits/rejected": 421100256.0, |
| "logps/chosen": -267.34557373817034, |
| "logps/rejected": -290.90634674922603, |
| "loss": 2.8455, |
| "rewards/chosen": 0.21449558968047613, |
| "rewards/margins": 2.5988345354767226, |
| "rewards/rejected": -2.3843389457962463, |
| "step": 830 |
| }, |
| { |
| "epoch": 0.8794660384766392, |
| "grad_norm": 53.03834533691406, |
| "kl": 0.0, |
| "learning_rate": 2.2162351615526544e-08, |
| "logits/chosen": 399964032.0, |
| "logits/rejected": 412186112.0, |
| "logps/chosen": -291.91339285714287, |
| "logps/rejected": -283.60141826923075, |
| "loss": 2.8146, |
| "rewards/chosen": 0.25672137548053076, |
| "rewards/margins": 2.538224906129569, |
| "rewards/rejected": -2.2815035306490383, |
| "step": 840 |
| }, |
| { |
| "epoch": 0.8899358722680277, |
| "grad_norm": 44.19123077392578, |
| "kl": 0.0, |
| "learning_rate": 1.8551935185811717e-08, |
| "logits/chosen": 369389504.0, |
| "logits/rejected": 379160128.0, |
| "logps/chosen": -273.960441468254, |
| "logps/rejected": -294.90163461538464, |
| "loss": 2.7471, |
| "rewards/chosen": 0.26350000775049603, |
| "rewards/margins": 2.7092262322096303, |
| "rewards/rejected": -2.4457262244591345, |
| "step": 850 |
| }, |
| { |
| "epoch": 0.9004057060594163, |
| "grad_norm": 69.50191497802734, |
| "kl": 0.0, |
| "learning_rate": 1.5251060028279612e-08, |
| "logits/chosen": 406441888.0, |
| "logits/rejected": 384275776.0, |
| "logps/chosen": -261.94189528593506, |
| "logps/rejected": -298.96228278041076, |
| "loss": 2.88, |
| "rewards/chosen": 0.2605755435996667, |
| "rewards/margins": 2.450452539921744, |
| "rewards/rejected": -2.189876996322077, |
| "step": 860 |
| }, |
| { |
| "epoch": 0.9108755398508048, |
| "grad_norm": 49.09358215332031, |
| "kl": 0.0, |
| "learning_rate": 1.2264140770878839e-08, |
| "logits/chosen": 381910016.0, |
| "logits/rejected": 408627904.0, |
| "logps/chosen": -288.8589045166403, |
| "logps/rejected": -287.52084938366716, |
| "loss": 2.86, |
| "rewards/chosen": 0.2764157931513718, |
| "rewards/margins": 2.49960226843787, |
| "rewards/rejected": -2.2231864752864983, |
| "step": 870 |
| }, |
| { |
| "epoch": 0.9213453736421934, |
| "grad_norm": 94.33980560302734, |
| "kl": 0.0, |
| "learning_rate": 9.59517215336922e-09, |
| "logits/chosen": 337365408.0, |
| "logits/rejected": 363262144.0, |
| "logps/chosen": -271.76117468701096, |
| "logps/rejected": -282.8595700078003, |
| "loss": 2.8254, |
| "rewards/chosen": 0.10245320494745819, |
| "rewards/margins": 2.6755982694219123, |
| "rewards/rejected": -2.573145064474454, |
| "step": 880 |
| }, |
| { |
| "epoch": 0.931815207433582, |
| "grad_norm": 67.45891571044922, |
| "kl": 0.0, |
| "learning_rate": 7.247723684711382e-09, |
| "logits/chosen": 388323680.0, |
| "logits/rejected": 380559360.0, |
| "logps/chosen": -261.3919588414634, |
| "logps/rejected": -287.53240184294873, |
| "loss": 2.7528, |
| "rewards/chosen": 0.4676312702458079, |
| "rewards/margins": 2.706891625877318, |
| "rewards/rejected": -2.2392603556315103, |
| "step": 890 |
| }, |
| { |
| "epoch": 0.9422850412249706, |
| "grad_norm": 61.053680419921875, |
| "kl": 0.0, |
| "learning_rate": 5.224934869164976e-09, |
| "logits/chosen": 389374336.0, |
| "logits/rejected": 416766144.0, |
| "logps/chosen": -281.80756179092384, |
| "logps/rejected": -292.0082249245852, |
| "loss": 2.9554, |
| "rewards/chosen": 0.13213771282177497, |
| "rewards/margins": 2.3010161904447957, |
| "rewards/rejected": -2.1688784776230206, |
| "step": 900 |
| }, |
| { |
| "epoch": 0.9527548750163591, |
| "grad_norm": 57.665740966796875, |
| "kl": 0.0, |
| "learning_rate": 3.529511007479946e-09, |
| "logits/chosen": 392336480.0, |
| "logits/rejected": 398449632.0, |
| "logps/chosen": -280.89854550691246, |
| "logps/rejected": -273.7118441971383, |
| "loss": 2.8584, |
| "rewards/chosen": 0.3160471879575293, |
| "rewards/margins": 2.478805146398507, |
| "rewards/rejected": -2.1627579584409777, |
| "step": 910 |
| }, |
| { |
| "epoch": 0.9632247088077477, |
| "grad_norm": 62.919960021972656, |
| "kl": 0.0, |
| "learning_rate": 2.1637195787966857e-09, |
| "logits/chosen": 370773184.0, |
| "logits/rejected": 438506944.0, |
| "logps/chosen": -283.6101524879615, |
| "logps/rejected": -277.2926179604262, |
| "loss": 2.7438, |
| "rewards/chosen": 0.5020495272371589, |
| "rewards/margins": 2.5911041864956252, |
| "rewards/rejected": -2.0890546592584665, |
| "step": 920 |
| }, |
| { |
| "epoch": 0.9736945425991362, |
| "grad_norm": 56.47111129760742, |
| "kl": 0.0, |
| "learning_rate": 1.1293872080934963e-09, |
| "logits/chosen": 367021120.0, |
| "logits/rejected": 405562304.0, |
| "logps/chosen": -274.2324472402597, |
| "logps/rejected": -289.800828313253, |
| "loss": 2.7894, |
| "rewards/chosen": 0.2850929359336952, |
| "rewards/margins": 2.6591384343335447, |
| "rewards/rejected": -2.3740454983998496, |
| "step": 930 |
| }, |
| { |
| "epoch": 0.9841643763905248, |
| "grad_norm": 55.910579681396484, |
| "kl": 0.0, |
| "learning_rate": 4.2789722323760546e-10, |
| "logits/chosen": 390189216.0, |
| "logits/rejected": 378433184.0, |
| "logps/chosen": -277.966340755988, |
| "logps/rejected": -284.6381229575163, |
| "loss": 2.7211, |
| "rewards/chosen": 0.5247520857942318, |
| "rewards/margins": 2.884224622367455, |
| "rewards/rejected": -2.359472536573223, |
| "step": 940 |
| }, |
| { |
| "epoch": 0.9946342101819133, |
| "grad_norm": 105.30035400390625, |
| "kl": 0.0, |
| "learning_rate": 6.018780490690822e-11, |
| "logits/chosen": 404358912.0, |
| "logits/rejected": 365349088.0, |
| "logps/chosen": -274.1581202651515, |
| "logps/rejected": -274.12162298387096, |
| "loss": 2.7314, |
| "rewards/chosen": 0.3820543924967448, |
| "rewards/margins": 2.859391448318317, |
| "rewards/rejected": -2.4773370558215726, |
| "step": 950 |
| }, |
| { |
| "epoch": 0.9998691270776077, |
| "step": 955, |
| "total_flos": 0.0, |
| "train_loss": 3.070508968892522, |
| "train_runtime": 9873.9747, |
| "train_samples_per_second": 12.383, |
| "train_steps_per_second": 0.097 |
| } |
| ], |
| "logging_steps": 10, |
| "max_steps": 955, |
| "num_input_tokens_seen": 0, |
| "num_train_epochs": 1, |
| "save_steps": 500, |
| "stateful_callbacks": { |
| "TrainerControl": { |
| "args": { |
| "should_epoch_stop": false, |
| "should_evaluate": false, |
| "should_log": false, |
| "should_save": false, |
| "should_training_stop": false |
| }, |
| "attributes": {} |
| } |
| }, |
| "total_flos": 0.0, |
| "train_batch_size": 4, |
| "trial_name": null, |
| "trial_params": null |
| } |
|
|
