| { |
| "best_global_step": null, |
| "best_metric": null, |
| "best_model_checkpoint": null, |
| "epoch": 0.9989528795811519, |
| "eval_steps": 200, |
| "global_step": 477, |
| "is_hyper_param_search": false, |
| "is_local_process_zero": true, |
| "is_world_process_zero": true, |
| "log_history": [ |
| { |
| "epoch": 0.0020942408376963353, |
| "grad_norm": 14.617609977722168, |
| "learning_rate": 0.0, |
| "logits/chosen": 2.203179359436035, |
| "logits/rejected": 2.035616397857666, |
| "logps/chosen": -257.4821472167969, |
| "logps/ref_chosen": -257.55841064453125, |
| "logps/ref_rejected": -199.84764099121094, |
| "logps/rejected": -199.93338012695312, |
| "loss": 5.5446, |
| "margin_dpo/margin_mean": 0.16199058294296265, |
| "margin_dpo/margin_std": 0.6907856464385986, |
| "step": 1 |
| }, |
| { |
| "epoch": 0.004188481675392671, |
| "grad_norm": 15.140374183654785, |
| "learning_rate": 1.0416666666666666e-08, |
| "logits/chosen": 2.1704792976379395, |
| "logits/rejected": 2.0754430294036865, |
| "logps/chosen": -224.03538513183594, |
| "logps/ref_chosen": -224.12454223632812, |
| "logps/ref_rejected": -182.62721252441406, |
| "logps/rejected": -182.67271423339844, |
| "loss": 5.5417, |
| "margin_dpo/margin_mean": 0.13464844226837158, |
| "margin_dpo/margin_std": 0.5429617166519165, |
| "step": 2 |
| }, |
| { |
| "epoch": 0.0062827225130890054, |
| "grad_norm": 14.625223159790039, |
| "learning_rate": 2.083333333333333e-08, |
| "logits/chosen": 2.4683523178100586, |
| "logits/rejected": 2.463977098464966, |
| "logps/chosen": -312.9666748046875, |
| "logps/ref_chosen": -312.8153991699219, |
| "logps/ref_rejected": -291.1138916015625, |
| "logps/rejected": -291.2332763671875, |
| "loss": 5.5426, |
| "margin_dpo/margin_mean": -0.03191244602203369, |
| "margin_dpo/margin_std": 0.6326964497566223, |
| "step": 3 |
| }, |
| { |
| "epoch": 0.008376963350785341, |
| "grad_norm": 15.790285110473633, |
| "learning_rate": 3.125e-08, |
| "logits/chosen": 1.5894497632980347, |
| "logits/rejected": 1.4774465560913086, |
| "logps/chosen": -310.7625427246094, |
| "logps/ref_chosen": -310.8699645996094, |
| "logps/ref_rejected": -323.95556640625, |
| "logps/rejected": -323.9718933105469, |
| "loss": 5.5437, |
| "margin_dpo/margin_mean": 0.12377279996871948, |
| "margin_dpo/margin_std": 0.9771984815597534, |
| "step": 4 |
| }, |
| { |
| "epoch": 0.010471204188481676, |
| "grad_norm": 15.793586730957031, |
| "learning_rate": 4.166666666666666e-08, |
| "logits/chosen": 1.5695815086364746, |
| "logits/rejected": 1.5709682703018188, |
| "logps/chosen": -303.8356628417969, |
| "logps/ref_chosen": -303.7280578613281, |
| "logps/ref_rejected": -262.055419921875, |
| "logps/rejected": -261.8935546875, |
| "loss": 5.548, |
| "margin_dpo/margin_mean": -0.26944446563720703, |
| "margin_dpo/margin_std": 0.66167151927948, |
| "step": 5 |
| }, |
| { |
| "epoch": 0.012565445026178011, |
| "grad_norm": 15.511699676513672, |
| "learning_rate": 5.208333333333333e-08, |
| "logits/chosen": 2.0192410945892334, |
| "logits/rejected": 1.9741183519363403, |
| "logps/chosen": -252.2058563232422, |
| "logps/ref_chosen": -252.3014373779297, |
| "logps/ref_rejected": -214.40451049804688, |
| "logps/rejected": -214.4804229736328, |
| "loss": 5.5507, |
| "margin_dpo/margin_mean": 0.1714714765548706, |
| "margin_dpo/margin_std": 0.6865968108177185, |
| "step": 6 |
| }, |
| { |
| "epoch": 0.014659685863874346, |
| "grad_norm": 15.63283634185791, |
| "learning_rate": 6.25e-08, |
| "logits/chosen": 2.191936492919922, |
| "logits/rejected": 2.0201575756073, |
| "logps/chosen": -248.16464233398438, |
| "logps/ref_chosen": -248.10345458984375, |
| "logps/ref_rejected": -204.55133056640625, |
| "logps/rejected": -204.63514709472656, |
| "loss": 5.5465, |
| "margin_dpo/margin_mean": 0.022650957107543945, |
| "margin_dpo/margin_std": 0.7195451855659485, |
| "step": 7 |
| }, |
| { |
| "epoch": 0.016753926701570682, |
| "grad_norm": 15.911747932434082, |
| "learning_rate": 7.291666666666667e-08, |
| "logits/chosen": 2.4633631706237793, |
| "logits/rejected": 2.229030132293701, |
| "logps/chosen": -446.24395751953125, |
| "logps/ref_chosen": -446.1068115234375, |
| "logps/ref_rejected": -316.3032531738281, |
| "logps/rejected": -316.33001708984375, |
| "loss": 5.5447, |
| "margin_dpo/margin_mean": -0.11035525798797607, |
| "margin_dpo/margin_std": 0.8465025424957275, |
| "step": 8 |
| }, |
| { |
| "epoch": 0.018848167539267015, |
| "grad_norm": 14.066997528076172, |
| "learning_rate": 8.333333333333333e-08, |
| "logits/chosen": 1.9973905086517334, |
| "logits/rejected": 1.8876209259033203, |
| "logps/chosen": -291.28857421875, |
| "logps/ref_chosen": -291.0896911621094, |
| "logps/ref_rejected": -298.3818054199219, |
| "logps/rejected": -298.3582763671875, |
| "loss": 5.5483, |
| "margin_dpo/margin_mean": -0.22240149974822998, |
| "margin_dpo/margin_std": 0.7139020562171936, |
| "step": 9 |
| }, |
| { |
| "epoch": 0.020942408376963352, |
| "grad_norm": 14.026876449584961, |
| "learning_rate": 9.375e-08, |
| "logits/chosen": 1.6050350666046143, |
| "logits/rejected": 1.755211591720581, |
| "logps/chosen": -221.44143676757812, |
| "logps/ref_chosen": -221.42408752441406, |
| "logps/ref_rejected": -210.35684204101562, |
| "logps/rejected": -210.39434814453125, |
| "loss": 5.544, |
| "margin_dpo/margin_mean": 0.02016240358352661, |
| "margin_dpo/margin_std": 0.5195479989051819, |
| "step": 10 |
| }, |
| { |
| "epoch": 0.023036649214659685, |
| "grad_norm": 15.404158592224121, |
| "learning_rate": 1.0416666666666667e-07, |
| "logits/chosen": 1.8669978380203247, |
| "logits/rejected": 1.7889609336853027, |
| "logps/chosen": -307.2198181152344, |
| "logps/ref_chosen": -307.2149658203125, |
| "logps/ref_rejected": -264.55902099609375, |
| "logps/rejected": -264.7065734863281, |
| "loss": 5.5427, |
| "margin_dpo/margin_mean": 0.1427026391029358, |
| "margin_dpo/margin_std": 0.9485504627227783, |
| "step": 11 |
| }, |
| { |
| "epoch": 0.025130890052356022, |
| "grad_norm": 14.81792163848877, |
| "learning_rate": 1.1458333333333332e-07, |
| "logits/chosen": 1.494691014289856, |
| "logits/rejected": 1.6338729858398438, |
| "logps/chosen": -273.935302734375, |
| "logps/ref_chosen": -273.97259521484375, |
| "logps/ref_rejected": -312.4557189941406, |
| "logps/rejected": -312.26611328125, |
| "loss": 5.5513, |
| "margin_dpo/margin_mean": -0.15232467651367188, |
| "margin_dpo/margin_std": 0.7628190517425537, |
| "step": 12 |
| }, |
| { |
| "epoch": 0.027225130890052355, |
| "grad_norm": 14.786741256713867, |
| "learning_rate": 1.25e-07, |
| "logits/chosen": 1.8189257383346558, |
| "logits/rejected": 1.8658004999160767, |
| "logps/chosen": -264.774658203125, |
| "logps/ref_chosen": -264.722412109375, |
| "logps/ref_rejected": -264.62823486328125, |
| "logps/rejected": -264.7838134765625, |
| "loss": 5.5457, |
| "margin_dpo/margin_mean": 0.10335606336593628, |
| "margin_dpo/margin_std": 0.7768966555595398, |
| "step": 13 |
| }, |
| { |
| "epoch": 0.02931937172774869, |
| "grad_norm": 15.321511268615723, |
| "learning_rate": 1.3541666666666666e-07, |
| "logits/chosen": 1.8423357009887695, |
| "logits/rejected": 1.6009153127670288, |
| "logps/chosen": -357.5430603027344, |
| "logps/ref_chosen": -357.3697509765625, |
| "logps/ref_rejected": -231.3351287841797, |
| "logps/rejected": -231.34188842773438, |
| "loss": 5.5436, |
| "margin_dpo/margin_mean": -0.16655707359313965, |
| "margin_dpo/margin_std": 0.6755635738372803, |
| "step": 14 |
| }, |
| { |
| "epoch": 0.031413612565445025, |
| "grad_norm": 16.096477508544922, |
| "learning_rate": 1.4583333333333335e-07, |
| "logits/chosen": 2.050579071044922, |
| "logits/rejected": 1.9528357982635498, |
| "logps/chosen": -282.3099670410156, |
| "logps/ref_chosen": -282.4208984375, |
| "logps/ref_rejected": -193.90872192382812, |
| "logps/rejected": -193.78834533691406, |
| "loss": 5.5457, |
| "margin_dpo/margin_mean": -0.009424567222595215, |
| "margin_dpo/margin_std": 0.5681266784667969, |
| "step": 15 |
| }, |
| { |
| "epoch": 0.033507853403141365, |
| "grad_norm": 16.60857391357422, |
| "learning_rate": 1.5624999999999999e-07, |
| "logits/chosen": 2.2264082431793213, |
| "logits/rejected": 1.9722710847854614, |
| "logps/chosen": -291.3759460449219, |
| "logps/ref_chosen": -291.56591796875, |
| "logps/ref_rejected": -252.4170684814453, |
| "logps/rejected": -252.54373168945312, |
| "loss": 5.54, |
| "margin_dpo/margin_mean": 0.31664133071899414, |
| "margin_dpo/margin_std": 0.7804574370384216, |
| "step": 16 |
| }, |
| { |
| "epoch": 0.0356020942408377, |
| "grad_norm": 15.15626049041748, |
| "learning_rate": 1.6666666666666665e-07, |
| "logits/chosen": 1.9703552722930908, |
| "logits/rejected": 1.9993352890014648, |
| "logps/chosen": -343.3455505371094, |
| "logps/ref_chosen": -343.4768981933594, |
| "logps/ref_rejected": -338.89654541015625, |
| "logps/rejected": -338.8592224121094, |
| "loss": 5.5409, |
| "margin_dpo/margin_mean": 0.09399676322937012, |
| "margin_dpo/margin_std": 0.5367782115936279, |
| "step": 17 |
| }, |
| { |
| "epoch": 0.03769633507853403, |
| "grad_norm": 15.167213439941406, |
| "learning_rate": 1.7708333333333334e-07, |
| "logits/chosen": 1.8425214290618896, |
| "logits/rejected": 1.8331950902938843, |
| "logps/chosen": -213.01934814453125, |
| "logps/ref_chosen": -213.05694580078125, |
| "logps/ref_rejected": -211.70962524414062, |
| "logps/rejected": -211.76414489746094, |
| "loss": 5.5491, |
| "margin_dpo/margin_mean": 0.09212470054626465, |
| "margin_dpo/margin_std": 0.6411672234535217, |
| "step": 18 |
| }, |
| { |
| "epoch": 0.039790575916230364, |
| "grad_norm": 14.854358673095703, |
| "learning_rate": 1.875e-07, |
| "logits/chosen": 2.0766916275024414, |
| "logits/rejected": 2.0941522121429443, |
| "logps/chosen": -240.00901794433594, |
| "logps/ref_chosen": -240.0670928955078, |
| "logps/ref_rejected": -246.15377807617188, |
| "logps/rejected": -246.24050903320312, |
| "loss": 5.5489, |
| "margin_dpo/margin_mean": 0.14478152990341187, |
| "margin_dpo/margin_std": 0.584217369556427, |
| "step": 19 |
| }, |
| { |
| "epoch": 0.041884816753926704, |
| "grad_norm": 15.4912748336792, |
| "learning_rate": 1.9791666666666664e-07, |
| "logits/chosen": 2.1966586112976074, |
| "logits/rejected": 1.9358861446380615, |
| "logps/chosen": -315.5570983886719, |
| "logps/ref_chosen": -315.71331787109375, |
| "logps/ref_rejected": -230.0822296142578, |
| "logps/rejected": -230.0750732421875, |
| "loss": 5.5455, |
| "margin_dpo/margin_mean": 0.14912045001983643, |
| "margin_dpo/margin_std": 0.5315914750099182, |
| "step": 20 |
| }, |
| { |
| "epoch": 0.04397905759162304, |
| "grad_norm": 15.429500579833984, |
| "learning_rate": 2.0833333333333333e-07, |
| "logits/chosen": 2.09773588180542, |
| "logits/rejected": 2.0702552795410156, |
| "logps/chosen": -279.3077697753906, |
| "logps/ref_chosen": -279.2261657714844, |
| "logps/ref_rejected": -300.1985168457031, |
| "logps/rejected": -300.22119140625, |
| "loss": 5.5468, |
| "margin_dpo/margin_mean": -0.05891883373260498, |
| "margin_dpo/margin_std": 0.8325001001358032, |
| "step": 21 |
| }, |
| { |
| "epoch": 0.04607329842931937, |
| "grad_norm": 13.734630584716797, |
| "learning_rate": 2.1875e-07, |
| "logits/chosen": 1.8216187953948975, |
| "logits/rejected": 1.9799120426177979, |
| "logps/chosen": -225.4229736328125, |
| "logps/ref_chosen": -225.4801788330078, |
| "logps/ref_rejected": -236.63134765625, |
| "logps/rejected": -236.60411071777344, |
| "loss": 5.5409, |
| "margin_dpo/margin_mean": 0.029949307441711426, |
| "margin_dpo/margin_std": 0.5145200490951538, |
| "step": 22 |
| }, |
| { |
| "epoch": 0.048167539267015703, |
| "grad_norm": 15.402115821838379, |
| "learning_rate": 2.2916666666666663e-07, |
| "logits/chosen": 1.9867033958435059, |
| "logits/rejected": 1.8609161376953125, |
| "logps/chosen": -340.4596862792969, |
| "logps/ref_chosen": -340.510986328125, |
| "logps/ref_rejected": -273.1431579589844, |
| "logps/rejected": -273.184814453125, |
| "loss": 5.5456, |
| "margin_dpo/margin_mean": 0.09300780296325684, |
| "margin_dpo/margin_std": 0.5188795924186707, |
| "step": 23 |
| }, |
| { |
| "epoch": 0.050261780104712044, |
| "grad_norm": 16.485750198364258, |
| "learning_rate": 2.3958333333333335e-07, |
| "logits/chosen": 1.7313284873962402, |
| "logits/rejected": 1.6817138195037842, |
| "logps/chosen": -274.0079040527344, |
| "logps/ref_chosen": -273.9709777832031, |
| "logps/ref_rejected": -269.8603210449219, |
| "logps/rejected": -269.9830017089844, |
| "loss": 5.5462, |
| "margin_dpo/margin_mean": 0.08572280406951904, |
| "margin_dpo/margin_std": 0.4962030053138733, |
| "step": 24 |
| }, |
| { |
| "epoch": 0.05235602094240838, |
| "grad_norm": 14.515819549560547, |
| "learning_rate": 2.5e-07, |
| "logits/chosen": 1.7567241191864014, |
| "logits/rejected": 1.772882342338562, |
| "logps/chosen": -245.420654296875, |
| "logps/ref_chosen": -245.38388061523438, |
| "logps/ref_rejected": -251.77703857421875, |
| "logps/rejected": -251.8808135986328, |
| "loss": 5.5402, |
| "margin_dpo/margin_mean": 0.06698936223983765, |
| "margin_dpo/margin_std": 0.7465457916259766, |
| "step": 25 |
| }, |
| { |
| "epoch": 0.05445026178010471, |
| "grad_norm": 15.561816215515137, |
| "learning_rate": 2.604166666666667e-07, |
| "logits/chosen": 1.6602405309677124, |
| "logits/rejected": 1.611204743385315, |
| "logps/chosen": -245.07839965820312, |
| "logps/ref_chosen": -245.162109375, |
| "logps/ref_rejected": -167.06671142578125, |
| "logps/rejected": -166.95631408691406, |
| "loss": 5.5441, |
| "margin_dpo/margin_mean": -0.026699483394622803, |
| "margin_dpo/margin_std": 0.7909866571426392, |
| "step": 26 |
| }, |
| { |
| "epoch": 0.05654450261780105, |
| "grad_norm": 15.185941696166992, |
| "learning_rate": 2.708333333333333e-07, |
| "logits/chosen": 2.148705244064331, |
| "logits/rejected": 1.9048577547073364, |
| "logps/chosen": -309.2626037597656, |
| "logps/ref_chosen": -309.4706115722656, |
| "logps/ref_rejected": -200.16006469726562, |
| "logps/rejected": -200.23269653320312, |
| "loss": 5.5469, |
| "margin_dpo/margin_mean": 0.2806363105773926, |
| "margin_dpo/margin_std": 0.6260923147201538, |
| "step": 27 |
| }, |
| { |
| "epoch": 0.05863874345549738, |
| "grad_norm": 15.434507369995117, |
| "learning_rate": 2.8125e-07, |
| "logits/chosen": 1.9996970891952515, |
| "logits/rejected": 2.1089255809783936, |
| "logps/chosen": -203.73443603515625, |
| "logps/ref_chosen": -203.72039794921875, |
| "logps/ref_rejected": -228.1062469482422, |
| "logps/rejected": -228.02944946289062, |
| "loss": 5.5409, |
| "margin_dpo/margin_mean": -0.09086447954177856, |
| "margin_dpo/margin_std": 0.3806726932525635, |
| "step": 28 |
| }, |
| { |
| "epoch": 0.060732984293193716, |
| "grad_norm": 14.699873924255371, |
| "learning_rate": 2.916666666666667e-07, |
| "logits/chosen": 2.243607997894287, |
| "logits/rejected": 1.9699711799621582, |
| "logps/chosen": -341.47991943359375, |
| "logps/ref_chosen": -341.7933349609375, |
| "logps/ref_rejected": -323.7848815917969, |
| "logps/rejected": -323.83416748046875, |
| "loss": 5.5414, |
| "margin_dpo/margin_mean": 0.3627087473869324, |
| "margin_dpo/margin_std": 0.9482086896896362, |
| "step": 29 |
| }, |
| { |
| "epoch": 0.06282722513089005, |
| "grad_norm": 14.436098098754883, |
| "learning_rate": 3.020833333333333e-07, |
| "logits/chosen": 1.4743300676345825, |
| "logits/rejected": 1.4441381692886353, |
| "logps/chosen": -239.34152221679688, |
| "logps/ref_chosen": -239.4767303466797, |
| "logps/ref_rejected": -228.0832977294922, |
| "logps/rejected": -228.0165252685547, |
| "loss": 5.5418, |
| "margin_dpo/margin_mean": 0.06841355562210083, |
| "margin_dpo/margin_std": 0.7110106348991394, |
| "step": 30 |
| }, |
| { |
| "epoch": 0.06492146596858639, |
| "grad_norm": 13.857452392578125, |
| "learning_rate": 3.1249999999999997e-07, |
| "logits/chosen": 1.6719400882720947, |
| "logits/rejected": 1.52069091796875, |
| "logps/chosen": -268.8196105957031, |
| "logps/ref_chosen": -268.9744567871094, |
| "logps/ref_rejected": -221.5098114013672, |
| "logps/rejected": -221.68231201171875, |
| "loss": 5.5392, |
| "margin_dpo/margin_mean": 0.3273264765739441, |
| "margin_dpo/margin_std": 0.8021472692489624, |
| "step": 31 |
| }, |
| { |
| "epoch": 0.06701570680628273, |
| "grad_norm": 15.621495246887207, |
| "learning_rate": 3.2291666666666666e-07, |
| "logits/chosen": 1.6164491176605225, |
| "logits/rejected": 1.4590275287628174, |
| "logps/chosen": -236.6236572265625, |
| "logps/ref_chosen": -236.76123046875, |
| "logps/ref_rejected": -191.0041046142578, |
| "logps/rejected": -190.91786193847656, |
| "loss": 5.5383, |
| "margin_dpo/margin_mean": 0.051319420337677, |
| "margin_dpo/margin_std": 0.562064528465271, |
| "step": 32 |
| }, |
| { |
| "epoch": 0.06910994764397906, |
| "grad_norm": 14.935791015625, |
| "learning_rate": 3.333333333333333e-07, |
| "logits/chosen": 1.937072515487671, |
| "logits/rejected": 1.866725206375122, |
| "logps/chosen": -258.4335021972656, |
| "logps/ref_chosen": -258.6623840332031, |
| "logps/ref_rejected": -233.15805053710938, |
| "logps/rejected": -233.19522094726562, |
| "loss": 5.5401, |
| "margin_dpo/margin_mean": 0.26607221364974976, |
| "margin_dpo/margin_std": 0.9210672974586487, |
| "step": 33 |
| }, |
| { |
| "epoch": 0.0712041884816754, |
| "grad_norm": 17.70219612121582, |
| "learning_rate": 3.4375e-07, |
| "logits/chosen": 2.076815128326416, |
| "logits/rejected": 2.0185177326202393, |
| "logps/chosen": -380.03729248046875, |
| "logps/ref_chosen": -380.25201416015625, |
| "logps/ref_rejected": -315.8236389160156, |
| "logps/rejected": -315.7915954589844, |
| "loss": 5.5395, |
| "margin_dpo/margin_mean": 0.18271714448928833, |
| "margin_dpo/margin_std": 0.7701175212860107, |
| "step": 34 |
| }, |
| { |
| "epoch": 0.07329842931937172, |
| "grad_norm": 13.645162582397461, |
| "learning_rate": 3.541666666666667e-07, |
| "logits/chosen": 1.5646406412124634, |
| "logits/rejected": 1.7504596710205078, |
| "logps/chosen": -245.80335998535156, |
| "logps/ref_chosen": -246.0772705078125, |
| "logps/ref_rejected": -317.1019592285156, |
| "logps/rejected": -317.00274658203125, |
| "loss": 5.54, |
| "margin_dpo/margin_mean": 0.17473018169403076, |
| "margin_dpo/margin_std": 0.7614114284515381, |
| "step": 35 |
| }, |
| { |
| "epoch": 0.07539267015706806, |
| "grad_norm": 17.520965576171875, |
| "learning_rate": 3.645833333333333e-07, |
| "logits/chosen": 1.7731884717941284, |
| "logits/rejected": 1.8305914402008057, |
| "logps/chosen": -343.9805908203125, |
| "logps/ref_chosen": -344.1368408203125, |
| "logps/ref_rejected": -343.6894836425781, |
| "logps/rejected": -343.47882080078125, |
| "loss": 5.5342, |
| "margin_dpo/margin_mean": -0.05438530445098877, |
| "margin_dpo/margin_std": 0.7112289071083069, |
| "step": 36 |
| }, |
| { |
| "epoch": 0.0774869109947644, |
| "grad_norm": 15.14476203918457, |
| "learning_rate": 3.75e-07, |
| "logits/chosen": 1.9591785669326782, |
| "logits/rejected": 1.9149752855300903, |
| "logps/chosen": -310.9266357421875, |
| "logps/ref_chosen": -311.3376770019531, |
| "logps/ref_rejected": -278.5052185058594, |
| "logps/rejected": -278.489990234375, |
| "loss": 5.5375, |
| "margin_dpo/margin_mean": 0.3958609700202942, |
| "margin_dpo/margin_std": 0.6456325054168701, |
| "step": 37 |
| }, |
| { |
| "epoch": 0.07958115183246073, |
| "grad_norm": 15.079659461975098, |
| "learning_rate": 3.8541666666666665e-07, |
| "logits/chosen": 2.1111977100372314, |
| "logits/rejected": 2.3584368228912354, |
| "logps/chosen": -193.07827758789062, |
| "logps/ref_chosen": -193.3851318359375, |
| "logps/ref_rejected": -234.6280975341797, |
| "logps/rejected": -234.42193603515625, |
| "loss": 5.5401, |
| "margin_dpo/margin_mean": 0.10068202018737793, |
| "margin_dpo/margin_std": 0.5997118353843689, |
| "step": 38 |
| }, |
| { |
| "epoch": 0.08167539267015707, |
| "grad_norm": 15.749566078186035, |
| "learning_rate": 3.958333333333333e-07, |
| "logits/chosen": 1.7943568229675293, |
| "logits/rejected": 1.8780990839004517, |
| "logps/chosen": -290.79742431640625, |
| "logps/ref_chosen": -291.5687255859375, |
| "logps/ref_rejected": -317.7392578125, |
| "logps/rejected": -317.748779296875, |
| "loss": 5.5255, |
| "margin_dpo/margin_mean": 0.7807860374450684, |
| "margin_dpo/margin_std": 1.0324419736862183, |
| "step": 39 |
| }, |
| { |
| "epoch": 0.08376963350785341, |
| "grad_norm": 15.053966522216797, |
| "learning_rate": 4.0625e-07, |
| "logits/chosen": 1.7152007818222046, |
| "logits/rejected": 1.685285210609436, |
| "logps/chosen": -211.45947265625, |
| "logps/ref_chosen": -211.951904296875, |
| "logps/ref_rejected": -166.82864379882812, |
| "logps/rejected": -166.58428955078125, |
| "loss": 5.528, |
| "margin_dpo/margin_mean": 0.24808716773986816, |
| "margin_dpo/margin_std": 0.9657536745071411, |
| "step": 40 |
| }, |
| { |
| "epoch": 0.08586387434554973, |
| "grad_norm": 15.568541526794434, |
| "learning_rate": 4.1666666666666667e-07, |
| "logits/chosen": 1.968687653541565, |
| "logits/rejected": 1.8429195880889893, |
| "logps/chosen": -300.13665771484375, |
| "logps/ref_chosen": -300.6400146484375, |
| "logps/ref_rejected": -224.77317810058594, |
| "logps/rejected": -224.72613525390625, |
| "loss": 5.535, |
| "margin_dpo/margin_mean": 0.45627307891845703, |
| "margin_dpo/margin_std": 0.7364793419837952, |
| "step": 41 |
| }, |
| { |
| "epoch": 0.08795811518324607, |
| "grad_norm": 14.581147193908691, |
| "learning_rate": 4.270833333333333e-07, |
| "logits/chosen": 2.1621668338775635, |
| "logits/rejected": 2.195481061935425, |
| "logps/chosen": -291.0929870605469, |
| "logps/ref_chosen": -291.4709167480469, |
| "logps/ref_rejected": -285.62982177734375, |
| "logps/rejected": -285.6851501464844, |
| "loss": 5.5294, |
| "margin_dpo/margin_mean": 0.43321943283081055, |
| "margin_dpo/margin_std": 1.0551120042800903, |
| "step": 42 |
| }, |
| { |
| "epoch": 0.09005235602094241, |
| "grad_norm": 15.62813663482666, |
| "learning_rate": 4.375e-07, |
| "logits/chosen": 1.9382034540176392, |
| "logits/rejected": 1.9245309829711914, |
| "logps/chosen": -313.7782897949219, |
| "logps/ref_chosen": -314.3768615722656, |
| "logps/ref_rejected": -246.80313110351562, |
| "logps/rejected": -246.7808380126953, |
| "loss": 5.5273, |
| "margin_dpo/margin_mean": 0.5762431621551514, |
| "margin_dpo/margin_std": 0.9067457914352417, |
| "step": 43 |
| }, |
| { |
| "epoch": 0.09214659685863874, |
| "grad_norm": 15.793681144714355, |
| "learning_rate": 4.479166666666667e-07, |
| "logits/chosen": 1.7762880325317383, |
| "logits/rejected": 1.7065317630767822, |
| "logps/chosen": -209.00802612304688, |
| "logps/ref_chosen": -209.8181915283203, |
| "logps/ref_rejected": -246.21340942382812, |
| "logps/rejected": -246.0368194580078, |
| "loss": 5.5223, |
| "margin_dpo/margin_mean": 0.633570671081543, |
| "margin_dpo/margin_std": 1.1882718801498413, |
| "step": 44 |
| }, |
| { |
| "epoch": 0.09424083769633508, |
| "grad_norm": 16.929059982299805, |
| "learning_rate": 4.5833333333333327e-07, |
| "logits/chosen": 1.7606732845306396, |
| "logits/rejected": 1.5792968273162842, |
| "logps/chosen": -308.1605224609375, |
| "logps/ref_chosen": -309.0930480957031, |
| "logps/ref_rejected": -269.3559265136719, |
| "logps/rejected": -268.9593200683594, |
| "loss": 5.5261, |
| "margin_dpo/margin_mean": 0.535961389541626, |
| "margin_dpo/margin_std": 1.0746005773544312, |
| "step": 45 |
| }, |
| { |
| "epoch": 0.09633507853403141, |
| "grad_norm": 16.489280700683594, |
| "learning_rate": 4.6874999999999996e-07, |
| "logits/chosen": 1.9223171472549438, |
| "logits/rejected": 1.9758403301239014, |
| "logps/chosen": -298.0412292480469, |
| "logps/ref_chosen": -298.72467041015625, |
| "logps/ref_rejected": -309.87786865234375, |
| "logps/rejected": -309.4717712402344, |
| "loss": 5.5236, |
| "margin_dpo/margin_mean": 0.277274489402771, |
| "margin_dpo/margin_std": 0.9340643882751465, |
| "step": 46 |
| }, |
| { |
| "epoch": 0.09842931937172775, |
| "grad_norm": 13.506661415100098, |
| "learning_rate": 4.791666666666667e-07, |
| "logits/chosen": 1.6691988706588745, |
| "logits/rejected": 2.0380465984344482, |
| "logps/chosen": -215.84332275390625, |
| "logps/ref_chosen": -216.43553161621094, |
| "logps/ref_rejected": -292.6329345703125, |
| "logps/rejected": -291.96148681640625, |
| "loss": 5.5293, |
| "margin_dpo/margin_mean": -0.0792464017868042, |
| "margin_dpo/margin_std": 1.1294535398483276, |
| "step": 47 |
| }, |
| { |
| "epoch": 0.10052356020942409, |
| "grad_norm": 14.86147403717041, |
| "learning_rate": 4.895833333333333e-07, |
| "logits/chosen": 2.211613178253174, |
| "logits/rejected": 2.186110496520996, |
| "logps/chosen": -234.05947875976562, |
| "logps/ref_chosen": -234.77496337890625, |
| "logps/ref_rejected": -240.41433715820312, |
| "logps/rejected": -240.24525451660156, |
| "loss": 5.5203, |
| "margin_dpo/margin_mean": 0.5463833212852478, |
| "margin_dpo/margin_std": 1.3220971822738647, |
| "step": 48 |
| }, |
| { |
| "epoch": 0.10261780104712041, |
| "grad_norm": 15.43526840209961, |
| "learning_rate": 5e-07, |
| "logits/chosen": 1.7962108850479126, |
| "logits/rejected": 1.9277849197387695, |
| "logps/chosen": -245.73326110839844, |
| "logps/ref_chosen": -246.7688446044922, |
| "logps/ref_rejected": -253.47378540039062, |
| "logps/rejected": -253.3438720703125, |
| "loss": 5.5203, |
| "margin_dpo/margin_mean": 0.9057276248931885, |
| "margin_dpo/margin_std": 1.4221221208572388, |
| "step": 49 |
| }, |
| { |
| "epoch": 0.10471204188481675, |
| "grad_norm": 15.111068725585938, |
| "learning_rate": 4.999932966293553e-07, |
| "logits/chosen": 2.2119665145874023, |
| "logits/rejected": 2.335810422897339, |
| "logps/chosen": -281.3116760253906, |
| "logps/ref_chosen": -282.61981201171875, |
| "logps/ref_rejected": -340.8515625, |
| "logps/rejected": -340.31781005859375, |
| "loss": 5.5284, |
| "margin_dpo/margin_mean": 0.7743173837661743, |
| "margin_dpo/margin_std": 1.4886585474014282, |
| "step": 50 |
| }, |
| { |
| "epoch": 0.1068062827225131, |
| "grad_norm": 14.817649841308594, |
| "learning_rate": 4.999731868769026e-07, |
| "logits/chosen": 1.637377381324768, |
| "logits/rejected": 1.7862030267715454, |
| "logps/chosen": -244.794677734375, |
| "logps/ref_chosen": -245.87562561035156, |
| "logps/ref_rejected": -309.7420654296875, |
| "logps/rejected": -309.6230773925781, |
| "loss": 5.5202, |
| "margin_dpo/margin_mean": 0.961925208568573, |
| "margin_dpo/margin_std": 1.9343087673187256, |
| "step": 51 |
| }, |
| { |
| "epoch": 0.10890052356020942, |
| "grad_norm": 17.035507202148438, |
| "learning_rate": 4.99939671821067e-07, |
| "logits/chosen": 1.8847734928131104, |
| "logits/rejected": 2.039155960083008, |
| "logps/chosen": -276.9980163574219, |
| "logps/ref_chosen": -278.3123474121094, |
| "logps/ref_rejected": -320.58203125, |
| "logps/rejected": -319.9336853027344, |
| "loss": 5.5067, |
| "margin_dpo/margin_mean": 0.6659917235374451, |
| "margin_dpo/margin_std": 1.508874773979187, |
| "step": 52 |
| }, |
| { |
| "epoch": 0.11099476439790576, |
| "grad_norm": 15.945631980895996, |
| "learning_rate": 4.998927532591591e-07, |
| "logits/chosen": 2.085860013961792, |
| "logits/rejected": 2.0801711082458496, |
| "logps/chosen": -331.2710266113281, |
| "logps/ref_chosen": -332.776123046875, |
| "logps/ref_rejected": -325.1794128417969, |
| "logps/rejected": -324.69622802734375, |
| "loss": 5.5144, |
| "margin_dpo/margin_mean": 1.0218517780303955, |
| "margin_dpo/margin_std": 1.590319037437439, |
| "step": 53 |
| }, |
| { |
| "epoch": 0.1130890052356021, |
| "grad_norm": 14.57484245300293, |
| "learning_rate": 4.998324337072792e-07, |
| "logits/chosen": 1.3913365602493286, |
| "logits/rejected": 1.4456019401550293, |
| "logps/chosen": -294.7577819824219, |
| "logps/ref_chosen": -296.2243347167969, |
| "logps/ref_rejected": -267.64251708984375, |
| "logps/rejected": -267.3682861328125, |
| "loss": 5.5131, |
| "margin_dpo/margin_mean": 1.1922770738601685, |
| "margin_dpo/margin_std": 1.47157621383667, |
| "step": 54 |
| }, |
| { |
| "epoch": 0.11518324607329843, |
| "grad_norm": 12.808218955993652, |
| "learning_rate": 4.997587164001815e-07, |
| "logits/chosen": 1.8794647455215454, |
| "logits/rejected": 1.8777508735656738, |
| "logps/chosen": -197.05091857910156, |
| "logps/ref_chosen": -198.1138916015625, |
| "logps/ref_rejected": -185.93772888183594, |
| "logps/rejected": -185.5297088623047, |
| "loss": 5.522, |
| "margin_dpo/margin_mean": 0.6549429893493652, |
| "margin_dpo/margin_std": 1.0751301050186157, |
| "step": 55 |
| }, |
| { |
| "epoch": 0.11727748691099477, |
| "grad_norm": 14.403154373168945, |
| "learning_rate": 4.996716052911017e-07, |
| "logits/chosen": 2.004265785217285, |
| "logits/rejected": 1.965603232383728, |
| "logps/chosen": -267.2569580078125, |
| "logps/ref_chosen": -268.8618469238281, |
| "logps/ref_rejected": -245.21348571777344, |
| "logps/rejected": -244.97555541992188, |
| "loss": 5.5085, |
| "margin_dpo/margin_mean": 1.3669579029083252, |
| "margin_dpo/margin_std": 1.768923282623291, |
| "step": 56 |
| }, |
| { |
| "epoch": 0.1193717277486911, |
| "grad_norm": 17.60223960876465, |
| "learning_rate": 4.99571105051544e-07, |
| "logits/chosen": 2.1416828632354736, |
| "logits/rejected": 1.8643951416015625, |
| "logps/chosen": -286.81622314453125, |
| "logps/ref_chosen": -288.4784851074219, |
| "logps/ref_rejected": -239.400146484375, |
| "logps/rejected": -238.46566772460938, |
| "loss": 5.4919, |
| "margin_dpo/margin_mean": 0.7277634739875793, |
| "margin_dpo/margin_std": 1.487809658050537, |
| "step": 57 |
| }, |
| { |
| "epoch": 0.12146596858638743, |
| "grad_norm": 14.885680198669434, |
| "learning_rate": 4.994572210710314e-07, |
| "logits/chosen": 1.8542314767837524, |
| "logits/rejected": 1.8795952796936035, |
| "logps/chosen": -276.6270446777344, |
| "logps/ref_chosen": -278.2837219238281, |
| "logps/ref_rejected": -262.5280456542969, |
| "logps/rejected": -262.4251708984375, |
| "loss": 5.5076, |
| "margin_dpo/margin_mean": 1.5538146495819092, |
| "margin_dpo/margin_std": 1.7451914548873901, |
| "step": 58 |
| }, |
| { |
| "epoch": 0.12356020942408377, |
| "grad_norm": 15.256675720214844, |
| "learning_rate": 4.993299594568162e-07, |
| "logits/chosen": 1.695129632949829, |
| "logits/rejected": 1.76768159866333, |
| "logps/chosen": -231.92245483398438, |
| "logps/ref_chosen": -232.77662658691406, |
| "logps/ref_rejected": -226.2711181640625, |
| "logps/rejected": -225.69354248046875, |
| "loss": 5.5103, |
| "margin_dpo/margin_mean": 0.27659308910369873, |
| "margin_dpo/margin_std": 1.5459599494934082, |
| "step": 59 |
| }, |
| { |
| "epoch": 0.1256544502617801, |
| "grad_norm": 14.66773509979248, |
| "learning_rate": 4.991893270335525e-07, |
| "logits/chosen": 1.8791348934173584, |
| "logits/rejected": 1.587320327758789, |
| "logps/chosen": -314.26800537109375, |
| "logps/ref_chosen": -315.6903991699219, |
| "logps/ref_rejected": -190.40899658203125, |
| "logps/rejected": -189.97193908691406, |
| "loss": 5.4951, |
| "margin_dpo/margin_mean": 0.9853799939155579, |
| "margin_dpo/margin_std": 1.7359843254089355, |
| "step": 60 |
| }, |
| { |
| "epoch": 0.12774869109947645, |
| "grad_norm": 15.011839866638184, |
| "learning_rate": 4.990353313429303e-07, |
| "logits/chosen": 2.034388542175293, |
| "logits/rejected": 2.065513849258423, |
| "logps/chosen": -249.50173950195312, |
| "logps/ref_chosen": -251.527099609375, |
| "logps/ref_rejected": -261.0340270996094, |
| "logps/rejected": -259.8365173339844, |
| "loss": 5.503, |
| "margin_dpo/margin_mean": 0.8278936147689819, |
| "margin_dpo/margin_std": 1.6765094995498657, |
| "step": 61 |
| }, |
| { |
| "epoch": 0.12984293193717278, |
| "grad_norm": 14.675267219543457, |
| "learning_rate": 4.988679806432711e-07, |
| "logits/chosen": 1.8862786293029785, |
| "logits/rejected": 1.8161289691925049, |
| "logps/chosen": -255.54144287109375, |
| "logps/ref_chosen": -257.3919982910156, |
| "logps/ref_rejected": -282.1814880371094, |
| "logps/rejected": -281.52008056640625, |
| "loss": 5.4967, |
| "margin_dpo/margin_mean": 1.189134955406189, |
| "margin_dpo/margin_std": 2.0515801906585693, |
| "step": 62 |
| }, |
| { |
| "epoch": 0.1319371727748691, |
| "grad_norm": 14.802937507629395, |
| "learning_rate": 4.986872839090852e-07, |
| "logits/chosen": 2.1043763160705566, |
| "logits/rejected": 2.2130091190338135, |
| "logps/chosen": -320.39398193359375, |
| "logps/ref_chosen": -322.24725341796875, |
| "logps/ref_rejected": -327.70892333984375, |
| "logps/rejected": -326.7320251464844, |
| "loss": 5.4938, |
| "margin_dpo/margin_mean": 0.876427948474884, |
| "margin_dpo/margin_std": 2.8320491313934326, |
| "step": 63 |
| }, |
| { |
| "epoch": 0.13403141361256546, |
| "grad_norm": 15.675727844238281, |
| "learning_rate": 4.9849325083059e-07, |
| "logits/chosen": 1.840427279472351, |
| "logits/rejected": 2.098795175552368, |
| "logps/chosen": -333.1174621582031, |
| "logps/ref_chosen": -335.7379455566406, |
| "logps/ref_rejected": -337.8742980957031, |
| "logps/rejected": -337.04913330078125, |
| "loss": 5.4794, |
| "margin_dpo/margin_mean": 1.795319676399231, |
| "margin_dpo/margin_std": 3.2046210765838623, |
| "step": 64 |
| }, |
| { |
| "epoch": 0.13612565445026178, |
| "grad_norm": 14.844298362731934, |
| "learning_rate": 4.982858918131906e-07, |
| "logits/chosen": 1.8770238161087036, |
| "logits/rejected": 1.9210056066513062, |
| "logps/chosen": -309.9880676269531, |
| "logps/ref_chosen": -312.36358642578125, |
| "logps/ref_rejected": -300.220947265625, |
| "logps/rejected": -298.6002502441406, |
| "loss": 5.5002, |
| "margin_dpo/margin_mean": 0.754831075668335, |
| "margin_dpo/margin_std": 2.1830434799194336, |
| "step": 65 |
| }, |
| { |
| "epoch": 0.1382198952879581, |
| "grad_norm": 14.79240894317627, |
| "learning_rate": 4.980652179769217e-07, |
| "logits/chosen": 1.8671329021453857, |
| "logits/rejected": 2.058912515640259, |
| "logps/chosen": -195.7026824951172, |
| "logps/ref_chosen": -198.186767578125, |
| "logps/ref_rejected": -248.18748474121094, |
| "logps/rejected": -247.15882873535156, |
| "loss": 5.4918, |
| "margin_dpo/margin_mean": 1.4554123878479004, |
| "margin_dpo/margin_std": 2.3160204887390137, |
| "step": 66 |
| }, |
| { |
| "epoch": 0.14031413612565444, |
| "grad_norm": 14.211318969726562, |
| "learning_rate": 4.978312411558517e-07, |
| "logits/chosen": 2.104246139526367, |
| "logits/rejected": 2.138582706451416, |
| "logps/chosen": -289.1455078125, |
| "logps/ref_chosen": -291.9940490722656, |
| "logps/ref_rejected": -269.945068359375, |
| "logps/rejected": -268.6217346191406, |
| "loss": 5.4991, |
| "margin_dpo/margin_mean": 1.5252022743225098, |
| "margin_dpo/margin_std": 2.973376989364624, |
| "step": 67 |
| }, |
| { |
| "epoch": 0.1424083769633508, |
| "grad_norm": 14.742173194885254, |
| "learning_rate": 4.975839738974473e-07, |
| "logits/chosen": 1.597095012664795, |
| "logits/rejected": 1.4467945098876953, |
| "logps/chosen": -287.52056884765625, |
| "logps/ref_chosen": -289.9323425292969, |
| "logps/ref_rejected": -225.7897491455078, |
| "logps/rejected": -225.21658325195312, |
| "loss": 5.4771, |
| "margin_dpo/margin_mean": 1.8386340141296387, |
| "margin_dpo/margin_std": 3.168964385986328, |
| "step": 68 |
| }, |
| { |
| "epoch": 0.14450261780104712, |
| "grad_norm": 14.997187614440918, |
| "learning_rate": 4.97323429461901e-07, |
| "logits/chosen": 2.1530022621154785, |
| "logits/rejected": 2.05501651763916, |
| "logps/chosen": -263.3707580566406, |
| "logps/ref_chosen": -266.7104797363281, |
| "logps/ref_rejected": -229.5946502685547, |
| "logps/rejected": -228.25326538085938, |
| "loss": 5.4597, |
| "margin_dpo/margin_mean": 1.9983257055282593, |
| "margin_dpo/margin_std": 3.47263503074646, |
| "step": 69 |
| }, |
| { |
| "epoch": 0.14659685863874344, |
| "grad_norm": 15.571727752685547, |
| "learning_rate": 4.970496218214204e-07, |
| "logits/chosen": 2.235600471496582, |
| "logits/rejected": 2.3245067596435547, |
| "logps/chosen": -265.7595520019531, |
| "logps/ref_chosen": -268.6711120605469, |
| "logps/ref_rejected": -261.61273193359375, |
| "logps/rejected": -260.5652160644531, |
| "loss": 5.4636, |
| "margin_dpo/margin_mean": 1.864060878753662, |
| "margin_dpo/margin_std": 2.7824549674987793, |
| "step": 70 |
| }, |
| { |
| "epoch": 0.1486910994764398, |
| "grad_norm": 14.973682403564453, |
| "learning_rate": 4.967625656594781e-07, |
| "logits/chosen": 1.9091215133666992, |
| "logits/rejected": 1.9661266803741455, |
| "logps/chosen": -241.69586181640625, |
| "logps/ref_chosen": -244.97821044921875, |
| "logps/ref_rejected": -263.7174377441406, |
| "logps/rejected": -262.50848388671875, |
| "loss": 5.469, |
| "margin_dpo/margin_mean": 2.0734434127807617, |
| "margin_dpo/margin_std": 4.373683929443359, |
| "step": 71 |
| }, |
| { |
| "epoch": 0.15078534031413612, |
| "grad_norm": 14.649141311645508, |
| "learning_rate": 4.964622763700252e-07, |
| "logits/chosen": 1.8277561664581299, |
| "logits/rejected": 1.8917427062988281, |
| "logps/chosen": -276.90264892578125, |
| "logps/ref_chosen": -280.0353698730469, |
| "logps/ref_rejected": -291.1289367675781, |
| "logps/rejected": -289.7200927734375, |
| "loss": 5.4668, |
| "margin_dpo/margin_mean": 1.7238655090332031, |
| "margin_dpo/margin_std": 3.216670274734497, |
| "step": 72 |
| }, |
| { |
| "epoch": 0.15287958115183245, |
| "grad_norm": 14.681085586547852, |
| "learning_rate": 4.961487700566646e-07, |
| "logits/chosen": 2.040257453918457, |
| "logits/rejected": 2.0101261138916016, |
| "logps/chosen": -237.6248321533203, |
| "logps/ref_chosen": -241.37384033203125, |
| "logps/ref_rejected": -227.28871154785156, |
| "logps/rejected": -224.7103271484375, |
| "loss": 5.4684, |
| "margin_dpo/margin_mean": 1.1706353425979614, |
| "margin_dpo/margin_std": 2.684138774871826, |
| "step": 73 |
| }, |
| { |
| "epoch": 0.1549738219895288, |
| "grad_norm": 16.057296752929688, |
| "learning_rate": 4.958220635317885e-07, |
| "logits/chosen": 1.7149076461791992, |
| "logits/rejected": 1.616335153579712, |
| "logps/chosen": -427.9046630859375, |
| "logps/ref_chosen": -432.6361389160156, |
| "logps/ref_rejected": -408.990478515625, |
| "logps/rejected": -406.4610595703125, |
| "loss": 5.4703, |
| "margin_dpo/margin_mean": 2.2020528316497803, |
| "margin_dpo/margin_std": 3.4136807918548584, |
| "step": 74 |
| }, |
| { |
| "epoch": 0.15706806282722513, |
| "grad_norm": 15.3256254196167, |
| "learning_rate": 4.954821743156767e-07, |
| "logits/chosen": 1.8307483196258545, |
| "logits/rejected": 1.8694071769714355, |
| "logps/chosen": -277.4227294921875, |
| "logps/ref_chosen": -282.2913513183594, |
| "logps/ref_rejected": -227.30093383789062, |
| "logps/rejected": -225.89971923828125, |
| "loss": 5.4384, |
| "margin_dpo/margin_mean": 3.4673845767974854, |
| "margin_dpo/margin_std": 3.8357293605804443, |
| "step": 75 |
| }, |
| { |
| "epoch": 0.15916230366492146, |
| "grad_norm": 16.74871253967285, |
| "learning_rate": 4.951291206355559e-07, |
| "logits/chosen": 1.9061857461929321, |
| "logits/rejected": 1.6594858169555664, |
| "logps/chosen": -272.63018798828125, |
| "logps/ref_chosen": -277.90081787109375, |
| "logps/ref_rejected": -214.1353302001953, |
| "logps/rejected": -211.89590454101562, |
| "loss": 5.431, |
| "margin_dpo/margin_mean": 3.031224250793457, |
| "margin_dpo/margin_std": 3.183443546295166, |
| "step": 76 |
| }, |
| { |
| "epoch": 0.1612565445026178, |
| "grad_norm": 18.260398864746094, |
| "learning_rate": 4.947629214246236e-07, |
| "logits/chosen": 2.142491102218628, |
| "logits/rejected": 2.1160454750061035, |
| "logps/chosen": -278.9680480957031, |
| "logps/ref_chosen": -283.3741455078125, |
| "logps/ref_rejected": -239.51246643066406, |
| "logps/rejected": -237.45001220703125, |
| "loss": 5.4527, |
| "margin_dpo/margin_mean": 2.3436222076416016, |
| "margin_dpo/margin_std": 3.3090319633483887, |
| "step": 77 |
| }, |
| { |
| "epoch": 0.16335078534031414, |
| "grad_norm": 14.633062362670898, |
| "learning_rate": 4.943835963210323e-07, |
| "logits/chosen": 1.6990811824798584, |
| "logits/rejected": 1.6937521696090698, |
| "logps/chosen": -202.76388549804688, |
| "logps/ref_chosen": -207.1702423095703, |
| "logps/ref_rejected": -196.26866149902344, |
| "logps/rejected": -194.35032653808594, |
| "loss": 5.4294, |
| "margin_dpo/margin_mean": 2.4880149364471436, |
| "margin_dpo/margin_std": 3.424355983734131, |
| "step": 78 |
| }, |
| { |
| "epoch": 0.16544502617801046, |
| "grad_norm": 16.774738311767578, |
| "learning_rate": 4.939911656668361e-07, |
| "logits/chosen": 1.9445700645446777, |
| "logits/rejected": 2.229759454727173, |
| "logps/chosen": -208.6917724609375, |
| "logps/ref_chosen": -212.90396118164062, |
| "logps/ref_rejected": -242.32528686523438, |
| "logps/rejected": -239.5742950439453, |
| "loss": 5.4268, |
| "margin_dpo/margin_mean": 1.461201786994934, |
| "margin_dpo/margin_std": 4.005527973175049, |
| "step": 79 |
| }, |
| { |
| "epoch": 0.16753926701570682, |
| "grad_norm": 15.637154579162598, |
| "learning_rate": 4.935856505068998e-07, |
| "logits/chosen": 1.3914299011230469, |
| "logits/rejected": 1.5492628812789917, |
| "logps/chosen": -251.85031127929688, |
| "logps/ref_chosen": -257.9057312011719, |
| "logps/ref_rejected": -246.391845703125, |
| "logps/rejected": -243.01177978515625, |
| "loss": 5.4504, |
| "margin_dpo/margin_mean": 2.6753690242767334, |
| "margin_dpo/margin_std": 3.9176571369171143, |
| "step": 80 |
| }, |
| { |
| "epoch": 0.16963350785340314, |
| "grad_norm": 14.57850456237793, |
| "learning_rate": 4.93167072587771e-07, |
| "logits/chosen": 2.009546995162964, |
| "logits/rejected": 2.2237491607666016, |
| "logps/chosen": -220.03717041015625, |
| "logps/ref_chosen": -226.68576049804688, |
| "logps/ref_rejected": -215.2713623046875, |
| "logps/rejected": -212.5276336669922, |
| "loss": 5.4326, |
| "margin_dpo/margin_mean": 3.9048891067504883, |
| "margin_dpo/margin_std": 3.982060432434082, |
| "step": 81 |
| }, |
| { |
| "epoch": 0.17172774869109947, |
| "grad_norm": 15.640838623046875, |
| "learning_rate": 4.92735454356513e-07, |
| "logits/chosen": 1.8449329137802124, |
| "logits/rejected": 1.773772954940796, |
| "logps/chosen": -290.084228515625, |
| "logps/ref_chosen": -296.12799072265625, |
| "logps/ref_rejected": -261.3748474121094, |
| "logps/rejected": -258.4228515625, |
| "loss": 5.4297, |
| "margin_dpo/margin_mean": 3.091761589050293, |
| "margin_dpo/margin_std": 4.757362365722656, |
| "step": 82 |
| }, |
| { |
| "epoch": 0.17382198952879582, |
| "grad_norm": 15.583915710449219, |
| "learning_rate": 4.922908189595017e-07, |
| "logits/chosen": 1.8198847770690918, |
| "logits/rejected": 1.8020501136779785, |
| "logps/chosen": -255.5862274169922, |
| "logps/ref_chosen": -261.39862060546875, |
| "logps/ref_rejected": -279.9942626953125, |
| "logps/rejected": -276.3531799316406, |
| "loss": 5.4115, |
| "margin_dpo/margin_mean": 2.1713221073150635, |
| "margin_dpo/margin_std": 4.908409118652344, |
| "step": 83 |
| }, |
| { |
| "epoch": 0.17591623036649215, |
| "grad_norm": 15.165980339050293, |
| "learning_rate": 4.918331902411841e-07, |
| "logits/chosen": 2.0596227645874023, |
| "logits/rejected": 1.9471057653427124, |
| "logps/chosen": -385.02862548828125, |
| "logps/ref_chosen": -392.54547119140625, |
| "logps/ref_rejected": -342.066162109375, |
| "logps/rejected": -336.90234375, |
| "loss": 5.4311, |
| "margin_dpo/margin_mean": 2.353001356124878, |
| "margin_dpo/margin_std": 5.491085052490234, |
| "step": 84 |
| }, |
| { |
| "epoch": 0.17801047120418848, |
| "grad_norm": 13.898218154907227, |
| "learning_rate": 4.913625927427995e-07, |
| "logits/chosen": 1.505142331123352, |
| "logits/rejected": 1.6710948944091797, |
| "logps/chosen": -186.115478515625, |
| "logps/ref_chosen": -192.9306640625, |
| "logps/ref_rejected": -231.5825653076172, |
| "logps/rejected": -227.84988403320312, |
| "loss": 5.4719, |
| "margin_dpo/margin_mean": 3.082519054412842, |
| "margin_dpo/margin_std": 4.139708995819092, |
| "step": 85 |
| }, |
| { |
| "epoch": 0.18010471204188483, |
| "grad_norm": 16.21003532409668, |
| "learning_rate": 4.908790517010636e-07, |
| "logits/chosen": 1.800257682800293, |
| "logits/rejected": 1.8198274374008179, |
| "logps/chosen": -306.5592346191406, |
| "logps/ref_chosen": -313.5525207519531, |
| "logps/ref_rejected": -285.59228515625, |
| "logps/rejected": -280.493896484375, |
| "loss": 5.4098, |
| "margin_dpo/margin_mean": 1.8949062824249268, |
| "margin_dpo/margin_std": 6.387627601623535, |
| "step": 86 |
| }, |
| { |
| "epoch": 0.18219895287958116, |
| "grad_norm": 15.313575744628906, |
| "learning_rate": 4.903825930468148e-07, |
| "logits/chosen": 1.5017904043197632, |
| "logits/rejected": 1.4683315753936768, |
| "logps/chosen": -227.59046936035156, |
| "logps/ref_chosen": -236.03445434570312, |
| "logps/ref_rejected": -225.67410278320312, |
| "logps/rejected": -221.80938720703125, |
| "loss": 5.3899, |
| "margin_dpo/margin_mean": 4.5792436599731445, |
| "margin_dpo/margin_std": 6.2218732833862305, |
| "step": 87 |
| }, |
| { |
| "epoch": 0.18429319371727748, |
| "grad_norm": 14.063505172729492, |
| "learning_rate": 4.898732434036243e-07, |
| "logits/chosen": 1.7088102102279663, |
| "logits/rejected": 1.6172915697097778, |
| "logps/chosen": -273.8514709472656, |
| "logps/ref_chosen": -280.1703186035156, |
| "logps/ref_rejected": -219.1881103515625, |
| "logps/rejected": -216.62831115722656, |
| "loss": 5.4232, |
| "margin_dpo/margin_mean": 3.7590301036834717, |
| "margin_dpo/margin_std": 6.66114616394043, |
| "step": 88 |
| }, |
| { |
| "epoch": 0.18638743455497384, |
| "grad_norm": 15.837873458862305, |
| "learning_rate": 4.893510300863676e-07, |
| "logits/chosen": 2.1236486434936523, |
| "logits/rejected": 2.1029891967773438, |
| "logps/chosen": -202.62815856933594, |
| "logps/ref_chosen": -211.3966827392578, |
| "logps/ref_rejected": -171.04954528808594, |
| "logps/rejected": -165.4285430908203, |
| "loss": 5.3997, |
| "margin_dpo/margin_mean": 3.1475319862365723, |
| "margin_dpo/margin_std": 4.286096572875977, |
| "step": 89 |
| }, |
| { |
| "epoch": 0.18848167539267016, |
| "grad_norm": 15.2637357711792, |
| "learning_rate": 4.8881598109976e-07, |
| "logits/chosen": 2.190295696258545, |
| "logits/rejected": 2.0822367668151855, |
| "logps/chosen": -271.2816467285156, |
| "logps/ref_chosen": -280.9217834472656, |
| "logps/ref_rejected": -245.75814819335938, |
| "logps/rejected": -239.31825256347656, |
| "loss": 5.4166, |
| "margin_dpo/margin_mean": 3.2002599239349365, |
| "margin_dpo/margin_std": 5.397155284881592, |
| "step": 90 |
| }, |
| { |
| "epoch": 0.1905759162303665, |
| "grad_norm": 14.667741775512695, |
| "learning_rate": 4.882681251368548e-07, |
| "logits/chosen": 1.3757317066192627, |
| "logits/rejected": 1.691314697265625, |
| "logps/chosen": -121.55278778076172, |
| "logps/ref_chosen": -130.23472595214844, |
| "logps/ref_rejected": -177.76895141601562, |
| "logps/rejected": -172.3560028076172, |
| "loss": 5.4142, |
| "margin_dpo/margin_mean": 3.2689881324768066, |
| "margin_dpo/margin_std": 4.278058052062988, |
| "step": 91 |
| }, |
| { |
| "epoch": 0.19267015706806281, |
| "grad_norm": 15.648965835571289, |
| "learning_rate": 4.877074915775048e-07, |
| "logits/chosen": 1.6639858484268188, |
| "logits/rejected": 1.4799772500991821, |
| "logps/chosen": -334.3116455078125, |
| "logps/ref_chosen": -344.4306335449219, |
| "logps/ref_rejected": -276.291748046875, |
| "logps/rejected": -270.984375, |
| "loss": 5.4004, |
| "margin_dpo/margin_mean": 4.811601161956787, |
| "margin_dpo/margin_std": 5.4291887283325195, |
| "step": 92 |
| }, |
| { |
| "epoch": 0.19476439790575917, |
| "grad_norm": 14.095191955566406, |
| "learning_rate": 4.871341104867864e-07, |
| "logits/chosen": 1.9756680727005005, |
| "logits/rejected": 1.923811674118042, |
| "logps/chosen": -196.8525390625, |
| "logps/ref_chosen": -206.1533660888672, |
| "logps/ref_rejected": -231.759033203125, |
| "logps/rejected": -227.26364135742188, |
| "loss": 5.408, |
| "margin_dpo/margin_mean": 4.805446147918701, |
| "margin_dpo/margin_std": 6.056692600250244, |
| "step": 93 |
| }, |
| { |
| "epoch": 0.1968586387434555, |
| "grad_norm": 15.55691909790039, |
| "learning_rate": 4.865480126133871e-07, |
| "logits/chosen": 1.7521295547485352, |
| "logits/rejected": 1.8241287469863892, |
| "logps/chosen": -250.36639404296875, |
| "logps/ref_chosen": -261.2528381347656, |
| "logps/ref_rejected": -269.2928771972656, |
| "logps/rejected": -263.65771484375, |
| "loss": 5.3919, |
| "margin_dpo/margin_mean": 5.2512712478637695, |
| "margin_dpo/margin_std": 7.996842384338379, |
| "step": 94 |
| }, |
| { |
| "epoch": 0.19895287958115182, |
| "grad_norm": 16.091550827026367, |
| "learning_rate": 4.859492293879573e-07, |
| "logits/chosen": 1.884320855140686, |
| "logits/rejected": 1.6460635662078857, |
| "logps/chosen": -334.9806823730469, |
| "logps/ref_chosen": -345.480224609375, |
| "logps/ref_rejected": -294.0064697265625, |
| "logps/rejected": -288.2855529785156, |
| "loss": 5.3719, |
| "margin_dpo/margin_mean": 4.778585433959961, |
| "margin_dpo/margin_std": 8.979715347290039, |
| "step": 95 |
| }, |
| { |
| "epoch": 0.20104712041884817, |
| "grad_norm": 15.211227416992188, |
| "learning_rate": 4.853377929214243e-07, |
| "logits/chosen": 1.442068099975586, |
| "logits/rejected": 1.3402963876724243, |
| "logps/chosen": -239.22625732421875, |
| "logps/ref_chosen": -249.85205078125, |
| "logps/ref_rejected": -274.1024169921875, |
| "logps/rejected": -266.5990295410156, |
| "loss": 5.3763, |
| "margin_dpo/margin_mean": 3.1224074363708496, |
| "margin_dpo/margin_std": 6.072546482086182, |
| "step": 96 |
| }, |
| { |
| "epoch": 0.2031413612565445, |
| "grad_norm": 15.3523588180542, |
| "learning_rate": 4.847137360032699e-07, |
| "logits/chosen": 1.682770013809204, |
| "logits/rejected": 1.789080262184143, |
| "logps/chosen": -224.3016815185547, |
| "logps/ref_chosen": -233.62025451660156, |
| "logps/ref_rejected": -258.32647705078125, |
| "logps/rejected": -253.4158935546875, |
| "loss": 5.3694, |
| "margin_dpo/margin_mean": 4.407979965209961, |
| "margin_dpo/margin_std": 6.528227806091309, |
| "step": 97 |
| }, |
| { |
| "epoch": 0.20523560209424083, |
| "grad_norm": 15.678691864013672, |
| "learning_rate": 4.84077092099773e-07, |
| "logits/chosen": 1.9161100387573242, |
| "logits/rejected": 2.1308605670928955, |
| "logps/chosen": -256.5081787109375, |
| "logps/ref_chosen": -267.27911376953125, |
| "logps/ref_rejected": -335.98284912109375, |
| "logps/rejected": -327.7537841796875, |
| "loss": 5.3668, |
| "margin_dpo/margin_mean": 2.5418522357940674, |
| "margin_dpo/margin_std": 7.70307731628418, |
| "step": 98 |
| }, |
| { |
| "epoch": 0.20732984293193718, |
| "grad_norm": 15.061153411865234, |
| "learning_rate": 4.834278953522137e-07, |
| "logits/chosen": 1.8618698120117188, |
| "logits/rejected": 1.8229163885116577, |
| "logps/chosen": -275.3375244140625, |
| "logps/ref_chosen": -285.90435791015625, |
| "logps/ref_rejected": -278.0072021484375, |
| "logps/rejected": -271.364013671875, |
| "loss": 5.3595, |
| "margin_dpo/margin_mean": 3.9236538410186768, |
| "margin_dpo/margin_std": 9.717151641845703, |
| "step": 99 |
| }, |
| { |
| "epoch": 0.2094240837696335, |
| "grad_norm": 15.537282943725586, |
| "learning_rate": 4.827661805750437e-07, |
| "logits/chosen": 1.6382941007614136, |
| "logits/rejected": 1.5340853929519653, |
| "logps/chosen": -327.0155944824219, |
| "logps/ref_chosen": -335.2471008300781, |
| "logps/ref_rejected": -304.7597351074219, |
| "logps/rejected": -300.1502380371094, |
| "loss": 5.3785, |
| "margin_dpo/margin_mean": 3.622096061706543, |
| "margin_dpo/margin_std": 7.071560859680176, |
| "step": 100 |
| }, |
| { |
| "epoch": 0.21151832460732983, |
| "grad_norm": 15.370798110961914, |
| "learning_rate": 4.820919832540181e-07, |
| "logits/chosen": 1.507110357284546, |
| "logits/rejected": 1.7560882568359375, |
| "logps/chosen": -262.4298400878906, |
| "logps/ref_chosen": -272.9364318847656, |
| "logps/ref_rejected": -271.82366943359375, |
| "logps/rejected": -268.6051330566406, |
| "loss": 5.3734, |
| "margin_dpo/margin_mean": 7.288076400756836, |
| "margin_dpo/margin_std": 8.229599952697754, |
| "step": 101 |
| }, |
| { |
| "epoch": 0.2136125654450262, |
| "grad_norm": 15.161721229553223, |
| "learning_rate": 4.814053395442932e-07, |
| "logits/chosen": 1.76149582862854, |
| "logits/rejected": 1.8778889179229736, |
| "logps/chosen": -151.35018920898438, |
| "logps/ref_chosen": -159.15536499023438, |
| "logps/ref_rejected": -191.47312927246094, |
| "logps/rejected": -188.49746704101562, |
| "loss": 5.305, |
| "margin_dpo/margin_mean": 4.829489707946777, |
| "margin_dpo/margin_std": 7.208276271820068, |
| "step": 102 |
| }, |
| { |
| "epoch": 0.2157068062827225, |
| "grad_norm": 15.669957160949707, |
| "learning_rate": 4.807062862684873e-07, |
| "logits/chosen": 2.092226505279541, |
| "logits/rejected": 2.202396869659424, |
| "logps/chosen": -291.3448486328125, |
| "logps/ref_chosen": -301.0699768066406, |
| "logps/ref_rejected": -306.12469482421875, |
| "logps/rejected": -298.96844482421875, |
| "loss": 5.3684, |
| "margin_dpo/margin_mean": 2.5688788890838623, |
| "margin_dpo/margin_std": 8.519815444946289, |
| "step": 103 |
| }, |
| { |
| "epoch": 0.21780104712041884, |
| "grad_norm": 14.514609336853027, |
| "learning_rate": 4.799948609147061e-07, |
| "logits/chosen": 1.875953197479248, |
| "logits/rejected": 1.7595632076263428, |
| "logps/chosen": -308.81158447265625, |
| "logps/ref_chosen": -316.44036865234375, |
| "logps/ref_rejected": -245.41790771484375, |
| "logps/rejected": -242.4790496826172, |
| "loss": 5.3826, |
| "margin_dpo/margin_mean": 4.689910411834717, |
| "margin_dpo/margin_std": 10.412727355957031, |
| "step": 104 |
| }, |
| { |
| "epoch": 0.2198952879581152, |
| "grad_norm": 17.934940338134766, |
| "learning_rate": 4.792711016345321e-07, |
| "logits/chosen": 1.8088258504867554, |
| "logits/rejected": 1.6915315389633179, |
| "logps/chosen": -253.7894744873047, |
| "logps/ref_chosen": -264.70599365234375, |
| "logps/ref_rejected": -232.14236450195312, |
| "logps/rejected": -229.86798095703125, |
| "loss": 5.241, |
| "margin_dpo/margin_mean": 8.642158508300781, |
| "margin_dpo/margin_std": 9.996341705322266, |
| "step": 105 |
| }, |
| { |
| "epoch": 0.22198952879581152, |
| "grad_norm": 16.461326599121094, |
| "learning_rate": 4.785350472409791e-07, |
| "logits/chosen": 1.8355200290679932, |
| "logits/rejected": 2.0365209579467773, |
| "logps/chosen": -274.2940673828125, |
| "logps/ref_chosen": -280.6784973144531, |
| "logps/ref_rejected": -353.0090026855469, |
| "logps/rejected": -352.0072021484375, |
| "loss": 5.3413, |
| "margin_dpo/margin_mean": 5.382654666900635, |
| "margin_dpo/margin_std": 9.296738624572754, |
| "step": 106 |
| }, |
| { |
| "epoch": 0.22408376963350785, |
| "grad_norm": 16.5199031829834, |
| "learning_rate": 4.777867372064105e-07, |
| "logits/chosen": 1.6165478229522705, |
| "logits/rejected": 1.5343804359436035, |
| "logps/chosen": -327.95794677734375, |
| "logps/ref_chosen": -336.91058349609375, |
| "logps/ref_rejected": -280.02325439453125, |
| "logps/rejected": -277.4742431640625, |
| "loss": 5.2697, |
| "margin_dpo/margin_mean": 6.403599262237549, |
| "margin_dpo/margin_std": 9.143040657043457, |
| "step": 107 |
| }, |
| { |
| "epoch": 0.2261780104712042, |
| "grad_norm": 16.170669555664062, |
| "learning_rate": 4.770262116604223e-07, |
| "logits/chosen": 1.8304221630096436, |
| "logits/rejected": 2.0288898944854736, |
| "logps/chosen": -224.6934356689453, |
| "logps/ref_chosen": -232.04891967773438, |
| "logps/ref_rejected": -248.3793487548828, |
| "logps/rejected": -246.96351623535156, |
| "loss": 5.2351, |
| "margin_dpo/margin_mean": 5.9396281242370605, |
| "margin_dpo/margin_std": 9.726874351501465, |
| "step": 108 |
| }, |
| { |
| "epoch": 0.22827225130890053, |
| "grad_norm": 17.59023094177246, |
| "learning_rate": 4.7625351138769166e-07, |
| "logits/chosen": 1.8960250616073608, |
| "logits/rejected": 1.919461965560913, |
| "logps/chosen": -236.6331329345703, |
| "logps/ref_chosen": -243.42401123046875, |
| "logps/ref_rejected": -276.1861877441406, |
| "logps/rejected": -274.469482421875, |
| "loss": 5.2323, |
| "margin_dpo/margin_mean": 5.074185371398926, |
| "margin_dpo/margin_std": 8.536012649536133, |
| "step": 109 |
| }, |
| { |
| "epoch": 0.23036649214659685, |
| "grad_norm": 15.948025703430176, |
| "learning_rate": 4.75468677825789e-07, |
| "logits/chosen": 1.6093004941940308, |
| "logits/rejected": 1.6397433280944824, |
| "logps/chosen": -234.94406127929688, |
| "logps/ref_chosen": -242.5493621826172, |
| "logps/ref_rejected": -195.59750366210938, |
| "logps/rejected": -193.1940155029297, |
| "loss": 5.246, |
| "margin_dpo/margin_mean": 5.201825141906738, |
| "margin_dpo/margin_std": 10.898710250854492, |
| "step": 110 |
| }, |
| { |
| "epoch": 0.2324607329842932, |
| "grad_norm": 18.407573699951172, |
| "learning_rate": 4.7467175306295647e-07, |
| "logits/chosen": 1.6897281408309937, |
| "logits/rejected": 1.7771556377410889, |
| "logps/chosen": -272.2618408203125, |
| "logps/ref_chosen": -279.930908203125, |
| "logps/ref_rejected": -281.9147644042969, |
| "logps/rejected": -282.97882080078125, |
| "loss": 5.223, |
| "margin_dpo/margin_mean": 8.733101844787598, |
| "margin_dpo/margin_std": 11.250627517700195, |
| "step": 111 |
| }, |
| { |
| "epoch": 0.23455497382198953, |
| "grad_norm": 14.606575012207031, |
| "learning_rate": 4.7386277983585053e-07, |
| "logits/chosen": 1.776769757270813, |
| "logits/rejected": 1.8782975673675537, |
| "logps/chosen": -243.624755859375, |
| "logps/ref_chosen": -246.89129638671875, |
| "logps/ref_rejected": -266.50506591796875, |
| "logps/rejected": -265.4157409667969, |
| "loss": 5.395, |
| "margin_dpo/margin_mean": 2.177186965942383, |
| "margin_dpo/margin_std": 10.55683422088623, |
| "step": 112 |
| }, |
| { |
| "epoch": 0.23664921465968586, |
| "grad_norm": 16.843202590942383, |
| "learning_rate": 4.7304180152725024e-07, |
| "logits/chosen": 1.4505597352981567, |
| "logits/rejected": 1.5904918909072876, |
| "logps/chosen": -269.72711181640625, |
| "logps/ref_chosen": -276.4613342285156, |
| "logps/ref_rejected": -341.7659912109375, |
| "logps/rejected": -342.48956298828125, |
| "loss": 5.1716, |
| "margin_dpo/margin_mean": 7.457816123962402, |
| "margin_dpo/margin_std": 12.05935287475586, |
| "step": 113 |
| }, |
| { |
| "epoch": 0.2387434554973822, |
| "grad_norm": 16.080974578857422, |
| "learning_rate": 4.7220886216373085e-07, |
| "logits/chosen": 1.6391416788101196, |
| "logits/rejected": 1.5624871253967285, |
| "logps/chosen": -247.58502197265625, |
| "logps/ref_chosen": -251.4463653564453, |
| "logps/ref_rejected": -210.03152465820312, |
| "logps/rejected": -213.17721557617188, |
| "loss": 5.3559, |
| "margin_dpo/margin_mean": 7.00706672668457, |
| "margin_dpo/margin_std": 8.853893280029297, |
| "step": 114 |
| }, |
| { |
| "epoch": 0.24083769633507854, |
| "grad_norm": 15.944089889526367, |
| "learning_rate": 4.7136400641330245e-07, |
| "logits/chosen": 1.8735270500183105, |
| "logits/rejected": 1.6029636859893799, |
| "logps/chosen": -253.3223876953125, |
| "logps/ref_chosen": -257.82574462890625, |
| "logps/ref_rejected": -192.41648864746094, |
| "logps/rejected": -191.51156616210938, |
| "loss": 5.3403, |
| "margin_dpo/margin_mean": 3.5983924865722656, |
| "margin_dpo/margin_std": 8.953197479248047, |
| "step": 115 |
| }, |
| { |
| "epoch": 0.24293193717277486, |
| "grad_norm": 16.291976928710938, |
| "learning_rate": 4.70507279583015e-07, |
| "logits/chosen": 1.695469856262207, |
| "logits/rejected": 1.8081481456756592, |
| "logps/chosen": -242.69943237304688, |
| "logps/ref_chosen": -248.17518615722656, |
| "logps/ref_rejected": -274.10870361328125, |
| "logps/rejected": -276.470703125, |
| "loss": 5.2459, |
| "margin_dpo/margin_mean": 7.837741851806641, |
| "margin_dpo/margin_std": 9.762773513793945, |
| "step": 116 |
| }, |
| { |
| "epoch": 0.2450261780104712, |
| "grad_norm": 16.642024993896484, |
| "learning_rate": 4.6963872761652834e-07, |
| "logits/chosen": 1.6590253114700317, |
| "logits/rejected": 1.4430992603302002, |
| "logps/chosen": -229.59909057617188, |
| "logps/ref_chosen": -235.29620361328125, |
| "logps/ref_rejected": -190.87095642089844, |
| "logps/rejected": -194.7079620361328, |
| "loss": 5.2344, |
| "margin_dpo/margin_mean": 9.53414535522461, |
| "margin_dpo/margin_std": 9.101675033569336, |
| "step": 117 |
| }, |
| { |
| "epoch": 0.24712041884816754, |
| "grad_norm": 20.776172637939453, |
| "learning_rate": 4.687583970916486e-07, |
| "logits/chosen": 1.6007872819900513, |
| "logits/rejected": 1.6555330753326416, |
| "logps/chosen": -256.0022277832031, |
| "logps/ref_chosen": -260.44781494140625, |
| "logps/ref_rejected": -308.3326416015625, |
| "logps/rejected": -313.2330627441406, |
| "loss": 5.2168, |
| "margin_dpo/margin_mean": 9.346000671386719, |
| "margin_dpo/margin_std": 13.443426132202148, |
| "step": 118 |
| }, |
| { |
| "epoch": 0.24921465968586387, |
| "grad_norm": 15.843025207519531, |
| "learning_rate": 4.6786633521783005e-07, |
| "logits/chosen": 1.9080662727355957, |
| "logits/rejected": 2.017760753631592, |
| "logps/chosen": -282.1200866699219, |
| "logps/ref_chosen": -286.9692687988281, |
| "logps/ref_rejected": -331.7510986328125, |
| "logps/rejected": -331.0477600097656, |
| "loss": 5.2789, |
| "margin_dpo/margin_mean": 4.145843982696533, |
| "margin_dpo/margin_std": 13.910889625549316, |
| "step": 119 |
| }, |
| { |
| "epoch": 0.2513089005235602, |
| "grad_norm": 16.17180061340332, |
| "learning_rate": 4.669625898336438e-07, |
| "logits/chosen": 1.9562854766845703, |
| "logits/rejected": 1.8660322427749634, |
| "logps/chosen": -278.0622253417969, |
| "logps/ref_chosen": -281.98077392578125, |
| "logps/ref_rejected": -283.52679443359375, |
| "logps/rejected": -288.8341369628906, |
| "loss": 5.2221, |
| "margin_dpo/margin_mean": 9.225922584533691, |
| "margin_dpo/margin_std": 13.011504173278809, |
| "step": 120 |
| }, |
| { |
| "epoch": 0.2534031413612565, |
| "grad_norm": 14.78394889831543, |
| "learning_rate": 4.6604720940421207e-07, |
| "logits/chosen": 1.1911287307739258, |
| "logits/rejected": 1.5024229288101196, |
| "logps/chosen": -144.0179443359375, |
| "logps/ref_chosen": -145.69662475585938, |
| "logps/ref_rejected": -195.21612548828125, |
| "logps/rejected": -199.57223510742188, |
| "loss": 5.3557, |
| "margin_dpo/margin_mean": 6.034780025482178, |
| "margin_dpo/margin_std": 9.371333122253418, |
| "step": 121 |
| }, |
| { |
| "epoch": 0.2554973821989529, |
| "grad_norm": 15.84109115600586, |
| "learning_rate": 4.651202430186092e-07, |
| "logits/chosen": 1.7703770399093628, |
| "logits/rejected": 2.0983457565307617, |
| "logps/chosen": -245.3528289794922, |
| "logps/ref_chosen": -252.1569366455078, |
| "logps/ref_rejected": -309.68548583984375, |
| "logps/rejected": -306.59942626953125, |
| "loss": 5.2869, |
| "margin_dpo/margin_mean": 3.718092441558838, |
| "margin_dpo/margin_std": 17.503427505493164, |
| "step": 122 |
| }, |
| { |
| "epoch": 0.25759162303664923, |
| "grad_norm": 18.21240997314453, |
| "learning_rate": 4.6418174038722924e-07, |
| "logits/chosen": 1.6439917087554932, |
| "logits/rejected": 1.4977948665618896, |
| "logps/chosen": -358.5697326660156, |
| "logps/ref_chosen": -366.5253601074219, |
| "logps/ref_rejected": -285.2503662109375, |
| "logps/rejected": -286.8184814453125, |
| "loss": 5.1686, |
| "margin_dpo/margin_mean": 9.523737907409668, |
| "margin_dpo/margin_std": 12.473346710205078, |
| "step": 123 |
| }, |
| { |
| "epoch": 0.25968586387434556, |
| "grad_norm": 16.816696166992188, |
| "learning_rate": 4.6323175183912023e-07, |
| "logits/chosen": 1.4895159006118774, |
| "logits/rejected": 1.6259382963180542, |
| "logps/chosen": -244.55775451660156, |
| "logps/ref_chosen": -251.4420623779297, |
| "logps/ref_rejected": -231.0302734375, |
| "logps/rejected": -230.74337768554688, |
| "loss": 5.2019, |
| "margin_dpo/margin_mean": 6.597394943237305, |
| "margin_dpo/margin_std": 15.111127853393555, |
| "step": 124 |
| }, |
| { |
| "epoch": 0.2617801047120419, |
| "grad_norm": 16.386260986328125, |
| "learning_rate": 4.6227032831928483e-07, |
| "logits/chosen": 1.6032323837280273, |
| "logits/rejected": 1.598075032234192, |
| "logps/chosen": -242.79583740234375, |
| "logps/ref_chosen": -248.3984375, |
| "logps/ref_rejected": -307.77557373046875, |
| "logps/rejected": -308.05059814453125, |
| "loss": 5.2189, |
| "margin_dpo/margin_mean": 5.877676010131836, |
| "margin_dpo/margin_std": 13.811307907104492, |
| "step": 125 |
| }, |
| { |
| "epoch": 0.2638743455497382, |
| "grad_norm": 16.69825553894043, |
| "learning_rate": 4.612975213859487e-07, |
| "logits/chosen": 1.7347309589385986, |
| "logits/rejected": 1.9158421754837036, |
| "logps/chosen": -291.75244140625, |
| "logps/ref_chosen": -295.82366943359375, |
| "logps/ref_rejected": -295.2666931152344, |
| "logps/rejected": -299.0240783691406, |
| "loss": 5.1658, |
| "margin_dpo/margin_mean": 7.8285651206970215, |
| "margin_dpo/margin_std": 13.573448181152344, |
| "step": 126 |
| }, |
| { |
| "epoch": 0.26596858638743454, |
| "grad_norm": 16.867650985717773, |
| "learning_rate": 4.603133832077953e-07, |
| "logits/chosen": 1.1577448844909668, |
| "logits/rejected": 1.106504201889038, |
| "logps/chosen": -273.8604736328125, |
| "logps/ref_chosen": -279.496337890625, |
| "logps/ref_rejected": -278.802978515625, |
| "logps/rejected": -282.5022277832031, |
| "loss": 5.1019, |
| "margin_dpo/margin_mean": 9.335136413574219, |
| "margin_dpo/margin_std": 13.274786949157715, |
| "step": 127 |
| }, |
| { |
| "epoch": 0.2680628272251309, |
| "grad_norm": 16.078149795532227, |
| "learning_rate": 4.5931796656116837e-07, |
| "logits/chosen": 1.4048773050308228, |
| "logits/rejected": 1.4003376960754395, |
| "logps/chosen": -258.86883544921875, |
| "logps/ref_chosen": -264.52252197265625, |
| "logps/ref_rejected": -239.76937866210938, |
| "logps/rejected": -247.31756591796875, |
| "loss": 5.057, |
| "margin_dpo/margin_mean": 13.201865196228027, |
| "margin_dpo/margin_std": 13.145772933959961, |
| "step": 128 |
| }, |
| { |
| "epoch": 0.27015706806282724, |
| "grad_norm": 16.511791229248047, |
| "learning_rate": 4.5831132482724193e-07, |
| "logits/chosen": 1.5169470310211182, |
| "logits/rejected": 1.6699875593185425, |
| "logps/chosen": -290.08258056640625, |
| "logps/ref_chosen": -296.95233154296875, |
| "logps/ref_rejected": -260.0984802246094, |
| "logps/rejected": -267.3520812988281, |
| "loss": 5.1034, |
| "margin_dpo/margin_mean": 14.12340259552002, |
| "margin_dpo/margin_std": 17.277435302734375, |
| "step": 129 |
| }, |
| { |
| "epoch": 0.27225130890052357, |
| "grad_norm": 21.19081687927246, |
| "learning_rate": 4.5729351198915705e-07, |
| "logits/chosen": 1.5894699096679688, |
| "logits/rejected": 1.8434865474700928, |
| "logps/chosen": -263.253173828125, |
| "logps/ref_chosen": -274.7286682128906, |
| "logps/ref_rejected": -325.187255859375, |
| "logps/rejected": -327.59503173828125, |
| "loss": 5.1333, |
| "margin_dpo/margin_mean": 13.883302688598633, |
| "margin_dpo/margin_std": 18.286108016967773, |
| "step": 130 |
| }, |
| { |
| "epoch": 0.2743455497382199, |
| "grad_norm": 17.56028175354004, |
| "learning_rate": 4.5626458262912735e-07, |
| "logits/chosen": 1.44374680519104, |
| "logits/rejected": 1.400553822517395, |
| "logps/chosen": -270.84796142578125, |
| "logps/ref_chosen": -279.3233642578125, |
| "logps/ref_rejected": -299.2681884765625, |
| "logps/rejected": -304.762451171875, |
| "loss": 5.2149, |
| "margin_dpo/margin_mean": 13.969644546508789, |
| "margin_dpo/margin_std": 20.493703842163086, |
| "step": 131 |
| }, |
| { |
| "epoch": 0.2764397905759162, |
| "grad_norm": 17.945659637451172, |
| "learning_rate": 4.5522459192551166e-07, |
| "logits/chosen": 1.5821537971496582, |
| "logits/rejected": 1.6266090869903564, |
| "logps/chosen": -281.5635681152344, |
| "logps/ref_chosen": -291.3346862792969, |
| "logps/ref_rejected": -283.13311767578125, |
| "logps/rejected": -291.1026306152344, |
| "loss": 5.1007, |
| "margin_dpo/margin_mean": 17.740650177001953, |
| "margin_dpo/margin_std": 17.511295318603516, |
| "step": 132 |
| }, |
| { |
| "epoch": 0.27853403141361255, |
| "grad_norm": 16.447092056274414, |
| "learning_rate": 4.541735956498554e-07, |
| "logits/chosen": 1.6146799325942993, |
| "logits/rejected": 1.5560858249664307, |
| "logps/chosen": -223.23194885253906, |
| "logps/ref_chosen": -233.71875, |
| "logps/ref_rejected": -216.53781127929688, |
| "logps/rejected": -223.18417358398438, |
| "loss": 5.0683, |
| "margin_dpo/margin_mean": 17.133150100708008, |
| "margin_dpo/margin_std": 14.197755813598633, |
| "step": 133 |
| }, |
| { |
| "epoch": 0.2806282722513089, |
| "grad_norm": 20.631309509277344, |
| "learning_rate": 4.5311165016389914e-07, |
| "logits/chosen": 1.920145869255066, |
| "logits/rejected": 1.981586217880249, |
| "logps/chosen": -348.9212951660156, |
| "logps/ref_chosen": -348.29547119140625, |
| "logps/ref_rejected": -343.04510498046875, |
| "logps/rejected": -351.0985412597656, |
| "loss": 5.22, |
| "margin_dpo/margin_mean": 7.427634239196777, |
| "margin_dpo/margin_std": 17.99872589111328, |
| "step": 134 |
| }, |
| { |
| "epoch": 0.28272251308900526, |
| "grad_norm": 17.117826461791992, |
| "learning_rate": 4.520388124165564e-07, |
| "logits/chosen": 1.1408627033233643, |
| "logits/rejected": 0.9298585057258606, |
| "logps/chosen": -226.50486755371094, |
| "logps/ref_chosen": -232.59129333496094, |
| "logps/ref_rejected": -175.74066162109375, |
| "logps/rejected": -181.21482849121094, |
| "loss": 5.0762, |
| "margin_dpo/margin_mean": 11.560598373413086, |
| "margin_dpo/margin_std": 14.716351509094238, |
| "step": 135 |
| }, |
| { |
| "epoch": 0.2848167539267016, |
| "grad_norm": 19.100990295410156, |
| "learning_rate": 4.5095513994085974e-07, |
| "logits/chosen": 1.0842311382293701, |
| "logits/rejected": 1.3095265626907349, |
| "logps/chosen": -183.53028869628906, |
| "logps/ref_chosen": -189.21795654296875, |
| "logps/ref_rejected": -191.75979614257812, |
| "logps/rejected": -200.99093627929688, |
| "loss": 5.1055, |
| "margin_dpo/margin_mean": 14.91882038116455, |
| "margin_dpo/margin_std": 16.662824630737305, |
| "step": 136 |
| }, |
| { |
| "epoch": 0.2869109947643979, |
| "grad_norm": 17.85831642150879, |
| "learning_rate": 4.498606908508753e-07, |
| "logits/chosen": 1.86328125, |
| "logits/rejected": 1.6945784091949463, |
| "logps/chosen": -356.1871032714844, |
| "logps/ref_chosen": -358.9820861816406, |
| "logps/ref_rejected": -277.2926330566406, |
| "logps/rejected": -286.5790710449219, |
| "loss": 5.1522, |
| "margin_dpo/margin_mean": 12.081487655639648, |
| "margin_dpo/margin_std": 15.93864631652832, |
| "step": 137 |
| }, |
| { |
| "epoch": 0.28900523560209423, |
| "grad_norm": 16.747636795043945, |
| "learning_rate": 4.487555238385862e-07, |
| "logits/chosen": 1.8756850957870483, |
| "logits/rejected": 1.9149004220962524, |
| "logps/chosen": -284.18572998046875, |
| "logps/ref_chosen": -283.7969055175781, |
| "logps/ref_rejected": -269.28643798828125, |
| "logps/rejected": -280.577880859375, |
| "loss": 5.1753, |
| "margin_dpo/margin_mean": 10.902585983276367, |
| "margin_dpo/margin_std": 22.474666595458984, |
| "step": 138 |
| }, |
| { |
| "epoch": 0.29109947643979056, |
| "grad_norm": 16.213245391845703, |
| "learning_rate": 4.476396981707453e-07, |
| "logits/chosen": 1.4325203895568848, |
| "logits/rejected": 1.5791009664535522, |
| "logps/chosen": -218.7724609375, |
| "logps/ref_chosen": -221.46124267578125, |
| "logps/ref_rejected": -234.3295440673828, |
| "logps/rejected": -236.3891143798828, |
| "loss": 5.2792, |
| "margin_dpo/margin_mean": 4.748367786407471, |
| "margin_dpo/margin_std": 21.647659301757812, |
| "step": 139 |
| }, |
| { |
| "epoch": 0.2931937172774869, |
| "grad_norm": 25.28333854675293, |
| "learning_rate": 4.4651327368569684e-07, |
| "logits/chosen": 1.5151917934417725, |
| "logits/rejected": 1.5757999420166016, |
| "logps/chosen": -237.8951873779297, |
| "logps/ref_chosen": -246.27151489257812, |
| "logps/ref_rejected": -255.00428771972656, |
| "logps/rejected": -261.1990661621094, |
| "loss": 5.0124, |
| "margin_dpo/margin_mean": 14.571114540100098, |
| "margin_dpo/margin_std": 15.991363525390625, |
| "step": 140 |
| }, |
| { |
| "epoch": 0.29528795811518327, |
| "grad_norm": 23.41806411743164, |
| "learning_rate": 4.453763107901675e-07, |
| "logits/chosen": 1.5907336473464966, |
| "logits/rejected": 1.6988296508789062, |
| "logps/chosen": -264.4815979003906, |
| "logps/ref_chosen": -267.79345703125, |
| "logps/ref_rejected": -295.4119873046875, |
| "logps/rejected": -308.6583557128906, |
| "loss": 5.0535, |
| "margin_dpo/margin_mean": 16.5582332611084, |
| "margin_dpo/margin_std": 21.787641525268555, |
| "step": 141 |
| }, |
| { |
| "epoch": 0.2973821989528796, |
| "grad_norm": 17.388578414916992, |
| "learning_rate": 4.4422887045602674e-07, |
| "logits/chosen": 1.8900976181030273, |
| "logits/rejected": 1.6236388683319092, |
| "logps/chosen": -341.6228942871094, |
| "logps/ref_chosen": -352.8658752441406, |
| "logps/ref_rejected": -219.5095672607422, |
| "logps/rejected": -223.31808471679688, |
| "loss": 5.0641, |
| "margin_dpo/margin_mean": 15.05146598815918, |
| "margin_dpo/margin_std": 16.49203109741211, |
| "step": 142 |
| }, |
| { |
| "epoch": 0.2994764397905759, |
| "grad_norm": 18.831209182739258, |
| "learning_rate": 4.4307101421701755e-07, |
| "logits/chosen": 1.4374781847000122, |
| "logits/rejected": 1.3258070945739746, |
| "logps/chosen": -327.2297058105469, |
| "logps/ref_chosen": -336.38482666015625, |
| "logps/ref_rejected": -213.85707092285156, |
| "logps/rejected": -229.42831420898438, |
| "loss": 4.9898, |
| "margin_dpo/margin_mean": 24.726341247558594, |
| "margin_dpo/margin_std": 23.126943588256836, |
| "step": 143 |
| }, |
| { |
| "epoch": 0.30157068062827225, |
| "grad_norm": 19.49022674560547, |
| "learning_rate": 4.419028041654559e-07, |
| "logits/chosen": 1.4996888637542725, |
| "logits/rejected": 1.4556653499603271, |
| "logps/chosen": -264.51727294921875, |
| "logps/ref_chosen": -274.0345458984375, |
| "logps/ref_rejected": -274.5603942871094, |
| "logps/rejected": -273.80316162109375, |
| "loss": 5.0793, |
| "margin_dpo/margin_mean": 8.760041236877441, |
| "margin_dpo/margin_std": 19.655494689941406, |
| "step": 144 |
| }, |
| { |
| "epoch": 0.3036649214659686, |
| "grad_norm": 16.73856544494629, |
| "learning_rate": 4.4072430294890166e-07, |
| "logits/chosen": 1.6811779737472534, |
| "logits/rejected": 1.7291994094848633, |
| "logps/chosen": -269.10491943359375, |
| "logps/ref_chosen": -274.1513366699219, |
| "logps/ref_rejected": -226.63064575195312, |
| "logps/rejected": -239.5631866455078, |
| "loss": 4.9405, |
| "margin_dpo/margin_mean": 17.978939056396484, |
| "margin_dpo/margin_std": 23.945186614990234, |
| "step": 145 |
| }, |
| { |
| "epoch": 0.3057591623036649, |
| "grad_norm": 29.0201358795166, |
| "learning_rate": 4.395355737667985e-07, |
| "logits/chosen": 1.4950841665267944, |
| "logits/rejected": 1.7295074462890625, |
| "logps/chosen": -227.69259643554688, |
| "logps/ref_chosen": -229.48269653320312, |
| "logps/ref_rejected": -249.7940216064453, |
| "logps/rejected": -259.9455261230469, |
| "loss": 5.0665, |
| "margin_dpo/margin_mean": 11.941591262817383, |
| "margin_dpo/margin_std": 17.751564025878906, |
| "step": 146 |
| }, |
| { |
| "epoch": 0.3078534031413613, |
| "grad_norm": 16.784557342529297, |
| "learning_rate": 4.3833668036708483e-07, |
| "logits/chosen": 1.4946480989456177, |
| "logits/rejected": 1.4648349285125732, |
| "logps/chosen": -284.0057067871094, |
| "logps/ref_chosen": -290.8128356933594, |
| "logps/ref_rejected": -218.97787475585938, |
| "logps/rejected": -229.5756072998047, |
| "loss": 5.1681, |
| "margin_dpo/margin_mean": 17.40483856201172, |
| "margin_dpo/margin_std": 24.006677627563477, |
| "step": 147 |
| }, |
| { |
| "epoch": 0.3099476439790576, |
| "grad_norm": 16.890840530395508, |
| "learning_rate": 4.3712768704277524e-07, |
| "logits/chosen": 1.5196683406829834, |
| "logits/rejected": 1.509922742843628, |
| "logps/chosen": -261.50762939453125, |
| "logps/ref_chosen": -263.70001220703125, |
| "logps/ref_rejected": -262.095703125, |
| "logps/rejected": -272.2943115234375, |
| "loss": 5.1353, |
| "margin_dpo/margin_mean": 12.390965461730957, |
| "margin_dpo/margin_std": 21.307296752929688, |
| "step": 148 |
| }, |
| { |
| "epoch": 0.31204188481675393, |
| "grad_norm": 18.739173889160156, |
| "learning_rate": 4.3590865862851263e-07, |
| "logits/chosen": 1.9116061925888062, |
| "logits/rejected": 1.7204910516738892, |
| "logps/chosen": -344.4569396972656, |
| "logps/ref_chosen": -350.6168518066406, |
| "logps/ref_rejected": -277.2320251464844, |
| "logps/rejected": -288.07904052734375, |
| "loss": 5.0166, |
| "margin_dpo/margin_mean": 17.006885528564453, |
| "margin_dpo/margin_std": 17.385805130004883, |
| "step": 149 |
| }, |
| { |
| "epoch": 0.31413612565445026, |
| "grad_norm": 17.36412811279297, |
| "learning_rate": 4.346796604970912e-07, |
| "logits/chosen": 1.934645652770996, |
| "logits/rejected": 1.848956823348999, |
| "logps/chosen": -261.2005920410156, |
| "logps/ref_chosen": -264.05096435546875, |
| "logps/ref_rejected": -286.02313232421875, |
| "logps/rejected": -298.2835693359375, |
| "loss": 5.0369, |
| "margin_dpo/margin_mean": 15.110857963562012, |
| "margin_dpo/margin_std": 18.34941291809082, |
| "step": 150 |
| }, |
| { |
| "epoch": 0.3162303664921466, |
| "grad_norm": 20.5943546295166, |
| "learning_rate": 4.3344075855595097e-07, |
| "logits/chosen": 1.3573246002197266, |
| "logits/rejected": 1.373565673828125, |
| "logps/chosen": -254.92498779296875, |
| "logps/ref_chosen": -257.74664306640625, |
| "logps/ref_rejected": -256.2339172363281, |
| "logps/rejected": -267.41278076171875, |
| "loss": 4.7848, |
| "margin_dpo/margin_mean": 14.000543594360352, |
| "margin_dpo/margin_std": 23.76491928100586, |
| "step": 151 |
| }, |
| { |
| "epoch": 0.3183246073298429, |
| "grad_norm": 21.099018096923828, |
| "learning_rate": 4.3219201924364323e-07, |
| "logits/chosen": 1.4018583297729492, |
| "logits/rejected": 1.803174376487732, |
| "logps/chosen": -245.9750213623047, |
| "logps/ref_chosen": -250.47512817382812, |
| "logps/ref_rejected": -322.36474609375, |
| "logps/rejected": -333.2466735839844, |
| "loss": 4.9817, |
| "margin_dpo/margin_mean": 15.381957054138184, |
| "margin_dpo/margin_std": 22.273956298828125, |
| "step": 152 |
| }, |
| { |
| "epoch": 0.3204188481675393, |
| "grad_norm": 22.535673141479492, |
| "learning_rate": 4.309335095262675e-07, |
| "logits/chosen": 1.5490094423294067, |
| "logits/rejected": 1.5208497047424316, |
| "logps/chosen": -235.2023162841797, |
| "logps/ref_chosen": -238.36544799804688, |
| "logps/ref_rejected": -215.78970336914062, |
| "logps/rejected": -236.40200805664062, |
| "loss": 4.6931, |
| "margin_dpo/margin_mean": 23.775440216064453, |
| "margin_dpo/margin_std": 23.90810775756836, |
| "step": 153 |
| }, |
| { |
| "epoch": 0.3225130890052356, |
| "grad_norm": 19.634624481201172, |
| "learning_rate": 4.2966529689388064e-07, |
| "logits/chosen": 1.213348627090454, |
| "logits/rejected": 1.2180352210998535, |
| "logps/chosen": -264.2608337402344, |
| "logps/ref_chosen": -259.7012939453125, |
| "logps/ref_rejected": -255.74172973632812, |
| "logps/rejected": -272.3033142089844, |
| "loss": 4.9699, |
| "margin_dpo/margin_mean": 12.002017974853516, |
| "margin_dpo/margin_std": 28.510345458984375, |
| "step": 154 |
| }, |
| { |
| "epoch": 0.32460732984293195, |
| "grad_norm": 19.311044692993164, |
| "learning_rate": 4.2838744935687716e-07, |
| "logits/chosen": 1.4307262897491455, |
| "logits/rejected": 1.4176700115203857, |
| "logps/chosen": -324.7783203125, |
| "logps/ref_chosen": -325.11517333984375, |
| "logps/ref_rejected": -288.08380126953125, |
| "logps/rejected": -307.0673828125, |
| "loss": 5.1244, |
| "margin_dpo/margin_mean": 19.320411682128906, |
| "margin_dpo/margin_std": 28.002426147460938, |
| "step": 155 |
| }, |
| { |
| "epoch": 0.3267015706806283, |
| "grad_norm": 19.045074462890625, |
| "learning_rate": 4.271000354423425e-07, |
| "logits/chosen": 1.6414060592651367, |
| "logits/rejected": 1.486697793006897, |
| "logps/chosen": -260.87078857421875, |
| "logps/ref_chosen": -263.62353515625, |
| "logps/ref_rejected": -183.94119262695312, |
| "logps/rejected": -202.97857666015625, |
| "loss": 4.8187, |
| "margin_dpo/margin_mean": 21.790143966674805, |
| "margin_dpo/margin_std": 19.9763240814209, |
| "step": 156 |
| }, |
| { |
| "epoch": 0.3287958115183246, |
| "grad_norm": 24.014020919799805, |
| "learning_rate": 4.258031241903777e-07, |
| "logits/chosen": 1.4358762502670288, |
| "logits/rejected": 1.5518109798431396, |
| "logps/chosen": -248.9981231689453, |
| "logps/ref_chosen": -237.6883087158203, |
| "logps/ref_rejected": -232.87484741210938, |
| "logps/rejected": -254.34902954101562, |
| "loss": 5.0507, |
| "margin_dpo/margin_mean": 10.164348602294922, |
| "margin_dpo/margin_std": 22.118553161621094, |
| "step": 157 |
| }, |
| { |
| "epoch": 0.3308900523560209, |
| "grad_norm": 19.902008056640625, |
| "learning_rate": 4.2449678515039743e-07, |
| "logits/chosen": 1.7699273824691772, |
| "logits/rejected": 1.8686857223510742, |
| "logps/chosen": -284.7595520019531, |
| "logps/ref_chosen": -279.62335205078125, |
| "logps/ref_rejected": -267.80615234375, |
| "logps/rejected": -285.53924560546875, |
| "loss": 4.9867, |
| "margin_dpo/margin_mean": 12.59688663482666, |
| "margin_dpo/margin_std": 24.365018844604492, |
| "step": 158 |
| }, |
| { |
| "epoch": 0.33298429319371725, |
| "grad_norm": 22.825597763061523, |
| "learning_rate": 4.2318108837739986e-07, |
| "logits/chosen": 1.553140640258789, |
| "logits/rejected": 1.439896583557129, |
| "logps/chosen": -303.68487548828125, |
| "logps/ref_chosen": -301.5324401855469, |
| "logps/ref_rejected": -263.529541015625, |
| "logps/rejected": -274.5115966796875, |
| "loss": 5.1446, |
| "margin_dpo/margin_mean": 8.82960319519043, |
| "margin_dpo/margin_std": 30.41036605834961, |
| "step": 159 |
| }, |
| { |
| "epoch": 0.33507853403141363, |
| "grad_norm": 20.597837448120117, |
| "learning_rate": 4.218561044282098e-07, |
| "logits/chosen": 1.9710590839385986, |
| "logits/rejected": 1.699224829673767, |
| "logps/chosen": -311.9967041015625, |
| "logps/ref_chosen": -314.1754455566406, |
| "logps/ref_rejected": -241.1903076171875, |
| "logps/rejected": -267.9695129394531, |
| "loss": 4.8583, |
| "margin_dpo/margin_mean": 28.957944869995117, |
| "margin_dpo/margin_std": 29.692768096923828, |
| "step": 160 |
| }, |
| { |
| "epoch": 0.33717277486910996, |
| "grad_norm": 25.702106475830078, |
| "learning_rate": 4.2052190435769554e-07, |
| "logits/chosen": 1.345297932624817, |
| "logits/rejected": 1.2094160318374634, |
| "logps/chosen": -268.9353942871094, |
| "logps/ref_chosen": -271.0775451660156, |
| "logps/ref_rejected": -212.71853637695312, |
| "logps/rejected": -228.42201232910156, |
| "loss": 4.8254, |
| "margin_dpo/margin_mean": 17.845624923706055, |
| "margin_dpo/margin_std": 25.553813934326172, |
| "step": 161 |
| }, |
| { |
| "epoch": 0.3392670157068063, |
| "grad_norm": 26.607454299926758, |
| "learning_rate": 4.1917855971495763e-07, |
| "logits/chosen": 1.6022093296051025, |
| "logits/rejected": 1.4934312105178833, |
| "logps/chosen": -293.98974609375, |
| "logps/ref_chosen": -296.7241516113281, |
| "logps/ref_rejected": -222.9241485595703, |
| "logps/rejected": -236.45635986328125, |
| "loss": 4.8946, |
| "margin_dpo/margin_mean": 16.266626358032227, |
| "margin_dpo/margin_std": 23.54033660888672, |
| "step": 162 |
| }, |
| { |
| "epoch": 0.3413612565445026, |
| "grad_norm": 30.976999282836914, |
| "learning_rate": 4.1782614253949255e-07, |
| "logits/chosen": 1.7216789722442627, |
| "logits/rejected": 1.7484244108200073, |
| "logps/chosen": -246.4569549560547, |
| "logps/ref_chosen": -249.64366149902344, |
| "logps/ref_rejected": -244.58258056640625, |
| "logps/rejected": -260.55218505859375, |
| "loss": 4.7557, |
| "margin_dpo/margin_mean": 19.156299591064453, |
| "margin_dpo/margin_std": 21.998430252075195, |
| "step": 163 |
| }, |
| { |
| "epoch": 0.34345549738219894, |
| "grad_norm": 22.384260177612305, |
| "learning_rate": 4.164647253573289e-07, |
| "logits/chosen": 1.4122521877288818, |
| "logits/rejected": 1.5924245119094849, |
| "logps/chosen": -214.8105926513672, |
| "logps/ref_chosen": -203.6176300048828, |
| "logps/ref_rejected": -216.5535888671875, |
| "logps/rejected": -240.29396057128906, |
| "loss": 4.8891, |
| "margin_dpo/margin_mean": 12.547422409057617, |
| "margin_dpo/margin_std": 23.612470626831055, |
| "step": 164 |
| }, |
| { |
| "epoch": 0.34554973821989526, |
| "grad_norm": 28.401796340942383, |
| "learning_rate": 4.1509438117713863e-07, |
| "logits/chosen": 2.1252005100250244, |
| "logits/rejected": 2.1613521575927734, |
| "logps/chosen": -350.52978515625, |
| "logps/ref_chosen": -344.1730651855469, |
| "logps/ref_rejected": -304.00128173828125, |
| "logps/rejected": -327.9066467285156, |
| "loss": 4.9931, |
| "margin_dpo/margin_mean": 17.548656463623047, |
| "margin_dpo/margin_std": 27.731534957885742, |
| "step": 165 |
| }, |
| { |
| "epoch": 0.34764397905759165, |
| "grad_norm": 19.331459045410156, |
| "learning_rate": 4.137151834863213e-07, |
| "logits/chosen": 1.646728277206421, |
| "logits/rejected": 1.640990138053894, |
| "logps/chosen": -242.68841552734375, |
| "logps/ref_chosen": -233.72891235351562, |
| "logps/ref_rejected": -208.29397583007812, |
| "logps/rejected": -224.4010467529297, |
| "loss": 5.0854, |
| "margin_dpo/margin_mean": 7.147580146789551, |
| "margin_dpo/margin_std": 26.665321350097656, |
| "step": 166 |
| }, |
| { |
| "epoch": 0.34973821989528797, |
| "grad_norm": 19.852638244628906, |
| "learning_rate": 4.123272062470633e-07, |
| "logits/chosen": 1.6361035108566284, |
| "logits/rejected": 1.411129117012024, |
| "logps/chosen": -327.1979064941406, |
| "logps/ref_chosen": -326.10198974609375, |
| "logps/ref_rejected": -232.0992889404297, |
| "logps/rejected": -256.6214294433594, |
| "loss": 4.9473, |
| "margin_dpo/margin_mean": 23.426191329956055, |
| "margin_dpo/margin_std": 28.530874252319336, |
| "step": 167 |
| }, |
| { |
| "epoch": 0.3518324607329843, |
| "grad_norm": 21.564373016357422, |
| "learning_rate": 4.1093052389237174e-07, |
| "logits/chosen": 1.3292641639709473, |
| "logits/rejected": 1.2153077125549316, |
| "logps/chosen": -246.62283325195312, |
| "logps/ref_chosen": -247.4376983642578, |
| "logps/ref_rejected": -216.68064880371094, |
| "logps/rejected": -241.31011962890625, |
| "loss": 4.7589, |
| "margin_dpo/margin_mean": 25.444313049316406, |
| "margin_dpo/margin_std": 17.723526000976562, |
| "step": 168 |
| }, |
| { |
| "epoch": 0.3539267015706806, |
| "grad_norm": 19.15829086303711, |
| "learning_rate": 4.0952521132208267e-07, |
| "logits/chosen": 1.6247047185897827, |
| "logits/rejected": 1.7833751440048218, |
| "logps/chosen": -281.30078125, |
| "logps/ref_chosen": -285.1272277832031, |
| "logps/ref_rejected": -279.10943603515625, |
| "logps/rejected": -302.5621032714844, |
| "loss": 4.632, |
| "margin_dpo/margin_mean": 27.279136657714844, |
| "margin_dpo/margin_std": 23.28731346130371, |
| "step": 169 |
| }, |
| { |
| "epoch": 0.35602094240837695, |
| "grad_norm": 24.02274513244629, |
| "learning_rate": 4.081113438988443e-07, |
| "logits/chosen": 1.5731761455535889, |
| "logits/rejected": 1.4810683727264404, |
| "logps/chosen": -357.42327880859375, |
| "logps/ref_chosen": -358.3712463378906, |
| "logps/ref_rejected": -245.13316345214844, |
| "logps/rejected": -264.8816223144531, |
| "loss": 4.7427, |
| "margin_dpo/margin_mean": 20.696434020996094, |
| "margin_dpo/margin_std": 30.21214485168457, |
| "step": 170 |
| }, |
| { |
| "epoch": 0.3581151832460733, |
| "grad_norm": 23.137300491333008, |
| "learning_rate": 4.0668899744407567e-07, |
| "logits/chosen": 1.5857964754104614, |
| "logits/rejected": 1.465027093887329, |
| "logps/chosen": -269.0282897949219, |
| "logps/ref_chosen": -273.9371337890625, |
| "logps/ref_rejected": -241.6103515625, |
| "logps/rejected": -259.7757263183594, |
| "loss": 4.7322, |
| "margin_dpo/margin_mean": 23.074222564697266, |
| "margin_dpo/margin_std": 30.494476318359375, |
| "step": 171 |
| }, |
| { |
| "epoch": 0.36020942408376966, |
| "grad_norm": 22.266416549682617, |
| "learning_rate": 4.0525824823390043e-07, |
| "logits/chosen": 1.6551828384399414, |
| "logits/rejected": 1.8315401077270508, |
| "logps/chosen": -254.328369140625, |
| "logps/ref_chosen": -255.1793975830078, |
| "logps/ref_rejected": -279.3556213378906, |
| "logps/rejected": -293.8561706542969, |
| "loss": 5.039, |
| "margin_dpo/margin_mean": 15.351570129394531, |
| "margin_dpo/margin_std": 22.528553009033203, |
| "step": 172 |
| }, |
| { |
| "epoch": 0.362303664921466, |
| "grad_norm": 24.233097076416016, |
| "learning_rate": 4.0381917299505686e-07, |
| "logits/chosen": 1.626520037651062, |
| "logits/rejected": 1.3303242921829224, |
| "logps/chosen": -338.2034606933594, |
| "logps/ref_chosen": -333.66375732421875, |
| "logps/ref_rejected": -275.1485290527344, |
| "logps/rejected": -300.3768310546875, |
| "loss": 4.8793, |
| "margin_dpo/margin_mean": 20.68860626220703, |
| "margin_dpo/margin_std": 28.740659713745117, |
| "step": 173 |
| }, |
| { |
| "epoch": 0.3643979057591623, |
| "grad_norm": 22.030344009399414, |
| "learning_rate": 4.0237184890078243e-07, |
| "logits/chosen": 1.9243297576904297, |
| "logits/rejected": 1.6874244213104248, |
| "logps/chosen": -354.771484375, |
| "logps/ref_chosen": -362.5843505859375, |
| "logps/ref_rejected": -250.0384521484375, |
| "logps/rejected": -277.5650939941406, |
| "loss": 4.6927, |
| "margin_dpo/margin_mean": 35.33952713012695, |
| "margin_dpo/margin_std": 33.1160774230957, |
| "step": 174 |
| }, |
| { |
| "epoch": 0.36649214659685864, |
| "grad_norm": 35.58210754394531, |
| "learning_rate": 4.00916353566676e-07, |
| "logits/chosen": 1.5620782375335693, |
| "logits/rejected": 1.598710536956787, |
| "logps/chosen": -242.1133270263672, |
| "logps/ref_chosen": -231.65187072753906, |
| "logps/ref_rejected": -264.08526611328125, |
| "logps/rejected": -294.81854248046875, |
| "loss": 4.8994, |
| "margin_dpo/margin_mean": 20.271865844726562, |
| "margin_dpo/margin_std": 31.47553062438965, |
| "step": 175 |
| }, |
| { |
| "epoch": 0.36858638743455496, |
| "grad_norm": 23.747568130493164, |
| "learning_rate": 3.994527650465352e-07, |
| "logits/chosen": 1.3475306034088135, |
| "logits/rejected": 1.4316266775131226, |
| "logps/chosen": -278.8919372558594, |
| "logps/ref_chosen": -271.37152099609375, |
| "logps/ref_rejected": -281.20074462890625, |
| "logps/rejected": -299.3853759765625, |
| "loss": 5.067, |
| "margin_dpo/margin_mean": 10.664226531982422, |
| "margin_dpo/margin_std": 33.30470657348633, |
| "step": 176 |
| }, |
| { |
| "epoch": 0.3706806282722513, |
| "grad_norm": 21.642898559570312, |
| "learning_rate": 3.979811618281705e-07, |
| "logits/chosen": 1.6163108348846436, |
| "logits/rejected": 1.4132215976715088, |
| "logps/chosen": -270.68548583984375, |
| "logps/ref_chosen": -266.7376403808594, |
| "logps/ref_rejected": -217.114990234375, |
| "logps/rejected": -240.8184356689453, |
| "loss": 5.1559, |
| "margin_dpo/margin_mean": 19.755634307861328, |
| "margin_dpo/margin_std": 33.71812438964844, |
| "step": 177 |
| }, |
| { |
| "epoch": 0.37277486910994767, |
| "grad_norm": 22.430517196655273, |
| "learning_rate": 3.9650162282919654e-07, |
| "logits/chosen": 1.463651180267334, |
| "logits/rejected": 1.5219404697418213, |
| "logps/chosen": -230.6317138671875, |
| "logps/ref_chosen": -230.67471313476562, |
| "logps/ref_rejected": -185.40577697753906, |
| "logps/rejected": -219.8003387451172, |
| "loss": 4.7678, |
| "margin_dpo/margin_mean": 34.437557220458984, |
| "margin_dpo/margin_std": 34.62123107910156, |
| "step": 178 |
| }, |
| { |
| "epoch": 0.374869109947644, |
| "grad_norm": 30.32175064086914, |
| "learning_rate": 3.9501422739279953e-07, |
| "logits/chosen": 1.3042542934417725, |
| "logits/rejected": 1.3105361461639404, |
| "logps/chosen": -272.2032470703125, |
| "logps/ref_chosen": -267.849853515625, |
| "logps/ref_rejected": -270.5272521972656, |
| "logps/rejected": -286.4674987792969, |
| "loss": 4.9431, |
| "margin_dpo/margin_mean": 11.586915969848633, |
| "margin_dpo/margin_std": 27.10576629638672, |
| "step": 179 |
| }, |
| { |
| "epoch": 0.3769633507853403, |
| "grad_norm": 41.640106201171875, |
| "learning_rate": 3.935190552834828e-07, |
| "logits/chosen": 1.7063815593719482, |
| "logits/rejected": 1.6488580703735352, |
| "logps/chosen": -302.65380859375, |
| "logps/ref_chosen": -296.4002685546875, |
| "logps/ref_rejected": -224.35203552246094, |
| "logps/rejected": -253.70095825195312, |
| "loss": 4.7457, |
| "margin_dpo/margin_mean": 23.0954647064209, |
| "margin_dpo/margin_std": 34.00359344482422, |
| "step": 180 |
| }, |
| { |
| "epoch": 0.37905759162303665, |
| "grad_norm": 30.104999542236328, |
| "learning_rate": 3.920161866827889e-07, |
| "logits/chosen": 1.1796238422393799, |
| "logits/rejected": 1.132272481918335, |
| "logps/chosen": -241.59747314453125, |
| "logps/ref_chosen": -243.10891723632812, |
| "logps/ref_rejected": -231.96902465820312, |
| "logps/rejected": -256.28497314453125, |
| "loss": 4.8516, |
| "margin_dpo/margin_mean": 25.827394485473633, |
| "margin_dpo/margin_std": 33.010704040527344, |
| "step": 181 |
| }, |
| { |
| "epoch": 0.381151832460733, |
| "grad_norm": 28.976200103759766, |
| "learning_rate": 3.90505702185e-07, |
| "logits/chosen": 1.584215521812439, |
| "logits/rejected": 1.6100966930389404, |
| "logps/chosen": -264.54351806640625, |
| "logps/ref_chosen": -263.5075988769531, |
| "logps/ref_rejected": -254.4083709716797, |
| "logps/rejected": -296.13641357421875, |
| "loss": 4.5773, |
| "margin_dpo/margin_mean": 40.692108154296875, |
| "margin_dpo/margin_std": 22.797809600830078, |
| "step": 182 |
| }, |
| { |
| "epoch": 0.3832460732984293, |
| "grad_norm": 36.45281982421875, |
| "learning_rate": 3.889876827928156e-07, |
| "logits/chosen": 1.0563912391662598, |
| "logits/rejected": 1.159271001815796, |
| "logps/chosen": -233.31439208984375, |
| "logps/ref_chosen": -220.9555206298828, |
| "logps/ref_rejected": -224.3114471435547, |
| "logps/rejected": -247.57742309570312, |
| "loss": 4.7918, |
| "margin_dpo/margin_mean": 10.90709114074707, |
| "margin_dpo/margin_std": 35.502681732177734, |
| "step": 183 |
| }, |
| { |
| "epoch": 0.38534031413612563, |
| "grad_norm": 24.624616622924805, |
| "learning_rate": 3.874622099130087e-07, |
| "logits/chosen": 1.666105031967163, |
| "logits/rejected": 1.6945122480392456, |
| "logps/chosen": -290.854736328125, |
| "logps/ref_chosen": -285.35125732421875, |
| "logps/ref_rejected": -282.2647705078125, |
| "logps/rejected": -324.21044921875, |
| "loss": 4.4107, |
| "margin_dpo/margin_mean": 36.442237854003906, |
| "margin_dpo/margin_std": 38.43600082397461, |
| "step": 184 |
| }, |
| { |
| "epoch": 0.387434554973822, |
| "grad_norm": 29.312538146972656, |
| "learning_rate": 3.859293653520604e-07, |
| "logits/chosen": 1.6671961545944214, |
| "logits/rejected": 1.7346235513687134, |
| "logps/chosen": -326.77490234375, |
| "logps/ref_chosen": -324.6773986816406, |
| "logps/ref_rejected": -275.9365539550781, |
| "logps/rejected": -308.9313659667969, |
| "loss": 4.8262, |
| "margin_dpo/margin_mean": 30.89735221862793, |
| "margin_dpo/margin_std": 34.63751220703125, |
| "step": 185 |
| }, |
| { |
| "epoch": 0.38952879581151834, |
| "grad_norm": 34.26539611816406, |
| "learning_rate": 3.8438923131177237e-07, |
| "logits/chosen": 1.594333291053772, |
| "logits/rejected": 1.5077753067016602, |
| "logps/chosen": -304.6080017089844, |
| "logps/ref_chosen": -287.4004211425781, |
| "logps/ref_rejected": -222.46803283691406, |
| "logps/rejected": -260.6602783203125, |
| "loss": 4.8356, |
| "margin_dpo/margin_mean": 20.984676361083984, |
| "margin_dpo/margin_std": 20.14511489868164, |
| "step": 186 |
| }, |
| { |
| "epoch": 0.39162303664921466, |
| "grad_norm": 25.851015090942383, |
| "learning_rate": 3.828418903848593e-07, |
| "logits/chosen": 1.441859245300293, |
| "logits/rejected": 1.5745567083358765, |
| "logps/chosen": -401.31182861328125, |
| "logps/ref_chosen": -378.8255310058594, |
| "logps/ref_rejected": -319.38116455078125, |
| "logps/rejected": -365.0159912109375, |
| "loss": 4.8863, |
| "margin_dpo/margin_mean": 23.14852523803711, |
| "margin_dpo/margin_std": 45.19081115722656, |
| "step": 187 |
| }, |
| { |
| "epoch": 0.393717277486911, |
| "grad_norm": 41.554141998291016, |
| "learning_rate": 3.812874255505191e-07, |
| "logits/chosen": 1.360278844833374, |
| "logits/rejected": 1.1752986907958984, |
| "logps/chosen": -250.5333251953125, |
| "logps/ref_chosen": -246.3994903564453, |
| "logps/ref_rejected": -204.85589599609375, |
| "logps/rejected": -239.05686950683594, |
| "loss": 4.8302, |
| "margin_dpo/margin_mean": 30.06714630126953, |
| "margin_dpo/margin_std": 34.51936340332031, |
| "step": 188 |
| }, |
| { |
| "epoch": 0.3958115183246073, |
| "grad_norm": 38.48931884765625, |
| "learning_rate": 3.797259201699833e-07, |
| "logits/chosen": 1.4543706178665161, |
| "logits/rejected": 1.5096098184585571, |
| "logps/chosen": -264.8511047363281, |
| "logps/ref_chosen": -264.7483825683594, |
| "logps/ref_rejected": -292.3799743652344, |
| "logps/rejected": -328.85107421875, |
| "loss": 4.6022, |
| "margin_dpo/margin_mean": 36.36838912963867, |
| "margin_dpo/margin_std": 26.913986206054688, |
| "step": 189 |
| }, |
| { |
| "epoch": 0.39790575916230364, |
| "grad_norm": 24.164396286010742, |
| "learning_rate": 3.781574579820464e-07, |
| "logits/chosen": 0.8813581466674805, |
| "logits/rejected": 0.9559296369552612, |
| "logps/chosen": -223.26422119140625, |
| "logps/ref_chosen": -211.2392120361328, |
| "logps/ref_rejected": -204.55384826660156, |
| "logps/rejected": -233.70541381835938, |
| "loss": 4.6669, |
| "margin_dpo/margin_mean": 17.12653350830078, |
| "margin_dpo/margin_std": 40.82097625732422, |
| "step": 190 |
| }, |
| { |
| "epoch": 0.4, |
| "grad_norm": 28.65985107421875, |
| "learning_rate": 3.765821230985757e-07, |
| "logits/chosen": 1.188876748085022, |
| "logits/rejected": 1.3144832849502563, |
| "logps/chosen": -177.3275604248047, |
| "logps/ref_chosen": -175.97952270507812, |
| "logps/ref_rejected": -206.85325622558594, |
| "logps/rejected": -228.22000122070312, |
| "loss": 4.7686, |
| "margin_dpo/margin_mean": 20.018733978271484, |
| "margin_dpo/margin_std": 31.200864791870117, |
| "step": 191 |
| }, |
| { |
| "epoch": 0.40209424083769635, |
| "grad_norm": 30.982559204101562, |
| "learning_rate": 3.75e-07, |
| "logits/chosen": 1.6706230640411377, |
| "logits/rejected": 1.854614496231079, |
| "logps/chosen": -253.131103515625, |
| "logps/ref_chosen": -241.5125732421875, |
| "logps/ref_rejected": -285.0710144042969, |
| "logps/rejected": -313.1866149902344, |
| "loss": 4.972, |
| "margin_dpo/margin_mean": 16.49706268310547, |
| "margin_dpo/margin_std": 54.19124221801758, |
| "step": 192 |
| }, |
| { |
| "epoch": 0.4041884816753927, |
| "grad_norm": 28.91083526611328, |
| "learning_rate": 3.734111735307796e-07, |
| "logits/chosen": 1.7183902263641357, |
| "logits/rejected": 1.5575065612792969, |
| "logps/chosen": -255.6170196533203, |
| "logps/ref_chosen": -247.06581115722656, |
| "logps/ref_rejected": -221.4132537841797, |
| "logps/rejected": -248.817138671875, |
| "loss": 4.8338, |
| "margin_dpo/margin_mean": 18.852684020996094, |
| "margin_dpo/margin_std": 29.74808120727539, |
| "step": 193 |
| }, |
| { |
| "epoch": 0.406282722513089, |
| "grad_norm": 35.922122955322266, |
| "learning_rate": 3.7181572889485623e-07, |
| "logits/chosen": 1.3995440006256104, |
| "logits/rejected": 1.4992262125015259, |
| "logps/chosen": -216.14686584472656, |
| "logps/ref_chosen": -208.60263061523438, |
| "logps/ref_rejected": -189.4849090576172, |
| "logps/rejected": -212.74192810058594, |
| "loss": 5.0447, |
| "margin_dpo/margin_mean": 15.712799072265625, |
| "margin_dpo/margin_std": 31.20469093322754, |
| "step": 194 |
| }, |
| { |
| "epoch": 0.4083769633507853, |
| "grad_norm": 31.717365264892578, |
| "learning_rate": 3.7021375165108377e-07, |
| "logits/chosen": 1.464687466621399, |
| "logits/rejected": 1.4596346616744995, |
| "logps/chosen": -287.36065673828125, |
| "logps/ref_chosen": -278.51275634765625, |
| "logps/ref_rejected": -298.09185791015625, |
| "logps/rejected": -318.23797607421875, |
| "loss": 5.0759, |
| "margin_dpo/margin_mean": 11.298222541809082, |
| "margin_dpo/margin_std": 27.541015625, |
| "step": 195 |
| }, |
| { |
| "epoch": 0.41047120418848165, |
| "grad_norm": 25.350812911987305, |
| "learning_rate": 3.6860532770864005e-07, |
| "logits/chosen": 1.1140937805175781, |
| "logits/rejected": 1.2928128242492676, |
| "logps/chosen": -213.8653564453125, |
| "logps/ref_chosen": -213.48568725585938, |
| "logps/ref_rejected": -216.8994903564453, |
| "logps/rejected": -244.5856170654297, |
| "loss": 4.6056, |
| "margin_dpo/margin_mean": 27.30645751953125, |
| "margin_dpo/margin_std": 27.342227935791016, |
| "step": 196 |
| }, |
| { |
| "epoch": 0.41256544502617803, |
| "grad_norm": 26.07918357849121, |
| "learning_rate": 3.6699054332241985e-07, |
| "logits/chosen": 1.4508588314056396, |
| "logits/rejected": 1.4278172254562378, |
| "logps/chosen": -255.36036682128906, |
| "logps/ref_chosen": -256.396728515625, |
| "logps/ref_rejected": -185.2763671875, |
| "logps/rejected": -232.59405517578125, |
| "loss": 4.3517, |
| "margin_dpo/margin_mean": 48.35406494140625, |
| "margin_dpo/margin_std": 30.625558853149414, |
| "step": 197 |
| }, |
| { |
| "epoch": 0.41465968586387436, |
| "grad_norm": 29.263551712036133, |
| "learning_rate": 3.653694850884091e-07, |
| "logits/chosen": 1.842124342918396, |
| "logits/rejected": 1.9477362632751465, |
| "logps/chosen": -362.33245849609375, |
| "logps/ref_chosen": -366.5196838378906, |
| "logps/ref_rejected": -361.7866516113281, |
| "logps/rejected": -392.13189697265625, |
| "loss": 4.6993, |
| "margin_dpo/margin_mean": 34.532501220703125, |
| "margin_dpo/margin_std": 40.5759162902832, |
| "step": 198 |
| }, |
| { |
| "epoch": 0.4167539267015707, |
| "grad_norm": 32.77777862548828, |
| "learning_rate": 3.6374223993904124e-07, |
| "logits/chosen": 0.9073523879051208, |
| "logits/rejected": 0.9206515550613403, |
| "logps/chosen": -210.94586181640625, |
| "logps/ref_chosen": -207.86968994140625, |
| "logps/ref_rejected": -184.52076721191406, |
| "logps/rejected": -226.15948486328125, |
| "loss": 4.7308, |
| "margin_dpo/margin_mean": 38.562538146972656, |
| "margin_dpo/margin_std": 28.093774795532227, |
| "step": 199 |
| }, |
| { |
| "epoch": 0.418848167539267, |
| "grad_norm": 28.612226486206055, |
| "learning_rate": 3.621088951385353e-07, |
| "logits/chosen": 1.3658336400985718, |
| "logits/rejected": 1.3859562873840332, |
| "logps/chosen": -281.68792724609375, |
| "logps/ref_chosen": -276.4098205566406, |
| "logps/ref_rejected": -252.23086547851562, |
| "logps/rejected": -272.3934326171875, |
| "loss": 4.9383, |
| "margin_dpo/margin_mean": 14.884419441223145, |
| "margin_dpo/margin_std": 53.33942794799805, |
| "step": 200 |
| }, |
| { |
| "epoch": 0.418848167539267, |
| "eval_logits/chosen": 1.4300137758255005, |
| "eval_logits/rejected": 1.4696903228759766, |
| "eval_logps/chosen": -287.022705078125, |
| "eval_logps/ref_chosen": -281.4588928222656, |
| "eval_logps/ref_rejected": -261.84954833984375, |
| "eval_logps/rejected": -295.6717529296875, |
| "eval_loss": 0.597048819065094, |
| "eval_margin_dpo/margin_mean": 28.258426666259766, |
| "eval_margin_dpo/margin_std": 39.02444076538086, |
| "eval_runtime": 93.548, |
| "eval_samples_per_second": 21.379, |
| "eval_steps_per_second": 1.336, |
| "step": 200 |
| }, |
| { |
| "epoch": 0.42094240837696334, |
| "grad_norm": 32.25981903076172, |
| "learning_rate": 3.604695382782159e-07, |
| "logits/chosen": 1.2966924905776978, |
| "logits/rejected": 1.4700032472610474, |
| "logps/chosen": -266.32220458984375, |
| "logps/ref_chosen": -265.32904052734375, |
| "logps/ref_rejected": -255.19529724121094, |
| "logps/rejected": -297.01544189453125, |
| "loss": 4.6702, |
| "margin_dpo/margin_mean": 40.82699203491211, |
| "margin_dpo/margin_std": 30.352630615234375, |
| "step": 201 |
| }, |
| { |
| "epoch": 0.42303664921465967, |
| "grad_norm": 35.75096130371094, |
| "learning_rate": 3.588242572718162e-07, |
| "logits/chosen": 1.6444792747497559, |
| "logits/rejected": 1.5493888854980469, |
| "logps/chosen": -278.0890197753906, |
| "logps/ref_chosen": -274.6075439453125, |
| "logps/ref_rejected": -219.9969940185547, |
| "logps/rejected": -251.83348083496094, |
| "loss": 4.7312, |
| "margin_dpo/margin_mean": 28.354969024658203, |
| "margin_dpo/margin_std": 44.327239990234375, |
| "step": 202 |
| }, |
| { |
| "epoch": 0.42513089005235605, |
| "grad_norm": 34.300376892089844, |
| "learning_rate": 3.571731403507635e-07, |
| "logits/chosen": 1.4360129833221436, |
| "logits/rejected": 1.3600637912750244, |
| "logps/chosen": -302.18707275390625, |
| "logps/ref_chosen": -295.6935119628906, |
| "logps/ref_rejected": -241.4007568359375, |
| "logps/rejected": -266.06201171875, |
| "loss": 4.8634, |
| "margin_dpo/margin_mean": 18.167736053466797, |
| "margin_dpo/margin_std": 24.508058547973633, |
| "step": 203 |
| }, |
| { |
| "epoch": 0.4272251308900524, |
| "grad_norm": 28.439380645751953, |
| "learning_rate": 3.5551627605944746e-07, |
| "logits/chosen": 2.044978141784668, |
| "logits/rejected": 1.9398431777954102, |
| "logps/chosen": -398.93328857421875, |
| "logps/ref_chosen": -392.3414611816406, |
| "logps/ref_rejected": -291.4375915527344, |
| "logps/rejected": -327.0179443359375, |
| "loss": 4.5824, |
| "margin_dpo/margin_mean": 28.98847007751465, |
| "margin_dpo/margin_std": 38.196693420410156, |
| "step": 204 |
| }, |
| { |
| "epoch": 0.4293193717277487, |
| "grad_norm": 27.83588981628418, |
| "learning_rate": 3.5385375325047163e-07, |
| "logits/chosen": 1.3867862224578857, |
| "logits/rejected": 1.657343864440918, |
| "logps/chosen": -191.40664672851562, |
| "logps/ref_chosen": -190.1780242919922, |
| "logps/ref_rejected": -275.8878479003906, |
| "logps/rejected": -311.2901306152344, |
| "loss": 4.6413, |
| "margin_dpo/margin_mean": 34.17367935180664, |
| "margin_dpo/margin_std": 32.90446090698242, |
| "step": 205 |
| }, |
| { |
| "epoch": 0.431413612565445, |
| "grad_norm": 30.838573455810547, |
| "learning_rate": 3.5218566107988867e-07, |
| "logits/chosen": 0.9244284629821777, |
| "logits/rejected": 1.1862902641296387, |
| "logps/chosen": -277.1954040527344, |
| "logps/ref_chosen": -278.95977783203125, |
| "logps/ref_rejected": -296.458984375, |
| "logps/rejected": -318.1335144042969, |
| "loss": 4.6764, |
| "margin_dpo/margin_mean": 23.43887710571289, |
| "margin_dpo/margin_std": 38.043148040771484, |
| "step": 206 |
| }, |
| { |
| "epoch": 0.43350785340314135, |
| "grad_norm": 31.335426330566406, |
| "learning_rate": 3.505120890024195e-07, |
| "logits/chosen": 1.658402681350708, |
| "logits/rejected": 1.8528728485107422, |
| "logps/chosen": -222.63587951660156, |
| "logps/ref_chosen": -219.367919921875, |
| "logps/ref_rejected": -231.6876678466797, |
| "logps/rejected": -260.1963195800781, |
| "loss": 4.9355, |
| "margin_dpo/margin_mean": 25.240697860717773, |
| "margin_dpo/margin_std": 46.75006866455078, |
| "step": 207 |
| }, |
| { |
| "epoch": 0.4356020942408377, |
| "grad_norm": 34.799400329589844, |
| "learning_rate": 3.4883312676665534e-07, |
| "logits/chosen": 1.4422191381454468, |
| "logits/rejected": 1.4274728298187256, |
| "logps/chosen": -308.7106628417969, |
| "logps/ref_chosen": -303.848388671875, |
| "logps/ref_rejected": -252.1853485107422, |
| "logps/rejected": -288.1015625, |
| "loss": 4.7313, |
| "margin_dpo/margin_mean": 31.05390739440918, |
| "margin_dpo/margin_std": 41.717647552490234, |
| "step": 208 |
| }, |
| { |
| "epoch": 0.437696335078534, |
| "grad_norm": 32.08354949951172, |
| "learning_rate": 3.4714886441024573e-07, |
| "logits/chosen": 1.48392915725708, |
| "logits/rejected": 1.2658922672271729, |
| "logps/chosen": -353.7491760253906, |
| "logps/ref_chosen": -347.6343688964844, |
| "logps/ref_rejected": -240.31988525390625, |
| "logps/rejected": -270.4415283203125, |
| "loss": 4.891, |
| "margin_dpo/margin_mean": 24.006847381591797, |
| "margin_dpo/margin_std": 38.43374252319336, |
| "step": 209 |
| }, |
| { |
| "epoch": 0.4397905759162304, |
| "grad_norm": 41.964515686035156, |
| "learning_rate": 3.454593922550693e-07, |
| "logits/chosen": 1.7486904859542847, |
| "logits/rejected": 1.899224042892456, |
| "logps/chosen": -230.97503662109375, |
| "logps/ref_chosen": -236.3311767578125, |
| "logps/ref_rejected": -289.6016845703125, |
| "logps/rejected": -317.5093688964844, |
| "loss": 4.7272, |
| "margin_dpo/margin_mean": 33.26382064819336, |
| "margin_dpo/margin_std": 33.677276611328125, |
| "step": 210 |
| }, |
| { |
| "epoch": 0.4418848167539267, |
| "grad_norm": 31.20326805114746, |
| "learning_rate": 3.4376480090239047e-07, |
| "logits/chosen": 1.3021446466445923, |
| "logits/rejected": 1.3574562072753906, |
| "logps/chosen": -205.39637756347656, |
| "logps/ref_chosen": -204.38107299804688, |
| "logps/ref_rejected": -212.449462890625, |
| "logps/rejected": -242.7283172607422, |
| "loss": 4.3731, |
| "margin_dpo/margin_mean": 29.263545989990234, |
| "margin_dpo/margin_std": 35.2406005859375, |
| "step": 211 |
| }, |
| { |
| "epoch": 0.44397905759162304, |
| "grad_norm": 35.141807556152344, |
| "learning_rate": 3.4206518122800055e-07, |
| "logits/chosen": 1.1048368215560913, |
| "logits/rejected": 1.1881780624389648, |
| "logps/chosen": -241.2005615234375, |
| "logps/ref_chosen": -231.28570556640625, |
| "logps/ref_rejected": -222.65725708007812, |
| "logps/rejected": -248.41322326660156, |
| "loss": 4.7702, |
| "margin_dpo/margin_mean": 15.841072082519531, |
| "margin_dpo/margin_std": 39.27621841430664, |
| "step": 212 |
| }, |
| { |
| "epoch": 0.44607329842931936, |
| "grad_norm": 27.873577117919922, |
| "learning_rate": 3.403606243773448e-07, |
| "logits/chosen": 1.5394573211669922, |
| "logits/rejected": 1.6554535627365112, |
| "logps/chosen": -336.94354248046875, |
| "logps/ref_chosen": -332.35968017578125, |
| "logps/ref_rejected": -329.94830322265625, |
| "logps/rejected": -365.0585021972656, |
| "loss": 4.7341, |
| "margin_dpo/margin_mean": 30.526296615600586, |
| "margin_dpo/margin_std": 37.60173034667969, |
| "step": 213 |
| }, |
| { |
| "epoch": 0.4481675392670157, |
| "grad_norm": 33.36276626586914, |
| "learning_rate": 3.3865122176063385e-07, |
| "logits/chosen": 1.8208783864974976, |
| "logits/rejected": 1.9227497577667236, |
| "logps/chosen": -320.77886962890625, |
| "logps/ref_chosen": -303.07257080078125, |
| "logps/ref_rejected": -310.52001953125, |
| "logps/rejected": -347.1795654296875, |
| "loss": 4.8278, |
| "margin_dpo/margin_mean": 18.953208923339844, |
| "margin_dpo/margin_std": 34.65628433227539, |
| "step": 214 |
| }, |
| { |
| "epoch": 0.450261780104712, |
| "grad_norm": 31.66336441040039, |
| "learning_rate": 3.3693706504794243e-07, |
| "logits/chosen": 2.0376367568969727, |
| "logits/rejected": 2.075817584991455, |
| "logps/chosen": -283.66595458984375, |
| "logps/ref_chosen": -286.654296875, |
| "logps/ref_rejected": -272.1281433105469, |
| "logps/rejected": -317.6420593261719, |
| "loss": 4.8219, |
| "margin_dpo/margin_mean": 48.50217819213867, |
| "margin_dpo/margin_std": 41.356727600097656, |
| "step": 215 |
| }, |
| { |
| "epoch": 0.4523560209424084, |
| "grad_norm": 51.738441467285156, |
| "learning_rate": 3.3521824616429284e-07, |
| "logits/chosen": 1.378481149673462, |
| "logits/rejected": 1.2707972526550293, |
| "logps/chosen": -364.43603515625, |
| "logps/ref_chosen": -351.34417724609375, |
| "logps/ref_rejected": -290.5171813964844, |
| "logps/rejected": -327.94488525390625, |
| "loss": 4.7106, |
| "margin_dpo/margin_mean": 24.335878372192383, |
| "margin_dpo/margin_std": 42.64997100830078, |
| "step": 216 |
| }, |
| { |
| "epoch": 0.4544502617801047, |
| "grad_norm": 35.788352966308594, |
| "learning_rate": 3.334948572847253e-07, |
| "logits/chosen": 1.5968372821807861, |
| "logits/rejected": 1.7124537229537964, |
| "logps/chosen": -279.4749755859375, |
| "logps/ref_chosen": -273.76788330078125, |
| "logps/ref_rejected": -286.2580261230469, |
| "logps/rejected": -343.501953125, |
| "loss": 4.4028, |
| "margin_dpo/margin_mean": 51.53682327270508, |
| "margin_dpo/margin_std": 38.95985412597656, |
| "step": 217 |
| }, |
| { |
| "epoch": 0.45654450261780105, |
| "grad_norm": 41.920623779296875, |
| "learning_rate": 3.317669908293554e-07, |
| "logits/chosen": 1.5484070777893066, |
| "logits/rejected": 1.8093584775924683, |
| "logps/chosen": -235.32321166992188, |
| "logps/ref_chosen": -219.74948120117188, |
| "logps/ref_rejected": -308.801025390625, |
| "logps/rejected": -354.8564147949219, |
| "loss": 4.5987, |
| "margin_dpo/margin_mean": 30.481698989868164, |
| "margin_dpo/margin_std": 39.370521545410156, |
| "step": 218 |
| }, |
| { |
| "epoch": 0.4586387434554974, |
| "grad_norm": 30.652027130126953, |
| "learning_rate": 3.300347394584172e-07, |
| "logits/chosen": 1.3850374221801758, |
| "logits/rejected": 1.4719693660736084, |
| "logps/chosen": -282.8400573730469, |
| "logps/ref_chosen": -264.65374755859375, |
| "logps/ref_rejected": -233.9711151123047, |
| "logps/rejected": -276.5810546875, |
| "loss": 4.6628, |
| "margin_dpo/margin_mean": 24.423654556274414, |
| "margin_dpo/margin_std": 40.1912841796875, |
| "step": 219 |
| }, |
| { |
| "epoch": 0.4607329842931937, |
| "grad_norm": 45.71303939819336, |
| "learning_rate": 3.2829819606729477e-07, |
| "logits/chosen": 1.9091517925262451, |
| "logits/rejected": 1.744594931602478, |
| "logps/chosen": -315.3508605957031, |
| "logps/ref_chosen": -295.8961486816406, |
| "logps/ref_rejected": -219.56228637695312, |
| "logps/rejected": -273.97418212890625, |
| "loss": 4.5451, |
| "margin_dpo/margin_mean": 34.957191467285156, |
| "margin_dpo/margin_std": 39.46710968017578, |
| "step": 220 |
| }, |
| { |
| "epoch": 0.46282722513089003, |
| "grad_norm": 29.89345932006836, |
| "learning_rate": 3.265574537815398e-07, |
| "logits/chosen": 1.0531387329101562, |
| "logits/rejected": 1.2520170211791992, |
| "logps/chosen": -302.5050048828125, |
| "logps/ref_chosen": -284.9080810546875, |
| "logps/ref_rejected": -310.0538330078125, |
| "logps/rejected": -356.16510009765625, |
| "loss": 4.8325, |
| "margin_dpo/margin_mean": 28.514326095581055, |
| "margin_dpo/margin_std": 32.273284912109375, |
| "step": 221 |
| }, |
| { |
| "epoch": 0.4649214659685864, |
| "grad_norm": 47.45641326904297, |
| "learning_rate": 3.248126059518784e-07, |
| "logits/chosen": 1.3722844123840332, |
| "logits/rejected": 1.2933783531188965, |
| "logps/chosen": -329.0756530761719, |
| "logps/ref_chosen": -308.44622802734375, |
| "logps/ref_rejected": -254.99667358398438, |
| "logps/rejected": -303.69677734375, |
| "loss": 4.5312, |
| "margin_dpo/margin_mean": 28.07069969177246, |
| "margin_dpo/margin_std": 30.664283752441406, |
| "step": 222 |
| }, |
| { |
| "epoch": 0.46701570680628274, |
| "grad_norm": 36.27118682861328, |
| "learning_rate": 3.230637461492043e-07, |
| "logits/chosen": 1.214386224746704, |
| "logits/rejected": 1.193519115447998, |
| "logps/chosen": -283.5272521972656, |
| "logps/ref_chosen": -258.5130310058594, |
| "logps/ref_rejected": -231.13885498046875, |
| "logps/rejected": -290.85296630859375, |
| "loss": 4.4404, |
| "margin_dpo/margin_mean": 34.699886322021484, |
| "margin_dpo/margin_std": 42.52618408203125, |
| "step": 223 |
| }, |
| { |
| "epoch": 0.46910994764397906, |
| "grad_norm": 36.553733825683594, |
| "learning_rate": 3.213109681595612e-07, |
| "logits/chosen": 1.2857964038848877, |
| "logits/rejected": 1.433445692062378, |
| "logps/chosen": -248.49815368652344, |
| "logps/ref_chosen": -234.55177307128906, |
| "logps/ref_rejected": -208.4610595703125, |
| "logps/rejected": -271.1744689941406, |
| "loss": 4.3795, |
| "margin_dpo/margin_mean": 48.76702117919922, |
| "margin_dpo/margin_std": 35.48724365234375, |
| "step": 224 |
| }, |
| { |
| "epoch": 0.4712041884816754, |
| "grad_norm": 40.446937561035156, |
| "learning_rate": 3.1955436597911315e-07, |
| "logits/chosen": 1.6013773679733276, |
| "logits/rejected": 1.722807765007019, |
| "logps/chosen": -360.8241882324219, |
| "logps/ref_chosen": -339.7688903808594, |
| "logps/ref_rejected": -347.96112060546875, |
| "logps/rejected": -397.2833251953125, |
| "loss": 4.9012, |
| "margin_dpo/margin_mean": 28.266937255859375, |
| "margin_dpo/margin_std": 49.008575439453125, |
| "step": 225 |
| }, |
| { |
| "epoch": 0.4732984293193717, |
| "grad_norm": 37.459991455078125, |
| "learning_rate": 3.1779403380910425e-07, |
| "logits/chosen": 0.7877386808395386, |
| "logits/rejected": 1.0014675855636597, |
| "logps/chosen": -225.65472412109375, |
| "logps/ref_chosen": -209.56515502929688, |
| "logps/ref_rejected": -207.83871459960938, |
| "logps/rejected": -261.1890563964844, |
| "loss": 4.8962, |
| "margin_dpo/margin_mean": 37.260780334472656, |
| "margin_dpo/margin_std": 42.05327606201172, |
| "step": 226 |
| }, |
| { |
| "epoch": 0.47539267015706804, |
| "grad_norm": 29.621501922607422, |
| "learning_rate": 3.160300660508064e-07, |
| "logits/chosen": 1.4312937259674072, |
| "logits/rejected": 1.644526481628418, |
| "logps/chosen": -278.422607421875, |
| "logps/ref_chosen": -252.69004821777344, |
| "logps/ref_rejected": -252.89427185058594, |
| "logps/rejected": -317.8539733886719, |
| "loss": 4.4608, |
| "margin_dpo/margin_mean": 39.22712707519531, |
| "margin_dpo/margin_std": 55.3231086730957, |
| "step": 227 |
| }, |
| { |
| "epoch": 0.4774869109947644, |
| "grad_norm": 46.8765869140625, |
| "learning_rate": 3.1426255730045695e-07, |
| "logits/chosen": 1.5183122158050537, |
| "logits/rejected": 1.6018824577331543, |
| "logps/chosen": -235.6670684814453, |
| "logps/ref_chosen": -210.62913513183594, |
| "logps/ref_rejected": -174.08975219726562, |
| "logps/rejected": -226.82781982421875, |
| "loss": 4.4722, |
| "margin_dpo/margin_mean": 27.70014762878418, |
| "margin_dpo/margin_std": 42.50754165649414, |
| "step": 228 |
| }, |
| { |
| "epoch": 0.47958115183246075, |
| "grad_norm": 39.29153823852539, |
| "learning_rate": 3.1249160234418644e-07, |
| "logits/chosen": 1.3761322498321533, |
| "logits/rejected": 1.3572083711624146, |
| "logps/chosen": -336.7579345703125, |
| "logps/ref_chosen": -315.1896057128906, |
| "logps/ref_rejected": -265.8664855957031, |
| "logps/rejected": -330.8331298828125, |
| "loss": 4.1991, |
| "margin_dpo/margin_mean": 43.39836883544922, |
| "margin_dpo/margin_std": 46.784549713134766, |
| "step": 229 |
| }, |
| { |
| "epoch": 0.4816753926701571, |
| "grad_norm": 39.0309944152832, |
| "learning_rate": 3.1071729615293424e-07, |
| "logits/chosen": 0.9634809494018555, |
| "logits/rejected": 0.9632886648178101, |
| "logps/chosen": -258.40020751953125, |
| "logps/ref_chosen": -240.54244995117188, |
| "logps/ref_rejected": -262.5657043457031, |
| "logps/rejected": -319.0147705078125, |
| "loss": 4.5649, |
| "margin_dpo/margin_mean": 38.59132385253906, |
| "margin_dpo/margin_std": 50.162498474121094, |
| "step": 230 |
| }, |
| { |
| "epoch": 0.4837696335078534, |
| "grad_norm": 55.88585662841797, |
| "learning_rate": 3.0893973387735683e-07, |
| "logits/chosen": 1.1088343858718872, |
| "logits/rejected": 1.1820017099380493, |
| "logps/chosen": -326.6155700683594, |
| "logps/ref_chosen": -290.8667907714844, |
| "logps/ref_rejected": -277.01739501953125, |
| "logps/rejected": -330.43194580078125, |
| "loss": 4.7561, |
| "margin_dpo/margin_mean": 17.66571807861328, |
| "margin_dpo/margin_std": 42.80667495727539, |
| "step": 231 |
| }, |
| { |
| "epoch": 0.48586387434554973, |
| "grad_norm": 43.03524398803711, |
| "learning_rate": 3.071590108427243e-07, |
| "logits/chosen": 1.3966903686523438, |
| "logits/rejected": 1.5778224468231201, |
| "logps/chosen": -285.2840270996094, |
| "logps/ref_chosen": -260.0438232421875, |
| "logps/ref_rejected": -261.63507080078125, |
| "logps/rejected": -320.8078308105469, |
| "loss": 4.5309, |
| "margin_dpo/margin_mean": 33.93254089355469, |
| "margin_dpo/margin_std": 37.87663269042969, |
| "step": 232 |
| }, |
| { |
| "epoch": 0.48795811518324606, |
| "grad_norm": 53.92399597167969, |
| "learning_rate": 3.05375222543809e-07, |
| "logits/chosen": 0.838955283164978, |
| "logits/rejected": 0.9410269856452942, |
| "logps/chosen": -240.427734375, |
| "logps/ref_chosen": -221.6608123779297, |
| "logps/ref_rejected": -261.16839599609375, |
| "logps/rejected": -328.1685485839844, |
| "loss": 4.5981, |
| "margin_dpo/margin_mean": 48.23321533203125, |
| "margin_dpo/margin_std": 38.929080963134766, |
| "step": 233 |
| }, |
| { |
| "epoch": 0.4900523560209424, |
| "grad_norm": 33.82901382446289, |
| "learning_rate": 3.035884646397637e-07, |
| "logits/chosen": 1.1958811283111572, |
| "logits/rejected": 1.237013816833496, |
| "logps/chosen": -297.06036376953125, |
| "logps/ref_chosen": -281.4861145019531, |
| "logps/ref_rejected": -276.58441162109375, |
| "logps/rejected": -340.1748046875, |
| "loss": 4.5243, |
| "margin_dpo/margin_mean": 48.01616668701172, |
| "margin_dpo/margin_std": 41.946895599365234, |
| "step": 234 |
| }, |
| { |
| "epoch": 0.49214659685863876, |
| "grad_norm": 44.3351936340332, |
| "learning_rate": 3.017988329489923e-07, |
| "logits/chosen": 1.6281356811523438, |
| "logits/rejected": 1.5668871402740479, |
| "logps/chosen": -301.54229736328125, |
| "logps/ref_chosen": -300.5598449707031, |
| "logps/ref_rejected": -259.905029296875, |
| "logps/rejected": -302.8565368652344, |
| "loss": 4.6976, |
| "margin_dpo/margin_mean": 41.96904754638672, |
| "margin_dpo/margin_std": 42.275962829589844, |
| "step": 235 |
| }, |
| { |
| "epoch": 0.4942408376963351, |
| "grad_norm": 32.59733963012695, |
| "learning_rate": 3.000064234440111e-07, |
| "logits/chosen": 1.2152283191680908, |
| "logits/rejected": 1.2443571090698242, |
| "logps/chosen": -282.0902099609375, |
| "logps/ref_chosen": -270.4844665527344, |
| "logps/ref_rejected": -231.67613220214844, |
| "logps/rejected": -282.9165954589844, |
| "loss": 4.4441, |
| "margin_dpo/margin_mean": 39.63469696044922, |
| "margin_dpo/margin_std": 43.977577209472656, |
| "step": 236 |
| }, |
| { |
| "epoch": 0.4963350785340314, |
| "grad_norm": 47.38420486450195, |
| "learning_rate": 2.9821133224630223e-07, |
| "logits/chosen": 1.266021490097046, |
| "logits/rejected": 1.4988112449645996, |
| "logps/chosen": -219.76870727539062, |
| "logps/ref_chosen": -194.99342346191406, |
| "logps/ref_rejected": -243.12779235839844, |
| "logps/rejected": -310.861083984375, |
| "loss": 4.5863, |
| "margin_dpo/margin_mean": 42.9580078125, |
| "margin_dpo/margin_std": 39.53506851196289, |
| "step": 237 |
| }, |
| { |
| "epoch": 0.49842931937172774, |
| "grad_norm": 39.1376838684082, |
| "learning_rate": 2.964136556211588e-07, |
| "logits/chosen": 1.1324211359024048, |
| "logits/rejected": 1.0826090574264526, |
| "logps/chosen": -261.50543212890625, |
| "logps/ref_chosen": -240.9060516357422, |
| "logps/ref_rejected": -205.97012329101562, |
| "logps/rejected": -254.07977294921875, |
| "loss": 4.446, |
| "margin_dpo/margin_mean": 27.510299682617188, |
| "margin_dpo/margin_std": 46.14052200317383, |
| "step": 238 |
| }, |
| { |
| "epoch": 0.5005235602094241, |
| "grad_norm": 40.104766845703125, |
| "learning_rate": 2.946134899725226e-07, |
| "logits/chosen": 1.3846328258514404, |
| "logits/rejected": 1.5582243204116821, |
| "logps/chosen": -303.406005859375, |
| "logps/ref_chosen": -277.0447998046875, |
| "logps/ref_rejected": -284.7602233886719, |
| "logps/rejected": -329.855712890625, |
| "loss": 4.701, |
| "margin_dpo/margin_mean": 18.734315872192383, |
| "margin_dpo/margin_std": 59.868492126464844, |
| "step": 239 |
| }, |
| { |
| "epoch": 0.5026178010471204, |
| "grad_norm": 38.7380485534668, |
| "learning_rate": 2.9281093183781403e-07, |
| "logits/chosen": 1.1157575845718384, |
| "logits/rejected": 1.0506106615066528, |
| "logps/chosen": -290.3915710449219, |
| "logps/ref_chosen": -285.29144287109375, |
| "logps/ref_rejected": -212.11915588378906, |
| "logps/rejected": -265.2477111816406, |
| "loss": 4.3702, |
| "margin_dpo/margin_mean": 48.02838134765625, |
| "margin_dpo/margin_std": 42.991214752197266, |
| "step": 240 |
| }, |
| { |
| "epoch": 0.5047120418848168, |
| "grad_norm": 41.1712646484375, |
| "learning_rate": 2.910060778827554e-07, |
| "logits/chosen": 1.3538631200790405, |
| "logits/rejected": 1.4909546375274658, |
| "logps/chosen": -260.87939453125, |
| "logps/ref_chosen": -254.9442901611328, |
| "logps/ref_rejected": -278.5121154785156, |
| "logps/rejected": -332.14483642578125, |
| "loss": 4.7755, |
| "margin_dpo/margin_mean": 47.697601318359375, |
| "margin_dpo/margin_std": 46.98115539550781, |
| "step": 241 |
| }, |
| { |
| "epoch": 0.506806282722513, |
| "grad_norm": 33.75428771972656, |
| "learning_rate": 2.891990248961871e-07, |
| "logits/chosen": 2.038219690322876, |
| "logits/rejected": 1.9361552000045776, |
| "logps/chosen": -274.6824645996094, |
| "logps/ref_chosen": -264.16876220703125, |
| "logps/ref_rejected": -215.627197265625, |
| "logps/rejected": -269.078125, |
| "loss": 4.4651, |
| "margin_dpo/margin_mean": 42.937225341796875, |
| "margin_dpo/margin_std": 52.65818786621094, |
| "step": 242 |
| }, |
| { |
| "epoch": 0.5089005235602094, |
| "grad_norm": 45.30524826049805, |
| "learning_rate": 2.873898697848762e-07, |
| "logits/chosen": 1.3483995199203491, |
| "logits/rejected": 1.3744844198226929, |
| "logps/chosen": -322.53204345703125, |
| "logps/ref_chosen": -313.7347106933594, |
| "logps/ref_rejected": -357.50054931640625, |
| "logps/rejected": -403.6512756347656, |
| "loss": 4.1712, |
| "margin_dpo/margin_mean": 37.353302001953125, |
| "margin_dpo/margin_std": 46.82036590576172, |
| "step": 243 |
| }, |
| { |
| "epoch": 0.5109947643979058, |
| "grad_norm": 35.50245666503906, |
| "learning_rate": 2.8557870956832133e-07, |
| "logits/chosen": 1.1742347478866577, |
| "logits/rejected": 0.9833606481552124, |
| "logps/chosen": -291.64044189453125, |
| "logps/ref_chosen": -265.0720520019531, |
| "logps/ref_rejected": -235.29541015625, |
| "logps/rejected": -293.6773376464844, |
| "loss": 4.2395, |
| "margin_dpo/margin_mean": 31.813528060913086, |
| "margin_dpo/margin_std": 38.89430236816406, |
| "step": 244 |
| }, |
| { |
| "epoch": 0.5130890052356021, |
| "grad_norm": 55.53532791137695, |
| "learning_rate": 2.837656413735479e-07, |
| "logits/chosen": 1.9007817506790161, |
| "logits/rejected": 1.6215357780456543, |
| "logps/chosen": -346.5862731933594, |
| "logps/ref_chosen": -338.6529235839844, |
| "logps/ref_rejected": -259.6473693847656, |
| "logps/rejected": -305.2576904296875, |
| "loss": 4.2236, |
| "margin_dpo/margin_mean": 37.676998138427734, |
| "margin_dpo/margin_std": 33.821868896484375, |
| "step": 245 |
| }, |
| { |
| "epoch": 0.5151832460732985, |
| "grad_norm": 36.23422622680664, |
| "learning_rate": 2.8195076242990116e-07, |
| "logits/chosen": 1.154848337173462, |
| "logits/rejected": 1.1089671850204468, |
| "logps/chosen": -273.8831787109375, |
| "logps/ref_chosen": -254.98756408691406, |
| "logps/ref_rejected": -201.20782470703125, |
| "logps/rejected": -253.29745483398438, |
| "loss": 4.8429, |
| "margin_dpo/margin_mean": 33.19401550292969, |
| "margin_dpo/margin_std": 46.2290153503418, |
| "step": 246 |
| }, |
| { |
| "epoch": 0.5172774869109947, |
| "grad_norm": 41.20656967163086, |
| "learning_rate": 2.801341700638307e-07, |
| "logits/chosen": 1.237385630607605, |
| "logits/rejected": 1.1006180047988892, |
| "logps/chosen": -284.4879455566406, |
| "logps/ref_chosen": -276.70361328125, |
| "logps/ref_rejected": -209.83523559570312, |
| "logps/rejected": -266.49542236328125, |
| "loss": 4.4147, |
| "margin_dpo/margin_mean": 48.87584686279297, |
| "margin_dpo/margin_std": 42.81962966918945, |
| "step": 247 |
| }, |
| { |
| "epoch": 0.5193717277486911, |
| "grad_norm": 57.703697204589844, |
| "learning_rate": 2.7831596169367227e-07, |
| "logits/chosen": 1.0914226770401, |
| "logits/rejected": 1.1984620094299316, |
| "logps/chosen": -258.7278747558594, |
| "logps/ref_chosen": -249.7368621826172, |
| "logps/ref_rejected": -230.7808837890625, |
| "logps/rejected": -274.591552734375, |
| "loss": 4.7502, |
| "margin_dpo/margin_mean": 34.819644927978516, |
| "margin_dpo/margin_std": 40.67426300048828, |
| "step": 248 |
| }, |
| { |
| "epoch": 0.5214659685863874, |
| "grad_norm": 47.22350311279297, |
| "learning_rate": 2.7649623482442274e-07, |
| "logits/chosen": 1.0606677532196045, |
| "logits/rejected": 1.1147487163543701, |
| "logps/chosen": -266.5928649902344, |
| "logps/ref_chosen": -229.43399047851562, |
| "logps/ref_rejected": -242.59182739257812, |
| "logps/rejected": -302.368896484375, |
| "loss": 4.5617, |
| "margin_dpo/margin_mean": 22.618181228637695, |
| "margin_dpo/margin_std": 44.011497497558594, |
| "step": 249 |
| }, |
| { |
| "epoch": 0.5235602094240838, |
| "grad_norm": 34.5158576965332, |
| "learning_rate": 2.7467508704251135e-07, |
| "logits/chosen": 1.6137490272521973, |
| "logits/rejected": 1.7355223894119263, |
| "logps/chosen": -386.4211120605469, |
| "logps/ref_chosen": -374.47015380859375, |
| "logps/ref_rejected": -397.1805114746094, |
| "logps/rejected": -455.82952880859375, |
| "loss": 4.6106, |
| "margin_dpo/margin_mean": 46.698097229003906, |
| "margin_dpo/margin_std": 53.3001823425293, |
| "step": 250 |
| }, |
| { |
| "epoch": 0.5256544502617801, |
| "grad_norm": 44.91852569580078, |
| "learning_rate": 2.7285261601056697e-07, |
| "logits/chosen": 1.0645577907562256, |
| "logits/rejected": 0.8425718545913696, |
| "logps/chosen": -355.9337463378906, |
| "logps/ref_chosen": -340.28240966796875, |
| "logps/ref_rejected": -255.56735229492188, |
| "logps/rejected": -305.7314147949219, |
| "loss": 4.5346, |
| "margin_dpo/margin_mean": 34.51277160644531, |
| "margin_dpo/margin_std": 47.6010627746582, |
| "step": 251 |
| }, |
| { |
| "epoch": 0.5277486910994764, |
| "grad_norm": 30.532474517822266, |
| "learning_rate": 2.7102891946217994e-07, |
| "logits/chosen": 1.4391117095947266, |
| "logits/rejected": 1.4691420793533325, |
| "logps/chosen": -215.19662475585938, |
| "logps/ref_chosen": -198.7939453125, |
| "logps/ref_rejected": -212.86849975585938, |
| "logps/rejected": -271.3706359863281, |
| "loss": 4.5578, |
| "margin_dpo/margin_mean": 42.09947967529297, |
| "margin_dpo/margin_std": 46.31159973144531, |
| "step": 252 |
| }, |
| { |
| "epoch": 0.5298429319371728, |
| "grad_norm": 45.59535598754883, |
| "learning_rate": 2.692040951966617e-07, |
| "logits/chosen": 1.448297142982483, |
| "logits/rejected": 1.3689329624176025, |
| "logps/chosen": -370.5470275878906, |
| "logps/ref_chosen": -343.3220520019531, |
| "logps/ref_rejected": -258.52044677734375, |
| "logps/rejected": -316.4342041015625, |
| "loss": 4.7989, |
| "margin_dpo/margin_mean": 30.688785552978516, |
| "margin_dpo/margin_std": 51.24434280395508, |
| "step": 253 |
| }, |
| { |
| "epoch": 0.5319371727748691, |
| "grad_norm": 37.76780700683594, |
| "learning_rate": 2.6737824107379947e-07, |
| "logits/chosen": 1.4605791568756104, |
| "logits/rejected": 1.3956043720245361, |
| "logps/chosen": -326.6246337890625, |
| "logps/ref_chosen": -300.8880310058594, |
| "logps/ref_rejected": -288.5895690917969, |
| "logps/rejected": -342.62518310546875, |
| "loss": 4.4466, |
| "margin_dpo/margin_mean": 28.299026489257812, |
| "margin_dpo/margin_std": 43.362815856933594, |
| "step": 254 |
| }, |
| { |
| "epoch": 0.5340314136125655, |
| "grad_norm": 38.82050323486328, |
| "learning_rate": 2.655514550086086e-07, |
| "logits/chosen": 1.3760805130004883, |
| "logits/rejected": 1.3785066604614258, |
| "logps/chosen": -309.2912902832031, |
| "logps/ref_chosen": -283.4182434082031, |
| "logps/ref_rejected": -317.677978515625, |
| "logps/rejected": -381.75701904296875, |
| "loss": 4.3792, |
| "margin_dpo/margin_mean": 38.20598220825195, |
| "margin_dpo/margin_std": 58.638553619384766, |
| "step": 255 |
| }, |
| { |
| "epoch": 0.5361256544502618, |
| "grad_norm": 37.75017166137695, |
| "learning_rate": 2.6372383496608186e-07, |
| "logits/chosen": 1.3811287879943848, |
| "logits/rejected": 1.417975902557373, |
| "logps/chosen": -352.6160583496094, |
| "logps/ref_chosen": -333.6951599121094, |
| "logps/ref_rejected": -302.9135437011719, |
| "logps/rejected": -374.9257507324219, |
| "loss": 4.4641, |
| "margin_dpo/margin_mean": 53.091312408447266, |
| "margin_dpo/margin_std": 62.08375930786133, |
| "step": 256 |
| }, |
| { |
| "epoch": 0.5382198952879581, |
| "grad_norm": 39.30986022949219, |
| "learning_rate": 2.618954789559356e-07, |
| "logits/chosen": 1.3786240816116333, |
| "logits/rejected": 1.5010215044021606, |
| "logps/chosen": -297.0694885253906, |
| "logps/ref_chosen": -269.2105712890625, |
| "logps/ref_rejected": -282.474365234375, |
| "logps/rejected": -354.8011474609375, |
| "loss": 4.3795, |
| "margin_dpo/margin_mean": 44.46790313720703, |
| "margin_dpo/margin_std": 52.83469772338867, |
| "step": 257 |
| }, |
| { |
| "epoch": 0.5403141361256545, |
| "grad_norm": 64.80396270751953, |
| "learning_rate": 2.600664850273538e-07, |
| "logits/chosen": 1.1780776977539062, |
| "logits/rejected": 1.358864665031433, |
| "logps/chosen": -304.0466613769531, |
| "logps/ref_chosen": -274.53314208984375, |
| "logps/ref_rejected": -284.3149108886719, |
| "logps/rejected": -365.9967346191406, |
| "loss": 4.2167, |
| "margin_dpo/margin_mean": 52.16826248168945, |
| "margin_dpo/margin_std": 58.13621139526367, |
| "step": 258 |
| }, |
| { |
| "epoch": 0.5424083769633508, |
| "grad_norm": 57.088375091552734, |
| "learning_rate": 2.582369512637302e-07, |
| "logits/chosen": 1.1932607889175415, |
| "logits/rejected": 1.141600489616394, |
| "logps/chosen": -255.16656494140625, |
| "logps/ref_chosen": -235.41139221191406, |
| "logps/ref_rejected": -217.746826171875, |
| "logps/rejected": -282.0968933105469, |
| "loss": 4.5131, |
| "margin_dpo/margin_mean": 44.59490966796875, |
| "margin_dpo/margin_std": 53.4549674987793, |
| "step": 259 |
| }, |
| { |
| "epoch": 0.5445026178010471, |
| "grad_norm": 65.07582092285156, |
| "learning_rate": 2.5640697577740815e-07, |
| "logits/chosen": 0.8414401412010193, |
| "logits/rejected": 0.9391928911209106, |
| "logps/chosen": -242.9241943359375, |
| "logps/ref_chosen": -224.4993133544922, |
| "logps/ref_rejected": -215.19839477539062, |
| "logps/rejected": -268.3463439941406, |
| "loss": 5.2773, |
| "margin_dpo/margin_mean": 34.72306442260742, |
| "margin_dpo/margin_std": 62.74570083618164, |
| "step": 260 |
| }, |
| { |
| "epoch": 0.5465968586387434, |
| "grad_norm": 62.26913070678711, |
| "learning_rate": 2.5457665670441937e-07, |
| "logits/chosen": 0.8039923906326294, |
| "logits/rejected": 0.6551789045333862, |
| "logps/chosen": -289.4314270019531, |
| "logps/ref_chosen": -251.2598114013672, |
| "logps/ref_rejected": -205.53323364257812, |
| "logps/rejected": -263.32464599609375, |
| "loss": 4.7681, |
| "margin_dpo/margin_mean": 19.61980438232422, |
| "margin_dpo/margin_std": 53.51505661010742, |
| "step": 261 |
| }, |
| { |
| "epoch": 0.5486910994764398, |
| "grad_norm": 57.36088562011719, |
| "learning_rate": 2.527460921992209e-07, |
| "logits/chosen": 1.5565768480300903, |
| "logits/rejected": 1.5727018117904663, |
| "logps/chosen": -370.4512634277344, |
| "logps/ref_chosen": -347.8548889160156, |
| "logps/ref_rejected": -309.43011474609375, |
| "logps/rejected": -387.045166015625, |
| "loss": 4.4059, |
| "margin_dpo/margin_mean": 55.01873016357422, |
| "margin_dpo/margin_std": 52.59794235229492, |
| "step": 262 |
| }, |
| { |
| "epoch": 0.5507853403141362, |
| "grad_norm": 80.06723022460938, |
| "learning_rate": 2.509153804294318e-07, |
| "logits/chosen": 1.2121027708053589, |
| "logits/rejected": 1.3596720695495605, |
| "logps/chosen": -301.9384765625, |
| "logps/ref_chosen": -261.0179443359375, |
| "logps/ref_rejected": -295.4287109375, |
| "logps/rejected": -357.9337463378906, |
| "loss": 4.7515, |
| "margin_dpo/margin_mean": 21.584484100341797, |
| "margin_dpo/margin_std": 49.02084732055664, |
| "step": 263 |
| }, |
| { |
| "epoch": 0.5528795811518324, |
| "grad_norm": 65.33440399169922, |
| "learning_rate": 2.4908461957056825e-07, |
| "logits/chosen": 1.4055628776550293, |
| "logits/rejected": 1.2200889587402344, |
| "logps/chosen": -321.2078857421875, |
| "logps/ref_chosen": -297.6844482421875, |
| "logps/ref_rejected": -205.72137451171875, |
| "logps/rejected": -284.78070068359375, |
| "loss": 4.2315, |
| "margin_dpo/margin_mean": 55.53590393066406, |
| "margin_dpo/margin_std": 46.41938018798828, |
| "step": 264 |
| }, |
| { |
| "epoch": 0.5549738219895288, |
| "grad_norm": 51.22663879394531, |
| "learning_rate": 2.4725390780077905e-07, |
| "logits/chosen": 1.3676010370254517, |
| "logits/rejected": 1.380630612373352, |
| "logps/chosen": -306.2537536621094, |
| "logps/ref_chosen": -285.8244323730469, |
| "logps/ref_rejected": -275.6885681152344, |
| "logps/rejected": -362.411865234375, |
| "loss": 4.4685, |
| "margin_dpo/margin_mean": 66.2939682006836, |
| "margin_dpo/margin_std": 51.13595199584961, |
| "step": 265 |
| }, |
| { |
| "epoch": 0.5570680628272251, |
| "grad_norm": 56.91117477416992, |
| "learning_rate": 2.454233432955807e-07, |
| "logits/chosen": 1.253815770149231, |
| "logits/rejected": 1.3310532569885254, |
| "logps/chosen": -280.5023193359375, |
| "logps/ref_chosen": -273.0467834472656, |
| "logps/ref_rejected": -291.18133544921875, |
| "logps/rejected": -342.8387145996094, |
| "loss": 4.314, |
| "margin_dpo/margin_mean": 44.201820373535156, |
| "margin_dpo/margin_std": 46.703495025634766, |
| "step": 266 |
| }, |
| { |
| "epoch": 0.5591623036649215, |
| "grad_norm": 44.81831741333008, |
| "learning_rate": 2.435930242225919e-07, |
| "logits/chosen": 1.1857373714447021, |
| "logits/rejected": 1.3162403106689453, |
| "logps/chosen": -294.0497131347656, |
| "logps/ref_chosen": -272.337890625, |
| "logps/ref_rejected": -279.97076416015625, |
| "logps/rejected": -351.3126525878906, |
| "loss": 4.6021, |
| "margin_dpo/margin_mean": 49.630027770996094, |
| "margin_dpo/margin_std": 56.707557678222656, |
| "step": 267 |
| }, |
| { |
| "epoch": 0.5612565445026177, |
| "grad_norm": 65.94684600830078, |
| "learning_rate": 2.4176304873626984e-07, |
| "logits/chosen": 1.1858967542648315, |
| "logits/rejected": 1.2295567989349365, |
| "logps/chosen": -257.2110900878906, |
| "logps/ref_chosen": -235.03692626953125, |
| "logps/ref_rejected": -245.3459014892578, |
| "logps/rejected": -307.6155090332031, |
| "loss": 4.4782, |
| "margin_dpo/margin_mean": 40.09546661376953, |
| "margin_dpo/margin_std": 52.132469177246094, |
| "step": 268 |
| }, |
| { |
| "epoch": 0.5633507853403141, |
| "grad_norm": 40.701568603515625, |
| "learning_rate": 2.399335149726463e-07, |
| "logits/chosen": 1.1352908611297607, |
| "logits/rejected": 1.3305590152740479, |
| "logps/chosen": -262.03607177734375, |
| "logps/ref_chosen": -240.3035430908203, |
| "logps/ref_rejected": -233.82675170898438, |
| "logps/rejected": -302.03057861328125, |
| "loss": 4.6077, |
| "margin_dpo/margin_mean": 46.471290588378906, |
| "margin_dpo/margin_std": 58.428775787353516, |
| "step": 269 |
| }, |
| { |
| "epoch": 0.5654450261780105, |
| "grad_norm": 73.9042739868164, |
| "learning_rate": 2.381045210440644e-07, |
| "logits/chosen": 1.550492286682129, |
| "logits/rejected": 1.8568247556686401, |
| "logps/chosen": -273.93243408203125, |
| "logps/ref_chosen": -249.420166015625, |
| "logps/ref_rejected": -279.5133972167969, |
| "logps/rejected": -334.2721862792969, |
| "loss": 4.5035, |
| "margin_dpo/margin_mean": 30.24651336669922, |
| "margin_dpo/margin_std": 66.87718200683594, |
| "step": 270 |
| }, |
| { |
| "epoch": 0.5675392670157068, |
| "grad_norm": 64.23003387451172, |
| "learning_rate": 2.3627616503391812e-07, |
| "logits/chosen": 1.003787636756897, |
| "logits/rejected": 1.0502243041992188, |
| "logps/chosen": -243.3548126220703, |
| "logps/ref_chosen": -227.45108032226562, |
| "logps/ref_rejected": -183.29275512695312, |
| "logps/rejected": -236.02288818359375, |
| "loss": 4.2933, |
| "margin_dpo/margin_mean": 36.826351165771484, |
| "margin_dpo/margin_std": 40.91915512084961, |
| "step": 271 |
| }, |
| { |
| "epoch": 0.5696335078534032, |
| "grad_norm": 54.780860900878906, |
| "learning_rate": 2.344485449913914e-07, |
| "logits/chosen": 1.5306094884872437, |
| "logits/rejected": 1.417227029800415, |
| "logps/chosen": -370.7244873046875, |
| "logps/ref_chosen": -360.17462158203125, |
| "logps/ref_rejected": -241.59568786621094, |
| "logps/rejected": -302.3287658691406, |
| "loss": 4.4468, |
| "margin_dpo/margin_mean": 50.18324661254883, |
| "margin_dpo/margin_std": 52.38308334350586, |
| "step": 272 |
| }, |
| { |
| "epoch": 0.5717277486910994, |
| "grad_norm": 76.68623352050781, |
| "learning_rate": 2.3262175892620062e-07, |
| "logits/chosen": 1.463561773300171, |
| "logits/rejected": 1.513543725013733, |
| "logps/chosen": -323.9863586425781, |
| "logps/ref_chosen": -309.366455078125, |
| "logps/ref_rejected": -271.2337951660156, |
| "logps/rejected": -335.74420166015625, |
| "loss": 4.3448, |
| "margin_dpo/margin_mean": 49.89052963256836, |
| "margin_dpo/margin_std": 69.81361389160156, |
| "step": 273 |
| }, |
| { |
| "epoch": 0.5738219895287958, |
| "grad_norm": 40.13818359375, |
| "learning_rate": 2.3079590480333827e-07, |
| "logits/chosen": 1.596007227897644, |
| "logits/rejected": 1.7439165115356445, |
| "logps/chosen": -304.8982238769531, |
| "logps/ref_chosen": -295.56866455078125, |
| "logps/ref_rejected": -253.984130859375, |
| "logps/rejected": -311.5342102050781, |
| "loss": 4.2593, |
| "margin_dpo/margin_mean": 48.22050476074219, |
| "margin_dpo/margin_std": 47.02198791503906, |
| "step": 274 |
| }, |
| { |
| "epoch": 0.5759162303664922, |
| "grad_norm": 51.986778259277344, |
| "learning_rate": 2.2897108053782e-07, |
| "logits/chosen": 0.9961601495742798, |
| "logits/rejected": 1.0950078964233398, |
| "logps/chosen": -251.74990844726562, |
| "logps/ref_chosen": -235.93154907226562, |
| "logps/ref_rejected": -230.19454956054688, |
| "logps/rejected": -288.5270080566406, |
| "loss": 4.061, |
| "margin_dpo/margin_mean": 42.51408386230469, |
| "margin_dpo/margin_std": 56.936180114746094, |
| "step": 275 |
| }, |
| { |
| "epoch": 0.5780104712041885, |
| "grad_norm": 51.089576721191406, |
| "learning_rate": 2.2714738398943308e-07, |
| "logits/chosen": 1.7104884386062622, |
| "logits/rejected": 1.6105390787124634, |
| "logps/chosen": -365.57635498046875, |
| "logps/ref_chosen": -357.3829650878906, |
| "logps/ref_rejected": -273.025146484375, |
| "logps/rejected": -322.5356140136719, |
| "loss": 4.1898, |
| "margin_dpo/margin_mean": 41.317039489746094, |
| "margin_dpo/margin_std": 46.224205017089844, |
| "step": 276 |
| }, |
| { |
| "epoch": 0.5801047120418849, |
| "grad_norm": 59.21977233886719, |
| "learning_rate": 2.2532491295748865e-07, |
| "logits/chosen": 1.0496208667755127, |
| "logits/rejected": 1.255394697189331, |
| "logps/chosen": -316.8267822265625, |
| "logps/ref_chosen": -289.98040771484375, |
| "logps/ref_rejected": -310.3972473144531, |
| "logps/rejected": -371.3232727050781, |
| "loss": 4.7638, |
| "margin_dpo/margin_mean": 34.079654693603516, |
| "margin_dpo/margin_std": 57.57683563232422, |
| "step": 277 |
| }, |
| { |
| "epoch": 0.5821989528795811, |
| "grad_norm": 52.5463981628418, |
| "learning_rate": 2.2350376517557726e-07, |
| "logits/chosen": 0.8676056861877441, |
| "logits/rejected": 0.8314589262008667, |
| "logps/chosen": -256.13165283203125, |
| "logps/ref_chosen": -237.13531494140625, |
| "logps/ref_rejected": -232.33502197265625, |
| "logps/rejected": -290.1609802246094, |
| "loss": 4.9549, |
| "margin_dpo/margin_mean": 38.829627990722656, |
| "margin_dpo/margin_std": 55.37847900390625, |
| "step": 278 |
| }, |
| { |
| "epoch": 0.5842931937172775, |
| "grad_norm": 45.49783706665039, |
| "learning_rate": 2.2168403830632769e-07, |
| "logits/chosen": 1.3541253805160522, |
| "logits/rejected": 1.4431573152542114, |
| "logps/chosen": -361.9897766113281, |
| "logps/ref_chosen": -354.13311767578125, |
| "logps/ref_rejected": -305.6336975097656, |
| "logps/rejected": -358.853271484375, |
| "loss": 4.1861, |
| "margin_dpo/margin_mean": 45.362972259521484, |
| "margin_dpo/margin_std": 40.74034118652344, |
| "step": 279 |
| }, |
| { |
| "epoch": 0.5863874345549738, |
| "grad_norm": 45.03684997558594, |
| "learning_rate": 2.1986582993616925e-07, |
| "logits/chosen": 1.337092638015747, |
| "logits/rejected": 1.3559750318527222, |
| "logps/chosen": -274.98260498046875, |
| "logps/ref_chosen": -268.2659912109375, |
| "logps/ref_rejected": -232.44114685058594, |
| "logps/rejected": -289.997314453125, |
| "loss": 4.4116, |
| "margin_dpo/margin_mean": 50.83951950073242, |
| "margin_dpo/margin_std": 63.36719512939453, |
| "step": 280 |
| }, |
| { |
| "epoch": 0.5884816753926702, |
| "grad_norm": 40.450538635253906, |
| "learning_rate": 2.1804923757009882e-07, |
| "logits/chosen": 1.3821660280227661, |
| "logits/rejected": 1.3343546390533447, |
| "logps/chosen": -287.6787414550781, |
| "logps/ref_chosen": -257.0721740722656, |
| "logps/ref_rejected": -248.18264770507812, |
| "logps/rejected": -319.9836730957031, |
| "loss": 4.3178, |
| "margin_dpo/margin_mean": 41.19449234008789, |
| "margin_dpo/margin_std": 56.606285095214844, |
| "step": 281 |
| }, |
| { |
| "epoch": 0.5905759162303665, |
| "grad_norm": 57.53801345825195, |
| "learning_rate": 2.1623435862645205e-07, |
| "logits/chosen": 1.5275428295135498, |
| "logits/rejected": 1.6021305322647095, |
| "logps/chosen": -293.01007080078125, |
| "logps/ref_chosen": -269.2411804199219, |
| "logps/ref_rejected": -323.8949279785156, |
| "logps/rejected": -384.69775390625, |
| "loss": 4.58, |
| "margin_dpo/margin_mean": 37.03391647338867, |
| "margin_dpo/margin_std": 60.21036148071289, |
| "step": 282 |
| }, |
| { |
| "epoch": 0.5926701570680628, |
| "grad_norm": 48.43301773071289, |
| "learning_rate": 2.1442129043167873e-07, |
| "logits/chosen": 1.0364420413970947, |
| "logits/rejected": 1.2966769933700562, |
| "logps/chosen": -279.5700378417969, |
| "logps/ref_chosen": -257.61688232421875, |
| "logps/ref_rejected": -234.8463134765625, |
| "logps/rejected": -297.4212646484375, |
| "loss": 4.4572, |
| "margin_dpo/margin_mean": 40.621826171875, |
| "margin_dpo/margin_std": 61.95793533325195, |
| "step": 283 |
| }, |
| { |
| "epoch": 0.5947643979057592, |
| "grad_norm": 77.20549011230469, |
| "learning_rate": 2.1261013021512378e-07, |
| "logits/chosen": 1.3976938724517822, |
| "logits/rejected": 1.3549877405166626, |
| "logps/chosen": -252.79287719726562, |
| "logps/ref_chosen": -228.94891357421875, |
| "logps/ref_rejected": -288.43804931640625, |
| "logps/rejected": -346.0372009277344, |
| "loss": 4.6112, |
| "margin_dpo/margin_mean": 33.755226135253906, |
| "margin_dpo/margin_std": 55.99586486816406, |
| "step": 284 |
| }, |
| { |
| "epoch": 0.5968586387434555, |
| "grad_norm": 57.3791389465332, |
| "learning_rate": 2.1080097510381294e-07, |
| "logits/chosen": 1.5715055465698242, |
| "logits/rejected": 1.4658746719360352, |
| "logps/chosen": -386.9960632324219, |
| "logps/ref_chosen": -364.84332275390625, |
| "logps/ref_rejected": -306.4946594238281, |
| "logps/rejected": -359.02520751953125, |
| "loss": 4.7905, |
| "margin_dpo/margin_mean": 30.377866744995117, |
| "margin_dpo/margin_std": 52.86358642578125, |
| "step": 285 |
| }, |
| { |
| "epoch": 0.5989528795811518, |
| "grad_norm": 37.026641845703125, |
| "learning_rate": 2.089939221172446e-07, |
| "logits/chosen": 1.36098051071167, |
| "logits/rejected": 1.4245069026947021, |
| "logps/chosen": -299.2890625, |
| "logps/ref_chosen": -269.2027893066406, |
| "logps/ref_rejected": -286.9102478027344, |
| "logps/rejected": -346.26043701171875, |
| "loss": 4.6962, |
| "margin_dpo/margin_mean": 29.26395034790039, |
| "margin_dpo/margin_std": 46.75328063964844, |
| "step": 286 |
| }, |
| { |
| "epoch": 0.6010471204188481, |
| "grad_norm": 58.61341094970703, |
| "learning_rate": 2.0718906816218595e-07, |
| "logits/chosen": 1.219170093536377, |
| "logits/rejected": 1.3217523097991943, |
| "logps/chosen": -259.4914855957031, |
| "logps/ref_chosen": -233.5873565673828, |
| "logps/ref_rejected": -230.03646850585938, |
| "logps/rejected": -291.0602722167969, |
| "loss": 4.594, |
| "margin_dpo/margin_mean": 35.11963653564453, |
| "margin_dpo/margin_std": 54.2941780090332, |
| "step": 287 |
| }, |
| { |
| "epoch": 0.6031413612565445, |
| "grad_norm": 46.509056091308594, |
| "learning_rate": 2.053865100274774e-07, |
| "logits/chosen": 1.5584362745285034, |
| "logits/rejected": 1.3865753412246704, |
| "logps/chosen": -412.9979553222656, |
| "logps/ref_chosen": -378.4530029296875, |
| "logps/ref_rejected": -302.7226257324219, |
| "logps/rejected": -366.6625061035156, |
| "loss": 4.5476, |
| "margin_dpo/margin_mean": 29.394969940185547, |
| "margin_dpo/margin_std": 42.88922119140625, |
| "step": 288 |
| }, |
| { |
| "epoch": 0.6052356020942409, |
| "grad_norm": 42.84352493286133, |
| "learning_rate": 2.035863443788411e-07, |
| "logits/chosen": 1.5689976215362549, |
| "logits/rejected": 1.5198795795440674, |
| "logps/chosen": -373.4714660644531, |
| "logps/ref_chosen": -342.27532958984375, |
| "logps/ref_rejected": -317.79638671875, |
| "logps/rejected": -370.5872802734375, |
| "loss": 4.944, |
| "margin_dpo/margin_mean": 21.594791412353516, |
| "margin_dpo/margin_std": 44.652915954589844, |
| "step": 289 |
| }, |
| { |
| "epoch": 0.6073298429319371, |
| "grad_norm": 58.85520935058594, |
| "learning_rate": 2.0178866775369774e-07, |
| "logits/chosen": 1.3218892812728882, |
| "logits/rejected": 1.2929930686950684, |
| "logps/chosen": -374.0101623535156, |
| "logps/ref_chosen": -348.39788818359375, |
| "logps/ref_rejected": -349.3028564453125, |
| "logps/rejected": -415.2039489746094, |
| "loss": 4.6543, |
| "margin_dpo/margin_mean": 40.28877258300781, |
| "margin_dpo/margin_std": 68.06195831298828, |
| "step": 290 |
| }, |
| { |
| "epoch": 0.6094240837696335, |
| "grad_norm": 39.78348922729492, |
| "learning_rate": 1.9999357655598891e-07, |
| "logits/chosen": 1.0092355012893677, |
| "logits/rejected": 1.146081805229187, |
| "logps/chosen": -268.5143737792969, |
| "logps/ref_chosen": -250.70835876464844, |
| "logps/ref_rejected": -240.2347869873047, |
| "logps/rejected": -312.80255126953125, |
| "loss": 4.2499, |
| "margin_dpo/margin_mean": 54.76176834106445, |
| "margin_dpo/margin_std": 55.031982421875, |
| "step": 291 |
| }, |
| { |
| "epoch": 0.6115183246073298, |
| "grad_norm": 56.82017135620117, |
| "learning_rate": 1.9820116705100775e-07, |
| "logits/chosen": 1.0279195308685303, |
| "logits/rejected": 1.0469276905059814, |
| "logps/chosen": -285.55364990234375, |
| "logps/ref_chosen": -277.9742431640625, |
| "logps/ref_rejected": -260.510986328125, |
| "logps/rejected": -321.3108825683594, |
| "loss": 4.4704, |
| "margin_dpo/margin_mean": 53.22050094604492, |
| "margin_dpo/margin_std": 38.52600860595703, |
| "step": 292 |
| }, |
| { |
| "epoch": 0.6136125654450262, |
| "grad_norm": 82.68972778320312, |
| "learning_rate": 1.9641153536023642e-07, |
| "logits/chosen": 1.8253206014633179, |
| "logits/rejected": 1.6558302640914917, |
| "logps/chosen": -322.6553039550781, |
| "logps/ref_chosen": -300.9186096191406, |
| "logps/ref_rejected": -257.6700439453125, |
| "logps/rejected": -320.5356750488281, |
| "loss": 4.4861, |
| "margin_dpo/margin_mean": 41.12899398803711, |
| "margin_dpo/margin_std": 47.92676544189453, |
| "step": 293 |
| }, |
| { |
| "epoch": 0.6157068062827226, |
| "grad_norm": 59.99162673950195, |
| "learning_rate": 1.9462477745619106e-07, |
| "logits/chosen": 1.151703953742981, |
| "logits/rejected": 1.2953401803970337, |
| "logps/chosen": -282.59271240234375, |
| "logps/ref_chosen": -266.8080139160156, |
| "logps/ref_rejected": -283.26959228515625, |
| "logps/rejected": -355.72576904296875, |
| "loss": 4.5619, |
| "margin_dpo/margin_mean": 56.671478271484375, |
| "margin_dpo/margin_std": 48.01198959350586, |
| "step": 294 |
| }, |
| { |
| "epoch": 0.6178010471204188, |
| "grad_norm": 43.26309585571289, |
| "learning_rate": 1.928409891572757e-07, |
| "logits/chosen": 1.2075080871582031, |
| "logits/rejected": 1.226868748664856, |
| "logps/chosen": -282.1271667480469, |
| "logps/ref_chosen": -240.19598388671875, |
| "logps/ref_rejected": -214.87818908691406, |
| "logps/rejected": -267.48272705078125, |
| "loss": 4.5008, |
| "margin_dpo/margin_mean": 10.673352241516113, |
| "margin_dpo/margin_std": 68.71654510498047, |
| "step": 295 |
| }, |
| { |
| "epoch": 0.6198952879581152, |
| "grad_norm": 41.60967254638672, |
| "learning_rate": 1.9106026612264315e-07, |
| "logits/chosen": 1.4175227880477905, |
| "logits/rejected": 1.595857858657837, |
| "logps/chosen": -236.48651123046875, |
| "logps/ref_chosen": -227.85513305664062, |
| "logps/ref_rejected": -256.476318359375, |
| "logps/rejected": -316.8334655761719, |
| "loss": 4.0503, |
| "margin_dpo/margin_mean": 51.72578430175781, |
| "margin_dpo/margin_std": 55.78533935546875, |
| "step": 296 |
| }, |
| { |
| "epoch": 0.6219895287958115, |
| "grad_norm": 49.98090744018555, |
| "learning_rate": 1.8928270384706582e-07, |
| "logits/chosen": 1.33966863155365, |
| "logits/rejected": 1.5027199983596802, |
| "logps/chosen": -248.77769470214844, |
| "logps/ref_chosen": -220.73609924316406, |
| "logps/ref_rejected": -272.24017333984375, |
| "logps/rejected": -329.5039367675781, |
| "loss": 4.359, |
| "margin_dpo/margin_mean": 29.222198486328125, |
| "margin_dpo/margin_std": 50.361183166503906, |
| "step": 297 |
| }, |
| { |
| "epoch": 0.6240837696335079, |
| "grad_norm": 70.29344940185547, |
| "learning_rate": 1.875083976558136e-07, |
| "logits/chosen": 1.4000345468521118, |
| "logits/rejected": 1.3036651611328125, |
| "logps/chosen": -363.42401123046875, |
| "logps/ref_chosen": -346.2327880859375, |
| "logps/ref_rejected": -285.7917785644531, |
| "logps/rejected": -350.0803527832031, |
| "loss": 4.2786, |
| "margin_dpo/margin_mean": 47.0973014831543, |
| "margin_dpo/margin_std": 58.554161071777344, |
| "step": 298 |
| }, |
| { |
| "epoch": 0.6261780104712041, |
| "grad_norm": 68.31159973144531, |
| "learning_rate": 1.8573744269954297e-07, |
| "logits/chosen": 1.3761045932769775, |
| "logits/rejected": 1.3730204105377197, |
| "logps/chosen": -297.95330810546875, |
| "logps/ref_chosen": -266.99658203125, |
| "logps/ref_rejected": -262.5125427246094, |
| "logps/rejected": -327.41265869140625, |
| "loss": 4.5174, |
| "margin_dpo/margin_mean": 33.94337463378906, |
| "margin_dpo/margin_std": 67.46819305419922, |
| "step": 299 |
| }, |
| { |
| "epoch": 0.6282722513089005, |
| "grad_norm": 41.25592803955078, |
| "learning_rate": 1.839699339491937e-07, |
| "logits/chosen": 1.0598005056381226, |
| "logits/rejected": 1.1422343254089355, |
| "logps/chosen": -306.2508239746094, |
| "logps/ref_chosen": -281.19525146484375, |
| "logps/ref_rejected": -288.6803894042969, |
| "logps/rejected": -364.2799377441406, |
| "loss": 4.5063, |
| "margin_dpo/margin_mean": 50.54399490356445, |
| "margin_dpo/margin_std": 57.26484298706055, |
| "step": 300 |
| }, |
| { |
| "epoch": 0.6303664921465969, |
| "grad_norm": 80.2366943359375, |
| "learning_rate": 1.8220596619089573e-07, |
| "logits/chosen": 1.5710808038711548, |
| "logits/rejected": 1.6003844738006592, |
| "logps/chosen": -304.22662353515625, |
| "logps/ref_chosen": -289.8253173828125, |
| "logps/ref_rejected": -327.8699645996094, |
| "logps/rejected": -389.2873840332031, |
| "loss": 4.5827, |
| "margin_dpo/margin_mean": 47.01613235473633, |
| "margin_dpo/margin_std": 53.770626068115234, |
| "step": 301 |
| }, |
| { |
| "epoch": 0.6324607329842932, |
| "grad_norm": 45.32533645629883, |
| "learning_rate": 1.8044563402088682e-07, |
| "logits/chosen": 1.390702486038208, |
| "logits/rejected": 1.5861480236053467, |
| "logps/chosen": -341.7214660644531, |
| "logps/ref_chosen": -307.1119079589844, |
| "logps/ref_rejected": -296.61785888671875, |
| "logps/rejected": -385.1257019042969, |
| "loss": 4.3462, |
| "margin_dpo/margin_mean": 53.89823532104492, |
| "margin_dpo/margin_std": 60.43596267700195, |
| "step": 302 |
| }, |
| { |
| "epoch": 0.6345549738219896, |
| "grad_norm": 35.210235595703125, |
| "learning_rate": 1.7868903184043885e-07, |
| "logits/chosen": 1.0414403676986694, |
| "logits/rejected": 1.2032232284545898, |
| "logps/chosen": -287.9827575683594, |
| "logps/ref_chosen": -261.281982421875, |
| "logps/ref_rejected": -287.9131164550781, |
| "logps/rejected": -370.0382385253906, |
| "loss": 4.4312, |
| "margin_dpo/margin_mean": 55.42430114746094, |
| "margin_dpo/margin_std": 58.83437728881836, |
| "step": 303 |
| }, |
| { |
| "epoch": 0.6366492146596858, |
| "grad_norm": 53.23714065551758, |
| "learning_rate": 1.7693625385079574e-07, |
| "logits/chosen": 1.1299974918365479, |
| "logits/rejected": 1.1754674911499023, |
| "logps/chosen": -317.24932861328125, |
| "logps/ref_chosen": -276.4831848144531, |
| "logps/ref_rejected": -257.2686462402344, |
| "logps/rejected": -332.5347595214844, |
| "loss": 4.6016, |
| "margin_dpo/margin_mean": 34.49999237060547, |
| "margin_dpo/margin_std": 42.753028869628906, |
| "step": 304 |
| }, |
| { |
| "epoch": 0.6387434554973822, |
| "grad_norm": 50.1977653503418, |
| "learning_rate": 1.7518739404812155e-07, |
| "logits/chosen": 1.1471208333969116, |
| "logits/rejected": 1.1764111518859863, |
| "logps/chosen": -272.23565673828125, |
| "logps/ref_chosen": -253.3165283203125, |
| "logps/ref_rejected": -225.20468139648438, |
| "logps/rejected": -278.5679931640625, |
| "loss": 4.0448, |
| "margin_dpo/margin_mean": 34.444190979003906, |
| "margin_dpo/margin_std": 46.56166076660156, |
| "step": 305 |
| }, |
| { |
| "epoch": 0.6408376963350786, |
| "grad_norm": 51.97341537475586, |
| "learning_rate": 1.7344254621846017e-07, |
| "logits/chosen": 1.2266101837158203, |
| "logits/rejected": 1.1241114139556885, |
| "logps/chosen": -338.9609069824219, |
| "logps/ref_chosen": -324.57122802734375, |
| "logps/ref_rejected": -299.1585693359375, |
| "logps/rejected": -352.9222412109375, |
| "loss": 4.306, |
| "margin_dpo/margin_mean": 39.374000549316406, |
| "margin_dpo/margin_std": 74.34258270263672, |
| "step": 306 |
| }, |
| { |
| "epoch": 0.6429319371727749, |
| "grad_norm": 45.46051025390625, |
| "learning_rate": 1.717018039327053e-07, |
| "logits/chosen": 1.1175193786621094, |
| "logits/rejected": 1.2578641176223755, |
| "logps/chosen": -320.50177001953125, |
| "logps/ref_chosen": -289.5794372558594, |
| "logps/ref_rejected": -262.92510986328125, |
| "logps/rejected": -347.1852722167969, |
| "loss": 4.1751, |
| "margin_dpo/margin_mean": 53.3377685546875, |
| "margin_dpo/margin_std": 65.59990692138672, |
| "step": 307 |
| }, |
| { |
| "epoch": 0.6450261780104712, |
| "grad_norm": 34.76543045043945, |
| "learning_rate": 1.699652605415828e-07, |
| "logits/chosen": 1.338348388671875, |
| "logits/rejected": 1.3202842473983765, |
| "logps/chosen": -348.8934020996094, |
| "logps/ref_chosen": -305.04351806640625, |
| "logps/ref_rejected": -305.0120849609375, |
| "logps/rejected": -384.90301513671875, |
| "loss": 4.6399, |
| "margin_dpo/margin_mean": 36.04100036621094, |
| "margin_dpo/margin_std": 60.608455657958984, |
| "step": 308 |
| }, |
| { |
| "epoch": 0.6471204188481675, |
| "grad_norm": 66.8311996459961, |
| "learning_rate": 1.6823300917064458e-07, |
| "logits/chosen": 1.619686484336853, |
| "logits/rejected": 1.3856267929077148, |
| "logps/chosen": -354.4423522949219, |
| "logps/ref_chosen": -316.80303955078125, |
| "logps/ref_rejected": -240.09307861328125, |
| "logps/rejected": -317.1116027832031, |
| "loss": 4.284, |
| "margin_dpo/margin_mean": 39.3791389465332, |
| "margin_dpo/margin_std": 59.34083938598633, |
| "step": 309 |
| }, |
| { |
| "epoch": 0.6492146596858639, |
| "grad_norm": 61.75398254394531, |
| "learning_rate": 1.6650514271527465e-07, |
| "logits/chosen": 1.2451605796813965, |
| "logits/rejected": 1.5067654848098755, |
| "logps/chosen": -289.0413818359375, |
| "logps/ref_chosen": -240.17652893066406, |
| "logps/ref_rejected": -242.7730712890625, |
| "logps/rejected": -332.27880859375, |
| "loss": 4.4389, |
| "margin_dpo/margin_mean": 40.6408576965332, |
| "margin_dpo/margin_std": 47.202674865722656, |
| "step": 310 |
| }, |
| { |
| "epoch": 0.6513089005235602, |
| "grad_norm": 44.972686767578125, |
| "learning_rate": 1.647817538357072e-07, |
| "logits/chosen": 1.1516824960708618, |
| "logits/rejected": 1.3081568479537964, |
| "logps/chosen": -300.0003662109375, |
| "logps/ref_chosen": -257.53515625, |
| "logps/ref_rejected": -249.1999053955078, |
| "logps/rejected": -334.6231689453125, |
| "loss": 4.4418, |
| "margin_dpo/margin_mean": 42.95802307128906, |
| "margin_dpo/margin_std": 56.176063537597656, |
| "step": 311 |
| }, |
| { |
| "epoch": 0.6534031413612565, |
| "grad_norm": 70.0779037475586, |
| "learning_rate": 1.6306293495205755e-07, |
| "logits/chosen": 1.374745488166809, |
| "logits/rejected": 1.4253482818603516, |
| "logps/chosen": -301.5932922363281, |
| "logps/ref_chosen": -261.98828125, |
| "logps/ref_rejected": -238.6123504638672, |
| "logps/rejected": -317.7529296875, |
| "loss": 4.3747, |
| "margin_dpo/margin_mean": 39.53556442260742, |
| "margin_dpo/margin_std": 68.29449462890625, |
| "step": 312 |
| }, |
| { |
| "epoch": 0.6554973821989529, |
| "grad_norm": 56.74006271362305, |
| "learning_rate": 1.6134877823936607e-07, |
| "logits/chosen": 1.480233073234558, |
| "logits/rejected": 1.6010148525238037, |
| "logps/chosen": -417.88751220703125, |
| "logps/ref_chosen": -380.5164794921875, |
| "logps/ref_rejected": -340.59722900390625, |
| "logps/rejected": -436.11846923828125, |
| "loss": 4.5172, |
| "margin_dpo/margin_mean": 58.15019607543945, |
| "margin_dpo/margin_std": 64.69535064697266, |
| "step": 313 |
| }, |
| { |
| "epoch": 0.6575916230366492, |
| "grad_norm": 52.93495559692383, |
| "learning_rate": 1.5963937562265522e-07, |
| "logits/chosen": 1.3546419143676758, |
| "logits/rejected": 1.3760360479354858, |
| "logps/chosen": -288.9587707519531, |
| "logps/ref_chosen": -254.8392791748047, |
| "logps/ref_rejected": -233.38494873046875, |
| "logps/rejected": -312.0257263183594, |
| "loss": 4.4718, |
| "margin_dpo/margin_mean": 44.52123260498047, |
| "margin_dpo/margin_std": 63.62104415893555, |
| "step": 314 |
| }, |
| { |
| "epoch": 0.6596858638743456, |
| "grad_norm": 41.35818862915039, |
| "learning_rate": 1.5793481877199943e-07, |
| "logits/chosen": 1.7810715436935425, |
| "logits/rejected": 1.7476561069488525, |
| "logps/chosen": -315.27471923828125, |
| "logps/ref_chosen": -287.1436767578125, |
| "logps/ref_rejected": -245.744873046875, |
| "logps/rejected": -311.196044921875, |
| "loss": 4.1818, |
| "margin_dpo/margin_mean": 37.32012939453125, |
| "margin_dpo/margin_std": 46.346778869628906, |
| "step": 315 |
| }, |
| { |
| "epoch": 0.6617801047120419, |
| "grad_norm": 60.255733489990234, |
| "learning_rate": 1.562351990976095e-07, |
| "logits/chosen": 0.9633012413978577, |
| "logits/rejected": 1.0967074632644653, |
| "logps/chosen": -310.6409912109375, |
| "logps/ref_chosen": -278.97003173828125, |
| "logps/ref_rejected": -268.5596618652344, |
| "logps/rejected": -364.9547119140625, |
| "loss": 4.1869, |
| "margin_dpo/margin_mean": 64.72406005859375, |
| "margin_dpo/margin_std": 64.02919006347656, |
| "step": 316 |
| }, |
| { |
| "epoch": 0.6638743455497382, |
| "grad_norm": 63.166786193847656, |
| "learning_rate": 1.5454060774493065e-07, |
| "logits/chosen": 1.2942625284194946, |
| "logits/rejected": 1.294532060623169, |
| "logps/chosen": -277.548095703125, |
| "logps/ref_chosen": -252.86656188964844, |
| "logps/ref_rejected": -236.70155334472656, |
| "logps/rejected": -304.314208984375, |
| "loss": 4.3267, |
| "margin_dpo/margin_mean": 42.931148529052734, |
| "margin_dpo/margin_std": 60.30817413330078, |
| "step": 317 |
| }, |
| { |
| "epoch": 0.6659685863874345, |
| "grad_norm": 59.5412712097168, |
| "learning_rate": 1.5285113558975427e-07, |
| "logits/chosen": 1.2442307472229004, |
| "logits/rejected": 1.4497402906417847, |
| "logps/chosen": -252.0952606201172, |
| "logps/ref_chosen": -217.34515380859375, |
| "logps/ref_rejected": -243.4803009033203, |
| "logps/rejected": -328.711669921875, |
| "loss": 4.2442, |
| "margin_dpo/margin_mean": 50.481266021728516, |
| "margin_dpo/margin_std": 49.45831298828125, |
| "step": 318 |
| }, |
| { |
| "epoch": 0.6680628272251309, |
| "grad_norm": 34.23768615722656, |
| "learning_rate": 1.5116687323334464e-07, |
| "logits/chosen": 1.0364283323287964, |
| "logits/rejected": 1.2878518104553223, |
| "logps/chosen": -290.0143737792969, |
| "logps/ref_chosen": -268.8816833496094, |
| "logps/ref_rejected": -275.4843444824219, |
| "logps/rejected": -347.4952697753906, |
| "loss": 4.0379, |
| "margin_dpo/margin_mean": 50.878273010253906, |
| "margin_dpo/margin_std": 42.63667678833008, |
| "step": 319 |
| }, |
| { |
| "epoch": 0.6701570680628273, |
| "grad_norm": 55.01826477050781, |
| "learning_rate": 1.4948791099758052e-07, |
| "logits/chosen": 1.7030011415481567, |
| "logits/rejected": 1.6658614873886108, |
| "logps/chosen": -328.2850341796875, |
| "logps/ref_chosen": -307.4996337890625, |
| "logps/ref_rejected": -251.08456420898438, |
| "logps/rejected": -320.5891418457031, |
| "loss": 4.3706, |
| "margin_dpo/margin_mean": 48.71923065185547, |
| "margin_dpo/margin_std": 54.48359680175781, |
| "step": 320 |
| }, |
| { |
| "epoch": 0.6722513089005235, |
| "grad_norm": 36.16436004638672, |
| "learning_rate": 1.478143389201113e-07, |
| "logits/chosen": 1.5252739191055298, |
| "logits/rejected": 1.3168452978134155, |
| "logps/chosen": -343.71514892578125, |
| "logps/ref_chosen": -309.8309631347656, |
| "logps/ref_rejected": -248.75213623046875, |
| "logps/rejected": -326.5061340332031, |
| "loss": 4.4452, |
| "margin_dpo/margin_mean": 43.869834899902344, |
| "margin_dpo/margin_std": 63.80504608154297, |
| "step": 321 |
| }, |
| { |
| "epoch": 0.6743455497382199, |
| "grad_norm": 43.79127502441406, |
| "learning_rate": 1.461462467495284e-07, |
| "logits/chosen": 0.9715927243232727, |
| "logits/rejected": 1.0002247095108032, |
| "logps/chosen": -323.4966735839844, |
| "logps/ref_chosen": -291.58843994140625, |
| "logps/ref_rejected": -265.43023681640625, |
| "logps/rejected": -339.85174560546875, |
| "loss": 4.1918, |
| "margin_dpo/margin_mean": 42.5133056640625, |
| "margin_dpo/margin_std": 48.27851486206055, |
| "step": 322 |
| }, |
| { |
| "epoch": 0.6764397905759162, |
| "grad_norm": 47.61728286743164, |
| "learning_rate": 1.4448372394055246e-07, |
| "logits/chosen": 1.0764468908309937, |
| "logits/rejected": 0.8316705822944641, |
| "logps/chosen": -385.0590515136719, |
| "logps/ref_chosen": -343.968017578125, |
| "logps/ref_rejected": -254.12161254882812, |
| "logps/rejected": -329.58868408203125, |
| "loss": 4.879, |
| "margin_dpo/margin_mean": 34.37602996826172, |
| "margin_dpo/margin_std": 67.8411636352539, |
| "step": 323 |
| }, |
| { |
| "epoch": 0.6785340314136126, |
| "grad_norm": 35.46821975708008, |
| "learning_rate": 1.428268596492364e-07, |
| "logits/chosen": 1.5448215007781982, |
| "logits/rejected": 1.5194947719573975, |
| "logps/chosen": -213.710693359375, |
| "logps/ref_chosen": -206.94500732421875, |
| "logps/ref_rejected": -262.6962890625, |
| "logps/rejected": -316.6614990234375, |
| "loss": 3.9753, |
| "margin_dpo/margin_mean": 47.19957733154297, |
| "margin_dpo/margin_std": 56.4971923828125, |
| "step": 324 |
| }, |
| { |
| "epoch": 0.680628272251309, |
| "grad_norm": 40.15785598754883, |
| "learning_rate": 1.4117574272818386e-07, |
| "logits/chosen": 1.4091088771820068, |
| "logits/rejected": 1.5492061376571655, |
| "logps/chosen": -311.66400146484375, |
| "logps/ref_chosen": -301.9862060546875, |
| "logps/ref_rejected": -333.42236328125, |
| "logps/rejected": -399.2699890136719, |
| "loss": 4.5302, |
| "margin_dpo/margin_mean": 56.1698112487793, |
| "margin_dpo/margin_std": 54.43414306640625, |
| "step": 325 |
| }, |
| { |
| "epoch": 0.6827225130890052, |
| "grad_norm": 52.91168975830078, |
| "learning_rate": 1.3953046172178413e-07, |
| "logits/chosen": 0.951869785785675, |
| "logits/rejected": 1.228202223777771, |
| "logps/chosen": -177.7034912109375, |
| "logps/ref_chosen": -164.46109008789062, |
| "logps/ref_rejected": -249.89413452148438, |
| "logps/rejected": -324.3243713378906, |
| "loss": 4.3905, |
| "margin_dpo/margin_mean": 61.18782043457031, |
| "margin_dpo/margin_std": 53.454124450683594, |
| "step": 326 |
| }, |
| { |
| "epoch": 0.6848167539267016, |
| "grad_norm": 42.79719543457031, |
| "learning_rate": 1.3789110486146468e-07, |
| "logits/chosen": 1.5188686847686768, |
| "logits/rejected": 1.4478236436843872, |
| "logps/chosen": -259.4933166503906, |
| "logps/ref_chosen": -246.3433837890625, |
| "logps/ref_rejected": -229.85508728027344, |
| "logps/rejected": -297.393798828125, |
| "loss": 4.1578, |
| "margin_dpo/margin_mean": 54.388763427734375, |
| "margin_dpo/margin_std": 67.07605743408203, |
| "step": 327 |
| }, |
| { |
| "epoch": 0.6869109947643979, |
| "grad_norm": 62.9756965637207, |
| "learning_rate": 1.362577600609588e-07, |
| "logits/chosen": 0.8666256666183472, |
| "logits/rejected": 0.9310898780822754, |
| "logps/chosen": -325.38824462890625, |
| "logps/ref_chosen": -305.82012939453125, |
| "logps/ref_rejected": -273.3159484863281, |
| "logps/rejected": -348.3529357910156, |
| "loss": 4.3427, |
| "margin_dpo/margin_mean": 55.4688720703125, |
| "margin_dpo/margin_std": 43.20487594604492, |
| "step": 328 |
| }, |
| { |
| "epoch": 0.6890052356020943, |
| "grad_norm": 51.00785827636719, |
| "learning_rate": 1.3463051491159093e-07, |
| "logits/chosen": 1.4473413228988647, |
| "logits/rejected": 1.7869523763656616, |
| "logps/chosen": -283.1918029785156, |
| "logps/ref_chosen": -258.7630615234375, |
| "logps/ref_rejected": -284.41131591796875, |
| "logps/rejected": -350.9872131347656, |
| "loss": 4.7045, |
| "margin_dpo/margin_mean": 42.1472053527832, |
| "margin_dpo/margin_std": 59.915836334228516, |
| "step": 329 |
| }, |
| { |
| "epoch": 0.6910994764397905, |
| "grad_norm": 47.54462432861328, |
| "learning_rate": 1.3300945667758012e-07, |
| "logits/chosen": 1.554024338722229, |
| "logits/rejected": 1.5038138628005981, |
| "logps/chosen": -360.6363830566406, |
| "logps/ref_chosen": -330.3982238769531, |
| "logps/ref_rejected": -274.9824523925781, |
| "logps/rejected": -335.7353515625, |
| "loss": 4.7699, |
| "margin_dpo/margin_mean": 30.514692306518555, |
| "margin_dpo/margin_std": 57.54186248779297, |
| "step": 330 |
| }, |
| { |
| "epoch": 0.6931937172774869, |
| "grad_norm": 43.25348663330078, |
| "learning_rate": 1.3139467229135998e-07, |
| "logits/chosen": 1.2080715894699097, |
| "logits/rejected": 1.1073570251464844, |
| "logps/chosen": -306.7835693359375, |
| "logps/ref_chosen": -279.2760009765625, |
| "logps/ref_rejected": -220.27761840820312, |
| "logps/rejected": -285.9614562988281, |
| "loss": 4.4308, |
| "margin_dpo/margin_mean": 38.1762580871582, |
| "margin_dpo/margin_std": 54.596893310546875, |
| "step": 331 |
| }, |
| { |
| "epoch": 0.6952879581151833, |
| "grad_norm": 40.57438278198242, |
| "learning_rate": 1.2978624834891626e-07, |
| "logits/chosen": 1.1713310480117798, |
| "logits/rejected": 1.2179394960403442, |
| "logps/chosen": -245.34036254882812, |
| "logps/ref_chosen": -226.70223999023438, |
| "logps/ref_rejected": -205.92601013183594, |
| "logps/rejected": -280.2979736328125, |
| "loss": 4.2023, |
| "margin_dpo/margin_mean": 55.733863830566406, |
| "margin_dpo/margin_std": 48.06736755371094, |
| "step": 332 |
| }, |
| { |
| "epoch": 0.6973821989528796, |
| "grad_norm": 51.89149475097656, |
| "learning_rate": 1.281842711051438e-07, |
| "logits/chosen": 1.1539713144302368, |
| "logits/rejected": 1.0611777305603027, |
| "logps/chosen": -303.71087646484375, |
| "logps/ref_chosen": -280.1510009765625, |
| "logps/ref_rejected": -231.2144012451172, |
| "logps/rejected": -309.7812194824219, |
| "loss": 4.5953, |
| "margin_dpo/margin_mean": 55.00693130493164, |
| "margin_dpo/margin_std": 62.069034576416016, |
| "step": 333 |
| }, |
| { |
| "epoch": 0.6994764397905759, |
| "grad_norm": 38.94709014892578, |
| "learning_rate": 1.2658882646922033e-07, |
| "logits/chosen": 1.1479655504226685, |
| "logits/rejected": 1.1974003314971924, |
| "logps/chosen": -290.0271301269531, |
| "logps/ref_chosen": -269.64227294921875, |
| "logps/ref_rejected": -260.0500793457031, |
| "logps/rejected": -314.5991516113281, |
| "loss": 4.4026, |
| "margin_dpo/margin_mean": 34.16423416137695, |
| "margin_dpo/margin_std": 44.75538635253906, |
| "step": 334 |
| }, |
| { |
| "epoch": 0.7015706806282722, |
| "grad_norm": 49.46840286254883, |
| "learning_rate": 1.2500000000000005e-07, |
| "logits/chosen": 1.2762084007263184, |
| "logits/rejected": 1.3634967803955078, |
| "logps/chosen": -351.87103271484375, |
| "logps/ref_chosen": -304.7079162597656, |
| "logps/ref_rejected": -269.1751403808594, |
| "logps/rejected": -330.24505615234375, |
| "loss": 4.5817, |
| "margin_dpo/margin_mean": 13.906787872314453, |
| "margin_dpo/margin_std": 66.35944366455078, |
| "step": 335 |
| }, |
| { |
| "epoch": 0.7036649214659686, |
| "grad_norm": 34.12943649291992, |
| "learning_rate": 1.2341787690142435e-07, |
| "logits/chosen": 1.5096263885498047, |
| "logits/rejected": 1.792067289352417, |
| "logps/chosen": -218.86050415039062, |
| "logps/ref_chosen": -210.38368225097656, |
| "logps/ref_rejected": -229.12037658691406, |
| "logps/rejected": -289.0048522949219, |
| "loss": 4.5624, |
| "margin_dpo/margin_mean": 51.40761947631836, |
| "margin_dpo/margin_std": 47.56304931640625, |
| "step": 336 |
| }, |
| { |
| "epoch": 0.7057591623036649, |
| "grad_norm": 52.81936264038086, |
| "learning_rate": 1.2184254201795363e-07, |
| "logits/chosen": 0.93461012840271, |
| "logits/rejected": 0.8486602306365967, |
| "logps/chosen": -299.5815734863281, |
| "logps/ref_chosen": -257.2767639160156, |
| "logps/ref_rejected": -297.5929260253906, |
| "logps/rejected": -374.0364074707031, |
| "loss": 4.4007, |
| "margin_dpo/margin_mean": 34.13860321044922, |
| "margin_dpo/margin_std": 52.06587219238281, |
| "step": 337 |
| }, |
| { |
| "epoch": 0.7078534031413612, |
| "grad_norm": 37.59018325805664, |
| "learning_rate": 1.202740798300168e-07, |
| "logits/chosen": 1.5364826917648315, |
| "logits/rejected": 1.5837008953094482, |
| "logps/chosen": -274.78564453125, |
| "logps/ref_chosen": -257.8255310058594, |
| "logps/ref_rejected": -216.51162719726562, |
| "logps/rejected": -298.27276611328125, |
| "loss": 4.1313, |
| "margin_dpo/margin_mean": 64.8010482788086, |
| "margin_dpo/margin_std": 52.778377532958984, |
| "step": 338 |
| }, |
| { |
| "epoch": 0.7099476439790576, |
| "grad_norm": 43.429386138916016, |
| "learning_rate": 1.1871257444948096e-07, |
| "logits/chosen": 1.5933047533035278, |
| "logits/rejected": 1.5398043394088745, |
| "logps/chosen": -267.3699645996094, |
| "logps/ref_chosen": -240.76815795898438, |
| "logps/ref_rejected": -244.97377014160156, |
| "logps/rejected": -315.84185791015625, |
| "loss": 4.1606, |
| "margin_dpo/margin_mean": 44.2662467956543, |
| "margin_dpo/margin_std": 55.80255126953125, |
| "step": 339 |
| }, |
| { |
| "epoch": 0.7120418848167539, |
| "grad_norm": 35.77497482299805, |
| "learning_rate": 1.1715810961514072e-07, |
| "logits/chosen": 0.9345113039016724, |
| "logits/rejected": 1.0999596118927002, |
| "logps/chosen": -204.77218627929688, |
| "logps/ref_chosen": -187.35751342773438, |
| "logps/ref_rejected": -232.0410614013672, |
| "logps/rejected": -292.81396484375, |
| "loss": 4.428, |
| "margin_dpo/margin_mean": 43.358245849609375, |
| "margin_dpo/margin_std": 77.42390441894531, |
| "step": 340 |
| }, |
| { |
| "epoch": 0.7141361256544503, |
| "grad_norm": 60.169677734375, |
| "learning_rate": 1.1561076868822755e-07, |
| "logits/chosen": 1.5765248537063599, |
| "logits/rejected": 1.8391637802124023, |
| "logps/chosen": -314.5309753417969, |
| "logps/ref_chosen": -283.4117736816406, |
| "logps/ref_rejected": -302.2451171875, |
| "logps/rejected": -380.4033203125, |
| "loss": 4.801, |
| "margin_dpo/margin_mean": 47.03903579711914, |
| "margin_dpo/margin_std": 53.40591049194336, |
| "step": 341 |
| }, |
| { |
| "epoch": 0.7162303664921466, |
| "grad_norm": 45.74888610839844, |
| "learning_rate": 1.1407063464793965e-07, |
| "logits/chosen": 1.175806999206543, |
| "logits/rejected": 1.3320945501327515, |
| "logps/chosen": -249.10618591308594, |
| "logps/ref_chosen": -221.50335693359375, |
| "logps/ref_rejected": -244.48382568359375, |
| "logps/rejected": -306.7912292480469, |
| "loss": 4.3427, |
| "margin_dpo/margin_mean": 34.704566955566406, |
| "margin_dpo/margin_std": 38.48919677734375, |
| "step": 342 |
| }, |
| { |
| "epoch": 0.7183246073298429, |
| "grad_norm": 33.789772033691406, |
| "learning_rate": 1.125377900869913e-07, |
| "logits/chosen": 1.5595121383666992, |
| "logits/rejected": 1.44582200050354, |
| "logps/chosen": -346.7395935058594, |
| "logps/ref_chosen": -340.46466064453125, |
| "logps/ref_rejected": -267.65313720703125, |
| "logps/rejected": -321.6587829589844, |
| "loss": 4.5708, |
| "margin_dpo/margin_mean": 47.7307243347168, |
| "margin_dpo/margin_std": 60.58958435058594, |
| "step": 343 |
| }, |
| { |
| "epoch": 0.7204188481675393, |
| "grad_norm": 31.717029571533203, |
| "learning_rate": 1.110123172071844e-07, |
| "logits/chosen": 1.2925912141799927, |
| "logits/rejected": 1.426027774810791, |
| "logps/chosen": -333.0621643066406, |
| "logps/ref_chosen": -310.25018310546875, |
| "logps/ref_rejected": -281.16302490234375, |
| "logps/rejected": -352.4907531738281, |
| "loss": 4.3502, |
| "margin_dpo/margin_mean": 48.51573181152344, |
| "margin_dpo/margin_std": 64.95976257324219, |
| "step": 344 |
| }, |
| { |
| "epoch": 0.7225130890052356, |
| "grad_norm": 51.54704284667969, |
| "learning_rate": 1.09494297815e-07, |
| "logits/chosen": 1.4484617710113525, |
| "logits/rejected": 1.585009217262268, |
| "logps/chosen": -307.612548828125, |
| "logps/ref_chosen": -284.6531066894531, |
| "logps/ref_rejected": -304.72369384765625, |
| "logps/rejected": -358.7782897949219, |
| "loss": 4.51, |
| "margin_dpo/margin_mean": 31.09515953063965, |
| "margin_dpo/margin_std": 56.86581039428711, |
| "step": 345 |
| }, |
| { |
| "epoch": 0.724607329842932, |
| "grad_norm": 65.52984619140625, |
| "learning_rate": 1.0798381331721107e-07, |
| "logits/chosen": 0.9799513220787048, |
| "logits/rejected": 1.088678002357483, |
| "logps/chosen": -310.8472595214844, |
| "logps/ref_chosen": -255.6278076171875, |
| "logps/ref_rejected": -237.61305236816406, |
| "logps/rejected": -325.5954284667969, |
| "loss": 4.375, |
| "margin_dpo/margin_mean": 32.76295471191406, |
| "margin_dpo/margin_std": 52.9566535949707, |
| "step": 346 |
| }, |
| { |
| "epoch": 0.7267015706806282, |
| "grad_norm": 57.053504943847656, |
| "learning_rate": 1.0648094471651722e-07, |
| "logits/chosen": 1.3673675060272217, |
| "logits/rejected": 1.3742492198944092, |
| "logps/chosen": -315.336181640625, |
| "logps/ref_chosen": -287.71807861328125, |
| "logps/ref_rejected": -276.2634582519531, |
| "logps/rejected": -353.022705078125, |
| "loss": 4.5065, |
| "margin_dpo/margin_mean": 49.14113998413086, |
| "margin_dpo/margin_std": 64.81655883789062, |
| "step": 347 |
| }, |
| { |
| "epoch": 0.7287958115183246, |
| "grad_norm": 41.55851745605469, |
| "learning_rate": 1.0498577260720048e-07, |
| "logits/chosen": 1.4351952075958252, |
| "logits/rejected": 1.5722769498825073, |
| "logps/chosen": -296.4559020996094, |
| "logps/ref_chosen": -285.63232421875, |
| "logps/ref_rejected": -264.0018615722656, |
| "logps/rejected": -319.1177978515625, |
| "loss": 4.6334, |
| "margin_dpo/margin_mean": 44.29237365722656, |
| "margin_dpo/margin_std": 51.82339096069336, |
| "step": 348 |
| }, |
| { |
| "epoch": 0.7308900523560209, |
| "grad_norm": 34.51556396484375, |
| "learning_rate": 1.0349837717080347e-07, |
| "logits/chosen": 1.2128405570983887, |
| "logits/rejected": 1.2827637195587158, |
| "logps/chosen": -378.03369140625, |
| "logps/ref_chosen": -347.98370361328125, |
| "logps/ref_rejected": -328.8855895996094, |
| "logps/rejected": -408.9764709472656, |
| "loss": 4.2865, |
| "margin_dpo/margin_mean": 50.040870666503906, |
| "margin_dpo/margin_std": 62.83095169067383, |
| "step": 349 |
| }, |
| { |
| "epoch": 0.7329842931937173, |
| "grad_norm": 51.1704216003418, |
| "learning_rate": 1.0201883817182949e-07, |
| "logits/chosen": 1.6027448177337646, |
| "logits/rejected": 1.452850103378296, |
| "logps/chosen": -300.945556640625, |
| "logps/ref_chosen": -292.4501647949219, |
| "logps/ref_rejected": -188.39346313476562, |
| "logps/rejected": -268.33087158203125, |
| "loss": 4.4481, |
| "margin_dpo/margin_mean": 71.44198608398438, |
| "margin_dpo/margin_std": 57.31879806518555, |
| "step": 350 |
| }, |
| { |
| "epoch": 0.7350785340314137, |
| "grad_norm": 54.72409439086914, |
| "learning_rate": 1.0054723495346482e-07, |
| "logits/chosen": 1.2379735708236694, |
| "logits/rejected": 1.2731664180755615, |
| "logps/chosen": -284.1971740722656, |
| "logps/ref_chosen": -267.4852294921875, |
| "logps/ref_rejected": -223.13552856445312, |
| "logps/rejected": -285.99163818359375, |
| "loss": 4.9619, |
| "margin_dpo/margin_mean": 46.14418029785156, |
| "margin_dpo/margin_std": 70.45701599121094, |
| "step": 351 |
| }, |
| { |
| "epoch": 0.7371727748691099, |
| "grad_norm": 65.03557586669922, |
| "learning_rate": 9.908364643332398e-08, |
| "logits/chosen": 1.2324098348617554, |
| "logits/rejected": 1.4873018264770508, |
| "logps/chosen": -282.592529296875, |
| "logps/ref_chosen": -257.07952880859375, |
| "logps/ref_rejected": -294.4090881347656, |
| "logps/rejected": -372.9225158691406, |
| "loss": 4.2427, |
| "margin_dpo/margin_mean": 53.00044631958008, |
| "margin_dpo/margin_std": 67.86283111572266, |
| "step": 352 |
| }, |
| { |
| "epoch": 0.7392670157068063, |
| "grad_norm": 39.74465560913086, |
| "learning_rate": 9.76281510992176e-08, |
| "logits/chosen": 1.1475257873535156, |
| "logits/rejected": 1.180965781211853, |
| "logps/chosen": -321.6578674316406, |
| "logps/ref_chosen": -290.9927062988281, |
| "logps/ref_rejected": -263.1128845214844, |
| "logps/rejected": -340.19183349609375, |
| "loss": 4.2739, |
| "margin_dpo/margin_mean": 46.413780212402344, |
| "margin_dpo/margin_std": 61.069339752197266, |
| "step": 353 |
| }, |
| { |
| "epoch": 0.7413612565445026, |
| "grad_norm": 48.98942184448242, |
| "learning_rate": 9.618082700494318e-08, |
| "logits/chosen": 1.084821343421936, |
| "logits/rejected": 1.1856131553649902, |
| "logps/chosen": -224.8162384033203, |
| "logps/ref_chosen": -196.65435791015625, |
| "logps/ref_rejected": -193.15533447265625, |
| "logps/rejected": -250.77032470703125, |
| "loss": 4.9672, |
| "margin_dpo/margin_mean": 29.453073501586914, |
| "margin_dpo/margin_std": 54.51377487182617, |
| "step": 354 |
| }, |
| { |
| "epoch": 0.743455497382199, |
| "grad_norm": 52.5598030090332, |
| "learning_rate": 9.474175176609956e-08, |
| "logits/chosen": 1.538980484008789, |
| "logits/rejected": 1.710750937461853, |
| "logps/chosen": -305.9091491699219, |
| "logps/ref_chosen": -277.7572937011719, |
| "logps/ref_rejected": -296.24908447265625, |
| "logps/rejected": -365.863525390625, |
| "loss": 4.304, |
| "margin_dpo/margin_mean": 41.46260452270508, |
| "margin_dpo/margin_std": 60.0238151550293, |
| "step": 355 |
| }, |
| { |
| "epoch": 0.7455497382198953, |
| "grad_norm": 38.38217544555664, |
| "learning_rate": 9.331100255592436e-08, |
| "logits/chosen": 1.1549817323684692, |
| "logits/rejected": 1.270320177078247, |
| "logps/chosen": -250.93751525878906, |
| "logps/ref_chosen": -228.735595703125, |
| "logps/ref_rejected": -288.4073486328125, |
| "logps/rejected": -340.65838623046875, |
| "loss": 4.4604, |
| "margin_dpo/margin_mean": 30.04913902282715, |
| "margin_dpo/margin_std": 50.18395233154297, |
| "step": 356 |
| }, |
| { |
| "epoch": 0.7476439790575916, |
| "grad_norm": 48.77327346801758, |
| "learning_rate": 9.18886561011557e-08, |
| "logits/chosen": 1.2281720638275146, |
| "logits/rejected": 1.2340593338012695, |
| "logps/chosen": -345.0635986328125, |
| "logps/ref_chosen": -327.5565185546875, |
| "logps/ref_rejected": -286.9888610839844, |
| "logps/rejected": -364.4182434082031, |
| "loss": 4.4785, |
| "margin_dpo/margin_mean": 59.92234802246094, |
| "margin_dpo/margin_std": 68.7721939086914, |
| "step": 357 |
| }, |
| { |
| "epoch": 0.749738219895288, |
| "grad_norm": 33.63121795654297, |
| "learning_rate": 9.047478867791731e-08, |
| "logits/chosen": 1.2939711809158325, |
| "logits/rejected": 1.3026853799819946, |
| "logps/chosen": -300.43011474609375, |
| "logps/ref_chosen": -275.9919738769531, |
| "logps/ref_rejected": -226.95779418945312, |
| "logps/rejected": -304.18017578125, |
| "loss": 4.326, |
| "margin_dpo/margin_mean": 52.78423309326172, |
| "margin_dpo/margin_std": 65.29212951660156, |
| "step": 358 |
| }, |
| { |
| "epoch": 0.7518324607329843, |
| "grad_norm": 41.51667022705078, |
| "learning_rate": 8.906947610762825e-08, |
| "logits/chosen": 1.193036675453186, |
| "logits/rejected": 1.3061870336532593, |
| "logps/chosen": -288.68792724609375, |
| "logps/ref_chosen": -265.4796447753906, |
| "logps/ref_rejected": -269.9594421386719, |
| "logps/rejected": -336.7837219238281, |
| "loss": 4.2962, |
| "margin_dpo/margin_mean": 43.615962982177734, |
| "margin_dpo/margin_std": 57.33641052246094, |
| "step": 359 |
| }, |
| { |
| "epoch": 0.7539267015706806, |
| "grad_norm": 41.91855239868164, |
| "learning_rate": 8.76727937529367e-08, |
| "logits/chosen": 1.4477754831314087, |
| "logits/rejected": 1.4021023511886597, |
| "logps/chosen": -364.541015625, |
| "logps/ref_chosen": -336.95709228515625, |
| "logps/ref_rejected": -275.51239013671875, |
| "logps/rejected": -352.3710632324219, |
| "loss": 4.4654, |
| "margin_dpo/margin_mean": 49.27482604980469, |
| "margin_dpo/margin_std": 65.66383361816406, |
| "step": 360 |
| }, |
| { |
| "epoch": 0.7560209424083769, |
| "grad_norm": 37.49740219116211, |
| "learning_rate": 8.628481651367875e-08, |
| "logits/chosen": 1.1130855083465576, |
| "logits/rejected": 1.3169794082641602, |
| "logps/chosen": -233.5477294921875, |
| "logps/ref_chosen": -223.0279541015625, |
| "logps/ref_rejected": -233.6653289794922, |
| "logps/rejected": -294.90289306640625, |
| "loss": 4.1403, |
| "margin_dpo/margin_mean": 50.717796325683594, |
| "margin_dpo/margin_std": 58.103668212890625, |
| "step": 361 |
| }, |
| { |
| "epoch": 0.7581151832460733, |
| "grad_norm": 75.54791259765625, |
| "learning_rate": 8.490561882286135e-08, |
| "logits/chosen": 1.167074203491211, |
| "logits/rejected": 1.180873155593872, |
| "logps/chosen": -337.9937744140625, |
| "logps/ref_chosen": -298.1035461425781, |
| "logps/ref_rejected": -230.74783325195312, |
| "logps/rejected": -312.3794250488281, |
| "loss": 4.3668, |
| "margin_dpo/margin_mean": 41.741363525390625, |
| "margin_dpo/margin_std": 74.15339660644531, |
| "step": 362 |
| }, |
| { |
| "epoch": 0.7602094240837697, |
| "grad_norm": 69.05213928222656, |
| "learning_rate": 8.353527464267104e-08, |
| "logits/chosen": 1.4149866104125977, |
| "logits/rejected": 1.3386046886444092, |
| "logps/chosen": -324.6624450683594, |
| "logps/ref_chosen": -315.25506591796875, |
| "logps/ref_rejected": -276.456298828125, |
| "logps/rejected": -352.84759521484375, |
| "loss": 4.1568, |
| "margin_dpo/margin_mean": 66.9839096069336, |
| "margin_dpo/margin_std": 64.92231750488281, |
| "step": 363 |
| }, |
| { |
| "epoch": 0.762303664921466, |
| "grad_norm": 31.97148895263672, |
| "learning_rate": 8.217385746050742e-08, |
| "logits/chosen": 1.5564781427383423, |
| "logits/rejected": 1.3156113624572754, |
| "logps/chosen": -380.2980041503906, |
| "logps/ref_chosen": -336.43798828125, |
| "logps/ref_rejected": -259.9676818847656, |
| "logps/rejected": -339.92596435546875, |
| "loss": 4.5886, |
| "margin_dpo/margin_mean": 36.0982666015625, |
| "margin_dpo/margin_std": 53.2963752746582, |
| "step": 364 |
| }, |
| { |
| "epoch": 0.7643979057591623, |
| "grad_norm": 53.09539794921875, |
| "learning_rate": 8.082144028504231e-08, |
| "logits/chosen": 1.0552072525024414, |
| "logits/rejected": 1.2801932096481323, |
| "logps/chosen": -227.60394287109375, |
| "logps/ref_chosen": -209.7356719970703, |
| "logps/ref_rejected": -294.5636291503906, |
| "logps/rejected": -367.4307861328125, |
| "loss": 4.2219, |
| "margin_dpo/margin_mean": 54.99891662597656, |
| "margin_dpo/margin_std": 50.34431457519531, |
| "step": 365 |
| }, |
| { |
| "epoch": 0.7664921465968586, |
| "grad_norm": 56.880043029785156, |
| "learning_rate": 7.947809564230445e-08, |
| "logits/chosen": 1.2870928049087524, |
| "logits/rejected": 1.2019919157028198, |
| "logps/chosen": -343.0006408691406, |
| "logps/ref_chosen": -312.77142333984375, |
| "logps/ref_rejected": -273.8427734375, |
| "logps/rejected": -375.21856689453125, |
| "loss": 4.2553, |
| "margin_dpo/margin_mean": 71.1466293334961, |
| "margin_dpo/margin_std": 62.22167205810547, |
| "step": 366 |
| }, |
| { |
| "epoch": 0.768586387434555, |
| "grad_norm": 47.9551887512207, |
| "learning_rate": 7.814389557179016e-08, |
| "logits/chosen": 1.7175925970077515, |
| "logits/rejected": 1.4891177415847778, |
| "logps/chosen": -329.87542724609375, |
| "logps/ref_chosen": -284.1925964355469, |
| "logps/ref_rejected": -208.8526611328125, |
| "logps/rejected": -294.8234558105469, |
| "loss": 4.1368, |
| "margin_dpo/margin_mean": 40.28790283203125, |
| "margin_dpo/margin_std": 68.52880859375, |
| "step": 367 |
| }, |
| { |
| "epoch": 0.7706806282722513, |
| "grad_norm": 32.45009231567383, |
| "learning_rate": 7.681891162260015e-08, |
| "logits/chosen": 1.7018953561782837, |
| "logits/rejected": 1.5844436883926392, |
| "logps/chosen": -376.4222412109375, |
| "logps/ref_chosen": -360.64459228515625, |
| "logps/ref_rejected": -297.281005859375, |
| "logps/rejected": -367.4183654785156, |
| "loss": 3.9894, |
| "margin_dpo/margin_mean": 54.35973358154297, |
| "margin_dpo/margin_std": 49.44243621826172, |
| "step": 368 |
| }, |
| { |
| "epoch": 0.7727748691099476, |
| "grad_norm": 85.05940246582031, |
| "learning_rate": 7.550321484960251e-08, |
| "logits/chosen": 1.4974400997161865, |
| "logits/rejected": 1.5112247467041016, |
| "logps/chosen": -364.4231872558594, |
| "logps/ref_chosen": -340.94610595703125, |
| "logps/ref_rejected": -285.1484069824219, |
| "logps/rejected": -367.0999450683594, |
| "loss": 4.5603, |
| "margin_dpo/margin_mean": 58.47446060180664, |
| "margin_dpo/margin_std": 59.77842712402344, |
| "step": 369 |
| }, |
| { |
| "epoch": 0.774869109947644, |
| "grad_norm": 36.800376892089844, |
| "learning_rate": 7.419687580962222e-08, |
| "logits/chosen": 1.2962076663970947, |
| "logits/rejected": 1.5088801383972168, |
| "logps/chosen": -313.3735046386719, |
| "logps/ref_chosen": -276.9629211425781, |
| "logps/ref_rejected": -274.93865966796875, |
| "logps/rejected": -353.8816223144531, |
| "loss": 4.1904, |
| "margin_dpo/margin_mean": 42.532344818115234, |
| "margin_dpo/margin_std": 52.544273376464844, |
| "step": 370 |
| }, |
| { |
| "epoch": 0.7769633507853403, |
| "grad_norm": 79.35717010498047, |
| "learning_rate": 7.289996455765748e-08, |
| "logits/chosen": 0.7718454599380493, |
| "logits/rejected": 1.0238559246063232, |
| "logps/chosen": -365.87713623046875, |
| "logps/ref_chosen": -323.23980712890625, |
| "logps/ref_rejected": -317.0935363769531, |
| "logps/rejected": -390.978759765625, |
| "loss": 4.4027, |
| "margin_dpo/margin_mean": 31.247907638549805, |
| "margin_dpo/margin_std": 54.091983795166016, |
| "step": 371 |
| }, |
| { |
| "epoch": 0.7790575916230367, |
| "grad_norm": 33.439125061035156, |
| "learning_rate": 7.161255064312283e-08, |
| "logits/chosen": 1.3146398067474365, |
| "logits/rejected": 1.290389060974121, |
| "logps/chosen": -338.93792724609375, |
| "logps/ref_chosen": -303.75262451171875, |
| "logps/ref_rejected": -205.62069702148438, |
| "logps/rejected": -309.5386962890625, |
| "loss": 4.0137, |
| "margin_dpo/margin_mean": 68.732666015625, |
| "margin_dpo/margin_std": 57.33296585083008, |
| "step": 372 |
| }, |
| { |
| "epoch": 0.7811518324607329, |
| "grad_norm": 45.49492263793945, |
| "learning_rate": 7.033470310611945e-08, |
| "logits/chosen": 1.4074095487594604, |
| "logits/rejected": 1.1533772945404053, |
| "logps/chosen": -377.86865234375, |
| "logps/ref_chosen": -346.5982666015625, |
| "logps/ref_rejected": -253.89280700683594, |
| "logps/rejected": -333.4144287109375, |
| "loss": 4.1968, |
| "margin_dpo/margin_mean": 48.25126266479492, |
| "margin_dpo/margin_std": 56.07286834716797, |
| "step": 373 |
| }, |
| { |
| "epoch": 0.7832460732984293, |
| "grad_norm": 57.437164306640625, |
| "learning_rate": 6.906649047373245e-08, |
| "logits/chosen": 1.4347639083862305, |
| "logits/rejected": 1.5623588562011719, |
| "logps/chosen": -292.1949768066406, |
| "logps/ref_chosen": -252.59971618652344, |
| "logps/ref_rejected": -249.61476135253906, |
| "logps/rejected": -319.74609375, |
| "loss": 4.7414, |
| "margin_dpo/margin_mean": 30.536096572875977, |
| "margin_dpo/margin_std": 58.64886474609375, |
| "step": 374 |
| }, |
| { |
| "epoch": 0.7853403141361257, |
| "grad_norm": 67.63172149658203, |
| "learning_rate": 6.780798075635675e-08, |
| "logits/chosen": 1.1573803424835205, |
| "logits/rejected": 1.0170570611953735, |
| "logps/chosen": -274.2476806640625, |
| "logps/ref_chosen": -247.3214569091797, |
| "logps/ref_rejected": -188.48236083984375, |
| "logps/rejected": -260.9528503417969, |
| "loss": 4.4518, |
| "margin_dpo/margin_mean": 45.54425811767578, |
| "margin_dpo/margin_std": 53.55020523071289, |
| "step": 375 |
| }, |
| { |
| "epoch": 0.787434554973822, |
| "grad_norm": 43.886695861816406, |
| "learning_rate": 6.655924144404906e-08, |
| "logits/chosen": 1.1326963901519775, |
| "logits/rejected": 1.3887975215911865, |
| "logps/chosen": -327.3310852050781, |
| "logps/ref_chosen": -272.513916015625, |
| "logps/ref_rejected": -305.7491760253906, |
| "logps/rejected": -412.55767822265625, |
| "loss": 4.5376, |
| "margin_dpo/margin_mean": 51.99131393432617, |
| "margin_dpo/margin_std": 69.22034454345703, |
| "step": 376 |
| }, |
| { |
| "epoch": 0.7895287958115184, |
| "grad_norm": 47.22002029418945, |
| "learning_rate": 6.532033950290885e-08, |
| "logits/chosen": 1.356651782989502, |
| "logits/rejected": 1.438635230064392, |
| "logps/chosen": -323.85699462890625, |
| "logps/ref_chosen": -298.3796081542969, |
| "logps/ref_rejected": -273.41839599609375, |
| "logps/rejected": -348.69940185546875, |
| "loss": 4.5118, |
| "margin_dpo/margin_mean": 49.80352020263672, |
| "margin_dpo/margin_std": 56.26287841796875, |
| "step": 377 |
| }, |
| { |
| "epoch": 0.7916230366492146, |
| "grad_norm": 43.87843704223633, |
| "learning_rate": 6.409134137148736e-08, |
| "logits/chosen": 1.4002658128738403, |
| "logits/rejected": 1.4882569313049316, |
| "logps/chosen": -305.824462890625, |
| "logps/ref_chosen": -286.3173522949219, |
| "logps/ref_rejected": -271.7178955078125, |
| "logps/rejected": -340.6441345214844, |
| "loss": 4.5644, |
| "margin_dpo/margin_mean": 49.41912078857422, |
| "margin_dpo/margin_std": 56.88557434082031, |
| "step": 378 |
| }, |
| { |
| "epoch": 0.793717277486911, |
| "grad_norm": 57.02334213256836, |
| "learning_rate": 6.28723129572247e-08, |
| "logits/chosen": 1.439416527748108, |
| "logits/rejected": 1.3806058168411255, |
| "logps/chosen": -271.30230712890625, |
| "logps/ref_chosen": -233.71743774414062, |
| "logps/ref_rejected": -206.68927001953125, |
| "logps/rejected": -284.8748474121094, |
| "loss": 4.5262, |
| "margin_dpo/margin_mean": 40.600711822509766, |
| "margin_dpo/margin_std": 50.349300384521484, |
| "step": 379 |
| }, |
| { |
| "epoch": 0.7958115183246073, |
| "grad_norm": 72.12355041503906, |
| "learning_rate": 6.166331963291519e-08, |
| "logits/chosen": 1.7105507850646973, |
| "logits/rejected": 1.5248544216156006, |
| "logps/chosen": -387.1551513671875, |
| "logps/ref_chosen": -356.8863525390625, |
| "logps/ref_rejected": -354.776123046875, |
| "logps/rejected": -424.4413757324219, |
| "loss": 4.4649, |
| "margin_dpo/margin_mean": 39.396446228027344, |
| "margin_dpo/margin_std": 53.38307189941406, |
| "step": 380 |
| }, |
| { |
| "epoch": 0.7979057591623037, |
| "grad_norm": 97.61827850341797, |
| "learning_rate": 6.046442623320145e-08, |
| "logits/chosen": 0.9507350325584412, |
| "logits/rejected": 1.0006842613220215, |
| "logps/chosen": -260.0545654296875, |
| "logps/ref_chosen": -235.81100463867188, |
| "logps/ref_rejected": -237.37062072753906, |
| "logps/rejected": -337.2359313964844, |
| "loss": 4.1799, |
| "margin_dpo/margin_mean": 75.62174987792969, |
| "margin_dpo/margin_std": 58.47461700439453, |
| "step": 381 |
| }, |
| { |
| "epoch": 0.8, |
| "grad_norm": 42.927345275878906, |
| "learning_rate": 5.9275697051098275e-08, |
| "logits/chosen": 1.312659740447998, |
| "logits/rejected": 1.322435975074768, |
| "logps/chosen": -294.5480651855469, |
| "logps/ref_chosen": -259.17388916015625, |
| "logps/ref_rejected": -230.83482360839844, |
| "logps/rejected": -323.2818298339844, |
| "loss": 3.9698, |
| "margin_dpo/margin_mean": 57.07281494140625, |
| "margin_dpo/margin_std": 65.82691955566406, |
| "step": 382 |
| }, |
| { |
| "epoch": 0.8020942408376963, |
| "grad_norm": 60.843833923339844, |
| "learning_rate": 5.809719583454414e-08, |
| "logits/chosen": 1.0891731977462769, |
| "logits/rejected": 1.3433376550674438, |
| "logps/chosen": -316.5683898925781, |
| "logps/ref_chosen": -269.8660583496094, |
| "logps/ref_rejected": -320.0415954589844, |
| "logps/rejected": -390.5964660644531, |
| "loss": 4.4135, |
| "margin_dpo/margin_mean": 23.852542877197266, |
| "margin_dpo/margin_std": 56.356651306152344, |
| "step": 383 |
| }, |
| { |
| "epoch": 0.8041884816753927, |
| "grad_norm": 70.86746978759766, |
| "learning_rate": 5.6928985782982524e-08, |
| "logits/chosen": 1.2213385105133057, |
| "logits/rejected": 1.5940483808517456, |
| "logps/chosen": -313.1070251464844, |
| "logps/ref_chosen": -280.7498779296875, |
| "logps/ref_rejected": -324.9134216308594, |
| "logps/rejected": -393.7688293457031, |
| "loss": 4.9522, |
| "margin_dpo/margin_mean": 36.498287200927734, |
| "margin_dpo/margin_std": 67.0685043334961, |
| "step": 384 |
| }, |
| { |
| "epoch": 0.806282722513089, |
| "grad_norm": 41.98042678833008, |
| "learning_rate": 5.57711295439732e-08, |
| "logits/chosen": 1.4527329206466675, |
| "logits/rejected": 1.5307879447937012, |
| "logps/chosen": -346.65673828125, |
| "logps/ref_chosen": -313.2212829589844, |
| "logps/ref_rejected": -256.9848937988281, |
| "logps/rejected": -341.84088134765625, |
| "loss": 4.4431, |
| "margin_dpo/margin_mean": 51.4205322265625, |
| "margin_dpo/margin_std": 66.022705078125, |
| "step": 385 |
| }, |
| { |
| "epoch": 0.8083769633507853, |
| "grad_norm": 48.37839126586914, |
| "learning_rate": 5.4623689209832484e-08, |
| "logits/chosen": 1.6142921447753906, |
| "logits/rejected": 1.77769935131073, |
| "logps/chosen": -376.63311767578125, |
| "logps/ref_chosen": -342.4034423828125, |
| "logps/ref_rejected": -319.1665954589844, |
| "logps/rejected": -405.9571838378906, |
| "loss": 4.1689, |
| "margin_dpo/margin_mean": 52.56089401245117, |
| "margin_dpo/margin_std": 56.50434875488281, |
| "step": 386 |
| }, |
| { |
| "epoch": 0.8104712041884817, |
| "grad_norm": 37.86534118652344, |
| "learning_rate": 5.3486726314303175e-08, |
| "logits/chosen": 1.3724387884140015, |
| "logits/rejected": 1.450311303138733, |
| "logps/chosen": -249.56488037109375, |
| "logps/ref_chosen": -209.16738891601562, |
| "logps/ref_rejected": -225.61949157714844, |
| "logps/rejected": -305.92340087890625, |
| "loss": 4.3007, |
| "margin_dpo/margin_mean": 39.906436920166016, |
| "margin_dpo/margin_std": 77.46627807617188, |
| "step": 387 |
| }, |
| { |
| "epoch": 0.812565445026178, |
| "grad_norm": 86.58211517333984, |
| "learning_rate": 5.2360301829254745e-08, |
| "logits/chosen": 1.78019118309021, |
| "logits/rejected": 1.7659128904342651, |
| "logps/chosen": -381.0487365722656, |
| "logps/ref_chosen": -342.5128173828125, |
| "logps/ref_rejected": -296.8653564453125, |
| "logps/rejected": -390.75970458984375, |
| "loss": 4.5827, |
| "margin_dpo/margin_mean": 55.35844421386719, |
| "margin_dpo/margin_std": 75.9105224609375, |
| "step": 388 |
| }, |
| { |
| "epoch": 0.8146596858638744, |
| "grad_norm": 73.34844207763672, |
| "learning_rate": 5.1244476161413806e-08, |
| "logits/chosen": 1.6132086515426636, |
| "logits/rejected": 1.4335089921951294, |
| "logps/chosen": -354.9220886230469, |
| "logps/ref_chosen": -336.53912353515625, |
| "logps/ref_rejected": -237.36383056640625, |
| "logps/rejected": -318.2511901855469, |
| "loss": 4.2202, |
| "margin_dpo/margin_mean": 62.504356384277344, |
| "margin_dpo/margin_std": 76.5676498413086, |
| "step": 389 |
| }, |
| { |
| "epoch": 0.8167539267015707, |
| "grad_norm": 72.32861328125, |
| "learning_rate": 5.013930914912476e-08, |
| "logits/chosen": 1.4496684074401855, |
| "logits/rejected": 1.6118597984313965, |
| "logps/chosen": -313.1777648925781, |
| "logps/ref_chosen": -275.41680908203125, |
| "logps/ref_rejected": -300.94329833984375, |
| "logps/rejected": -397.3767395019531, |
| "loss": 4.4604, |
| "margin_dpo/margin_mean": 58.67253112792969, |
| "margin_dpo/margin_std": 57.53920364379883, |
| "step": 390 |
| }, |
| { |
| "epoch": 0.818848167539267, |
| "grad_norm": 38.00588607788086, |
| "learning_rate": 4.904486005914027e-08, |
| "logits/chosen": 1.421841025352478, |
| "logits/rejected": 1.3273773193359375, |
| "logps/chosen": -301.99920654296875, |
| "logps/ref_chosen": -249.42276000976562, |
| "logps/ref_rejected": -187.71572875976562, |
| "logps/rejected": -270.2522888183594, |
| "loss": 4.2838, |
| "margin_dpo/margin_mean": 29.960115432739258, |
| "margin_dpo/margin_std": 43.918495178222656, |
| "step": 391 |
| }, |
| { |
| "epoch": 0.8209424083769633, |
| "grad_norm": 45.547096252441406, |
| "learning_rate": 4.796118758344353e-08, |
| "logits/chosen": 1.0780835151672363, |
| "logits/rejected": 1.0823677778244019, |
| "logps/chosen": -326.24456787109375, |
| "logps/ref_chosen": -290.30438232421875, |
| "logps/ref_rejected": -262.2787780761719, |
| "logps/rejected": -357.97216796875, |
| "loss": 3.8296, |
| "margin_dpo/margin_mean": 59.75324249267578, |
| "margin_dpo/margin_std": 60.47087860107422, |
| "step": 392 |
| }, |
| { |
| "epoch": 0.8230366492146597, |
| "grad_norm": 53.403141021728516, |
| "learning_rate": 4.688834983610082e-08, |
| "logits/chosen": 1.3256309032440186, |
| "logits/rejected": 1.1313904523849487, |
| "logps/chosen": -356.47021484375, |
| "logps/ref_chosen": -317.2633972167969, |
| "logps/ref_rejected": -237.91380310058594, |
| "logps/rejected": -326.9722900390625, |
| "loss": 4.5399, |
| "margin_dpo/margin_mean": 49.85166931152344, |
| "margin_dpo/margin_std": 60.655738830566406, |
| "step": 393 |
| }, |
| { |
| "epoch": 0.8251308900523561, |
| "grad_norm": 44.783817291259766, |
| "learning_rate": 4.582640435014459e-08, |
| "logits/chosen": 1.5566421747207642, |
| "logits/rejected": 1.6724714040756226, |
| "logps/chosen": -406.4788818359375, |
| "logps/ref_chosen": -377.4843444824219, |
| "logps/ref_rejected": -298.2265319824219, |
| "logps/rejected": -381.8688659667969, |
| "loss": 4.4044, |
| "margin_dpo/margin_mean": 54.6478271484375, |
| "margin_dpo/margin_std": 60.59336853027344, |
| "step": 394 |
| }, |
| { |
| "epoch": 0.8272251308900523, |
| "grad_norm": 43.559539794921875, |
| "learning_rate": 4.477540807448832e-08, |
| "logits/chosen": 1.251692771911621, |
| "logits/rejected": 1.2946186065673828, |
| "logps/chosen": -310.1583557128906, |
| "logps/ref_chosen": -281.3030090332031, |
| "logps/ref_rejected": -272.98968505859375, |
| "logps/rejected": -343.1010437011719, |
| "loss": 4.3814, |
| "margin_dpo/margin_mean": 41.256004333496094, |
| "margin_dpo/margin_std": 55.48289108276367, |
| "step": 395 |
| }, |
| { |
| "epoch": 0.8293193717277487, |
| "grad_norm": 173.74839782714844, |
| "learning_rate": 4.373541737087263e-08, |
| "logits/chosen": 1.4800870418548584, |
| "logits/rejected": 1.4773489236831665, |
| "logps/chosen": -338.620849609375, |
| "logps/ref_chosen": -295.05364990234375, |
| "logps/ref_rejected": -255.04061889648438, |
| "logps/rejected": -347.4020080566406, |
| "loss": 4.402, |
| "margin_dpo/margin_mean": 48.79420471191406, |
| "margin_dpo/margin_std": 69.13119506835938, |
| "step": 396 |
| }, |
| { |
| "epoch": 0.831413612565445, |
| "grad_norm": 48.23088836669922, |
| "learning_rate": 4.270648801084295e-08, |
| "logits/chosen": 1.4665961265563965, |
| "logits/rejected": 1.5797699689865112, |
| "logps/chosen": -311.6304016113281, |
| "logps/ref_chosen": -288.0824890136719, |
| "logps/ref_rejected": -270.2839050292969, |
| "logps/rejected": -332.85223388671875, |
| "loss": 4.47, |
| "margin_dpo/margin_mean": 39.020355224609375, |
| "margin_dpo/margin_std": 48.824378967285156, |
| "step": 397 |
| }, |
| { |
| "epoch": 0.8335078534031414, |
| "grad_norm": 92.45420837402344, |
| "learning_rate": 4.168867517275806e-08, |
| "logits/chosen": 1.2100859880447388, |
| "logits/rejected": 1.4831223487854004, |
| "logps/chosen": -297.94940185546875, |
| "logps/ref_chosen": -252.48330688476562, |
| "logps/ref_rejected": -273.87139892578125, |
| "logps/rejected": -354.4050598144531, |
| "loss": 4.9205, |
| "margin_dpo/margin_mean": 35.06761932373047, |
| "margin_dpo/margin_std": 80.69606018066406, |
| "step": 398 |
| }, |
| { |
| "epoch": 0.8356020942408376, |
| "grad_norm": 82.78327941894531, |
| "learning_rate": 4.0682033438831584e-08, |
| "logits/chosen": 1.3988953828811646, |
| "logits/rejected": 1.5033973455429077, |
| "logps/chosen": -330.5037536621094, |
| "logps/ref_chosen": -277.4305419921875, |
| "logps/ref_rejected": -271.1197204589844, |
| "logps/rejected": -356.5560607910156, |
| "loss": 4.4852, |
| "margin_dpo/margin_mean": 32.36311340332031, |
| "margin_dpo/margin_std": 66.02288818359375, |
| "step": 399 |
| }, |
| { |
| "epoch": 0.837696335078534, |
| "grad_norm": 37.495018005371094, |
| "learning_rate": 3.968661679220467e-08, |
| "logits/chosen": 1.2931967973709106, |
| "logits/rejected": 1.240352988243103, |
| "logps/chosen": -299.0211181640625, |
| "logps/ref_chosen": -266.20025634765625, |
| "logps/ref_rejected": -217.865966796875, |
| "logps/rejected": -301.7166748046875, |
| "loss": 4.2739, |
| "margin_dpo/margin_mean": 51.02983093261719, |
| "margin_dpo/margin_std": 63.865108489990234, |
| "step": 400 |
| }, |
| { |
| "epoch": 0.837696335078534, |
| "eval_logits/chosen": 1.193334937095642, |
| "eval_logits/rejected": 1.2366639375686646, |
| "eval_logps/chosen": -316.0413513183594, |
| "eval_logps/ref_chosen": -281.4588928222656, |
| "eval_logps/ref_rejected": -261.84954833984375, |
| "eval_logps/rejected": -345.1450500488281, |
| "eval_loss": 0.5601758360862732, |
| "eval_margin_dpo/margin_mean": 48.71305465698242, |
| "eval_margin_dpo/margin_std": 68.15460205078125, |
| "eval_runtime": 92.6746, |
| "eval_samples_per_second": 21.581, |
| "eval_steps_per_second": 1.349, |
| "step": 400 |
| }, |
| { |
| "epoch": 0.8397905759162304, |
| "grad_norm": 42.00017547607422, |
| "learning_rate": 3.8702478614051345e-08, |
| "logits/chosen": 1.1790591478347778, |
| "logits/rejected": 1.3527344465255737, |
| "logps/chosen": -331.7237854003906, |
| "logps/ref_chosen": -293.3692932128906, |
| "logps/ref_rejected": -299.9609069824219, |
| "logps/rejected": -400.61529541015625, |
| "loss": 3.9949, |
| "margin_dpo/margin_mean": 62.299903869628906, |
| "margin_dpo/margin_std": 44.7177619934082, |
| "step": 401 |
| }, |
| { |
| "epoch": 0.8418848167539267, |
| "grad_norm": 58.57191848754883, |
| "learning_rate": 3.772967168071517e-08, |
| "logits/chosen": 1.4228373765945435, |
| "logits/rejected": 1.3410090208053589, |
| "logps/chosen": -332.11737060546875, |
| "logps/ref_chosen": -279.55889892578125, |
| "logps/ref_rejected": -259.6942138671875, |
| "logps/rejected": -331.76983642578125, |
| "loss": 4.3682, |
| "margin_dpo/margin_mean": 19.51715660095215, |
| "margin_dpo/margin_std": 81.67544555664062, |
| "step": 402 |
| }, |
| { |
| "epoch": 0.8439790575916231, |
| "grad_norm": 46.684837341308594, |
| "learning_rate": 3.676824816087978e-08, |
| "logits/chosen": 1.5173556804656982, |
| "logits/rejected": 1.6118801832199097, |
| "logps/chosen": -406.491455078125, |
| "logps/ref_chosen": -372.2436828613281, |
| "logps/ref_rejected": -285.5693359375, |
| "logps/rejected": -390.88140869140625, |
| "loss": 3.7148, |
| "margin_dpo/margin_mean": 71.0643081665039, |
| "margin_dpo/margin_std": 51.123374938964844, |
| "step": 403 |
| }, |
| { |
| "epoch": 0.8460732984293193, |
| "grad_norm": 38.361358642578125, |
| "learning_rate": 3.581825961277074e-08, |
| "logits/chosen": 1.4560322761535645, |
| "logits/rejected": 1.349548578262329, |
| "logps/chosen": -372.926513671875, |
| "logps/ref_chosen": -328.00860595703125, |
| "logps/ref_rejected": -278.35009765625, |
| "logps/rejected": -352.3278503417969, |
| "loss": 4.4734, |
| "margin_dpo/margin_mean": 29.05979347229004, |
| "margin_dpo/margin_std": 75.3277816772461, |
| "step": 404 |
| }, |
| { |
| "epoch": 0.8481675392670157, |
| "grad_norm": 50.2591438293457, |
| "learning_rate": 3.487975698139084e-08, |
| "logits/chosen": 1.3894712924957275, |
| "logits/rejected": 1.5616024732589722, |
| "logps/chosen": -270.36468505859375, |
| "logps/ref_chosen": -228.44178771972656, |
| "logps/ref_rejected": -244.33395385742188, |
| "logps/rejected": -330.79925537109375, |
| "loss": 4.2189, |
| "margin_dpo/margin_mean": 44.542388916015625, |
| "margin_dpo/margin_std": 59.48860168457031, |
| "step": 405 |
| }, |
| { |
| "epoch": 0.8502617801047121, |
| "grad_norm": 63.63072204589844, |
| "learning_rate": 3.3952790595787986e-08, |
| "logits/chosen": 1.3200132846832275, |
| "logits/rejected": 1.2781833410263062, |
| "logps/chosen": -353.1100158691406, |
| "logps/ref_chosen": -321.44598388671875, |
| "logps/ref_rejected": -268.0445861816406, |
| "logps/rejected": -349.9507751464844, |
| "loss": 4.6261, |
| "margin_dpo/margin_mean": 50.24217987060547, |
| "margin_dpo/margin_std": 67.54312896728516, |
| "step": 406 |
| }, |
| { |
| "epoch": 0.8523560209424084, |
| "grad_norm": 53.97494888305664, |
| "learning_rate": 3.303741016635614e-08, |
| "logits/chosen": 1.3076931238174438, |
| "logits/rejected": 1.1266769170761108, |
| "logps/chosen": -309.05596923828125, |
| "logps/ref_chosen": -247.0522918701172, |
| "logps/ref_rejected": -186.8645782470703, |
| "logps/rejected": -270.8433837890625, |
| "loss": 4.3225, |
| "margin_dpo/margin_mean": 21.975143432617188, |
| "margin_dpo/margin_std": 54.5270881652832, |
| "step": 407 |
| }, |
| { |
| "epoch": 0.8544502617801047, |
| "grad_norm": 53.08890151977539, |
| "learning_rate": 3.2133664782169944e-08, |
| "logits/chosen": 1.1703412532806396, |
| "logits/rejected": 1.2470866441726685, |
| "logps/chosen": -253.00314331054688, |
| "logps/ref_chosen": -213.5513458251953, |
| "logps/ref_rejected": -267.9826354980469, |
| "logps/rejected": -361.67535400390625, |
| "loss": 4.2575, |
| "margin_dpo/margin_mean": 54.240936279296875, |
| "margin_dpo/margin_std": 45.82322692871094, |
| "step": 408 |
| }, |
| { |
| "epoch": 0.856544502617801, |
| "grad_norm": 48.80266571044922, |
| "learning_rate": 3.12416029083514e-08, |
| "logits/chosen": 1.4997256994247437, |
| "logits/rejected": 1.6531615257263184, |
| "logps/chosen": -320.41851806640625, |
| "logps/ref_chosen": -280.0785217285156, |
| "logps/ref_rejected": -304.26910400390625, |
| "logps/rejected": -392.4356994628906, |
| "loss": 4.3055, |
| "margin_dpo/margin_mean": 47.82660675048828, |
| "margin_dpo/margin_std": 75.04901885986328, |
| "step": 409 |
| }, |
| { |
| "epoch": 0.8586387434554974, |
| "grad_norm": 63.39336013793945, |
| "learning_rate": 3.036127238347164e-08, |
| "logits/chosen": 1.5989128351211548, |
| "logits/rejected": 1.5641684532165527, |
| "logps/chosen": -292.51336669921875, |
| "logps/ref_chosen": -260.9378662109375, |
| "logps/ref_rejected": -296.7695007324219, |
| "logps/rejected": -368.9515380859375, |
| "loss": 4.4542, |
| "margin_dpo/margin_mean": 40.606529235839844, |
| "margin_dpo/margin_std": 67.39288330078125, |
| "step": 410 |
| }, |
| { |
| "epoch": 0.8607329842931937, |
| "grad_norm": 40.665950775146484, |
| "learning_rate": 2.9492720416985e-08, |
| "logits/chosen": 1.3025436401367188, |
| "logits/rejected": 1.4121453762054443, |
| "logps/chosen": -354.9593200683594, |
| "logps/ref_chosen": -330.0611877441406, |
| "logps/ref_rejected": -310.21368408203125, |
| "logps/rejected": -398.6880187988281, |
| "loss": 4.1892, |
| "margin_dpo/margin_mean": 63.576229095458984, |
| "margin_dpo/margin_std": 51.235904693603516, |
| "step": 411 |
| }, |
| { |
| "epoch": 0.86282722513089, |
| "grad_norm": 55.58880615234375, |
| "learning_rate": 2.863599358669755e-08, |
| "logits/chosen": 1.1845945119857788, |
| "logits/rejected": 1.3782296180725098, |
| "logps/chosen": -288.62408447265625, |
| "logps/ref_chosen": -254.76255798339844, |
| "logps/ref_rejected": -315.16156005859375, |
| "logps/rejected": -383.30535888671875, |
| "loss": 4.4912, |
| "margin_dpo/margin_mean": 34.28225326538086, |
| "margin_dpo/margin_std": 53.833274841308594, |
| "step": 412 |
| }, |
| { |
| "epoch": 0.8649214659685864, |
| "grad_norm": 81.09705352783203, |
| "learning_rate": 2.7791137836269158e-08, |
| "logits/chosen": 1.3580859899520874, |
| "logits/rejected": 1.3456140756607056, |
| "logps/chosen": -294.88671875, |
| "logps/ref_chosen": -260.4962463378906, |
| "logps/ref_rejected": -250.442626953125, |
| "logps/rejected": -319.46563720703125, |
| "loss": 4.5037, |
| "margin_dpo/margin_mean": 34.632568359375, |
| "margin_dpo/margin_std": 63.56281280517578, |
| "step": 413 |
| }, |
| { |
| "epoch": 0.8670157068062827, |
| "grad_norm": 51.592769622802734, |
| "learning_rate": 2.6958198472749717e-08, |
| "logits/chosen": 1.6022746562957764, |
| "logits/rejected": 1.5306450128555298, |
| "logps/chosen": -428.2533264160156, |
| "logps/ref_chosen": -391.94610595703125, |
| "logps/ref_rejected": -287.8221130371094, |
| "logps/rejected": -345.431396484375, |
| "loss": 4.3943, |
| "margin_dpo/margin_mean": 21.30208396911621, |
| "margin_dpo/margin_std": 81.18660736083984, |
| "step": 414 |
| }, |
| { |
| "epoch": 0.8691099476439791, |
| "grad_norm": 49.326297760009766, |
| "learning_rate": 2.613722016414943e-08, |
| "logits/chosen": 0.8005275726318359, |
| "logits/rejected": 0.8713321685791016, |
| "logps/chosen": -245.8915557861328, |
| "logps/ref_chosen": -223.82870483398438, |
| "logps/ref_rejected": -206.082763671875, |
| "logps/rejected": -291.24188232421875, |
| "loss": 4.0595, |
| "margin_dpo/margin_mean": 63.09626007080078, |
| "margin_dpo/margin_std": 63.13758087158203, |
| "step": 415 |
| }, |
| { |
| "epoch": 0.8712041884816754, |
| "grad_norm": 48.53297424316406, |
| "learning_rate": 2.5328246937043525e-08, |
| "logits/chosen": 1.264346718788147, |
| "logits/rejected": 1.331539273262024, |
| "logps/chosen": -316.0765686035156, |
| "logps/ref_chosen": -303.91656494140625, |
| "logps/ref_rejected": -267.0210266113281, |
| "logps/rejected": -343.7052917480469, |
| "loss": 4.154, |
| "margin_dpo/margin_mean": 64.5242919921875, |
| "margin_dpo/margin_std": 58.65679168701172, |
| "step": 416 |
| }, |
| { |
| "epoch": 0.8732984293193717, |
| "grad_norm": 43.959312438964844, |
| "learning_rate": 2.4531322174210973e-08, |
| "logits/chosen": 1.207326054573059, |
| "logits/rejected": 1.3471075296401978, |
| "logps/chosen": -298.7107849121094, |
| "logps/ref_chosen": -249.8788604736328, |
| "logps/ref_rejected": -210.48683166503906, |
| "logps/rejected": -290.503662109375, |
| "loss": 4.4556, |
| "margin_dpo/margin_mean": 31.184890747070312, |
| "margin_dpo/margin_std": 68.83541107177734, |
| "step": 417 |
| }, |
| { |
| "epoch": 0.875392670157068, |
| "grad_norm": 57.87160110473633, |
| "learning_rate": 2.3746488612308295e-08, |
| "logits/chosen": 1.1209101676940918, |
| "logits/rejected": 1.000208854675293, |
| "logps/chosen": -411.3262023925781, |
| "logps/ref_chosen": -355.4482421875, |
| "logps/ref_rejected": -344.9490661621094, |
| "logps/rejected": -433.7691650390625, |
| "loss": 4.5932, |
| "margin_dpo/margin_mean": 32.94209671020508, |
| "margin_dpo/margin_std": 72.00234985351562, |
| "step": 418 |
| }, |
| { |
| "epoch": 0.8774869109947644, |
| "grad_norm": 55.62238311767578, |
| "learning_rate": 2.297378833957761e-08, |
| "logits/chosen": 1.6920902729034424, |
| "logits/rejected": 1.618727445602417, |
| "logps/chosen": -433.79205322265625, |
| "logps/ref_chosen": -381.6947021484375, |
| "logps/ref_rejected": -322.436767578125, |
| "logps/rejected": -426.5209045410156, |
| "loss": 4.2079, |
| "margin_dpo/margin_mean": 51.986717224121094, |
| "margin_dpo/margin_std": 64.11946868896484, |
| "step": 419 |
| }, |
| { |
| "epoch": 0.8795811518324608, |
| "grad_norm": 42.748905181884766, |
| "learning_rate": 2.2213262793589482e-08, |
| "logits/chosen": 1.0813815593719482, |
| "logits/rejected": 1.1282932758331299, |
| "logps/chosen": -290.3770446777344, |
| "logps/ref_chosen": -255.09539794921875, |
| "logps/ref_rejected": -254.58306884765625, |
| "logps/rejected": -328.9988708496094, |
| "loss": 4.2003, |
| "margin_dpo/margin_mean": 39.13414001464844, |
| "margin_dpo/margin_std": 68.49993896484375, |
| "step": 420 |
| }, |
| { |
| "epoch": 0.881675392670157, |
| "grad_norm": 62.450592041015625, |
| "learning_rate": 2.1464952759020856e-08, |
| "logits/chosen": 1.2170071601867676, |
| "logits/rejected": 1.0278328657150269, |
| "logps/chosen": -303.1274719238281, |
| "logps/ref_chosen": -280.7524719238281, |
| "logps/ref_rejected": -187.39218139648438, |
| "logps/rejected": -256.837646484375, |
| "loss": 4.0246, |
| "margin_dpo/margin_mean": 47.07042694091797, |
| "margin_dpo/margin_std": 56.84251403808594, |
| "step": 421 |
| }, |
| { |
| "epoch": 0.8837696335078534, |
| "grad_norm": 56.18370056152344, |
| "learning_rate": 2.07288983654679e-08, |
| "logits/chosen": 1.3475362062454224, |
| "logits/rejected": 1.4008029699325562, |
| "logps/chosen": -307.1512145996094, |
| "logps/ref_chosen": -278.18890380859375, |
| "logps/ref_rejected": -250.71591186523438, |
| "logps/rejected": -318.3471984863281, |
| "loss": 4.6777, |
| "margin_dpo/margin_mean": 38.6689453125, |
| "margin_dpo/margin_std": 72.85610961914062, |
| "step": 422 |
| }, |
| { |
| "epoch": 0.8858638743455497, |
| "grad_norm": 64.5829849243164, |
| "learning_rate": 2.0005139085293942e-08, |
| "logits/chosen": 1.3403944969177246, |
| "logits/rejected": 1.4666098356246948, |
| "logps/chosen": -301.6260986328125, |
| "logps/ref_chosen": -281.21820068359375, |
| "logps/ref_rejected": -296.73907470703125, |
| "logps/rejected": -367.79888916015625, |
| "loss": 4.1139, |
| "margin_dpo/margin_mean": 50.65192413330078, |
| "margin_dpo/margin_std": 72.86007690429688, |
| "step": 423 |
| }, |
| { |
| "epoch": 0.8879581151832461, |
| "grad_norm": 43.40376281738281, |
| "learning_rate": 1.9293713731512673e-08, |
| "logits/chosen": 1.1475857496261597, |
| "logits/rejected": 1.005727767944336, |
| "logps/chosen": -361.1445617675781, |
| "logps/ref_chosen": -339.32550048828125, |
| "logps/ref_rejected": -274.3222351074219, |
| "logps/rejected": -354.5352478027344, |
| "loss": 4.1116, |
| "margin_dpo/margin_mean": 58.39393615722656, |
| "margin_dpo/margin_std": 50.71818542480469, |
| "step": 424 |
| }, |
| { |
| "epoch": 0.8900523560209425, |
| "grad_norm": 46.304969787597656, |
| "learning_rate": 1.8594660455706763e-08, |
| "logits/chosen": 1.2751210927963257, |
| "logits/rejected": 1.5024828910827637, |
| "logps/chosen": -283.94232177734375, |
| "logps/ref_chosen": -254.7490997314453, |
| "logps/ref_rejected": -266.1432800292969, |
| "logps/rejected": -349.154541015625, |
| "loss": 4.3925, |
| "margin_dpo/margin_mean": 53.81803894042969, |
| "margin_dpo/margin_std": 49.791236877441406, |
| "step": 425 |
| }, |
| { |
| "epoch": 0.8921465968586387, |
| "grad_norm": 41.32174301147461, |
| "learning_rate": 1.7908016745981856e-08, |
| "logits/chosen": 0.896172285079956, |
| "logits/rejected": 1.0513912439346313, |
| "logps/chosen": -284.55767822265625, |
| "logps/ref_chosen": -264.97216796875, |
| "logps/ref_rejected": -244.05935668945312, |
| "logps/rejected": -316.9269714355469, |
| "loss": 4.2069, |
| "margin_dpo/margin_mean": 53.282081604003906, |
| "margin_dpo/margin_std": 63.170196533203125, |
| "step": 426 |
| }, |
| { |
| "epoch": 0.8942408376963351, |
| "grad_norm": 42.068763732910156, |
| "learning_rate": 1.7233819424956247e-08, |
| "logits/chosen": 1.2293376922607422, |
| "logits/rejected": 1.2114195823669434, |
| "logps/chosen": -329.5334167480469, |
| "logps/ref_chosen": -301.6879577636719, |
| "logps/ref_rejected": -265.6783752441406, |
| "logps/rejected": -380.6156311035156, |
| "loss": 3.7144, |
| "margin_dpo/margin_mean": 87.09181213378906, |
| "margin_dpo/margin_std": 52.67485046386719, |
| "step": 427 |
| }, |
| { |
| "epoch": 0.8963350785340314, |
| "grad_norm": 55.713130950927734, |
| "learning_rate": 1.6572104647786245e-08, |
| "logits/chosen": 1.5100287199020386, |
| "logits/rejected": 1.70786714553833, |
| "logps/chosen": -415.50616455078125, |
| "logps/ref_chosen": -376.43316650390625, |
| "logps/ref_rejected": -334.24676513671875, |
| "logps/rejected": -440.6484375, |
| "loss": 3.7971, |
| "margin_dpo/margin_mean": 67.32867431640625, |
| "margin_dpo/margin_std": 66.57231140136719, |
| "step": 428 |
| }, |
| { |
| "epoch": 0.8984293193717278, |
| "grad_norm": 70.71985626220703, |
| "learning_rate": 1.5922907900227017e-08, |
| "logits/chosen": 1.4469355344772339, |
| "logits/rejected": 1.4792401790618896, |
| "logps/chosen": -247.02073669433594, |
| "logps/ref_chosen": -218.89503479003906, |
| "logps/ref_rejected": -236.546630859375, |
| "logps/rejected": -322.55419921875, |
| "loss": 4.4589, |
| "margin_dpo/margin_mean": 57.881866455078125, |
| "margin_dpo/margin_std": 76.20121002197266, |
| "step": 429 |
| }, |
| { |
| "epoch": 0.900523560209424, |
| "grad_norm": 92.4472427368164, |
| "learning_rate": 1.5286263996730026e-08, |
| "logits/chosen": 1.3988643884658813, |
| "logits/rejected": 1.5516958236694336, |
| "logps/chosen": -318.7999267578125, |
| "logps/ref_chosen": -281.9652099609375, |
| "logps/ref_rejected": -266.40411376953125, |
| "logps/rejected": -329.5230712890625, |
| "loss": 4.4918, |
| "margin_dpo/margin_mean": 26.284305572509766, |
| "margin_dpo/margin_std": 64.46965789794922, |
| "step": 430 |
| }, |
| { |
| "epoch": 0.9026178010471204, |
| "grad_norm": 42.31532287597656, |
| "learning_rate": 1.4662207078575684e-08, |
| "logits/chosen": 1.7694166898727417, |
| "logits/rejected": 1.8366968631744385, |
| "logps/chosen": -332.827880859375, |
| "logps/ref_chosen": -286.10888671875, |
| "logps/ref_rejected": -274.0017395019531, |
| "logps/rejected": -361.9835510253906, |
| "loss": 4.5107, |
| "margin_dpo/margin_mean": 41.2628173828125, |
| "margin_dpo/margin_std": 59.571441650390625, |
| "step": 431 |
| }, |
| { |
| "epoch": 0.9047120418848168, |
| "grad_norm": 37.992881774902344, |
| "learning_rate": 1.40507706120426e-08, |
| "logits/chosen": 1.4250589609146118, |
| "logits/rejected": 1.6473501920700073, |
| "logps/chosen": -348.860595703125, |
| "logps/ref_chosen": -316.9443359375, |
| "logps/ref_rejected": -358.3585510253906, |
| "logps/rejected": -438.4804382324219, |
| "loss": 4.1495, |
| "margin_dpo/margin_mean": 48.20561981201172, |
| "margin_dpo/margin_std": 72.83356475830078, |
| "step": 432 |
| }, |
| { |
| "epoch": 0.9068062827225131, |
| "grad_norm": 56.42523193359375, |
| "learning_rate": 1.345198738661285e-08, |
| "logits/chosen": 1.288971185684204, |
| "logits/rejected": 1.2402310371398926, |
| "logps/chosen": -312.3082275390625, |
| "logps/ref_chosen": -282.2297668457031, |
| "logps/ref_rejected": -258.5012512207031, |
| "logps/rejected": -328.95513916015625, |
| "loss": 4.5137, |
| "margin_dpo/margin_mean": 40.375389099121094, |
| "margin_dpo/margin_std": 55.306190490722656, |
| "step": 433 |
| }, |
| { |
| "epoch": 0.9089005235602095, |
| "grad_norm": 34.96892547607422, |
| "learning_rate": 1.2865889513213628e-08, |
| "logits/chosen": 1.7188708782196045, |
| "logits/rejected": 1.7233338356018066, |
| "logps/chosen": -339.9571533203125, |
| "logps/ref_chosen": -313.5975646972656, |
| "logps/ref_rejected": -293.44097900390625, |
| "logps/rejected": -374.973876953125, |
| "loss": 4.2523, |
| "margin_dpo/margin_mean": 55.173301696777344, |
| "margin_dpo/margin_std": 61.733543395996094, |
| "step": 434 |
| }, |
| { |
| "epoch": 0.9109947643979057, |
| "grad_norm": 84.76171875, |
| "learning_rate": 1.2292508422495157e-08, |
| "logits/chosen": 1.584928035736084, |
| "logits/rejected": 1.718322515487671, |
| "logps/chosen": -211.07582092285156, |
| "logps/ref_chosen": -191.58889770507812, |
| "logps/ref_rejected": -206.38133239746094, |
| "logps/rejected": -269.05389404296875, |
| "loss": 4.3653, |
| "margin_dpo/margin_mean": 43.18561553955078, |
| "margin_dpo/margin_std": 55.161441802978516, |
| "step": 435 |
| }, |
| { |
| "epoch": 0.9130890052356021, |
| "grad_norm": 57.35254669189453, |
| "learning_rate": 1.1731874863145142e-08, |
| "logits/chosen": 1.098213791847229, |
| "logits/rejected": 1.1476788520812988, |
| "logps/chosen": -349.2528076171875, |
| "logps/ref_chosen": -329.4399719238281, |
| "logps/ref_rejected": -310.59783935546875, |
| "logps/rejected": -374.02459716796875, |
| "loss": 4.8351, |
| "margin_dpo/margin_mean": 43.61388397216797, |
| "margin_dpo/margin_std": 60.17463302612305, |
| "step": 436 |
| }, |
| { |
| "epoch": 0.9151832460732985, |
| "grad_norm": 33.499061584472656, |
| "learning_rate": 1.118401890024001e-08, |
| "logits/chosen": 1.4528710842132568, |
| "logits/rejected": 1.5862656831741333, |
| "logps/chosen": -284.3026123046875, |
| "logps/ref_chosen": -245.99761962890625, |
| "logps/ref_rejected": -340.40283203125, |
| "logps/rejected": -442.78497314453125, |
| "loss": 4.1697, |
| "margin_dpo/margin_mean": 64.07716369628906, |
| "margin_dpo/margin_std": 62.12276077270508, |
| "step": 437 |
| }, |
| { |
| "epoch": 0.9172774869109948, |
| "grad_norm": 139.29515075683594, |
| "learning_rate": 1.06489699136324e-08, |
| "logits/chosen": 1.1451233625411987, |
| "logits/rejected": 1.288071870803833, |
| "logps/chosen": -289.2056884765625, |
| "logps/ref_chosen": -264.5708923339844, |
| "logps/ref_rejected": -263.8647155761719, |
| "logps/rejected": -315.4420471191406, |
| "loss": 5.1452, |
| "margin_dpo/margin_mean": 26.942535400390625, |
| "margin_dpo/margin_std": 64.0194320678711, |
| "step": 438 |
| }, |
| { |
| "epoch": 0.9193717277486911, |
| "grad_norm": 45.723548889160156, |
| "learning_rate": 1.0126756596375685e-08, |
| "logits/chosen": 1.449920892715454, |
| "logits/rejected": 1.4370176792144775, |
| "logps/chosen": -275.9388427734375, |
| "logps/ref_chosen": -236.2272491455078, |
| "logps/ref_rejected": -260.26531982421875, |
| "logps/rejected": -337.4900207519531, |
| "loss": 4.4005, |
| "margin_dpo/margin_mean": 37.513099670410156, |
| "margin_dpo/margin_std": 69.3291244506836, |
| "step": 439 |
| }, |
| { |
| "epoch": 0.9214659685863874, |
| "grad_norm": 51.19523620605469, |
| "learning_rate": 9.617406953185136e-09, |
| "logits/chosen": 1.4269630908966064, |
| "logits/rejected": 1.2528316974639893, |
| "logps/chosen": -450.7151794433594, |
| "logps/ref_chosen": -402.7833557128906, |
| "logps/ref_rejected": -285.3439636230469, |
| "logps/rejected": -360.1721496582031, |
| "loss": 4.77, |
| "margin_dpo/margin_mean": 26.89635467529297, |
| "margin_dpo/margin_std": 57.31180191040039, |
| "step": 440 |
| }, |
| { |
| "epoch": 0.9235602094240838, |
| "grad_norm": 45.438751220703125, |
| "learning_rate": 9.12094829893642e-09, |
| "logits/chosen": 1.4254412651062012, |
| "logits/rejected": 1.6387895345687866, |
| "logps/chosen": -382.6518859863281, |
| "logps/ref_chosen": -348.18212890625, |
| "logps/ref_rejected": -375.537353515625, |
| "logps/rejected": -471.38824462890625, |
| "loss": 4.0819, |
| "margin_dpo/margin_mean": 61.381141662597656, |
| "margin_dpo/margin_std": 54.50360870361328, |
| "step": 441 |
| }, |
| { |
| "epoch": 0.9256544502617801, |
| "grad_norm": 53.282371520996094, |
| "learning_rate": 8.637407257200496e-09, |
| "logits/chosen": 1.2402985095977783, |
| "logits/rejected": 1.3638098239898682, |
| "logps/chosen": -275.3695373535156, |
| "logps/ref_chosen": -232.696044921875, |
| "logps/ref_rejected": -204.60752868652344, |
| "logps/rejected": -279.0164489746094, |
| "loss": 4.6929, |
| "margin_dpo/margin_mean": 31.735416412353516, |
| "margin_dpo/margin_std": 70.92927551269531, |
| "step": 442 |
| }, |
| { |
| "epoch": 0.9277486910994764, |
| "grad_norm": 41.01181411743164, |
| "learning_rate": 8.166809758815895e-09, |
| "logits/chosen": 1.3078192472457886, |
| "logits/rejected": 1.3253602981567383, |
| "logps/chosen": -330.3334655761719, |
| "logps/ref_chosen": -275.13873291015625, |
| "logps/ref_rejected": -264.1988830566406, |
| "logps/rejected": -356.9568176269531, |
| "loss": 4.5058, |
| "margin_dpo/margin_mean": 37.56320571899414, |
| "margin_dpo/margin_std": 54.74559783935547, |
| "step": 443 |
| }, |
| { |
| "epoch": 0.9298429319371728, |
| "grad_norm": 51.980506896972656, |
| "learning_rate": 7.709181040498253e-09, |
| "logits/chosen": 0.7922985553741455, |
| "logits/rejected": 0.9772998690605164, |
| "logps/chosen": -340.13336181640625, |
| "logps/ref_chosen": -305.7708740234375, |
| "logps/ref_rejected": -292.68731689453125, |
| "logps/rejected": -366.3361511230469, |
| "loss": 4.3732, |
| "margin_dpo/margin_mean": 39.28638458251953, |
| "margin_dpo/margin_std": 50.5058479309082, |
| "step": 444 |
| }, |
| { |
| "epoch": 0.9319371727748691, |
| "grad_norm": 50.610992431640625, |
| "learning_rate": 7.2645456434869965e-09, |
| "logits/chosen": 1.336693286895752, |
| "logits/rejected": 1.4031049013137817, |
| "logps/chosen": -257.5855712890625, |
| "logps/ref_chosen": -240.496337890625, |
| "logps/ref_rejected": -226.0730743408203, |
| "logps/rejected": -288.43756103515625, |
| "loss": 4.6998, |
| "margin_dpo/margin_mean": 45.27527618408203, |
| "margin_dpo/margin_std": 74.52619934082031, |
| "step": 445 |
| }, |
| { |
| "epoch": 0.9340314136125655, |
| "grad_norm": 62.949615478515625, |
| "learning_rate": 6.832927412229017e-09, |
| "logits/chosen": 1.3089869022369385, |
| "logits/rejected": 1.283385992050171, |
| "logps/chosen": -267.8168029785156, |
| "logps/ref_chosen": -244.18284606933594, |
| "logps/ref_rejected": -211.3776397705078, |
| "logps/rejected": -278.58465576171875, |
| "loss": 4.3051, |
| "margin_dpo/margin_mean": 43.57305145263672, |
| "margin_dpo/margin_std": 57.471893310546875, |
| "step": 446 |
| }, |
| { |
| "epoch": 0.9361256544502617, |
| "grad_norm": 52.591312408447266, |
| "learning_rate": 6.414349493100129e-09, |
| "logits/chosen": 1.3533639907836914, |
| "logits/rejected": 1.4261730909347534, |
| "logps/chosen": -258.5954895019531, |
| "logps/ref_chosen": -237.29592895507812, |
| "logps/ref_rejected": -246.57034301757812, |
| "logps/rejected": -316.0662841796875, |
| "loss": 4.0452, |
| "margin_dpo/margin_mean": 48.1964225769043, |
| "margin_dpo/margin_std": 65.75294494628906, |
| "step": 447 |
| }, |
| { |
| "epoch": 0.9382198952879581, |
| "grad_norm": 46.954933166503906, |
| "learning_rate": 6.0088343331638756e-09, |
| "logits/chosen": 1.6584538221359253, |
| "logits/rejected": 1.7075550556182861, |
| "logps/chosen": -336.6059265136719, |
| "logps/ref_chosen": -308.6024475097656, |
| "logps/ref_rejected": -277.87921142578125, |
| "logps/rejected": -346.0470275878906, |
| "loss": 4.3113, |
| "margin_dpo/margin_mean": 40.164310455322266, |
| "margin_dpo/margin_std": 57.308650970458984, |
| "step": 448 |
| }, |
| { |
| "epoch": 0.9403141361256544, |
| "grad_norm": 54.72856140136719, |
| "learning_rate": 5.616403678967624e-09, |
| "logits/chosen": 1.8703218698501587, |
| "logits/rejected": 1.5918076038360596, |
| "logps/chosen": -386.6697998046875, |
| "logps/ref_chosen": -376.94281005859375, |
| "logps/ref_rejected": -267.11236572265625, |
| "logps/rejected": -342.6089782714844, |
| "loss": 4.1562, |
| "margin_dpo/margin_mean": 65.76960754394531, |
| "margin_dpo/margin_std": 61.83976745605469, |
| "step": 449 |
| }, |
| { |
| "epoch": 0.9424083769633508, |
| "grad_norm": 65.85668182373047, |
| "learning_rate": 5.2370785753763356e-09, |
| "logits/chosen": 1.6628509759902954, |
| "logits/rejected": 1.4311879873275757, |
| "logps/chosen": -327.4361572265625, |
| "logps/ref_chosen": -312.619384765625, |
| "logps/ref_rejected": -215.62857055664062, |
| "logps/rejected": -281.2243957519531, |
| "loss": 4.5566, |
| "margin_dpo/margin_mean": 50.77903366088867, |
| "margin_dpo/margin_std": 56.17271041870117, |
| "step": 450 |
| }, |
| { |
| "epoch": 0.9445026178010472, |
| "grad_norm": 55.37830352783203, |
| "learning_rate": 4.8708793644441086e-09, |
| "logits/chosen": 1.4510207176208496, |
| "logits/rejected": 1.5802251100540161, |
| "logps/chosen": -330.12811279296875, |
| "logps/ref_chosen": -296.6983947753906, |
| "logps/ref_rejected": -312.4747619628906, |
| "logps/rejected": -392.5194396972656, |
| "loss": 4.3406, |
| "margin_dpo/margin_mean": 46.6149787902832, |
| "margin_dpo/margin_std": 63.69156265258789, |
| "step": 451 |
| }, |
| { |
| "epoch": 0.9465968586387434, |
| "grad_norm": 50.63416290283203, |
| "learning_rate": 4.517825684323323e-09, |
| "logits/chosen": 1.3996615409851074, |
| "logits/rejected": 1.581308364868164, |
| "logps/chosen": -329.9542541503906, |
| "logps/ref_chosen": -294.50958251953125, |
| "logps/ref_rejected": -295.56097412109375, |
| "logps/rejected": -362.0646667480469, |
| "loss": 4.3711, |
| "margin_dpo/margin_mean": 31.059064865112305, |
| "margin_dpo/margin_std": 63.657875061035156, |
| "step": 452 |
| }, |
| { |
| "epoch": 0.9486910994764398, |
| "grad_norm": 44.21456527709961, |
| "learning_rate": 4.1779364682113794e-09, |
| "logits/chosen": 1.4765185117721558, |
| "logits/rejected": 1.6153349876403809, |
| "logps/chosen": -341.7298278808594, |
| "logps/ref_chosen": -308.21917724609375, |
| "logps/ref_rejected": -328.357421875, |
| "logps/rejected": -414.666748046875, |
| "loss": 4.1702, |
| "margin_dpo/margin_mean": 52.798675537109375, |
| "margin_dpo/margin_std": 59.44575881958008, |
| "step": 453 |
| }, |
| { |
| "epoch": 0.9507853403141361, |
| "grad_norm": 36.86773681640625, |
| "learning_rate": 3.851229943335393e-09, |
| "logits/chosen": 1.8053613901138306, |
| "logits/rejected": 1.7590644359588623, |
| "logps/chosen": -365.978759765625, |
| "logps/ref_chosen": -332.5453796386719, |
| "logps/ref_rejected": -267.130615234375, |
| "logps/rejected": -339.3133544921875, |
| "loss": 4.3145, |
| "margin_dpo/margin_mean": 38.749393463134766, |
| "margin_dpo/margin_std": 65.05109405517578, |
| "step": 454 |
| }, |
| { |
| "epoch": 0.9528795811518325, |
| "grad_norm": 49.20968246459961, |
| "learning_rate": 3.5377236299748147e-09, |
| "logits/chosen": 1.3919376134872437, |
| "logits/rejected": 1.5252500772476196, |
| "logps/chosen": -261.4302978515625, |
| "logps/ref_chosen": -240.13719177246094, |
| "logps/ref_rejected": -233.28451538085938, |
| "logps/rejected": -313.81097412109375, |
| "loss": 4.5531, |
| "margin_dpo/margin_mean": 59.233402252197266, |
| "margin_dpo/margin_std": 58.3806037902832, |
| "step": 455 |
| }, |
| { |
| "epoch": 0.9549738219895288, |
| "grad_norm": 42.70205307006836, |
| "learning_rate": 3.2374343405217884e-09, |
| "logits/chosen": 1.635735034942627, |
| "logits/rejected": 1.8064732551574707, |
| "logps/chosen": -370.0901184082031, |
| "logps/ref_chosen": -334.82666015625, |
| "logps/ref_rejected": -313.20404052734375, |
| "logps/rejected": -386.83837890625, |
| "loss": 4.1309, |
| "margin_dpo/margin_mean": 38.370887756347656, |
| "margin_dpo/margin_std": 76.44358825683594, |
| "step": 456 |
| }, |
| { |
| "epoch": 0.9570680628272251, |
| "grad_norm": 44.188438415527344, |
| "learning_rate": 2.9503781785795713e-09, |
| "logits/chosen": 1.4283052682876587, |
| "logits/rejected": 1.370866298675537, |
| "logps/chosen": -319.96136474609375, |
| "logps/ref_chosen": -299.60650634765625, |
| "logps/ref_rejected": -263.5287780761719, |
| "logps/rejected": -360.9966125488281, |
| "loss": 4.2728, |
| "margin_dpo/margin_mean": 77.11296844482422, |
| "margin_dpo/margin_std": 65.07032775878906, |
| "step": 457 |
| }, |
| { |
| "epoch": 0.9591623036649215, |
| "grad_norm": 53.56594467163086, |
| "learning_rate": 2.6765705380989432e-09, |
| "logits/chosen": 1.6065071821212769, |
| "logits/rejected": 1.4925872087478638, |
| "logps/chosen": -309.9053955078125, |
| "logps/ref_chosen": -272.7044372558594, |
| "logps/ref_rejected": -235.6636962890625, |
| "logps/rejected": -300.68353271484375, |
| "loss": 4.3749, |
| "margin_dpo/margin_mean": 27.818876266479492, |
| "margin_dpo/margin_std": 55.92146301269531, |
| "step": 458 |
| }, |
| { |
| "epoch": 0.9612565445026178, |
| "grad_norm": 38.86277389526367, |
| "learning_rate": 2.416026102552732e-09, |
| "logits/chosen": 1.428498387336731, |
| "logits/rejected": 1.3140032291412354, |
| "logps/chosen": -328.8189697265625, |
| "logps/ref_chosen": -280.32196044921875, |
| "logps/ref_rejected": -217.7216339111328, |
| "logps/rejected": -297.50244140625, |
| "loss": 4.8445, |
| "margin_dpo/margin_mean": 31.283798217773438, |
| "margin_dpo/margin_std": 54.594871520996094, |
| "step": 459 |
| }, |
| { |
| "epoch": 0.9633507853403142, |
| "grad_norm": 91.57032012939453, |
| "learning_rate": 2.168758844148272e-09, |
| "logits/chosen": 1.443167805671692, |
| "logits/rejected": 1.4732171297073364, |
| "logps/chosen": -426.33984375, |
| "logps/ref_chosen": -387.5949401855469, |
| "logps/ref_rejected": -289.61505126953125, |
| "logps/rejected": -362.8568115234375, |
| "loss": 4.8551, |
| "margin_dpo/margin_mean": 34.49686050415039, |
| "margin_dpo/margin_std": 72.81037902832031, |
| "step": 460 |
| }, |
| { |
| "epoch": 0.9654450261780104, |
| "grad_norm": 57.414451599121094, |
| "learning_rate": 1.9347820230782295e-09, |
| "logits/chosen": 1.5660542249679565, |
| "logits/rejected": 1.520784616470337, |
| "logps/chosen": -268.8662414550781, |
| "logps/ref_chosen": -247.67520141601562, |
| "logps/ref_rejected": -227.18458557128906, |
| "logps/rejected": -311.7144775390625, |
| "loss": 4.5036, |
| "margin_dpo/margin_mean": 63.33882141113281, |
| "margin_dpo/margin_std": 73.68379974365234, |
| "step": 461 |
| }, |
| { |
| "epoch": 0.9675392670157068, |
| "grad_norm": 49.87493896484375, |
| "learning_rate": 1.7141081868094209e-09, |
| "logits/chosen": 1.4014174938201904, |
| "logits/rejected": 1.3472931385040283, |
| "logps/chosen": -378.84088134765625, |
| "logps/ref_chosen": -350.8253173828125, |
| "logps/ref_rejected": -262.5743713378906, |
| "logps/rejected": -355.7335205078125, |
| "loss": 4.116, |
| "margin_dpo/margin_mean": 65.14358520507812, |
| "margin_dpo/margin_std": 64.9459457397461, |
| "step": 462 |
| }, |
| { |
| "epoch": 0.9696335078534032, |
| "grad_norm": 84.11515808105469, |
| "learning_rate": 1.5067491694100153e-09, |
| "logits/chosen": 1.2984429597854614, |
| "logits/rejected": 1.3639535903930664, |
| "logps/chosen": -271.6904602050781, |
| "logps/ref_chosen": -229.31683349609375, |
| "logps/ref_rejected": -227.2904815673828, |
| "logps/rejected": -293.2787170410156, |
| "loss": 4.4943, |
| "margin_dpo/margin_mean": 23.614595413208008, |
| "margin_dpo/margin_std": 70.91625213623047, |
| "step": 463 |
| }, |
| { |
| "epoch": 0.9717277486910995, |
| "grad_norm": 51.687442779541016, |
| "learning_rate": 1.3127160909147672e-09, |
| "logits/chosen": 1.7068381309509277, |
| "logits/rejected": 1.6888086795806885, |
| "logps/chosen": -248.65789794921875, |
| "logps/ref_chosen": -226.55776977539062, |
| "logps/ref_rejected": -208.3471221923828, |
| "logps/rejected": -285.44244384765625, |
| "loss": 4.4572, |
| "margin_dpo/margin_mean": 54.995182037353516, |
| "margin_dpo/margin_std": 61.694210052490234, |
| "step": 464 |
| }, |
| { |
| "epoch": 0.9738219895287958, |
| "grad_norm": 37.04083251953125, |
| "learning_rate": 1.1320193567288527e-09, |
| "logits/chosen": 1.1953487396240234, |
| "logits/rejected": 1.2031378746032715, |
| "logps/chosen": -311.39080810546875, |
| "logps/ref_chosen": -287.9401550292969, |
| "logps/ref_rejected": -305.5926818847656, |
| "logps/rejected": -382.921630859375, |
| "loss": 4.0925, |
| "margin_dpo/margin_mean": 53.878334045410156, |
| "margin_dpo/margin_std": 47.030704498291016, |
| "step": 465 |
| }, |
| { |
| "epoch": 0.9759162303664921, |
| "grad_norm": 40.89207458496094, |
| "learning_rate": 9.64668657069706e-10, |
| "logits/chosen": 1.327850580215454, |
| "logits/rejected": 1.417038917541504, |
| "logps/chosen": -233.49664306640625, |
| "logps/ref_chosen": -224.32131958007812, |
| "logps/ref_rejected": -205.79212951660156, |
| "logps/rejected": -285.48529052734375, |
| "loss": 3.8998, |
| "margin_dpo/margin_mean": 70.51785278320312, |
| "margin_dpo/margin_std": 54.0175895690918, |
| "step": 466 |
| }, |
| { |
| "epoch": 0.9780104712041885, |
| "grad_norm": 66.9071273803711, |
| "learning_rate": 8.106729664475176e-10, |
| "logits/chosen": 0.6929614543914795, |
| "logits/rejected": 0.9394963979721069, |
| "logps/chosen": -260.9166259765625, |
| "logps/ref_chosen": -227.0828094482422, |
| "logps/ref_rejected": -285.5081481933594, |
| "logps/rejected": -364.97393798828125, |
| "loss": 4.4527, |
| "margin_dpo/margin_mean": 45.63193893432617, |
| "margin_dpo/margin_std": 61.40578079223633, |
| "step": 467 |
| }, |
| { |
| "epoch": 0.9801047120418848, |
| "grad_norm": 43.40538787841797, |
| "learning_rate": 6.700405431837585e-10, |
| "logits/chosen": 1.376441478729248, |
| "logits/rejected": 1.1604515314102173, |
| "logps/chosen": -349.08599853515625, |
| "logps/ref_chosen": -314.6758117675781, |
| "logps/ref_rejected": -290.43023681640625, |
| "logps/rejected": -357.7455749511719, |
| "loss": 4.5586, |
| "margin_dpo/margin_mean": 32.90521240234375, |
| "margin_dpo/margin_std": 61.592552185058594, |
| "step": 468 |
| }, |
| { |
| "epoch": 0.9821989528795811, |
| "grad_norm": 48.64304733276367, |
| "learning_rate": 5.427789289685347e-10, |
| "logits/chosen": 1.2290468215942383, |
| "logits/rejected": 1.205275058746338, |
| "logps/chosen": -290.6042175292969, |
| "logps/ref_chosen": -269.7442321777344, |
| "logps/ref_rejected": -237.1964874267578, |
| "logps/rejected": -307.8065185546875, |
| "loss": 4.2722, |
| "margin_dpo/margin_mean": 49.750064849853516, |
| "margin_dpo/margin_std": 63.16607666015625, |
| "step": 469 |
| }, |
| { |
| "epoch": 0.9842931937172775, |
| "grad_norm": 48.205284118652344, |
| "learning_rate": 4.288949484559934e-10, |
| "logits/chosen": 0.8344168066978455, |
| "logits/rejected": 0.8336673378944397, |
| "logps/chosen": -342.15057373046875, |
| "logps/ref_chosen": -326.9454650878906, |
| "logps/ref_rejected": -283.81768798828125, |
| "logps/rejected": -360.6470031738281, |
| "loss": 4.0516, |
| "margin_dpo/margin_mean": 61.624237060546875, |
| "margin_dpo/margin_std": 61.775753021240234, |
| "step": 470 |
| }, |
| { |
| "epoch": 0.9863874345549738, |
| "grad_norm": 58.954227447509766, |
| "learning_rate": 3.2839470889836627e-10, |
| "logits/chosen": 1.4265129566192627, |
| "logits/rejected": 1.394487977027893, |
| "logps/chosen": -337.0513000488281, |
| "logps/ref_chosen": -309.4604797363281, |
| "logps/ref_rejected": -292.0646057128906, |
| "logps/rejected": -356.0199890136719, |
| "loss": 4.5083, |
| "margin_dpo/margin_mean": 36.36451721191406, |
| "margin_dpo/margin_std": 59.396270751953125, |
| "step": 471 |
| }, |
| { |
| "epoch": 0.9884816753926702, |
| "grad_norm": 61.63766098022461, |
| "learning_rate": 2.412835998185092e-10, |
| "logits/chosen": 1.1254520416259766, |
| "logits/rejected": 1.199691891670227, |
| "logps/chosen": -210.82077026367188, |
| "logps/ref_chosen": -185.00701904296875, |
| "logps/ref_rejected": -208.1576385498047, |
| "logps/rejected": -272.89483642578125, |
| "loss": 4.1475, |
| "margin_dpo/margin_mean": 38.92341613769531, |
| "margin_dpo/margin_std": 71.17210388183594, |
| "step": 472 |
| }, |
| { |
| "epoch": 0.9905759162303664, |
| "grad_norm": 56.59166717529297, |
| "learning_rate": 1.6756629272085544e-10, |
| "logits/chosen": 1.3166826963424683, |
| "logits/rejected": 1.1217594146728516, |
| "logps/chosen": -350.744140625, |
| "logps/ref_chosen": -330.0291442871094, |
| "logps/ref_rejected": -224.92051696777344, |
| "logps/rejected": -302.14898681640625, |
| "loss": 4.1337, |
| "margin_dpo/margin_mean": 56.51344299316406, |
| "margin_dpo/margin_std": 51.90734100341797, |
| "step": 473 |
| }, |
| { |
| "epoch": 0.9926701570680628, |
| "grad_norm": 59.311004638671875, |
| "learning_rate": 1.072467408408384e-10, |
| "logits/chosen": 1.2557741403579712, |
| "logits/rejected": 1.3725204467773438, |
| "logps/chosen": -355.0524597167969, |
| "logps/ref_chosen": -315.9046936035156, |
| "logps/ref_rejected": -340.7234191894531, |
| "logps/rejected": -411.844482421875, |
| "loss": 4.5535, |
| "margin_dpo/margin_mean": 31.97334861755371, |
| "margin_dpo/margin_std": 64.91607666015625, |
| "step": 474 |
| }, |
| { |
| "epoch": 0.9947643979057592, |
| "grad_norm": 54.42702102661133, |
| "learning_rate": 6.032817893297793e-11, |
| "logits/chosen": 0.872796356678009, |
| "logits/rejected": 0.9358187913894653, |
| "logps/chosen": -227.73483276367188, |
| "logps/ref_chosen": -202.84310913085938, |
| "logps/ref_rejected": -175.70704650878906, |
| "logps/rejected": -250.8938751220703, |
| "loss": 4.4903, |
| "margin_dpo/margin_mean": 50.29510498046875, |
| "margin_dpo/margin_std": 74.1104965209961, |
| "step": 475 |
| }, |
| { |
| "epoch": 0.9968586387434555, |
| "grad_norm": 50.788307189941406, |
| "learning_rate": 2.6813123097352287e-11, |
| "logits/chosen": 1.07370924949646, |
| "logits/rejected": 1.213090181350708, |
| "logps/chosen": -292.42132568359375, |
| "logps/ref_chosen": -276.843505859375, |
| "logps/ref_rejected": -309.07757568359375, |
| "logps/rejected": -368.9504089355469, |
| "loss": 4.2622, |
| "margin_dpo/margin_mean": 44.29502868652344, |
| "margin_dpo/margin_std": 54.164615631103516, |
| "step": 476 |
| }, |
| { |
| "epoch": 0.9989528795811519, |
| "grad_norm": 40.021400451660156, |
| "learning_rate": 6.7033706447061635e-12, |
| "logits/chosen": 0.8247851729393005, |
| "logits/rejected": 0.9125658869743347, |
| "logps/chosen": -297.199951171875, |
| "logps/ref_chosen": -262.76971435546875, |
| "logps/ref_rejected": -272.2499694824219, |
| "logps/rejected": -356.56658935546875, |
| "loss": 4.216, |
| "margin_dpo/margin_mean": 49.88636779785156, |
| "margin_dpo/margin_std": 77.48868560791016, |
| "step": 477 |
| }, |
| { |
| "epoch": 0.9989528795811519, |
| "step": 477, |
| "total_flos": 0.0, |
| "train_loss": 4.779813265150698, |
| "train_runtime": 7822.2821, |
| "train_samples_per_second": 7.815, |
| "train_steps_per_second": 0.061 |
| } |
| ], |
| "logging_steps": 1, |
| "max_steps": 477, |
| "num_input_tokens_seen": 0, |
| "num_train_epochs": 1, |
| "save_steps": 200, |
| "stateful_callbacks": { |
| "TrainerControl": { |
| "args": { |
| "should_epoch_stop": false, |
| "should_evaluate": false, |
| "should_log": false, |
| "should_save": true, |
| "should_training_stop": true |
| }, |
| "attributes": {} |
| } |
| }, |
| "total_flos": 0.0, |
| "train_batch_size": 4, |
| "trial_name": null, |
| "trial_params": null |
| } |
|
|
