| { |
| "best_global_step": null, |
| "best_metric": null, |
| "best_model_checkpoint": null, |
| "epoch": 3.9951721918249117, |
| "eval_steps": 500, |
| "global_step": 3104, |
| "is_hyper_param_search": false, |
| "is_local_process_zero": true, |
| "is_world_process_zero": true, |
| "log_history": [ |
| { |
| "epoch": 0.012874155133569359, |
| "grad_norm": 116.00975162348479, |
| "learning_rate": 9.971005154639174e-07, |
| "logits/chosen": -0.4733520448207855, |
| "logits/rejected": -0.5006790161132812, |
| "logps/chosen": -357.125, |
| "logps/rejected": -361.4624938964844, |
| "loss": 0.6745, |
| "rewards/accuracies": 0.3499999940395355, |
| "rewards/chosen": -0.2167530059814453, |
| "rewards/margins": 0.06282500922679901, |
| "rewards/rejected": -0.279653936624527, |
| "step": 10 |
| }, |
| { |
| "epoch": 0.025748310267138717, |
| "grad_norm": 110.36198541397852, |
| "learning_rate": 9.938788659793814e-07, |
| "logits/chosen": -0.4485534727573395, |
| "logits/rejected": -0.47062987089157104, |
| "logps/chosen": -325.48748779296875, |
| "logps/rejected": -322.1000061035156, |
| "loss": 0.5977, |
| "rewards/accuracies": 0.606249988079071, |
| "rewards/chosen": -0.7079528570175171, |
| "rewards/margins": 0.3867950439453125, |
| "rewards/rejected": -1.094751000404358, |
| "step": 20 |
| }, |
| { |
| "epoch": 0.03862246540070808, |
| "grad_norm": 123.52649986895027, |
| "learning_rate": 9.906572164948455e-07, |
| "logits/chosen": -0.7004241943359375, |
| "logits/rejected": -0.7022460699081421, |
| "logps/chosen": -400.625, |
| "logps/rejected": -400.2749938964844, |
| "loss": 0.6372, |
| "rewards/accuracies": 0.6000000238418579, |
| "rewards/chosen": -1.50439453125, |
| "rewards/margins": 0.4994445741176605, |
| "rewards/rejected": -2.0038084983825684, |
| "step": 30 |
| }, |
| { |
| "epoch": 0.051496620534277435, |
| "grad_norm": 98.330089952775, |
| "learning_rate": 9.874355670103093e-07, |
| "logits/chosen": -0.606005847454071, |
| "logits/rejected": -0.628643810749054, |
| "logps/chosen": -425.7749938964844, |
| "logps/rejected": -373.6000061035156, |
| "loss": 0.707, |
| "rewards/accuracies": 0.5562499761581421, |
| "rewards/chosen": -1.1126892566680908, |
| "rewards/margins": 0.47386473417282104, |
| "rewards/rejected": -1.5862548351287842, |
| "step": 40 |
| }, |
| { |
| "epoch": 0.0643707756678468, |
| "grad_norm": 108.78565495841251, |
| "learning_rate": 9.84213917525773e-07, |
| "logits/chosen": -0.47911375761032104, |
| "logits/rejected": -0.4762206971645355, |
| "logps/chosen": -360.20001220703125, |
| "logps/rejected": -352.7250061035156, |
| "loss": 0.6266, |
| "rewards/accuracies": 0.606249988079071, |
| "rewards/chosen": 0.02710571326315403, |
| "rewards/margins": 0.42482298612594604, |
| "rewards/rejected": -0.3973388671875, |
| "step": 50 |
| }, |
| { |
| "epoch": 0.07724493080141616, |
| "grad_norm": 86.86103704117889, |
| "learning_rate": 9.80992268041237e-07, |
| "logits/chosen": -0.47004395723342896, |
| "logits/rejected": -0.4599609375, |
| "logps/chosen": -362.01251220703125, |
| "logps/rejected": -316.54998779296875, |
| "loss": 0.6111, |
| "rewards/accuracies": 0.59375, |
| "rewards/chosen": 0.381765753030777, |
| "rewards/margins": 0.4092163145542145, |
| "rewards/rejected": -0.02763671800494194, |
| "step": 60 |
| }, |
| { |
| "epoch": 0.09011908593498552, |
| "grad_norm": 112.58148578622469, |
| "learning_rate": 9.777706185567009e-07, |
| "logits/chosen": -0.47901612520217896, |
| "logits/rejected": -0.5212951898574829, |
| "logps/chosen": -381.6499938964844, |
| "logps/rejected": -362.29998779296875, |
| "loss": 0.6341, |
| "rewards/accuracies": 0.5687500238418579, |
| "rewards/chosen": 0.11572265625, |
| "rewards/margins": 0.47606199979782104, |
| "rewards/rejected": -0.360748291015625, |
| "step": 70 |
| }, |
| { |
| "epoch": 0.10299324106855487, |
| "grad_norm": 112.67612692520709, |
| "learning_rate": 9.74548969072165e-07, |
| "logits/chosen": -0.3992675840854645, |
| "logits/rejected": -0.4010772705078125, |
| "logps/chosen": -336.23748779296875, |
| "logps/rejected": -324.8500061035156, |
| "loss": 0.6137, |
| "rewards/accuracies": 0.59375, |
| "rewards/chosen": 0.11996765434741974, |
| "rewards/margins": 0.48876953125, |
| "rewards/rejected": -0.3689514100551605, |
| "step": 80 |
| }, |
| { |
| "epoch": 0.11586739620212423, |
| "grad_norm": 120.71710686325964, |
| "learning_rate": 9.71327319587629e-07, |
| "logits/chosen": -0.4455810487270355, |
| "logits/rejected": -0.46094971895217896, |
| "logps/chosen": -368.4624938964844, |
| "logps/rejected": -340.8812561035156, |
| "loss": 0.601, |
| "rewards/accuracies": 0.612500011920929, |
| "rewards/chosen": 0.15057983994483948, |
| "rewards/margins": 0.4910644590854645, |
| "rewards/rejected": -0.33942872285842896, |
| "step": 90 |
| }, |
| { |
| "epoch": 0.1287415513356936, |
| "grad_norm": 94.09238987623964, |
| "learning_rate": 9.681056701030927e-07, |
| "logits/chosen": -0.4274749755859375, |
| "logits/rejected": -0.4272094666957855, |
| "logps/chosen": -349.5874938964844, |
| "logps/rejected": -326.48748779296875, |
| "loss": 0.6003, |
| "rewards/accuracies": 0.6312500238418579, |
| "rewards/chosen": 0.292459100484848, |
| "rewards/margins": 0.49652099609375, |
| "rewards/rejected": -0.20349006354808807, |
| "step": 100 |
| }, |
| { |
| "epoch": 0.14161570646926294, |
| "grad_norm": 115.3027666657839, |
| "learning_rate": 9.648840206185567e-07, |
| "logits/chosen": -0.4821411073207855, |
| "logits/rejected": -0.473165899515152, |
| "logps/chosen": -364.82501220703125, |
| "logps/rejected": -361.45001220703125, |
| "loss": 0.6459, |
| "rewards/accuracies": 0.59375, |
| "rewards/chosen": 0.7324981689453125, |
| "rewards/margins": 0.547314465045929, |
| "rewards/rejected": 0.18525390326976776, |
| "step": 110 |
| }, |
| { |
| "epoch": 0.15448986160283232, |
| "grad_norm": 100.58566673282141, |
| "learning_rate": 9.616623711340205e-07, |
| "logits/chosen": -0.4615234434604645, |
| "logits/rejected": -0.46809083223342896, |
| "logps/chosen": -337.25, |
| "logps/rejected": -339.4375, |
| "loss": 0.6278, |
| "rewards/accuracies": 0.5874999761581421, |
| "rewards/chosen": 0.8613036870956421, |
| "rewards/margins": 0.5265868902206421, |
| "rewards/rejected": 0.3347229063510895, |
| "step": 120 |
| }, |
| { |
| "epoch": 0.16736401673640167, |
| "grad_norm": 122.76745910102257, |
| "learning_rate": 9.584407216494846e-07, |
| "logits/chosen": -0.5855712890625, |
| "logits/rejected": -0.6137939691543579, |
| "logps/chosen": -402.11248779296875, |
| "logps/rejected": -360.8500061035156, |
| "loss": 0.5982, |
| "rewards/accuracies": 0.668749988079071, |
| "rewards/chosen": 1.0935547351837158, |
| "rewards/margins": 0.5986083745956421, |
| "rewards/rejected": 0.495025634765625, |
| "step": 130 |
| }, |
| { |
| "epoch": 0.18023817186997104, |
| "grad_norm": 106.63724432186719, |
| "learning_rate": 9.552190721649484e-07, |
| "logits/chosen": -0.5129760503768921, |
| "logits/rejected": -0.5257323980331421, |
| "logps/chosen": -352.5874938964844, |
| "logps/rejected": -346.38751220703125, |
| "loss": 0.6439, |
| "rewards/accuracies": 0.6187499761581421, |
| "rewards/chosen": 0.80364990234375, |
| "rewards/margins": 0.48191529512405396, |
| "rewards/rejected": 0.32114869356155396, |
| "step": 140 |
| }, |
| { |
| "epoch": 0.1931123270035404, |
| "grad_norm": 108.47394171237873, |
| "learning_rate": 9.519974226804123e-07, |
| "logits/chosen": -0.6437743902206421, |
| "logits/rejected": -0.65399169921875, |
| "logps/chosen": -382.125, |
| "logps/rejected": -388.0249938964844, |
| "loss": 0.6183, |
| "rewards/accuracies": 0.637499988079071, |
| "rewards/chosen": 0.25493162870407104, |
| "rewards/margins": 0.700610339641571, |
| "rewards/rejected": -0.44627076387405396, |
| "step": 150 |
| }, |
| { |
| "epoch": 0.20598648213710974, |
| "grad_norm": 114.07304846157687, |
| "learning_rate": 9.487757731958762e-07, |
| "logits/chosen": -0.6397460699081421, |
| "logits/rejected": -0.612835705280304, |
| "logps/chosen": -345.11248779296875, |
| "logps/rejected": -337.36248779296875, |
| "loss": 0.536, |
| "rewards/accuracies": 0.699999988079071, |
| "rewards/chosen": 0.21904297173023224, |
| "rewards/margins": 0.951220691204071, |
| "rewards/rejected": -0.733203113079071, |
| "step": 160 |
| }, |
| { |
| "epoch": 0.21886063727067911, |
| "grad_norm": 105.78503442333401, |
| "learning_rate": 9.455541237113401e-07, |
| "logits/chosen": -0.642993152141571, |
| "logits/rejected": -0.634033203125, |
| "logps/chosen": -407.07501220703125, |
| "logps/rejected": -365.3999938964844, |
| "loss": 0.6362, |
| "rewards/accuracies": 0.637499988079071, |
| "rewards/chosen": 0.16254882514476776, |
| "rewards/margins": 0.850903332233429, |
| "rewards/rejected": -0.688751220703125, |
| "step": 170 |
| }, |
| { |
| "epoch": 0.23173479240424846, |
| "grad_norm": 98.33327737276883, |
| "learning_rate": 9.423324742268041e-07, |
| "logits/chosen": -0.6112426519393921, |
| "logits/rejected": -0.61279296875, |
| "logps/chosen": -366.9624938964844, |
| "logps/rejected": -348.95001220703125, |
| "loss": 0.6003, |
| "rewards/accuracies": 0.6312500238418579, |
| "rewards/chosen": 0.25787353515625, |
| "rewards/margins": 0.7079833745956421, |
| "rewards/rejected": -0.4504150450229645, |
| "step": 180 |
| }, |
| { |
| "epoch": 0.24460894753781784, |
| "grad_norm": 120.00275335834641, |
| "learning_rate": 9.39110824742268e-07, |
| "logits/chosen": -0.632250964641571, |
| "logits/rejected": -0.651928722858429, |
| "logps/chosen": -372.3125, |
| "logps/rejected": -362.79998779296875, |
| "loss": 0.5571, |
| "rewards/accuracies": 0.6812499761581421, |
| "rewards/chosen": 0.01650390587747097, |
| "rewards/margins": 0.7537597417831421, |
| "rewards/rejected": -0.7377685308456421, |
| "step": 190 |
| }, |
| { |
| "epoch": 0.2574831026713872, |
| "grad_norm": 129.37774954834526, |
| "learning_rate": 9.358891752577319e-07, |
| "logits/chosen": -0.5298446416854858, |
| "logits/rejected": -0.5588653683662415, |
| "logps/chosen": -385.6000061035156, |
| "logps/rejected": -359.0, |
| "loss": 0.6273, |
| "rewards/accuracies": 0.6187499761581421, |
| "rewards/chosen": -0.3814453184604645, |
| "rewards/margins": 0.734661877155304, |
| "rewards/rejected": -1.115972876548767, |
| "step": 200 |
| }, |
| { |
| "epoch": 0.27035725780495656, |
| "grad_norm": 85.46600016889145, |
| "learning_rate": 9.326675257731958e-07, |
| "logits/chosen": -0.5501617193222046, |
| "logits/rejected": -0.557861328125, |
| "logps/chosen": -375.79998779296875, |
| "logps/rejected": -389.61248779296875, |
| "loss": 0.5897, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": -0.4184021055698395, |
| "rewards/margins": 0.7344604730606079, |
| "rewards/rejected": -1.152307152748108, |
| "step": 210 |
| }, |
| { |
| "epoch": 0.2832314129385259, |
| "grad_norm": 112.06031531752737, |
| "learning_rate": 9.294458762886597e-07, |
| "logits/chosen": -0.61865234375, |
| "logits/rejected": -0.6116698980331421, |
| "logps/chosen": -375.4437561035156, |
| "logps/rejected": -348.0, |
| "loss": 0.6132, |
| "rewards/accuracies": 0.637499988079071, |
| "rewards/chosen": -0.16201171278953552, |
| "rewards/margins": 0.8055419921875, |
| "rewards/rejected": -0.9680572748184204, |
| "step": 220 |
| }, |
| { |
| "epoch": 0.29610556807209526, |
| "grad_norm": 131.6342917904006, |
| "learning_rate": 9.262242268041238e-07, |
| "logits/chosen": -0.6041504144668579, |
| "logits/rejected": -0.5826416015625, |
| "logps/chosen": -366.8374938964844, |
| "logps/rejected": -360.5249938964844, |
| "loss": 0.6912, |
| "rewards/accuracies": 0.5625, |
| "rewards/chosen": -0.23549041152000427, |
| "rewards/margins": 0.52764892578125, |
| "rewards/rejected": -0.7628723382949829, |
| "step": 230 |
| }, |
| { |
| "epoch": 0.30897972320566464, |
| "grad_norm": 90.71845088558837, |
| "learning_rate": 9.230025773195877e-07, |
| "logits/chosen": -0.47716981172561646, |
| "logits/rejected": -0.47523194551467896, |
| "logps/chosen": -393.6000061035156, |
| "logps/rejected": -380.0, |
| "loss": 0.6737, |
| "rewards/accuracies": 0.6625000238418579, |
| "rewards/chosen": -0.11795654147863388, |
| "rewards/margins": 0.4956298768520355, |
| "rewards/rejected": -0.614117443561554, |
| "step": 240 |
| }, |
| { |
| "epoch": 0.321853878339234, |
| "grad_norm": 108.95199143251729, |
| "learning_rate": 9.197809278350515e-07, |
| "logits/chosen": -0.5445922613143921, |
| "logits/rejected": -0.555126965045929, |
| "logps/chosen": -348.6499938964844, |
| "logps/rejected": -335.07501220703125, |
| "loss": 0.5944, |
| "rewards/accuracies": 0.6812499761581421, |
| "rewards/chosen": 0.34312742948532104, |
| "rewards/margins": 0.5995849370956421, |
| "rewards/rejected": -0.256674200296402, |
| "step": 250 |
| }, |
| { |
| "epoch": 0.33472803347280333, |
| "grad_norm": 117.30565370500288, |
| "learning_rate": 9.165592783505154e-07, |
| "logits/chosen": -0.5303589105606079, |
| "logits/rejected": -0.531933605670929, |
| "logps/chosen": -383.2875061035156, |
| "logps/rejected": -357.375, |
| "loss": 0.6335, |
| "rewards/accuracies": 0.5874999761581421, |
| "rewards/chosen": 0.18229980766773224, |
| "rewards/margins": 0.515429675579071, |
| "rewards/rejected": -0.3332763612270355, |
| "step": 260 |
| }, |
| { |
| "epoch": 0.3476021886063727, |
| "grad_norm": 137.214632537694, |
| "learning_rate": 9.133376288659793e-07, |
| "logits/chosen": -0.49357300996780396, |
| "logits/rejected": -0.524523913860321, |
| "logps/chosen": -333.70001220703125, |
| "logps/rejected": -317.45001220703125, |
| "loss": 0.6348, |
| "rewards/accuracies": 0.637499988079071, |
| "rewards/chosen": 0.3096069395542145, |
| "rewards/margins": 0.714184582233429, |
| "rewards/rejected": -0.40443724393844604, |
| "step": 270 |
| }, |
| { |
| "epoch": 0.3604763437399421, |
| "grad_norm": 120.58309781518359, |
| "learning_rate": 9.101159793814432e-07, |
| "logits/chosen": -0.547167956829071, |
| "logits/rejected": -0.542285144329071, |
| "logps/chosen": -345.1625061035156, |
| "logps/rejected": -328.2875061035156, |
| "loss": 0.61, |
| "rewards/accuracies": 0.6812499761581421, |
| "rewards/chosen": 0.11395873874425888, |
| "rewards/margins": 0.7199951410293579, |
| "rewards/rejected": -0.6070495843887329, |
| "step": 280 |
| }, |
| { |
| "epoch": 0.3733504988735114, |
| "grad_norm": 134.39026346308177, |
| "learning_rate": 9.068943298969072e-07, |
| "logits/chosen": -0.5337455868721008, |
| "logits/rejected": -0.548297107219696, |
| "logps/chosen": -339.92498779296875, |
| "logps/rejected": -339.0249938964844, |
| "loss": 0.7029, |
| "rewards/accuracies": 0.6000000238418579, |
| "rewards/chosen": -0.17380675673484802, |
| "rewards/margins": 0.6114867925643921, |
| "rewards/rejected": -0.7854369878768921, |
| "step": 290 |
| }, |
| { |
| "epoch": 0.3862246540070808, |
| "grad_norm": 102.9863094116885, |
| "learning_rate": 9.036726804123711e-07, |
| "logits/chosen": -0.522229015827179, |
| "logits/rejected": -0.558056652545929, |
| "logps/chosen": -399.5, |
| "logps/rejected": -381.29998779296875, |
| "loss": 0.5618, |
| "rewards/accuracies": 0.675000011920929, |
| "rewards/chosen": 0.040252685546875, |
| "rewards/margins": 0.8593994379043579, |
| "rewards/rejected": -0.818774402141571, |
| "step": 300 |
| }, |
| { |
| "epoch": 0.39909880914065016, |
| "grad_norm": 75.27946455181142, |
| "learning_rate": 9.00451030927835e-07, |
| "logits/chosen": -0.594006359577179, |
| "logits/rejected": -0.5999511480331421, |
| "logps/chosen": -377.1625061035156, |
| "logps/rejected": -372.5, |
| "loss": 0.5782, |
| "rewards/accuracies": 0.7250000238418579, |
| "rewards/chosen": 0.4391235411167145, |
| "rewards/margins": 0.9082275629043579, |
| "rewards/rejected": -0.4693969786167145, |
| "step": 310 |
| }, |
| { |
| "epoch": 0.4119729642742195, |
| "grad_norm": 109.37234543670532, |
| "learning_rate": 8.97229381443299e-07, |
| "logits/chosen": -0.559466540813446, |
| "logits/rejected": -0.5611419677734375, |
| "logps/chosen": -402.92498779296875, |
| "logps/rejected": -364.25, |
| "loss": 0.6529, |
| "rewards/accuracies": 0.574999988079071, |
| "rewards/chosen": 0.15833130478858948, |
| "rewards/margins": 0.510876476764679, |
| "rewards/rejected": -0.3524169921875, |
| "step": 320 |
| }, |
| { |
| "epoch": 0.42484711940778885, |
| "grad_norm": 100.35729116251687, |
| "learning_rate": 8.940077319587629e-07, |
| "logits/chosen": -0.49317628145217896, |
| "logits/rejected": -0.510180652141571, |
| "logps/chosen": -340.0625, |
| "logps/rejected": -362.82501220703125, |
| "loss": 0.63, |
| "rewards/accuracies": 0.625, |
| "rewards/chosen": 0.11295776069164276, |
| "rewards/margins": 0.605303943157196, |
| "rewards/rejected": -0.49190062284469604, |
| "step": 330 |
| }, |
| { |
| "epoch": 0.43772127454135823, |
| "grad_norm": 118.0041581111788, |
| "learning_rate": 8.907860824742268e-07, |
| "logits/chosen": -0.5267089605331421, |
| "logits/rejected": -0.513476550579071, |
| "logps/chosen": -382.75, |
| "logps/rejected": -366.42498779296875, |
| "loss": 0.6493, |
| "rewards/accuracies": 0.643750011920929, |
| "rewards/chosen": -0.2607360780239105, |
| "rewards/margins": 0.6520751714706421, |
| "rewards/rejected": -0.912457287311554, |
| "step": 340 |
| }, |
| { |
| "epoch": 0.4505954296749276, |
| "grad_norm": 107.54174475375848, |
| "learning_rate": 8.875644329896907e-07, |
| "logits/chosen": -0.4590210020542145, |
| "logits/rejected": -0.4604553282260895, |
| "logps/chosen": -368.58123779296875, |
| "logps/rejected": -370.70001220703125, |
| "loss": 0.6031, |
| "rewards/accuracies": 0.612500011920929, |
| "rewards/chosen": -0.18199463188648224, |
| "rewards/margins": 0.683074951171875, |
| "rewards/rejected": -0.864489734172821, |
| "step": 350 |
| }, |
| { |
| "epoch": 0.4634695848084969, |
| "grad_norm": 129.6562462083683, |
| "learning_rate": 8.843427835051546e-07, |
| "logits/chosen": -0.5956786870956421, |
| "logits/rejected": -0.5789794921875, |
| "logps/chosen": -377.17498779296875, |
| "logps/rejected": -343.67498779296875, |
| "loss": 0.6192, |
| "rewards/accuracies": 0.6000000238418579, |
| "rewards/chosen": -0.30079346895217896, |
| "rewards/margins": 0.771191418170929, |
| "rewards/rejected": -1.0708221197128296, |
| "step": 360 |
| }, |
| { |
| "epoch": 0.4763437399420663, |
| "grad_norm": 78.5251638434667, |
| "learning_rate": 8.811211340206185e-07, |
| "logits/chosen": -0.504956066608429, |
| "logits/rejected": -0.5179077386856079, |
| "logps/chosen": -404.6499938964844, |
| "logps/rejected": -383.6499938964844, |
| "loss": 0.6394, |
| "rewards/accuracies": 0.637499988079071, |
| "rewards/chosen": -0.8715881109237671, |
| "rewards/margins": 0.7264404296875, |
| "rewards/rejected": -1.597412109375, |
| "step": 370 |
| }, |
| { |
| "epoch": 0.4892178950756357, |
| "grad_norm": 115.40461395946903, |
| "learning_rate": 8.778994845360824e-07, |
| "logits/chosen": -0.4976440370082855, |
| "logits/rejected": -0.461700439453125, |
| "logps/chosen": -400.70001220703125, |
| "logps/rejected": -386.2250061035156, |
| "loss": 0.7111, |
| "rewards/accuracies": 0.606249988079071, |
| "rewards/chosen": -0.793865978717804, |
| "rewards/margins": 0.489105224609375, |
| "rewards/rejected": -1.283532738685608, |
| "step": 380 |
| }, |
| { |
| "epoch": 0.502092050209205, |
| "grad_norm": 86.37675981578106, |
| "learning_rate": 8.746778350515463e-07, |
| "logits/chosen": -0.46751707792282104, |
| "logits/rejected": -0.482351690530777, |
| "logps/chosen": -363.92498779296875, |
| "logps/rejected": -333.98748779296875, |
| "loss": 0.5009, |
| "rewards/accuracies": 0.675000011920929, |
| "rewards/chosen": 0.16302490234375, |
| "rewards/margins": 0.8822997808456421, |
| "rewards/rejected": -0.7193847894668579, |
| "step": 390 |
| }, |
| { |
| "epoch": 0.5149662053427744, |
| "grad_norm": 81.10071006473835, |
| "learning_rate": 8.714561855670102e-07, |
| "logits/chosen": -0.5183471441268921, |
| "logits/rejected": -0.504687488079071, |
| "logps/chosen": -394.875, |
| "logps/rejected": -363.125, |
| "loss": 0.5786, |
| "rewards/accuracies": 0.675000011920929, |
| "rewards/chosen": 0.15723876655101776, |
| "rewards/margins": 0.764453113079071, |
| "rewards/rejected": -0.6068481206893921, |
| "step": 400 |
| }, |
| { |
| "epoch": 0.5278403604763438, |
| "grad_norm": 116.45510610225128, |
| "learning_rate": 8.682345360824743e-07, |
| "logits/chosen": -0.47703248262405396, |
| "logits/rejected": -0.48649901151657104, |
| "logps/chosen": -344.875, |
| "logps/rejected": -347.63751220703125, |
| "loss": 0.682, |
| "rewards/accuracies": 0.5874999761581421, |
| "rewards/chosen": -0.3260864317417145, |
| "rewards/margins": 0.641326904296875, |
| "rewards/rejected": -0.967907726764679, |
| "step": 410 |
| }, |
| { |
| "epoch": 0.5407145156099131, |
| "grad_norm": 112.50781335140714, |
| "learning_rate": 8.650128865979382e-07, |
| "logits/chosen": -0.43253785371780396, |
| "logits/rejected": -0.44200438261032104, |
| "logps/chosen": -358.8125, |
| "logps/rejected": -372.32501220703125, |
| "loss": 0.6378, |
| "rewards/accuracies": 0.625, |
| "rewards/chosen": -0.4115051329135895, |
| "rewards/margins": 0.6463562250137329, |
| "rewards/rejected": -1.0579345226287842, |
| "step": 420 |
| }, |
| { |
| "epoch": 0.5535886707434825, |
| "grad_norm": 120.49044437617238, |
| "learning_rate": 8.617912371134021e-07, |
| "logits/chosen": -0.4475952088832855, |
| "logits/rejected": -0.41944581270217896, |
| "logps/chosen": -357.125, |
| "logps/rejected": -344.79998779296875, |
| "loss": 0.6222, |
| "rewards/accuracies": 0.606249988079071, |
| "rewards/chosen": -0.23563233017921448, |
| "rewards/margins": 0.6660400629043579, |
| "rewards/rejected": -0.9013671875, |
| "step": 430 |
| }, |
| { |
| "epoch": 0.5664628258770518, |
| "grad_norm": 68.29113701212785, |
| "learning_rate": 8.585695876288659e-07, |
| "logits/chosen": -0.4893554747104645, |
| "logits/rejected": -0.5026305913925171, |
| "logps/chosen": -372.04998779296875, |
| "logps/rejected": -373.0, |
| "loss": 0.5066, |
| "rewards/accuracies": 0.71875, |
| "rewards/chosen": 0.12487487494945526, |
| "rewards/margins": 0.988940417766571, |
| "rewards/rejected": -0.863964855670929, |
| "step": 440 |
| }, |
| { |
| "epoch": 0.5793369810106211, |
| "grad_norm": 89.51903868981923, |
| "learning_rate": 8.553479381443298e-07, |
| "logits/chosen": -0.4515014588832855, |
| "logits/rejected": -0.47959595918655396, |
| "logps/chosen": -356.125, |
| "logps/rejected": -362.67498779296875, |
| "loss": 0.6334, |
| "rewards/accuracies": 0.6499999761581421, |
| "rewards/chosen": 0.608264148235321, |
| "rewards/margins": 0.724169909954071, |
| "rewards/rejected": -0.11616210639476776, |
| "step": 450 |
| }, |
| { |
| "epoch": 0.5922111361441905, |
| "grad_norm": 148.43372901427, |
| "learning_rate": 8.521262886597937e-07, |
| "logits/chosen": -0.529223620891571, |
| "logits/rejected": -0.54193115234375, |
| "logps/chosen": -421.82501220703125, |
| "logps/rejected": -414.875, |
| "loss": 0.5713, |
| "rewards/accuracies": 0.6812499761581421, |
| "rewards/chosen": 0.897900402545929, |
| "rewards/margins": 0.8597167730331421, |
| "rewards/rejected": 0.03883666917681694, |
| "step": 460 |
| }, |
| { |
| "epoch": 0.6050852912777599, |
| "grad_norm": 118.08132341606118, |
| "learning_rate": 8.489046391752577e-07, |
| "logits/chosen": -0.569384753704071, |
| "logits/rejected": -0.568408191204071, |
| "logps/chosen": -380.57501220703125, |
| "logps/rejected": -394.17498779296875, |
| "loss": 0.7024, |
| "rewards/accuracies": 0.5874999761581421, |
| "rewards/chosen": 0.3481384217739105, |
| "rewards/margins": 0.4853881895542145, |
| "rewards/rejected": -0.13858643174171448, |
| "step": 470 |
| }, |
| { |
| "epoch": 0.6179594464113293, |
| "grad_norm": 107.65250644411428, |
| "learning_rate": 8.456829896907216e-07, |
| "logits/chosen": -0.539965808391571, |
| "logits/rejected": -0.523364245891571, |
| "logps/chosen": -373.3500061035156, |
| "logps/rejected": -360.3999938964844, |
| "loss": 0.5671, |
| "rewards/accuracies": 0.65625, |
| "rewards/chosen": 0.508160412311554, |
| "rewards/margins": 0.902575671672821, |
| "rewards/rejected": -0.3946289122104645, |
| "step": 480 |
| }, |
| { |
| "epoch": 0.6308336015448986, |
| "grad_norm": 230.1997356494601, |
| "learning_rate": 8.424613402061855e-07, |
| "logits/chosen": -0.504443347454071, |
| "logits/rejected": -0.5317474603652954, |
| "logps/chosen": -366.2875061035156, |
| "logps/rejected": -358.7749938964844, |
| "loss": 0.5907, |
| "rewards/accuracies": 0.612500011920929, |
| "rewards/chosen": 0.33641356229782104, |
| "rewards/margins": 0.6843627691268921, |
| "rewards/rejected": -0.347940057516098, |
| "step": 490 |
| }, |
| { |
| "epoch": 0.643707756678468, |
| "grad_norm": 124.54339481905843, |
| "learning_rate": 8.392396907216494e-07, |
| "logits/chosen": -0.5900634527206421, |
| "logits/rejected": -0.597729504108429, |
| "logps/chosen": -365.23748779296875, |
| "logps/rejected": -349.32501220703125, |
| "loss": 0.7723, |
| "rewards/accuracies": 0.5625, |
| "rewards/chosen": -0.802197277545929, |
| "rewards/margins": 0.4662719666957855, |
| "rewards/rejected": -1.267785668373108, |
| "step": 500 |
| }, |
| { |
| "epoch": 0.6565819118120373, |
| "grad_norm": 132.57625659875265, |
| "learning_rate": 8.360180412371134e-07, |
| "logits/chosen": -0.5092223882675171, |
| "logits/rejected": -0.5284179449081421, |
| "logps/chosen": -409.3999938964844, |
| "logps/rejected": -370.70001220703125, |
| "loss": 0.6064, |
| "rewards/accuracies": 0.65625, |
| "rewards/chosen": -0.33637696504592896, |
| "rewards/margins": 0.8779052495956421, |
| "rewards/rejected": -1.2140624523162842, |
| "step": 510 |
| }, |
| { |
| "epoch": 0.6694560669456067, |
| "grad_norm": 109.23063994206017, |
| "learning_rate": 8.327963917525774e-07, |
| "logits/chosen": -0.4924560487270355, |
| "logits/rejected": -0.4965576231479645, |
| "logps/chosen": -377.0, |
| "logps/rejected": -366.48748779296875, |
| "loss": 0.5947, |
| "rewards/accuracies": 0.6812499761581421, |
| "rewards/chosen": 0.23482970893383026, |
| "rewards/margins": 0.7789062261581421, |
| "rewards/rejected": -0.543811023235321, |
| "step": 520 |
| }, |
| { |
| "epoch": 0.682330222079176, |
| "grad_norm": 79.28047130384992, |
| "learning_rate": 8.295747422680413e-07, |
| "logits/chosen": -0.4707702696323395, |
| "logits/rejected": -0.494821161031723, |
| "logps/chosen": -339.92498779296875, |
| "logps/rejected": -329.95001220703125, |
| "loss": 0.5824, |
| "rewards/accuracies": 0.6812499761581421, |
| "rewards/chosen": 0.066864013671875, |
| "rewards/margins": 0.8806396722793579, |
| "rewards/rejected": -0.813763439655304, |
| "step": 530 |
| }, |
| { |
| "epoch": 0.6952043772127454, |
| "grad_norm": 80.78969993556497, |
| "learning_rate": 8.263530927835051e-07, |
| "logits/chosen": -0.565600574016571, |
| "logits/rejected": -0.5763915777206421, |
| "logps/chosen": -416.6000061035156, |
| "logps/rejected": -380.0, |
| "loss": 0.6185, |
| "rewards/accuracies": 0.675000011920929, |
| "rewards/chosen": -0.05002441257238388, |
| "rewards/margins": 0.8234497308731079, |
| "rewards/rejected": -0.872814953327179, |
| "step": 540 |
| }, |
| { |
| "epoch": 0.7080785323463148, |
| "grad_norm": 103.61303464709924, |
| "learning_rate": 8.23131443298969e-07, |
| "logits/chosen": -0.4691925048828125, |
| "logits/rejected": -0.47784423828125, |
| "logps/chosen": -354.1000061035156, |
| "logps/rejected": -342.17498779296875, |
| "loss": 0.5404, |
| "rewards/accuracies": 0.65625, |
| "rewards/chosen": 0.3255248963832855, |
| "rewards/margins": 0.8783203363418579, |
| "rewards/rejected": -0.5528930425643921, |
| "step": 550 |
| }, |
| { |
| "epoch": 0.7209526874798842, |
| "grad_norm": 121.89695070499967, |
| "learning_rate": 8.199097938144329e-07, |
| "logits/chosen": -0.4993652403354645, |
| "logits/rejected": -0.5168091058731079, |
| "logps/chosen": -370.8500061035156, |
| "logps/rejected": -361.6000061035156, |
| "loss": 0.604, |
| "rewards/accuracies": 0.668749988079071, |
| "rewards/chosen": 0.34550780057907104, |
| "rewards/margins": 0.926831066608429, |
| "rewards/rejected": -0.5806182622909546, |
| "step": 560 |
| }, |
| { |
| "epoch": 0.7338268426134535, |
| "grad_norm": 110.86609527894251, |
| "learning_rate": 8.166881443298968e-07, |
| "logits/chosen": -0.5541015863418579, |
| "logits/rejected": -0.5427612066268921, |
| "logps/chosen": -364.0249938964844, |
| "logps/rejected": -360.2250061035156, |
| "loss": 0.6308, |
| "rewards/accuracies": 0.6499999761581421, |
| "rewards/chosen": 0.39935302734375, |
| "rewards/margins": 0.878186047077179, |
| "rewards/rejected": -0.47841185331344604, |
| "step": 570 |
| }, |
| { |
| "epoch": 0.7467009977470228, |
| "grad_norm": 110.69885173772958, |
| "learning_rate": 8.134664948453608e-07, |
| "logits/chosen": -0.50360107421875, |
| "logits/rejected": -0.50262451171875, |
| "logps/chosen": -397.36248779296875, |
| "logps/rejected": -381.8999938964844, |
| "loss": 0.6482, |
| "rewards/accuracies": 0.6312500238418579, |
| "rewards/chosen": 0.6587890386581421, |
| "rewards/margins": 0.916259765625, |
| "rewards/rejected": -0.2574401795864105, |
| "step": 580 |
| }, |
| { |
| "epoch": 0.7595751528805922, |
| "grad_norm": 74.35379476153639, |
| "learning_rate": 8.102448453608247e-07, |
| "logits/chosen": -0.4267639219760895, |
| "logits/rejected": -0.45759278535842896, |
| "logps/chosen": -381.20001220703125, |
| "logps/rejected": -344.88751220703125, |
| "loss": 0.5601, |
| "rewards/accuracies": 0.6499999761581421, |
| "rewards/chosen": 1.430151343345642, |
| "rewards/margins": 0.877062976360321, |
| "rewards/rejected": 0.552661120891571, |
| "step": 590 |
| }, |
| { |
| "epoch": 0.7724493080141616, |
| "grad_norm": 111.98976593700215, |
| "learning_rate": 8.070231958762887e-07, |
| "logits/chosen": -0.44500732421875, |
| "logits/rejected": -0.42840576171875, |
| "logps/chosen": -367.3999938964844, |
| "logps/rejected": -347.79998779296875, |
| "loss": 0.652, |
| "rewards/accuracies": 0.6312500238418579, |
| "rewards/chosen": 0.9703918695449829, |
| "rewards/margins": 0.7120361328125, |
| "rewards/rejected": 0.257925420999527, |
| "step": 600 |
| }, |
| { |
| "epoch": 0.7853234631477309, |
| "grad_norm": 103.96984933499336, |
| "learning_rate": 8.038015463917526e-07, |
| "logits/chosen": -0.4539550840854645, |
| "logits/rejected": -0.4577895998954773, |
| "logps/chosen": -348.79998779296875, |
| "logps/rejected": -364.6499938964844, |
| "loss": 0.607, |
| "rewards/accuracies": 0.6312500238418579, |
| "rewards/chosen": 0.05695800855755806, |
| "rewards/margins": 0.9610961675643921, |
| "rewards/rejected": -0.9044739007949829, |
| "step": 610 |
| }, |
| { |
| "epoch": 0.7981976182813003, |
| "grad_norm": 83.82621463499343, |
| "learning_rate": 8.005798969072165e-07, |
| "logits/chosen": -0.44810181856155396, |
| "logits/rejected": -0.44012451171875, |
| "logps/chosen": -387.7250061035156, |
| "logps/rejected": -382.75, |
| "loss": 0.7625, |
| "rewards/accuracies": 0.5687500238418579, |
| "rewards/chosen": -0.9187866449356079, |
| "rewards/margins": 0.5645202398300171, |
| "rewards/rejected": -1.4832274913787842, |
| "step": 620 |
| }, |
| { |
| "epoch": 0.8110717734148697, |
| "grad_norm": 136.60454494529628, |
| "learning_rate": 7.973582474226804e-07, |
| "logits/chosen": -0.53546142578125, |
| "logits/rejected": -0.5382217168807983, |
| "logps/chosen": -415.1000061035156, |
| "logps/rejected": -393.125, |
| "loss": 0.6367, |
| "rewards/accuracies": 0.6187499761581421, |
| "rewards/chosen": -1.3832824230194092, |
| "rewards/margins": 0.589917004108429, |
| "rewards/rejected": -1.9736328125, |
| "step": 630 |
| }, |
| { |
| "epoch": 0.823945928548439, |
| "grad_norm": 175.185405939881, |
| "learning_rate": 7.941365979381443e-07, |
| "logits/chosen": -0.522747814655304, |
| "logits/rejected": -0.5485290288925171, |
| "logps/chosen": -378.4750061035156, |
| "logps/rejected": -362.3999938964844, |
| "loss": 0.5862, |
| "rewards/accuracies": 0.625, |
| "rewards/chosen": -0.9291747808456421, |
| "rewards/margins": 0.790759265422821, |
| "rewards/rejected": -1.7187988758087158, |
| "step": 640 |
| }, |
| { |
| "epoch": 0.8368200836820083, |
| "grad_norm": 93.82500243107654, |
| "learning_rate": 7.909149484536082e-07, |
| "logits/chosen": -0.540942370891571, |
| "logits/rejected": -0.540112316608429, |
| "logps/chosen": -406.8999938964844, |
| "logps/rejected": -394.3999938964844, |
| "loss": 0.6338, |
| "rewards/accuracies": 0.612500011920929, |
| "rewards/chosen": -0.9627624750137329, |
| "rewards/margins": 0.6733764410018921, |
| "rewards/rejected": -1.6355712413787842, |
| "step": 650 |
| }, |
| { |
| "epoch": 0.8496942388155777, |
| "grad_norm": 98.8982278348208, |
| "learning_rate": 7.876932989690721e-07, |
| "logits/chosen": -0.467041015625, |
| "logits/rejected": -0.4739440977573395, |
| "logps/chosen": -358.5, |
| "logps/rejected": -358.92498779296875, |
| "loss": 0.6224, |
| "rewards/accuracies": 0.643750011920929, |
| "rewards/chosen": -0.10512085258960724, |
| "rewards/margins": 0.875134289264679, |
| "rewards/rejected": -0.979418933391571, |
| "step": 660 |
| }, |
| { |
| "epoch": 0.8625683939491471, |
| "grad_norm": 81.26508421612064, |
| "learning_rate": 7.84471649484536e-07, |
| "logits/chosen": -0.4732421934604645, |
| "logits/rejected": -0.46550291776657104, |
| "logps/chosen": -358.42498779296875, |
| "logps/rejected": -339.2749938964844, |
| "loss": 0.669, |
| "rewards/accuracies": 0.6187499761581421, |
| "rewards/chosen": 0.03513794019818306, |
| "rewards/margins": 0.5534912347793579, |
| "rewards/rejected": -0.5187774896621704, |
| "step": 670 |
| }, |
| { |
| "epoch": 0.8754425490827165, |
| "grad_norm": 96.38179368759864, |
| "learning_rate": 7.812499999999999e-07, |
| "logits/chosen": -0.502551257610321, |
| "logits/rejected": -0.5140380859375, |
| "logps/chosen": -387.92498779296875, |
| "logps/rejected": -371.48748779296875, |
| "loss": 0.5181, |
| "rewards/accuracies": 0.706250011920929, |
| "rewards/chosen": 0.40931397676467896, |
| "rewards/margins": 1.079833984375, |
| "rewards/rejected": -0.670947253704071, |
| "step": 680 |
| }, |
| { |
| "epoch": 0.8883167042162858, |
| "grad_norm": 101.76727637730922, |
| "learning_rate": 7.780283505154638e-07, |
| "logits/chosen": -0.40287476778030396, |
| "logits/rejected": -0.38833314180374146, |
| "logps/chosen": -343.7875061035156, |
| "logps/rejected": -352.8125, |
| "loss": 0.6171, |
| "rewards/accuracies": 0.65625, |
| "rewards/chosen": -0.24252013862133026, |
| "rewards/margins": 0.8604491949081421, |
| "rewards/rejected": -1.102075219154358, |
| "step": 690 |
| }, |
| { |
| "epoch": 0.9011908593498552, |
| "grad_norm": 122.00950637295834, |
| "learning_rate": 7.748067010309279e-07, |
| "logits/chosen": -0.42670899629592896, |
| "logits/rejected": -0.44072264432907104, |
| "logps/chosen": -361.5, |
| "logps/rejected": -335.3500061035156, |
| "loss": 0.7121, |
| "rewards/accuracies": 0.612500011920929, |
| "rewards/chosen": 0.06911621242761612, |
| "rewards/margins": 0.7628173828125, |
| "rewards/rejected": -0.694226086139679, |
| "step": 700 |
| }, |
| { |
| "epoch": 0.9140650144834245, |
| "grad_norm": 127.45981012366637, |
| "learning_rate": 7.715850515463918e-07, |
| "logits/chosen": -0.4952026307582855, |
| "logits/rejected": -0.501721203327179, |
| "logps/chosen": -383.5874938964844, |
| "logps/rejected": -377.6499938964844, |
| "loss": 0.6536, |
| "rewards/accuracies": 0.6187499761581421, |
| "rewards/chosen": 0.18529053032398224, |
| "rewards/margins": 0.780322253704071, |
| "rewards/rejected": -0.596270740032196, |
| "step": 710 |
| }, |
| { |
| "epoch": 0.9269391696169939, |
| "grad_norm": 137.90708858880924, |
| "learning_rate": 7.683634020618557e-07, |
| "logits/chosen": -0.484619140625, |
| "logits/rejected": -0.5116943120956421, |
| "logps/chosen": -368.2749938964844, |
| "logps/rejected": -354.4750061035156, |
| "loss": 0.5936, |
| "rewards/accuracies": 0.637499988079071, |
| "rewards/chosen": 0.6229461431503296, |
| "rewards/margins": 0.652905285358429, |
| "rewards/rejected": -0.03063049353659153, |
| "step": 720 |
| }, |
| { |
| "epoch": 0.9398133247505632, |
| "grad_norm": 109.52357137017624, |
| "learning_rate": 7.651417525773195e-07, |
| "logits/chosen": -0.572705090045929, |
| "logits/rejected": -0.5957580804824829, |
| "logps/chosen": -402.57501220703125, |
| "logps/rejected": -377.2250061035156, |
| "loss": 0.6558, |
| "rewards/accuracies": 0.612500011920929, |
| "rewards/chosen": 0.385833740234375, |
| "rewards/margins": 0.52545166015625, |
| "rewards/rejected": -0.13958129286766052, |
| "step": 730 |
| }, |
| { |
| "epoch": 0.9526874798841326, |
| "grad_norm": 133.00416812765525, |
| "learning_rate": 7.619201030927834e-07, |
| "logits/chosen": -0.4806762635707855, |
| "logits/rejected": -0.48743897676467896, |
| "logps/chosen": -347.0, |
| "logps/rejected": -332.6499938964844, |
| "loss": 0.6252, |
| "rewards/accuracies": 0.6187499761581421, |
| "rewards/chosen": 0.3818115293979645, |
| "rewards/margins": 0.603198230266571, |
| "rewards/rejected": -0.22109374403953552, |
| "step": 740 |
| }, |
| { |
| "epoch": 0.965561635017702, |
| "grad_norm": 140.99897302512156, |
| "learning_rate": 7.586984536082474e-07, |
| "logits/chosen": -0.45958250761032104, |
| "logits/rejected": -0.4542907774448395, |
| "logps/chosen": -391.1875, |
| "logps/rejected": -387.6000061035156, |
| "loss": 0.7327, |
| "rewards/accuracies": 0.581250011920929, |
| "rewards/chosen": -0.12029419094324112, |
| "rewards/margins": 0.48200684785842896, |
| "rewards/rejected": -0.602307140827179, |
| "step": 750 |
| }, |
| { |
| "epoch": 0.9784357901512714, |
| "grad_norm": 131.274271707173, |
| "learning_rate": 7.554768041237113e-07, |
| "logits/chosen": -0.39448851346969604, |
| "logits/rejected": -0.3970092833042145, |
| "logps/chosen": -376.48748779296875, |
| "logps/rejected": -361.95001220703125, |
| "loss": 0.6513, |
| "rewards/accuracies": 0.6312500238418579, |
| "rewards/chosen": 0.16361084580421448, |
| "rewards/margins": 0.576507568359375, |
| "rewards/rejected": -0.41328126192092896, |
| "step": 760 |
| }, |
| { |
| "epoch": 0.9913099452848407, |
| "grad_norm": 73.69391656660149, |
| "learning_rate": 7.522551546391752e-07, |
| "logits/chosen": -0.412353515625, |
| "logits/rejected": -0.41024476289749146, |
| "logps/chosen": -370.9750061035156, |
| "logps/rejected": -381.3500061035156, |
| "loss": 0.6321, |
| "rewards/accuracies": 0.6499999761581421, |
| "rewards/chosen": 0.28642576932907104, |
| "rewards/margins": 0.6205993890762329, |
| "rewards/rejected": -0.33460694551467896, |
| "step": 770 |
| }, |
| { |
| "epoch": 1.0038622465400708, |
| "grad_norm": 7.6687016456195645, |
| "learning_rate": 7.490335051546391e-07, |
| "logits/chosen": -0.4482797384262085, |
| "logits/rejected": -0.45645689964294434, |
| "logps/chosen": -348.0769348144531, |
| "logps/rejected": -349.5641174316406, |
| "loss": 0.4166, |
| "rewards/accuracies": 0.7371794581413269, |
| "rewards/chosen": 1.3597506284713745, |
| "rewards/margins": 1.9606996774673462, |
| "rewards/rejected": -0.5973307490348816, |
| "step": 780 |
| }, |
| { |
| "epoch": 1.0167364016736402, |
| "grad_norm": 9.230599114177751, |
| "learning_rate": 7.45811855670103e-07, |
| "logits/chosen": -0.44798582792282104, |
| "logits/rejected": -0.4685913026332855, |
| "logps/chosen": -367.0249938964844, |
| "logps/rejected": -393.32501220703125, |
| "loss": 0.067, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 2.8109374046325684, |
| "rewards/margins": 6.264452934265137, |
| "rewards/rejected": -3.4593749046325684, |
| "step": 790 |
| }, |
| { |
| "epoch": 1.0296105568072096, |
| "grad_norm": 16.61098369972291, |
| "learning_rate": 7.42590206185567e-07, |
| "logits/chosen": -0.5721801519393921, |
| "logits/rejected": -0.559155285358429, |
| "logps/chosen": -319.0249938964844, |
| "logps/rejected": -415.9750061035156, |
| "loss": 0.0528, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": 2.3069825172424316, |
| "rewards/margins": 7.700585842132568, |
| "rewards/rejected": -5.3922119140625, |
| "step": 800 |
| }, |
| { |
| "epoch": 1.042484711940779, |
| "grad_norm": 22.829431572502866, |
| "learning_rate": 7.39368556701031e-07, |
| "logits/chosen": -0.627636730670929, |
| "logits/rejected": -0.6467529535293579, |
| "logps/chosen": -364.125, |
| "logps/rejected": -428.7250061035156, |
| "loss": 0.0828, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": 1.1227538585662842, |
| "rewards/margins": 7.400000095367432, |
| "rewards/rejected": -6.280077934265137, |
| "step": 810 |
| }, |
| { |
| "epoch": 1.0553588670743483, |
| "grad_norm": 4.36402073822104, |
| "learning_rate": 7.361469072164949e-07, |
| "logits/chosen": -0.7441650629043579, |
| "logits/rejected": -0.72998046875, |
| "logps/chosen": -376.57501220703125, |
| "logps/rejected": -452.7250061035156, |
| "loss": 0.0561, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": 0.10989990085363388, |
| "rewards/margins": 8.282812118530273, |
| "rewards/rejected": -8.182031631469727, |
| "step": 820 |
| }, |
| { |
| "epoch": 1.0682330222079175, |
| "grad_norm": 9.44908390049402, |
| "learning_rate": 7.329252577319587e-07, |
| "logits/chosen": -0.865307629108429, |
| "logits/rejected": -0.8622802495956421, |
| "logps/chosen": -369.57501220703125, |
| "logps/rejected": -472.8500061035156, |
| "loss": 0.0743, |
| "rewards/accuracies": 0.9312499761581421, |
| "rewards/chosen": -0.3714187741279602, |
| "rewards/margins": 9.832812309265137, |
| "rewards/rejected": -10.209375381469727, |
| "step": 830 |
| }, |
| { |
| "epoch": 1.0811071773414869, |
| "grad_norm": 2.5940939792945583, |
| "learning_rate": 7.297036082474226e-07, |
| "logits/chosen": -0.903515636920929, |
| "logits/rejected": -0.876757800579071, |
| "logps/chosen": -378.2749938964844, |
| "logps/rejected": -444.3999938964844, |
| "loss": 0.0859, |
| "rewards/accuracies": 0.9312499761581421, |
| "rewards/chosen": -1.336401343345642, |
| "rewards/margins": 9.068163871765137, |
| "rewards/rejected": -10.41015625, |
| "step": 840 |
| }, |
| { |
| "epoch": 1.0939813324750562, |
| "grad_norm": 4.93864942706677, |
| "learning_rate": 7.264819587628865e-07, |
| "logits/chosen": -0.8818359375, |
| "logits/rejected": -0.883740246295929, |
| "logps/chosen": -376.1499938964844, |
| "logps/rejected": -451.67498779296875, |
| "loss": 0.1076, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": -1.2569091320037842, |
| "rewards/margins": 8.835156440734863, |
| "rewards/rejected": -10.096484184265137, |
| "step": 850 |
| }, |
| { |
| "epoch": 1.1068554876086256, |
| "grad_norm": 1.1622572843562746, |
| "learning_rate": 7.232603092783504e-07, |
| "logits/chosen": -1.0085937976837158, |
| "logits/rejected": -0.991894543170929, |
| "logps/chosen": -382.2250061035156, |
| "logps/rejected": -464.25, |
| "loss": 0.0571, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -1.985925316810608, |
| "rewards/margins": 9.11328125, |
| "rewards/rejected": -11.096875190734863, |
| "step": 860 |
| }, |
| { |
| "epoch": 1.119729642742195, |
| "grad_norm": 13.207891586411098, |
| "learning_rate": 7.200386597938144e-07, |
| "logits/chosen": -0.960888683795929, |
| "logits/rejected": -0.9482421875, |
| "logps/chosen": -401.70001220703125, |
| "logps/rejected": -449.07501220703125, |
| "loss": 0.083, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -1.0959961414337158, |
| "rewards/margins": 9.560937881469727, |
| "rewards/rejected": -10.660937309265137, |
| "step": 870 |
| }, |
| { |
| "epoch": 1.1326037978757644, |
| "grad_norm": 26.57295769679731, |
| "learning_rate": 7.168170103092783e-07, |
| "logits/chosen": -0.85919189453125, |
| "logits/rejected": -0.8564453125, |
| "logps/chosen": -392.76251220703125, |
| "logps/rejected": -458.79998779296875, |
| "loss": 0.069, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -0.5528320074081421, |
| "rewards/margins": 9.028905868530273, |
| "rewards/rejected": -9.583593368530273, |
| "step": 880 |
| }, |
| { |
| "epoch": 1.1454779530093337, |
| "grad_norm": 5.906974482408663, |
| "learning_rate": 7.135953608247423e-07, |
| "logits/chosen": -0.826647937297821, |
| "logits/rejected": -0.8048461675643921, |
| "logps/chosen": -388.04998779296875, |
| "logps/rejected": -440.17498779296875, |
| "loss": 0.065, |
| "rewards/accuracies": 0.9312499761581421, |
| "rewards/chosen": -0.10826416313648224, |
| "rewards/margins": 8.711718559265137, |
| "rewards/rejected": -8.819726943969727, |
| "step": 890 |
| }, |
| { |
| "epoch": 1.1583521081429031, |
| "grad_norm": 24.927906669623493, |
| "learning_rate": 7.103737113402062e-07, |
| "logits/chosen": -0.8695923089981079, |
| "logits/rejected": -0.856518566608429, |
| "logps/chosen": -403.0874938964844, |
| "logps/rejected": -490.04998779296875, |
| "loss": 0.0603, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -0.8900390863418579, |
| "rewards/margins": 10.569531440734863, |
| "rewards/rejected": -11.462499618530273, |
| "step": 900 |
| }, |
| { |
| "epoch": 1.1712262632764725, |
| "grad_norm": 4.506062930490332, |
| "learning_rate": 7.071520618556701e-07, |
| "logits/chosen": -0.9596923589706421, |
| "logits/rejected": -0.9530273675918579, |
| "logps/chosen": -391.23748779296875, |
| "logps/rejected": -456.5249938964844, |
| "loss": 0.078, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": -2.318310499191284, |
| "rewards/margins": 9.502344131469727, |
| "rewards/rejected": -11.823827743530273, |
| "step": 910 |
| }, |
| { |
| "epoch": 1.1841004184100419, |
| "grad_norm": 22.19667225060241, |
| "learning_rate": 7.039304123711341e-07, |
| "logits/chosen": -0.8713623285293579, |
| "logits/rejected": -0.860217273235321, |
| "logps/chosen": -418.04998779296875, |
| "logps/rejected": -501.32501220703125, |
| "loss": 0.0381, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -1.0840332508087158, |
| "rewards/margins": 10.9453125, |
| "rewards/rejected": -12.029687881469727, |
| "step": 920 |
| }, |
| { |
| "epoch": 1.1969745735436113, |
| "grad_norm": 3.0097726188152327, |
| "learning_rate": 7.007087628865979e-07, |
| "logits/chosen": -0.8865722417831421, |
| "logits/rejected": -0.886523425579071, |
| "logps/chosen": -364.0687561035156, |
| "logps/rejected": -432.4375, |
| "loss": 0.0699, |
| "rewards/accuracies": 0.9312499761581421, |
| "rewards/chosen": -1.1237884759902954, |
| "rewards/margins": 9.000781059265137, |
| "rewards/rejected": -10.123632431030273, |
| "step": 930 |
| }, |
| { |
| "epoch": 1.2098487286771806, |
| "grad_norm": 122.1654239956535, |
| "learning_rate": 6.974871134020618e-07, |
| "logits/chosen": -0.9017089605331421, |
| "logits/rejected": -0.8979247808456421, |
| "logps/chosen": -372.01251220703125, |
| "logps/rejected": -438.875, |
| "loss": 0.0782, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -0.824169933795929, |
| "rewards/margins": 10.292187690734863, |
| "rewards/rejected": -11.118749618530273, |
| "step": 940 |
| }, |
| { |
| "epoch": 1.22272288381075, |
| "grad_norm": 0.49727599372317804, |
| "learning_rate": 6.942654639175257e-07, |
| "logits/chosen": -0.8161590695381165, |
| "logits/rejected": -0.813720703125, |
| "logps/chosen": -371.48126220703125, |
| "logps/rejected": -474.71875, |
| "loss": 0.0646, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.615966796875, |
| "rewards/margins": 10.194140434265137, |
| "rewards/rejected": -10.8134765625, |
| "step": 950 |
| }, |
| { |
| "epoch": 1.2355970389443192, |
| "grad_norm": 15.351228026815852, |
| "learning_rate": 6.910438144329896e-07, |
| "logits/chosen": -0.916015625, |
| "logits/rejected": -0.920654296875, |
| "logps/chosen": -386.25, |
| "logps/rejected": -462.25, |
| "loss": 0.0591, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -1.6080443859100342, |
| "rewards/margins": 10.084375381469727, |
| "rewards/rejected": -11.686718940734863, |
| "step": 960 |
| }, |
| { |
| "epoch": 1.2484711940778888, |
| "grad_norm": 73.16765440550557, |
| "learning_rate": 6.878221649484535e-07, |
| "logits/chosen": -0.9052734375, |
| "logits/rejected": -0.92333984375, |
| "logps/chosen": -403.8687438964844, |
| "logps/rejected": -481.8999938964844, |
| "loss": 0.1255, |
| "rewards/accuracies": 0.90625, |
| "rewards/chosen": -3.6683106422424316, |
| "rewards/margins": 9.447851181030273, |
| "rewards/rejected": -13.109375, |
| "step": 970 |
| }, |
| { |
| "epoch": 1.261345349211458, |
| "grad_norm": 1.552151037291679, |
| "learning_rate": 6.846005154639176e-07, |
| "logits/chosen": -0.928955078125, |
| "logits/rejected": -0.9185546636581421, |
| "logps/chosen": -415.54998779296875, |
| "logps/rejected": -502.17498779296875, |
| "loss": 0.0278, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -4.029528617858887, |
| "rewards/margins": 11.318750381469727, |
| "rewards/rejected": -15.346875190734863, |
| "step": 980 |
| }, |
| { |
| "epoch": 1.2742195043450273, |
| "grad_norm": 155.41470997153405, |
| "learning_rate": 6.813788659793815e-07, |
| "logits/chosen": -0.9188476800918579, |
| "logits/rejected": -0.9298309087753296, |
| "logps/chosen": -495.7250061035156, |
| "logps/rejected": -532.6500244140625, |
| "loss": 0.0846, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -3.2472167015075684, |
| "rewards/margins": 10.753125190734863, |
| "rewards/rejected": -14.003125190734863, |
| "step": 990 |
| }, |
| { |
| "epoch": 1.2870936594785967, |
| "grad_norm": 5.116040863492018, |
| "learning_rate": 6.781572164948454e-07, |
| "logits/chosen": -0.8142944574356079, |
| "logits/rejected": -0.812969982624054, |
| "logps/chosen": -384.875, |
| "logps/rejected": -453.6499938964844, |
| "loss": 0.0471, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -2.236328125, |
| "rewards/margins": 10.518750190734863, |
| "rewards/rejected": -12.74609375, |
| "step": 1000 |
| }, |
| { |
| "epoch": 1.299967814612166, |
| "grad_norm": 203.17549940484994, |
| "learning_rate": 6.749355670103093e-07, |
| "logits/chosen": -0.8174377679824829, |
| "logits/rejected": -0.8206436038017273, |
| "logps/chosen": -357.5, |
| "logps/rejected": -430.3500061035156, |
| "loss": 0.0952, |
| "rewards/accuracies": 0.9312499761581421, |
| "rewards/chosen": -1.4882080554962158, |
| "rewards/margins": 9.531641006469727, |
| "rewards/rejected": -11.017187118530273, |
| "step": 1010 |
| }, |
| { |
| "epoch": 1.3128419697457354, |
| "grad_norm": 3.2265856708286482, |
| "learning_rate": 6.717139175257731e-07, |
| "logits/chosen": -0.848876953125, |
| "logits/rejected": -0.840283215045929, |
| "logps/chosen": -351.4375, |
| "logps/rejected": -410.82501220703125, |
| "loss": 0.045, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -0.6333678960800171, |
| "rewards/margins": 9.462499618530273, |
| "rewards/rejected": -10.099609375, |
| "step": 1020 |
| }, |
| { |
| "epoch": 1.3257161248793048, |
| "grad_norm": 2.48479318064024, |
| "learning_rate": 6.68492268041237e-07, |
| "logits/chosen": -0.9139159917831421, |
| "logits/rejected": -0.905322253704071, |
| "logps/chosen": -373.0249938964844, |
| "logps/rejected": -462.32501220703125, |
| "loss": 0.0602, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 0.2511962950229645, |
| "rewards/margins": 10.485937118530273, |
| "rewards/rejected": -10.232812881469727, |
| "step": 1030 |
| }, |
| { |
| "epoch": 1.3385902800128742, |
| "grad_norm": 11.744877027581841, |
| "learning_rate": 6.65270618556701e-07, |
| "logits/chosen": -0.8927978277206421, |
| "logits/rejected": -0.901123046875, |
| "logps/chosen": -406.7749938964844, |
| "logps/rejected": -482.57501220703125, |
| "loss": 0.048, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": 0.47990721464157104, |
| "rewards/margins": 10.22265625, |
| "rewards/rejected": -9.742968559265137, |
| "step": 1040 |
| }, |
| { |
| "epoch": 1.3514644351464435, |
| "grad_norm": 22.161078888328298, |
| "learning_rate": 6.620489690721649e-07, |
| "logits/chosen": -0.8483642339706421, |
| "logits/rejected": -0.837329089641571, |
| "logps/chosen": -371.6875, |
| "logps/rejected": -451.1499938964844, |
| "loss": 0.0586, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": 0.47760313749313354, |
| "rewards/margins": 9.533594131469727, |
| "rewards/rejected": -9.053906440734863, |
| "step": 1050 |
| }, |
| { |
| "epoch": 1.364338590280013, |
| "grad_norm": 49.17969665142029, |
| "learning_rate": 6.588273195876288e-07, |
| "logits/chosen": -0.885449230670929, |
| "logits/rejected": -0.8734496831893921, |
| "logps/chosen": -358.5249938964844, |
| "logps/rejected": -416.32501220703125, |
| "loss": 0.0744, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 0.206329345703125, |
| "rewards/margins": 9.153905868530273, |
| "rewards/rejected": -8.94921875, |
| "step": 1060 |
| }, |
| { |
| "epoch": 1.3772127454135823, |
| "grad_norm": 1.3865788823811889, |
| "learning_rate": 6.556056701030927e-07, |
| "logits/chosen": -0.9468749761581421, |
| "logits/rejected": -0.9432128667831421, |
| "logps/chosen": -324.04998779296875, |
| "logps/rejected": -409.1000061035156, |
| "loss": 0.0565, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -0.23262938857078552, |
| "rewards/margins": 9.378125190734863, |
| "rewards/rejected": -9.6171875, |
| "step": 1070 |
| }, |
| { |
| "epoch": 1.3900869005471517, |
| "grad_norm": 220.96584927000097, |
| "learning_rate": 6.523840206185567e-07, |
| "logits/chosen": -0.973876953125, |
| "logits/rejected": -0.957104504108429, |
| "logps/chosen": -389.20001220703125, |
| "logps/rejected": -452.5249938964844, |
| "loss": 0.0817, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": 0.05045776441693306, |
| "rewards/margins": 9.750781059265137, |
| "rewards/rejected": -9.705469131469727, |
| "step": 1080 |
| }, |
| { |
| "epoch": 1.4029610556807208, |
| "grad_norm": 2.741340879707575, |
| "learning_rate": 6.491623711340206e-07, |
| "logits/chosen": -0.9906250238418579, |
| "logits/rejected": -0.981518566608429, |
| "logps/chosen": -397.70001220703125, |
| "logps/rejected": -498.95001220703125, |
| "loss": 0.0657, |
| "rewards/accuracies": 0.9312499761581421, |
| "rewards/chosen": -0.05647582933306694, |
| "rewards/margins": 9.798437118530273, |
| "rewards/rejected": -9.861328125, |
| "step": 1090 |
| }, |
| { |
| "epoch": 1.4158352108142904, |
| "grad_norm": 5.290171237298657, |
| "learning_rate": 6.459407216494846e-07, |
| "logits/chosen": -0.9808593988418579, |
| "logits/rejected": -0.9601074457168579, |
| "logps/chosen": -389.8500061035156, |
| "logps/rejected": -472.0, |
| "loss": 0.0383, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": 0.26752930879592896, |
| "rewards/margins": 10.471094131469727, |
| "rewards/rejected": -10.202343940734863, |
| "step": 1100 |
| }, |
| { |
| "epoch": 1.4287093659478596, |
| "grad_norm": 4.170559858681645, |
| "learning_rate": 6.427190721649485e-07, |
| "logits/chosen": -0.9833984375, |
| "logits/rejected": -0.999316394329071, |
| "logps/chosen": -395.375, |
| "logps/rejected": -460.20001220703125, |
| "loss": 0.0696, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.895581066608429, |
| "rewards/margins": 10.228906631469727, |
| "rewards/rejected": -11.122655868530273, |
| "step": 1110 |
| }, |
| { |
| "epoch": 1.441583521081429, |
| "grad_norm": 75.26395674949998, |
| "learning_rate": 6.394974226804123e-07, |
| "logits/chosen": -0.964770495891571, |
| "logits/rejected": -0.931689441204071, |
| "logps/chosen": -374.3999938964844, |
| "logps/rejected": -486.3999938964844, |
| "loss": 0.0676, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -2.9120116233825684, |
| "rewards/margins": 10.5078125, |
| "rewards/rejected": -13.420312881469727, |
| "step": 1120 |
| }, |
| { |
| "epoch": 1.4544576762149983, |
| "grad_norm": 5.547685368185955, |
| "learning_rate": 6.362757731958762e-07, |
| "logits/chosen": -0.963085949420929, |
| "logits/rejected": -0.958935558795929, |
| "logps/chosen": -373.1499938964844, |
| "logps/rejected": -480.1000061035156, |
| "loss": 0.0545, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -2.208447217941284, |
| "rewards/margins": 10.94921875, |
| "rewards/rejected": -13.160937309265137, |
| "step": 1130 |
| }, |
| { |
| "epoch": 1.4673318313485677, |
| "grad_norm": 10.491415939641222, |
| "learning_rate": 6.330541237113401e-07, |
| "logits/chosen": -0.9105468988418579, |
| "logits/rejected": -0.888720691204071, |
| "logps/chosen": -378.5, |
| "logps/rejected": -473.0, |
| "loss": 0.0594, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -1.3115966320037842, |
| "rewards/margins": 10.66796875, |
| "rewards/rejected": -11.981249809265137, |
| "step": 1140 |
| }, |
| { |
| "epoch": 1.480205986482137, |
| "grad_norm": 5.10762368180044, |
| "learning_rate": 6.29832474226804e-07, |
| "logits/chosen": -0.921875, |
| "logits/rejected": -0.9062744379043579, |
| "logps/chosen": -368.5, |
| "logps/rejected": -478.4750061035156, |
| "loss": 0.0382, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -1.3436768054962158, |
| "rewards/margins": 10.7734375, |
| "rewards/rejected": -12.12109375, |
| "step": 1150 |
| }, |
| { |
| "epoch": 1.4930801416157065, |
| "grad_norm": 9.900405690491036, |
| "learning_rate": 6.26610824742268e-07, |
| "logits/chosen": -0.923291027545929, |
| "logits/rejected": -0.899157702922821, |
| "logps/chosen": -381.17498779296875, |
| "logps/rejected": -462.75, |
| "loss": 0.0467, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -2.11822509765625, |
| "rewards/margins": 10.829687118530273, |
| "rewards/rejected": -12.944531440734863, |
| "step": 1160 |
| }, |
| { |
| "epoch": 1.5059542967492758, |
| "grad_norm": 6.152643520140975, |
| "learning_rate": 6.23389175257732e-07, |
| "logits/chosen": -1.017480492591858, |
| "logits/rejected": -0.987963855266571, |
| "logps/chosen": -414.2749938964844, |
| "logps/rejected": -506.70001220703125, |
| "loss": 0.0393, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -4.409295558929443, |
| "rewards/margins": 10.576562881469727, |
| "rewards/rejected": -14.982812881469727, |
| "step": 1170 |
| }, |
| { |
| "epoch": 1.5188284518828452, |
| "grad_norm": 10.576249252330557, |
| "learning_rate": 6.201675257731959e-07, |
| "logits/chosen": -0.940014660358429, |
| "logits/rejected": -0.9420166015625, |
| "logps/chosen": -388.92498779296875, |
| "logps/rejected": -498.70001220703125, |
| "loss": 0.0621, |
| "rewards/accuracies": 0.9312499761581421, |
| "rewards/chosen": -3.6192870140075684, |
| "rewards/margins": 11.6328125, |
| "rewards/rejected": -15.253125190734863, |
| "step": 1180 |
| }, |
| { |
| "epoch": 1.5317026070164146, |
| "grad_norm": 5.858898211376708, |
| "learning_rate": 6.169458762886598e-07, |
| "logits/chosen": -0.971435546875, |
| "logits/rejected": -0.972216784954071, |
| "logps/chosen": -375.6625061035156, |
| "logps/rejected": -452.1875, |
| "loss": 0.0492, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -3.7707762718200684, |
| "rewards/margins": 10.260546684265137, |
| "rewards/rejected": -14.032812118530273, |
| "step": 1190 |
| }, |
| { |
| "epoch": 1.544576762149984, |
| "grad_norm": 73.088561052754, |
| "learning_rate": 6.137242268041237e-07, |
| "logits/chosen": -1.0227539539337158, |
| "logits/rejected": -0.9654296636581421, |
| "logps/chosen": -447.9750061035156, |
| "logps/rejected": -547.875, |
| "loss": 0.0675, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -4.362378120422363, |
| "rewards/margins": 12.967968940734863, |
| "rewards/rejected": -17.332813262939453, |
| "step": 1200 |
| }, |
| { |
| "epoch": 1.5574509172835533, |
| "grad_norm": 16.087749078441842, |
| "learning_rate": 6.105025773195877e-07, |
| "logits/chosen": -1.08251953125, |
| "logits/rejected": -1.072607398033142, |
| "logps/chosen": -428.125, |
| "logps/rejected": -541.25, |
| "loss": 0.0971, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -5.455664157867432, |
| "rewards/margins": 12.564844131469727, |
| "rewards/rejected": -18.026561737060547, |
| "step": 1210 |
| }, |
| { |
| "epoch": 1.5703250724171225, |
| "grad_norm": 8.015976908256388, |
| "learning_rate": 6.072809278350515e-07, |
| "logits/chosen": -0.999829113483429, |
| "logits/rejected": -1.026025414466858, |
| "logps/chosen": -411.4125061035156, |
| "logps/rejected": -488.7749938964844, |
| "loss": 0.0738, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -4.011401176452637, |
| "rewards/margins": 10.776171684265137, |
| "rewards/rejected": -14.782812118530273, |
| "step": 1220 |
| }, |
| { |
| "epoch": 1.583199227550692, |
| "grad_norm": 7.496227160228813, |
| "learning_rate": 6.040592783505154e-07, |
| "logits/chosen": -0.8304198980331421, |
| "logits/rejected": -0.83154296875, |
| "logps/chosen": -335.81561279296875, |
| "logps/rejected": -454.4437561035156, |
| "loss": 0.109, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": -2.8844237327575684, |
| "rewards/margins": 9.571484565734863, |
| "rewards/rejected": -12.45703125, |
| "step": 1230 |
| }, |
| { |
| "epoch": 1.5960733826842612, |
| "grad_norm": 38.899724862220474, |
| "learning_rate": 6.008376288659793e-07, |
| "logits/chosen": -0.9081786870956421, |
| "logits/rejected": -0.9267822504043579, |
| "logps/chosen": -393.25, |
| "logps/rejected": -473.04998779296875, |
| "loss": 0.0442, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -3.959277391433716, |
| "rewards/margins": 11.373437881469727, |
| "rewards/rejected": -15.329687118530273, |
| "step": 1240 |
| }, |
| { |
| "epoch": 1.6089475378178308, |
| "grad_norm": 56.23438612247682, |
| "learning_rate": 5.976159793814432e-07, |
| "logits/chosen": -0.971386730670929, |
| "logits/rejected": -0.9666992425918579, |
| "logps/chosen": -406.29998779296875, |
| "logps/rejected": -507.6499938964844, |
| "loss": 0.0303, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -2.40478515625, |
| "rewards/margins": 11.353124618530273, |
| "rewards/rejected": -13.760156631469727, |
| "step": 1250 |
| }, |
| { |
| "epoch": 1.6218216929514, |
| "grad_norm": 10.42344864534372, |
| "learning_rate": 5.943943298969071e-07, |
| "logits/chosen": -0.994335949420929, |
| "logits/rejected": -0.992724597454071, |
| "logps/chosen": -393.125, |
| "logps/rejected": -502.125, |
| "loss": 0.0554, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -2.322924852371216, |
| "rewards/margins": 11.29296875, |
| "rewards/rejected": -13.6171875, |
| "step": 1260 |
| }, |
| { |
| "epoch": 1.6346958480849694, |
| "grad_norm": 24.851501417189294, |
| "learning_rate": 5.911726804123712e-07, |
| "logits/chosen": -0.967041015625, |
| "logits/rejected": -0.92529296875, |
| "logps/chosen": -361.57501220703125, |
| "logps/rejected": -458.0249938964844, |
| "loss": 0.0577, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": -1.720422387123108, |
| "rewards/margins": 11.080469131469727, |
| "rewards/rejected": -12.807031631469727, |
| "step": 1270 |
| }, |
| { |
| "epoch": 1.6475700032185387, |
| "grad_norm": 1.9460817231647556, |
| "learning_rate": 5.879510309278351e-07, |
| "logits/chosen": -0.9542480707168579, |
| "logits/rejected": -0.949999988079071, |
| "logps/chosen": -398.4750061035156, |
| "logps/rejected": -488.1000061035156, |
| "loss": 0.1017, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": -0.5497802495956421, |
| "rewards/margins": 11.510156631469727, |
| "rewards/rejected": -12.067187309265137, |
| "step": 1280 |
| }, |
| { |
| "epoch": 1.6604441583521081, |
| "grad_norm": 22.436709932766853, |
| "learning_rate": 5.84729381443299e-07, |
| "logits/chosen": -0.920214831829071, |
| "logits/rejected": -0.917529284954071, |
| "logps/chosen": -394.7250061035156, |
| "logps/rejected": -464.8999938964844, |
| "loss": 0.0585, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -2.0501708984375, |
| "rewards/margins": 11.30859375, |
| "rewards/rejected": -13.360937118530273, |
| "step": 1290 |
| }, |
| { |
| "epoch": 1.6733183134856775, |
| "grad_norm": 2.922267266125603, |
| "learning_rate": 5.815077319587629e-07, |
| "logits/chosen": -0.810717761516571, |
| "logits/rejected": -0.8197021484375, |
| "logps/chosen": -421.3500061035156, |
| "logps/rejected": -514.25, |
| "loss": 0.0395, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -1.7827637195587158, |
| "rewards/margins": 12.157617568969727, |
| "rewards/rejected": -13.939844131469727, |
| "step": 1300 |
| }, |
| { |
| "epoch": 1.6861924686192469, |
| "grad_norm": 20.574767289402832, |
| "learning_rate": 5.782860824742268e-07, |
| "logits/chosen": -0.9783569574356079, |
| "logits/rejected": -0.9566406011581421, |
| "logps/chosen": -409.6000061035156, |
| "logps/rejected": -500.79998779296875, |
| "loss": 0.104, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -2.32958984375, |
| "rewards/margins": 11.189844131469727, |
| "rewards/rejected": -13.518750190734863, |
| "step": 1310 |
| }, |
| { |
| "epoch": 1.6990666237528163, |
| "grad_norm": 110.28911944115465, |
| "learning_rate": 5.750644329896906e-07, |
| "logits/chosen": -0.8671630620956421, |
| "logits/rejected": -0.854736328125, |
| "logps/chosen": -427.9750061035156, |
| "logps/rejected": -514.0750122070312, |
| "loss": 0.0576, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -2.6690430641174316, |
| "rewards/margins": 10.728124618530273, |
| "rewards/rejected": -13.391406059265137, |
| "step": 1320 |
| }, |
| { |
| "epoch": 1.7119407788863856, |
| "grad_norm": 29.883782295799218, |
| "learning_rate": 5.718427835051546e-07, |
| "logits/chosen": -0.869580090045929, |
| "logits/rejected": -0.8749023675918579, |
| "logps/chosen": -384.375, |
| "logps/rejected": -489.75, |
| "loss": 0.0525, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": -1.936621069908142, |
| "rewards/margins": 10.360937118530273, |
| "rewards/rejected": -12.296875, |
| "step": 1330 |
| }, |
| { |
| "epoch": 1.724814934019955, |
| "grad_norm": 15.825926255473947, |
| "learning_rate": 5.686211340206185e-07, |
| "logits/chosen": -0.896435558795929, |
| "logits/rejected": -0.895214855670929, |
| "logps/chosen": -377.75, |
| "logps/rejected": -477.75, |
| "loss": 0.0428, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -1.439355492591858, |
| "rewards/margins": 10.345312118530273, |
| "rewards/rejected": -11.776562690734863, |
| "step": 1340 |
| }, |
| { |
| "epoch": 1.7376890891535242, |
| "grad_norm": 8.470103187327458, |
| "learning_rate": 5.653994845360824e-07, |
| "logits/chosen": -0.949902355670929, |
| "logits/rejected": -0.9499267339706421, |
| "logps/chosen": -375.375, |
| "logps/rejected": -446.95001220703125, |
| "loss": 0.0519, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -1.849633812904358, |
| "rewards/margins": 9.740625381469727, |
| "rewards/rejected": -11.599218368530273, |
| "step": 1350 |
| }, |
| { |
| "epoch": 1.7505632442870938, |
| "grad_norm": 4.5563431355585875, |
| "learning_rate": 5.621778350515464e-07, |
| "logits/chosen": -0.952929675579071, |
| "logits/rejected": -0.928955078125, |
| "logps/chosen": -417.29998779296875, |
| "logps/rejected": -505.6499938964844, |
| "loss": 0.0394, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -2.0886597633361816, |
| "rewards/margins": 11.784375190734863, |
| "rewards/rejected": -13.874218940734863, |
| "step": 1360 |
| }, |
| { |
| "epoch": 1.763437399420663, |
| "grad_norm": 9.103762775061622, |
| "learning_rate": 5.589561855670103e-07, |
| "logits/chosen": -0.9593261480331421, |
| "logits/rejected": -0.9452148675918579, |
| "logps/chosen": -423.32501220703125, |
| "logps/rejected": -505.6000061035156, |
| "loss": 0.049, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -3.0106263160705566, |
| "rewards/margins": 10.846094131469727, |
| "rewards/rejected": -13.854687690734863, |
| "step": 1370 |
| }, |
| { |
| "epoch": 1.7763115545542325, |
| "grad_norm": 1.691523363024812, |
| "learning_rate": 5.557345360824743e-07, |
| "logits/chosen": -0.845349133014679, |
| "logits/rejected": -0.8228515386581421, |
| "logps/chosen": -371.42498779296875, |
| "logps/rejected": -481.4624938964844, |
| "loss": 0.1147, |
| "rewards/accuracies": 0.9125000238418579, |
| "rewards/chosen": -2.849853515625, |
| "rewards/margins": 10.446874618530273, |
| "rewards/rejected": -13.291406631469727, |
| "step": 1380 |
| }, |
| { |
| "epoch": 1.7891857096878017, |
| "grad_norm": 14.771932390824606, |
| "learning_rate": 5.525128865979382e-07, |
| "logits/chosen": -0.84832763671875, |
| "logits/rejected": -0.824292004108429, |
| "logps/chosen": -376.57501220703125, |
| "logps/rejected": -491.3999938964844, |
| "loss": 0.065, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -1.8124511241912842, |
| "rewards/margins": 10.788281440734863, |
| "rewards/rejected": -12.591405868530273, |
| "step": 1390 |
| }, |
| { |
| "epoch": 1.8020598648213713, |
| "grad_norm": 29.299452797327557, |
| "learning_rate": 5.492912371134021e-07, |
| "logits/chosen": -0.990966796875, |
| "logits/rejected": -0.987255871295929, |
| "logps/chosen": -412.0, |
| "logps/rejected": -511.8500061035156, |
| "loss": 0.0333, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -1.2932891845703125, |
| "rewards/margins": 10.651562690734863, |
| "rewards/rejected": -11.946874618530273, |
| "step": 1400 |
| }, |
| { |
| "epoch": 1.8149340199549404, |
| "grad_norm": 100.26192553543316, |
| "learning_rate": 5.460695876288659e-07, |
| "logits/chosen": -0.902539074420929, |
| "logits/rejected": -0.887866199016571, |
| "logps/chosen": -407.54998779296875, |
| "logps/rejected": -509.25, |
| "loss": 0.0592, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -1.274084448814392, |
| "rewards/margins": 11.475781440734863, |
| "rewards/rejected": -12.744531631469727, |
| "step": 1410 |
| }, |
| { |
| "epoch": 1.8278081750885098, |
| "grad_norm": 35.25113627842931, |
| "learning_rate": 5.428479381443298e-07, |
| "logits/chosen": -0.954638659954071, |
| "logits/rejected": -0.9381347894668579, |
| "logps/chosen": -356.9750061035156, |
| "logps/rejected": -439.45001220703125, |
| "loss": 0.0425, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -1.6203124523162842, |
| "rewards/margins": 10.171875, |
| "rewards/rejected": -11.791406631469727, |
| "step": 1420 |
| }, |
| { |
| "epoch": 1.8406823302220792, |
| "grad_norm": 0.6778421877980854, |
| "learning_rate": 5.396262886597937e-07, |
| "logits/chosen": -0.9102538824081421, |
| "logits/rejected": -0.8841797113418579, |
| "logps/chosen": -375.8999938964844, |
| "logps/rejected": -465.1499938964844, |
| "loss": 0.0219, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -0.5498291254043579, |
| "rewards/margins": 10.825780868530273, |
| "rewards/rejected": -11.375, |
| "step": 1430 |
| }, |
| { |
| "epoch": 1.8535564853556485, |
| "grad_norm": 21.24551771606512, |
| "learning_rate": 5.364046391752577e-07, |
| "logits/chosen": -0.9066406488418579, |
| "logits/rejected": -0.921826183795929, |
| "logps/chosen": -396.8999938964844, |
| "logps/rejected": -491.1000061035156, |
| "loss": 0.0572, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -1.164526343345642, |
| "rewards/margins": 11.449999809265137, |
| "rewards/rejected": -12.615625381469727, |
| "step": 1440 |
| }, |
| { |
| "epoch": 1.866430640489218, |
| "grad_norm": 10.543090540151306, |
| "learning_rate": 5.331829896907216e-07, |
| "logits/chosen": -0.901171863079071, |
| "logits/rejected": -0.892016589641571, |
| "logps/chosen": -429.82501220703125, |
| "logps/rejected": -509.70001220703125, |
| "loss": 0.0678, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -1.7703644037246704, |
| "rewards/margins": 10.507031440734863, |
| "rewards/rejected": -12.272656440734863, |
| "step": 1450 |
| }, |
| { |
| "epoch": 1.8793047956227873, |
| "grad_norm": 0.44776982375157787, |
| "learning_rate": 5.299613402061856e-07, |
| "logits/chosen": -0.9541991949081421, |
| "logits/rejected": -0.9314941167831421, |
| "logps/chosen": -406.8999938964844, |
| "logps/rejected": -504.7749938964844, |
| "loss": 0.0515, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -1.4205474853515625, |
| "rewards/margins": 11.315625190734863, |
| "rewards/rejected": -12.733983993530273, |
| "step": 1460 |
| }, |
| { |
| "epoch": 1.8921789507563567, |
| "grad_norm": 64.17339300757935, |
| "learning_rate": 5.267396907216495e-07, |
| "logits/chosen": -0.9212890863418579, |
| "logits/rejected": -0.9178222417831421, |
| "logps/chosen": -389.32501220703125, |
| "logps/rejected": -481.3999938964844, |
| "loss": 0.0504, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -2.74786376953125, |
| "rewards/margins": 10.51953125, |
| "rewards/rejected": -13.26953125, |
| "step": 1470 |
| }, |
| { |
| "epoch": 1.9050531058899258, |
| "grad_norm": 2.3569087767370966, |
| "learning_rate": 5.235180412371134e-07, |
| "logits/chosen": -0.915698230266571, |
| "logits/rejected": -0.88592529296875, |
| "logps/chosen": -435.17498779296875, |
| "logps/rejected": -532.1500244140625, |
| "loss": 0.0603, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -2.54071044921875, |
| "rewards/margins": 12.057031631469727, |
| "rewards/rejected": -14.594531059265137, |
| "step": 1480 |
| }, |
| { |
| "epoch": 1.9179272610234954, |
| "grad_norm": 32.07335603062137, |
| "learning_rate": 5.202963917525773e-07, |
| "logits/chosen": -0.949511706829071, |
| "logits/rejected": -0.9261718988418579, |
| "logps/chosen": -400.42498779296875, |
| "logps/rejected": -488.1499938964844, |
| "loss": 0.0346, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -2.5046019554138184, |
| "rewards/margins": 11.435155868530273, |
| "rewards/rejected": -13.943750381469727, |
| "step": 1490 |
| }, |
| { |
| "epoch": 1.9308014161570646, |
| "grad_norm": 5.079861293118151, |
| "learning_rate": 5.170747422680413e-07, |
| "logits/chosen": -0.9255126714706421, |
| "logits/rejected": -0.9140869379043579, |
| "logps/chosen": -393.5, |
| "logps/rejected": -460.95001220703125, |
| "loss": 0.0647, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -2.2625489234924316, |
| "rewards/margins": 11.5546875, |
| "rewards/rejected": -13.8203125, |
| "step": 1500 |
| }, |
| { |
| "epoch": 1.9436755712906342, |
| "grad_norm": 13.730196804403965, |
| "learning_rate": 5.138530927835051e-07, |
| "logits/chosen": -0.964599609375, |
| "logits/rejected": -0.9576171636581421, |
| "logps/chosen": -381.73748779296875, |
| "logps/rejected": -464.57501220703125, |
| "loss": 0.096, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": -1.831457495689392, |
| "rewards/margins": 10.2421875, |
| "rewards/rejected": -12.07421875, |
| "step": 1510 |
| }, |
| { |
| "epoch": 1.9565497264242033, |
| "grad_norm": 1.9292744667507238, |
| "learning_rate": 5.10631443298969e-07, |
| "logits/chosen": -0.919140636920929, |
| "logits/rejected": -0.880786120891571, |
| "logps/chosen": -382.9375, |
| "logps/rejected": -459.5, |
| "loss": 0.055, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -0.831347644329071, |
| "rewards/margins": 10.905468940734863, |
| "rewards/rejected": -11.734375, |
| "step": 1520 |
| }, |
| { |
| "epoch": 1.969423881557773, |
| "grad_norm": 11.449713018559374, |
| "learning_rate": 5.074097938144329e-07, |
| "logits/chosen": -0.9642578363418579, |
| "logits/rejected": -0.9501708745956421, |
| "logps/chosen": -385.29998779296875, |
| "logps/rejected": -506.04998779296875, |
| "loss": 0.0454, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -0.8114013671875, |
| "rewards/margins": 11.5625, |
| "rewards/rejected": -12.375781059265137, |
| "step": 1530 |
| }, |
| { |
| "epoch": 1.982298036691342, |
| "grad_norm": 3.350678145456376, |
| "learning_rate": 5.041881443298968e-07, |
| "logits/chosen": -0.921582043170929, |
| "logits/rejected": -0.912402331829071, |
| "logps/chosen": -406.29998779296875, |
| "logps/rejected": -441.32501220703125, |
| "loss": 0.0492, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -1.100073218345642, |
| "rewards/margins": 10.561718940734863, |
| "rewards/rejected": -11.6640625, |
| "step": 1540 |
| }, |
| { |
| "epoch": 1.9951721918249115, |
| "grad_norm": 54.70846216282573, |
| "learning_rate": 5.009664948453608e-07, |
| "logits/chosen": -0.922558605670929, |
| "logits/rejected": -0.917529284954071, |
| "logps/chosen": -389.5249938964844, |
| "logps/rejected": -470.75, |
| "loss": 0.065, |
| "rewards/accuracies": 0.9312499761581421, |
| "rewards/chosen": -1.89117431640625, |
| "rewards/margins": 11.014062881469727, |
| "rewards/rejected": -12.916406631469727, |
| "step": 1550 |
| }, |
| { |
| "epoch": 2.0077244930801417, |
| "grad_norm": 0.27387321212084004, |
| "learning_rate": 4.977448453608248e-07, |
| "logits/chosen": -0.9893704652786255, |
| "logits/rejected": -0.9759489893913269, |
| "logps/chosen": -374.28204345703125, |
| "logps/rejected": -480.025634765625, |
| "loss": 0.0246, |
| "rewards/accuracies": 0.9679487347602844, |
| "rewards/chosen": -1.1674273014068604, |
| "rewards/margins": 12.660256385803223, |
| "rewards/rejected": -13.831730842590332, |
| "step": 1560 |
| }, |
| { |
| "epoch": 2.020598648213711, |
| "grad_norm": 16.195642178522633, |
| "learning_rate": 4.945231958762887e-07, |
| "logits/chosen": -0.8692626953125, |
| "logits/rejected": -0.855395495891571, |
| "logps/chosen": -364.4125061035156, |
| "logps/rejected": -511.4375, |
| "loss": 0.0406, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": -1.5969727039337158, |
| "rewards/margins": 13.655077934265137, |
| "rewards/rejected": -15.242968559265137, |
| "step": 1570 |
| }, |
| { |
| "epoch": 2.0334728033472804, |
| "grad_norm": 0.011066699938703848, |
| "learning_rate": 4.913015463917525e-07, |
| "logits/chosen": -0.9894043207168579, |
| "logits/rejected": -0.9693359136581421, |
| "logps/chosen": -391.7124938964844, |
| "logps/rejected": -511.3500061035156, |
| "loss": 0.0429, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": -2.259082078933716, |
| "rewards/margins": 13.080469131469727, |
| "rewards/rejected": -15.342187881469727, |
| "step": 1580 |
| }, |
| { |
| "epoch": 2.0463469584808496, |
| "grad_norm": 0.30088485969690315, |
| "learning_rate": 4.880798969072165e-07, |
| "logits/chosen": -0.9302002191543579, |
| "logits/rejected": -0.9267578125, |
| "logps/chosen": -375.5, |
| "logps/rejected": -490.8999938964844, |
| "loss": 0.0177, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -2.5155029296875, |
| "rewards/margins": 15.078125, |
| "rewards/rejected": -17.590625762939453, |
| "step": 1590 |
| }, |
| { |
| "epoch": 2.059221113614419, |
| "grad_norm": 3.7766222698166008, |
| "learning_rate": 4.848582474226804e-07, |
| "logits/chosen": -0.990429699420929, |
| "logits/rejected": -0.9600585699081421, |
| "logps/chosen": -389.95001220703125, |
| "logps/rejected": -542.9000244140625, |
| "loss": 0.0144, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -3.7989258766174316, |
| "rewards/margins": 15.314062118530273, |
| "rewards/rejected": -19.110937118530273, |
| "step": 1600 |
| }, |
| { |
| "epoch": 2.0720952687479883, |
| "grad_norm": 1.5230275244877616, |
| "learning_rate": 4.816365979381443e-07, |
| "logits/chosen": -0.985791027545929, |
| "logits/rejected": -0.965527355670929, |
| "logps/chosen": -407.125, |
| "logps/rejected": -549.5499877929688, |
| "loss": 0.0247, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -3.044628858566284, |
| "rewards/margins": 15.612500190734863, |
| "rewards/rejected": -18.653125762939453, |
| "step": 1610 |
| }, |
| { |
| "epoch": 2.084969423881558, |
| "grad_norm": 0.5241685764089725, |
| "learning_rate": 4.784149484536082e-07, |
| "logits/chosen": -0.8857177495956421, |
| "logits/rejected": -0.888867199420929, |
| "logps/chosen": -411.79998779296875, |
| "logps/rejected": -524.1500244140625, |
| "loss": 0.0325, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -5.372949123382568, |
| "rewards/margins": 14.409375190734863, |
| "rewards/rejected": -19.782812118530273, |
| "step": 1620 |
| }, |
| { |
| "epoch": 2.097843579015127, |
| "grad_norm": 14.457877642201927, |
| "learning_rate": 4.7519329896907214e-07, |
| "logits/chosen": -1.0594604015350342, |
| "logits/rejected": -1.0399048328399658, |
| "logps/chosen": -423.45001220703125, |
| "logps/rejected": -568.0499877929688, |
| "loss": 0.0244, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -6.598046779632568, |
| "rewards/margins": 15.84375, |
| "rewards/rejected": -22.443750381469727, |
| "step": 1630 |
| }, |
| { |
| "epoch": 2.1107177341486967, |
| "grad_norm": 18.76607789875824, |
| "learning_rate": 4.7197164948453605e-07, |
| "logits/chosen": -1.090917944908142, |
| "logits/rejected": -1.0798828601837158, |
| "logps/chosen": -449.1499938964844, |
| "logps/rejected": -559.8499755859375, |
| "loss": 0.0335, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -5.502295017242432, |
| "rewards/margins": 15.567187309265137, |
| "rewards/rejected": -21.0546875, |
| "step": 1640 |
| }, |
| { |
| "epoch": 2.123591889282266, |
| "grad_norm": 4.069282822417778, |
| "learning_rate": 4.6874999999999996e-07, |
| "logits/chosen": -1.083837866783142, |
| "logits/rejected": -1.050439476966858, |
| "logps/chosen": -424.1499938964844, |
| "logps/rejected": -553.5999755859375, |
| "loss": 0.0162, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -4.673583984375, |
| "rewards/margins": 15.3671875, |
| "rewards/rejected": -20.049999237060547, |
| "step": 1650 |
| }, |
| { |
| "epoch": 2.136466044415835, |
| "grad_norm": 0.019867844889664508, |
| "learning_rate": 4.655283505154639e-07, |
| "logits/chosen": -1.0280640125274658, |
| "logits/rejected": -1.0054931640625, |
| "logps/chosen": -395.95001220703125, |
| "logps/rejected": -527.4500122070312, |
| "loss": 0.0228, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -4.825781345367432, |
| "rewards/margins": 15.746874809265137, |
| "rewards/rejected": -20.573436737060547, |
| "step": 1660 |
| }, |
| { |
| "epoch": 2.1493401995494046, |
| "grad_norm": 1.405391631885844, |
| "learning_rate": 4.6230670103092783e-07, |
| "logits/chosen": -1.0292479991912842, |
| "logits/rejected": -1.001928687095642, |
| "logps/chosen": -449.2250061035156, |
| "logps/rejected": -581.4500122070312, |
| "loss": 0.0242, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -3.895703077316284, |
| "rewards/margins": 15.307812690734863, |
| "rewards/rejected": -19.189062118530273, |
| "step": 1670 |
| }, |
| { |
| "epoch": 2.1622143546829737, |
| "grad_norm": 0.5519313995694172, |
| "learning_rate": 4.590850515463917e-07, |
| "logits/chosen": -0.94366455078125, |
| "logits/rejected": -0.9423980712890625, |
| "logps/chosen": -431.5, |
| "logps/rejected": -548.125, |
| "loss": 0.0294, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -4.086621284484863, |
| "rewards/margins": 14.803906440734863, |
| "rewards/rejected": -18.889062881469727, |
| "step": 1680 |
| }, |
| { |
| "epoch": 2.1750885098165433, |
| "grad_norm": 0.04917387233529229, |
| "learning_rate": 4.5586340206185565e-07, |
| "logits/chosen": -1.0073730945587158, |
| "logits/rejected": -0.983837902545929, |
| "logps/chosen": -393.65625, |
| "logps/rejected": -515.4375, |
| "loss": 0.0547, |
| "rewards/accuracies": 0.925000011920929, |
| "rewards/chosen": -3.69677734375, |
| "rewards/margins": 14.767187118530273, |
| "rewards/rejected": -18.470312118530273, |
| "step": 1690 |
| }, |
| { |
| "epoch": 2.1879626649501125, |
| "grad_norm": 1.1593323599908398, |
| "learning_rate": 4.5264175257731956e-07, |
| "logits/chosen": -1.046777367591858, |
| "logits/rejected": -1.0193359851837158, |
| "logps/chosen": -396.07501220703125, |
| "logps/rejected": -525.0750122070312, |
| "loss": 0.0483, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -4.259375095367432, |
| "rewards/margins": 14.039843559265137, |
| "rewards/rejected": -18.298437118530273, |
| "step": 1700 |
| }, |
| { |
| "epoch": 2.200836820083682, |
| "grad_norm": 0.30503012975802324, |
| "learning_rate": 4.4942010309278347e-07, |
| "logits/chosen": -0.9892578125, |
| "logits/rejected": -0.9864746332168579, |
| "logps/chosen": -385.6499938964844, |
| "logps/rejected": -523.0, |
| "loss": 0.0286, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -4.869189262390137, |
| "rewards/margins": 15.048437118530273, |
| "rewards/rejected": -19.910938262939453, |
| "step": 1710 |
| }, |
| { |
| "epoch": 2.2137109752172512, |
| "grad_norm": 0.025660689139265877, |
| "learning_rate": 4.4619845360824744e-07, |
| "logits/chosen": -0.982128918170929, |
| "logits/rejected": -0.9969726800918579, |
| "logps/chosen": -435.82501220703125, |
| "logps/rejected": -571.4500122070312, |
| "loss": 0.035, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -5.135912895202637, |
| "rewards/margins": 15.172656059265137, |
| "rewards/rejected": -20.3046875, |
| "step": 1720 |
| }, |
| { |
| "epoch": 2.226585130350821, |
| "grad_norm": 140.50599875108497, |
| "learning_rate": 4.429768041237113e-07, |
| "logits/chosen": -0.911694347858429, |
| "logits/rejected": -0.8985351324081421, |
| "logps/chosen": -404.07501220703125, |
| "logps/rejected": -538.2999877929688, |
| "loss": 0.0316, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -5.711035251617432, |
| "rewards/margins": 14.632031440734863, |
| "rewards/rejected": -20.345312118530273, |
| "step": 1730 |
| }, |
| { |
| "epoch": 2.23945928548439, |
| "grad_norm": 0.1313501575125143, |
| "learning_rate": 4.397551546391752e-07, |
| "logits/chosen": -0.933483898639679, |
| "logits/rejected": -0.944042980670929, |
| "logps/chosen": -427.20001220703125, |
| "logps/rejected": -553.0, |
| "loss": 0.0365, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -6.276562690734863, |
| "rewards/margins": 14.622655868530273, |
| "rewards/rejected": -20.8984375, |
| "step": 1740 |
| }, |
| { |
| "epoch": 2.2523334406179596, |
| "grad_norm": 0.7371999610473159, |
| "learning_rate": 4.3653350515463917e-07, |
| "logits/chosen": -0.9942382574081421, |
| "logits/rejected": -0.9849609136581421, |
| "logps/chosen": -419.70001220703125, |
| "logps/rejected": -582.0999755859375, |
| "loss": 0.0326, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -6.280077934265137, |
| "rewards/margins": 16.040624618530273, |
| "rewards/rejected": -22.314062118530273, |
| "step": 1750 |
| }, |
| { |
| "epoch": 2.2652075957515287, |
| "grad_norm": 0.129869449494833, |
| "learning_rate": 4.333118556701031e-07, |
| "logits/chosen": -0.933154284954071, |
| "logits/rejected": -0.91748046875, |
| "logps/chosen": -430.04998779296875, |
| "logps/rejected": -566.4500122070312, |
| "loss": 0.042, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": -4.696862697601318, |
| "rewards/margins": 15.498437881469727, |
| "rewards/rejected": -20.201562881469727, |
| "step": 1760 |
| }, |
| { |
| "epoch": 2.2780817508850983, |
| "grad_norm": 0.13607345195503057, |
| "learning_rate": 4.3009020618556704e-07, |
| "logits/chosen": -1.015039086341858, |
| "logits/rejected": -0.995849609375, |
| "logps/chosen": -451.54998779296875, |
| "logps/rejected": -543.4500122070312, |
| "loss": 0.0244, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -4.693749904632568, |
| "rewards/margins": 14.237500190734863, |
| "rewards/rejected": -18.942188262939453, |
| "step": 1770 |
| }, |
| { |
| "epoch": 2.2909559060186675, |
| "grad_norm": 0.39155247794928777, |
| "learning_rate": 4.268685567010309e-07, |
| "logits/chosen": -0.974365234375, |
| "logits/rejected": -0.960986316204071, |
| "logps/chosen": -401.3999938964844, |
| "logps/rejected": -522.4000244140625, |
| "loss": 0.0177, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -3.511425733566284, |
| "rewards/margins": 15.060937881469727, |
| "rewards/rejected": -18.578125, |
| "step": 1780 |
| }, |
| { |
| "epoch": 2.3038300611522367, |
| "grad_norm": 0.08585169933868184, |
| "learning_rate": 4.236469072164948e-07, |
| "logits/chosen": -0.908154308795929, |
| "logits/rejected": -0.895312488079071, |
| "logps/chosen": -442.70001220703125, |
| "logps/rejected": -591.0, |
| "loss": 0.0183, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -3.9928956031799316, |
| "rewards/margins": 15.760937690734863, |
| "rewards/rejected": -19.754688262939453, |
| "step": 1790 |
| }, |
| { |
| "epoch": 2.3167042162858063, |
| "grad_norm": 1.0309690238634248, |
| "learning_rate": 4.2042525773195877e-07, |
| "logits/chosen": -0.9856933355331421, |
| "logits/rejected": -0.9870361089706421, |
| "logps/chosen": -444.04998779296875, |
| "logps/rejected": -564.2999877929688, |
| "loss": 0.0266, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -5.415820121765137, |
| "rewards/margins": 15.154687881469727, |
| "rewards/rejected": -20.568750381469727, |
| "step": 1800 |
| }, |
| { |
| "epoch": 2.329578371419376, |
| "grad_norm": 0.3147659868365795, |
| "learning_rate": 4.172036082474227e-07, |
| "logits/chosen": -0.878552258014679, |
| "logits/rejected": -0.864392101764679, |
| "logps/chosen": -458.3500061035156, |
| "logps/rejected": -572.8499755859375, |
| "loss": 0.0491, |
| "rewards/accuracies": 0.9312499761581421, |
| "rewards/chosen": -5.186669826507568, |
| "rewards/margins": 14.610937118530273, |
| "rewards/rejected": -19.798437118530273, |
| "step": 1810 |
| }, |
| { |
| "epoch": 2.342452526552945, |
| "grad_norm": 2.027945212441702, |
| "learning_rate": 4.1398195876288654e-07, |
| "logits/chosen": -0.954394519329071, |
| "logits/rejected": -0.9604736566543579, |
| "logps/chosen": -451.29998779296875, |
| "logps/rejected": -590.4500122070312, |
| "loss": 0.0253, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -5.277050971984863, |
| "rewards/margins": 14.571874618530273, |
| "rewards/rejected": -19.850000381469727, |
| "step": 1820 |
| }, |
| { |
| "epoch": 2.355326681686514, |
| "grad_norm": 3.6674065556267053, |
| "learning_rate": 4.107603092783505e-07, |
| "logits/chosen": -0.9901367425918579, |
| "logits/rejected": -0.9849609136581421, |
| "logps/chosen": -399.57501220703125, |
| "logps/rejected": -510.20001220703125, |
| "loss": 0.0399, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -4.4237060546875, |
| "rewards/margins": 13.792187690734863, |
| "rewards/rejected": -18.212499618530273, |
| "step": 1830 |
| }, |
| { |
| "epoch": 2.3682008368200838, |
| "grad_norm": 0.011220754857104063, |
| "learning_rate": 4.075386597938144e-07, |
| "logits/chosen": -0.9344238042831421, |
| "logits/rejected": -0.917895495891571, |
| "logps/chosen": -363.42498779296875, |
| "logps/rejected": -516.7000122070312, |
| "loss": 0.0226, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -4.707543849945068, |
| "rewards/margins": 15.096875190734863, |
| "rewards/rejected": -19.815624237060547, |
| "step": 1840 |
| }, |
| { |
| "epoch": 2.381074991953653, |
| "grad_norm": 1.084213236167663, |
| "learning_rate": 4.043170103092783e-07, |
| "logits/chosen": -1.003729224205017, |
| "logits/rejected": -0.9970871210098267, |
| "logps/chosen": -422.0, |
| "logps/rejected": -577.9500122070312, |
| "loss": 0.018, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -4.445825099945068, |
| "rewards/margins": 15.790624618530273, |
| "rewards/rejected": -20.237499237060547, |
| "step": 1850 |
| }, |
| { |
| "epoch": 2.3939491470872225, |
| "grad_norm": 0.11138110362978013, |
| "learning_rate": 4.010953608247423e-07, |
| "logits/chosen": -1.015112280845642, |
| "logits/rejected": -1.0192139148712158, |
| "logps/chosen": -434.6000061035156, |
| "logps/rejected": -530.125, |
| "loss": 0.0205, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -4.493237495422363, |
| "rewards/margins": 15.137499809265137, |
| "rewards/rejected": -19.627344131469727, |
| "step": 1860 |
| }, |
| { |
| "epoch": 2.4068233022207917, |
| "grad_norm": 0.07209992509377328, |
| "learning_rate": 3.9787371134020614e-07, |
| "logits/chosen": -1.021484375, |
| "logits/rejected": -1.012841820716858, |
| "logps/chosen": -408.1875, |
| "logps/rejected": -553.0999755859375, |
| "loss": 0.0226, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -4.288866996765137, |
| "rewards/margins": 15.574999809265137, |
| "rewards/rejected": -19.860937118530273, |
| "step": 1870 |
| }, |
| { |
| "epoch": 2.4196974573543613, |
| "grad_norm": 26.658700203141652, |
| "learning_rate": 3.9465206185567005e-07, |
| "logits/chosen": -0.9209960699081421, |
| "logits/rejected": -0.8855956792831421, |
| "logps/chosen": -436.0, |
| "logps/rejected": -571.6749877929688, |
| "loss": 0.0188, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -4.043432712554932, |
| "rewards/margins": 15.83984375, |
| "rewards/rejected": -19.890625, |
| "step": 1880 |
| }, |
| { |
| "epoch": 2.4325716124879304, |
| "grad_norm": 10.340676509370857, |
| "learning_rate": 3.91430412371134e-07, |
| "logits/chosen": -0.9385986328125, |
| "logits/rejected": -0.912792980670929, |
| "logps/chosen": -400.6000061035156, |
| "logps/rejected": -534.9249877929688, |
| "loss": 0.0357, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -4.141528129577637, |
| "rewards/margins": 14.765625, |
| "rewards/rejected": -18.901561737060547, |
| "step": 1890 |
| }, |
| { |
| "epoch": 2.4454457676215, |
| "grad_norm": 0.1837428615330857, |
| "learning_rate": 3.882087628865979e-07, |
| "logits/chosen": -0.942370593547821, |
| "logits/rejected": -0.9447265863418579, |
| "logps/chosen": -406.9937438964844, |
| "logps/rejected": -544.5250244140625, |
| "loss": 0.0395, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": -3.229174852371216, |
| "rewards/margins": 15.078125, |
| "rewards/rejected": -18.3046875, |
| "step": 1900 |
| }, |
| { |
| "epoch": 2.458319922755069, |
| "grad_norm": 0.06528086713553832, |
| "learning_rate": 3.8498711340206184e-07, |
| "logits/chosen": -1.0470702648162842, |
| "logits/rejected": -1.0382812023162842, |
| "logps/chosen": -407.82501220703125, |
| "logps/rejected": -526.8499755859375, |
| "loss": 0.0141, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -3.409912109375, |
| "rewards/margins": 14.985937118530273, |
| "rewards/rejected": -18.401561737060547, |
| "step": 1910 |
| }, |
| { |
| "epoch": 2.4711940778886383, |
| "grad_norm": 12.890073513058875, |
| "learning_rate": 3.8176546391752575e-07, |
| "logits/chosen": -1.028564453125, |
| "logits/rejected": -1.0325927734375, |
| "logps/chosen": -411.04998779296875, |
| "logps/rejected": -546.2000122070312, |
| "loss": 0.0356, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -3.9456543922424316, |
| "rewards/margins": 15.208593368530273, |
| "rewards/rejected": -19.146875381469727, |
| "step": 1920 |
| }, |
| { |
| "epoch": 2.484068233022208, |
| "grad_norm": 2.4135445503560464, |
| "learning_rate": 3.7854381443298966e-07, |
| "logits/chosen": -1.056909203529358, |
| "logits/rejected": -1.024682641029358, |
| "logps/chosen": -431.5249938964844, |
| "logps/rejected": -563.25, |
| "loss": 0.0294, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -4.433886528015137, |
| "rewards/margins": 15.984375, |
| "rewards/rejected": -20.426563262939453, |
| "step": 1930 |
| }, |
| { |
| "epoch": 2.4969423881557775, |
| "grad_norm": 4.32141413981977, |
| "learning_rate": 3.7532216494845357e-07, |
| "logits/chosen": -1.046728491783142, |
| "logits/rejected": -1.02490234375, |
| "logps/chosen": -403.0249938964844, |
| "logps/rejected": -542.0499877929688, |
| "loss": 0.0218, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -4.49169921875, |
| "rewards/margins": 15.235937118530273, |
| "rewards/rejected": -19.725000381469727, |
| "step": 1940 |
| }, |
| { |
| "epoch": 2.5098165432893467, |
| "grad_norm": 1.4656812942309294, |
| "learning_rate": 3.7210051546391753e-07, |
| "logits/chosen": -1.004663109779358, |
| "logits/rejected": -0.996167004108429, |
| "logps/chosen": -426.2124938964844, |
| "logps/rejected": -551.2999877929688, |
| "loss": 0.0264, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -5.308349609375, |
| "rewards/margins": 14.5546875, |
| "rewards/rejected": -19.862499237060547, |
| "step": 1950 |
| }, |
| { |
| "epoch": 2.522690698422916, |
| "grad_norm": 0.06357133072392389, |
| "learning_rate": 3.6887886597938144e-07, |
| "logits/chosen": -1.0270264148712158, |
| "logits/rejected": -0.9864746332168579, |
| "logps/chosen": -407.70001220703125, |
| "logps/rejected": -538.8499755859375, |
| "loss": 0.0233, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -5.082373142242432, |
| "rewards/margins": 14.225000381469727, |
| "rewards/rejected": -19.306249618530273, |
| "step": 1960 |
| }, |
| { |
| "epoch": 2.5355648535564854, |
| "grad_norm": 0.17219392427815344, |
| "learning_rate": 3.656572164948453e-07, |
| "logits/chosen": -1.095312476158142, |
| "logits/rejected": -1.09765625, |
| "logps/chosen": -421.2875061035156, |
| "logps/rejected": -564.3499755859375, |
| "loss": 0.0244, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -4.148681640625, |
| "rewards/margins": 15.556249618530273, |
| "rewards/rejected": -19.692188262939453, |
| "step": 1970 |
| }, |
| { |
| "epoch": 2.5484390086900546, |
| "grad_norm": 0.02503924044752003, |
| "learning_rate": 3.6243556701030926e-07, |
| "logits/chosen": -1.0810546875, |
| "logits/rejected": -1.093164086341858, |
| "logps/chosen": -416.79998779296875, |
| "logps/rejected": -556.8499755859375, |
| "loss": 0.011, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -4.688818454742432, |
| "rewards/margins": 15.245312690734863, |
| "rewards/rejected": -19.932811737060547, |
| "step": 1980 |
| }, |
| { |
| "epoch": 2.561313163823624, |
| "grad_norm": 0.055260003017054, |
| "learning_rate": 3.5921391752577317e-07, |
| "logits/chosen": -1.043115258216858, |
| "logits/rejected": -1.0253417491912842, |
| "logps/chosen": -403.07501220703125, |
| "logps/rejected": -537.7750244140625, |
| "loss": 0.0178, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -5.153515815734863, |
| "rewards/margins": 15.246874809265137, |
| "rewards/rejected": -20.393749237060547, |
| "step": 1990 |
| }, |
| { |
| "epoch": 2.5741873189571933, |
| "grad_norm": 1.4980599504146637, |
| "learning_rate": 3.5599226804123713e-07, |
| "logits/chosen": -1.001977562904358, |
| "logits/rejected": -0.985791027545929, |
| "logps/chosen": -403.82501220703125, |
| "logps/rejected": -505.79998779296875, |
| "loss": 0.0391, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": -4.720019340515137, |
| "rewards/margins": 13.964062690734863, |
| "rewards/rejected": -18.684375762939453, |
| "step": 2000 |
| }, |
| { |
| "epoch": 2.587061474090763, |
| "grad_norm": 85.99081877050921, |
| "learning_rate": 3.5277061855670104e-07, |
| "logits/chosen": -0.9850708246231079, |
| "logits/rejected": -0.9672485589981079, |
| "logps/chosen": -401.7875061035156, |
| "logps/rejected": -523.0750122070312, |
| "loss": 0.0526, |
| "rewards/accuracies": 0.9312499761581421, |
| "rewards/chosen": -3.9986329078674316, |
| "rewards/margins": 14.18359375, |
| "rewards/rejected": -18.178905487060547, |
| "step": 2010 |
| }, |
| { |
| "epoch": 2.599935629224332, |
| "grad_norm": 181.98998012124002, |
| "learning_rate": 3.495489690721649e-07, |
| "logits/chosen": -1.027734398841858, |
| "logits/rejected": -1.01318359375, |
| "logps/chosen": -444.2749938964844, |
| "logps/rejected": -566.0499877929688, |
| "loss": 0.0324, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -4.080468654632568, |
| "rewards/margins": 14.451562881469727, |
| "rewards/rejected": -18.526561737060547, |
| "step": 2020 |
| }, |
| { |
| "epoch": 2.6128097843579017, |
| "grad_norm": 3.7256787279315344, |
| "learning_rate": 3.4632731958762886e-07, |
| "logits/chosen": -1.019921898841858, |
| "logits/rejected": -0.9986816644668579, |
| "logps/chosen": -391.9375, |
| "logps/rejected": -520.6749877929688, |
| "loss": 0.0273, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -3.239990234375, |
| "rewards/margins": 14.225000381469727, |
| "rewards/rejected": -17.467187881469727, |
| "step": 2030 |
| }, |
| { |
| "epoch": 2.625683939491471, |
| "grad_norm": 0.6934846653919562, |
| "learning_rate": 3.4310567010309277e-07, |
| "logits/chosen": -1.037451148033142, |
| "logits/rejected": -1.0004150867462158, |
| "logps/chosen": -409.70001220703125, |
| "logps/rejected": -569.7000122070312, |
| "loss": 0.023, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -3.6155762672424316, |
| "rewards/margins": 16.117969512939453, |
| "rewards/rejected": -19.7265625, |
| "step": 2040 |
| }, |
| { |
| "epoch": 2.63855809462504, |
| "grad_norm": 1.084526871715855, |
| "learning_rate": 3.398840206185567e-07, |
| "logits/chosen": -1.072167992591858, |
| "logits/rejected": -1.0823242664337158, |
| "logps/chosen": -437.3125, |
| "logps/rejected": -534.2249755859375, |
| "loss": 0.0156, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -4.259228706359863, |
| "rewards/margins": 14.357030868530273, |
| "rewards/rejected": -18.607812881469727, |
| "step": 2050 |
| }, |
| { |
| "epoch": 2.6514322497586096, |
| "grad_norm": 0.13588939136218478, |
| "learning_rate": 3.3666237113402065e-07, |
| "logits/chosen": -0.9833984375, |
| "logits/rejected": -0.9649413824081421, |
| "logps/chosen": -417.70001220703125, |
| "logps/rejected": -553.5750122070312, |
| "loss": 0.027, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -4.4521484375, |
| "rewards/margins": 15.2890625, |
| "rewards/rejected": -19.746875762939453, |
| "step": 2060 |
| }, |
| { |
| "epoch": 2.664306404892179, |
| "grad_norm": 18.71919913613174, |
| "learning_rate": 3.334407216494845e-07, |
| "logits/chosen": -0.9654175043106079, |
| "logits/rejected": -0.951892077922821, |
| "logps/chosen": -411.0, |
| "logps/rejected": -526.0499877929688, |
| "loss": 0.0241, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -5.281836032867432, |
| "rewards/margins": 14.720312118530273, |
| "rewards/rejected": -19.996875762939453, |
| "step": 2070 |
| }, |
| { |
| "epoch": 2.6771805600257483, |
| "grad_norm": 0.1687335570953075, |
| "learning_rate": 3.302190721649484e-07, |
| "logits/chosen": -1.0874512195587158, |
| "logits/rejected": -1.0557129383087158, |
| "logps/chosen": -466.07501220703125, |
| "logps/rejected": -609.9000244140625, |
| "loss": 0.0229, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -5.810986518859863, |
| "rewards/margins": 15.785937309265137, |
| "rewards/rejected": -21.600000381469727, |
| "step": 2080 |
| }, |
| { |
| "epoch": 2.6900547151593175, |
| "grad_norm": 0.07496715000168404, |
| "learning_rate": 3.269974226804124e-07, |
| "logits/chosen": -1.061132788658142, |
| "logits/rejected": -1.0348632335662842, |
| "logps/chosen": -433.5, |
| "logps/rejected": -542.5499877929688, |
| "loss": 0.0264, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -5.439917087554932, |
| "rewards/margins": 14.901562690734863, |
| "rewards/rejected": -20.337499618530273, |
| "step": 2090 |
| }, |
| { |
| "epoch": 2.702928870292887, |
| "grad_norm": 3.342920359584719, |
| "learning_rate": 3.237757731958763e-07, |
| "logits/chosen": -1.0269043445587158, |
| "logits/rejected": -1.0263183116912842, |
| "logps/chosen": -451.07501220703125, |
| "logps/rejected": -550.3499755859375, |
| "loss": 0.0223, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -5.473876953125, |
| "rewards/margins": 14.681249618530273, |
| "rewards/rejected": -20.171875, |
| "step": 2100 |
| }, |
| { |
| "epoch": 2.7158030254264562, |
| "grad_norm": 0.41399028262679977, |
| "learning_rate": 3.205541237113402e-07, |
| "logits/chosen": -0.9803222417831421, |
| "logits/rejected": -0.970996081829071, |
| "logps/chosen": -416.45001220703125, |
| "logps/rejected": -556.625, |
| "loss": 0.0272, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -4.261034965515137, |
| "rewards/margins": 15.196093559265137, |
| "rewards/rejected": -19.4609375, |
| "step": 2110 |
| }, |
| { |
| "epoch": 2.728677180560026, |
| "grad_norm": 0.08328554014051173, |
| "learning_rate": 3.173324742268041e-07, |
| "logits/chosen": -1.0111510753631592, |
| "logits/rejected": -1.000097632408142, |
| "logps/chosen": -394.2250061035156, |
| "logps/rejected": -528.5250244140625, |
| "loss": 0.0367, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -3.413928270339966, |
| "rewards/margins": 14.673437118530273, |
| "rewards/rejected": -18.082813262939453, |
| "step": 2120 |
| }, |
| { |
| "epoch": 2.741551335693595, |
| "grad_norm": 0.997200008758714, |
| "learning_rate": 3.14110824742268e-07, |
| "logits/chosen": -0.987011730670929, |
| "logits/rejected": -0.9897216558456421, |
| "logps/chosen": -425.61248779296875, |
| "logps/rejected": -536.1500244140625, |
| "loss": 0.0334, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -2.638232469558716, |
| "rewards/margins": 13.8203125, |
| "rewards/rejected": -16.453125, |
| "step": 2130 |
| }, |
| { |
| "epoch": 2.7544254908271646, |
| "grad_norm": 0.07908592939561318, |
| "learning_rate": 3.1088917525773193e-07, |
| "logits/chosen": -1.01739501953125, |
| "logits/rejected": -1.0194091796875, |
| "logps/chosen": -412.67498779296875, |
| "logps/rejected": -543.7000122070312, |
| "loss": 0.0141, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -3.637744188308716, |
| "rewards/margins": 14.646875381469727, |
| "rewards/rejected": -18.279687881469727, |
| "step": 2140 |
| }, |
| { |
| "epoch": 2.7672996459607337, |
| "grad_norm": 0.31312077632852253, |
| "learning_rate": 3.076675257731959e-07, |
| "logits/chosen": -1.015039086341858, |
| "logits/rejected": -0.9869629144668579, |
| "logps/chosen": -412.2250061035156, |
| "logps/rejected": -571.9000244140625, |
| "loss": 0.0135, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -3.989062547683716, |
| "rewards/margins": 15.939062118530273, |
| "rewards/rejected": -19.928125381469727, |
| "step": 2150 |
| }, |
| { |
| "epoch": 2.7801738010943033, |
| "grad_norm": 0.5372266843183616, |
| "learning_rate": 3.0444587628865975e-07, |
| "logits/chosen": -0.9476562738418579, |
| "logits/rejected": -0.9249511957168579, |
| "logps/chosen": -433.79998779296875, |
| "logps/rejected": -548.9000244140625, |
| "loss": 0.0245, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -4.148535251617432, |
| "rewards/margins": 15.0546875, |
| "rewards/rejected": -19.196874618530273, |
| "step": 2160 |
| }, |
| { |
| "epoch": 2.7930479562278725, |
| "grad_norm": 0.21085049221881774, |
| "learning_rate": 3.0122422680412366e-07, |
| "logits/chosen": -1.0446288585662842, |
| "logits/rejected": -1.030908226966858, |
| "logps/chosen": -430.75, |
| "logps/rejected": -559.0999755859375, |
| "loss": 0.0361, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -4.50244140625, |
| "rewards/margins": 15.458593368530273, |
| "rewards/rejected": -19.962499618530273, |
| "step": 2170 |
| }, |
| { |
| "epoch": 2.8059221113614417, |
| "grad_norm": 0.07088843316040844, |
| "learning_rate": 2.980025773195876e-07, |
| "logits/chosen": -1.0450928211212158, |
| "logits/rejected": -1.036767601966858, |
| "logps/chosen": -428.4375, |
| "logps/rejected": -547.5499877929688, |
| "loss": 0.0176, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -4.658154487609863, |
| "rewards/margins": 15.4296875, |
| "rewards/rejected": -20.071874618530273, |
| "step": 2180 |
| }, |
| { |
| "epoch": 2.8187962664950112, |
| "grad_norm": 0.7000631568785479, |
| "learning_rate": 2.9478092783505153e-07, |
| "logits/chosen": -1.0032227039337158, |
| "logits/rejected": -1.0255615711212158, |
| "logps/chosen": -431.125, |
| "logps/rejected": -551.9000244140625, |
| "loss": 0.0408, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -4.879492282867432, |
| "rewards/margins": 14.317968368530273, |
| "rewards/rejected": -19.201562881469727, |
| "step": 2190 |
| }, |
| { |
| "epoch": 2.831670421628581, |
| "grad_norm": 10.34328122014099, |
| "learning_rate": 2.915592783505155e-07, |
| "logits/chosen": -0.921124279499054, |
| "logits/rejected": -0.9173828363418579, |
| "logps/chosen": -395.45001220703125, |
| "logps/rejected": -533.5999755859375, |
| "loss": 0.0529, |
| "rewards/accuracies": 0.9312499761581421, |
| "rewards/chosen": -5.427599906921387, |
| "rewards/margins": 14.371874809265137, |
| "rewards/rejected": -19.810937881469727, |
| "step": 2200 |
| }, |
| { |
| "epoch": 2.84454457676215, |
| "grad_norm": 0.08482399874585603, |
| "learning_rate": 2.8833762886597935e-07, |
| "logits/chosen": -1.068017601966858, |
| "logits/rejected": -1.060449242591858, |
| "logps/chosen": -420.1000061035156, |
| "logps/rejected": -585.4500122070312, |
| "loss": 0.0273, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -6.1865234375, |
| "rewards/margins": 15.414843559265137, |
| "rewards/rejected": -21.610937118530273, |
| "step": 2210 |
| }, |
| { |
| "epoch": 2.857418731895719, |
| "grad_norm": 0.3961966887737209, |
| "learning_rate": 2.8511597938144326e-07, |
| "logits/chosen": -1.036230444908142, |
| "logits/rejected": -1.0049316883087158, |
| "logps/chosen": -466.8999938964844, |
| "logps/rejected": -608.5499877929688, |
| "loss": 0.0307, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -6.053515434265137, |
| "rewards/margins": 15.393750190734863, |
| "rewards/rejected": -21.439062118530273, |
| "step": 2220 |
| }, |
| { |
| "epoch": 2.8702928870292888, |
| "grad_norm": 0.09519495396041948, |
| "learning_rate": 2.818943298969072e-07, |
| "logits/chosen": -1.0175049304962158, |
| "logits/rejected": -1.0093994140625, |
| "logps/chosen": -416.4750061035156, |
| "logps/rejected": -567.5999755859375, |
| "loss": 0.0275, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -5.609961032867432, |
| "rewards/margins": 15.884374618530273, |
| "rewards/rejected": -21.490625381469727, |
| "step": 2230 |
| }, |
| { |
| "epoch": 2.883167042162858, |
| "grad_norm": 0.5232996096645688, |
| "learning_rate": 2.7867268041237114e-07, |
| "logits/chosen": -0.9981933832168579, |
| "logits/rejected": -0.9861816167831421, |
| "logps/chosen": -430.7250061035156, |
| "logps/rejected": -597.5999755859375, |
| "loss": 0.0181, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -5.131201267242432, |
| "rewards/margins": 17.307811737060547, |
| "rewards/rejected": -22.4453125, |
| "step": 2240 |
| }, |
| { |
| "epoch": 2.8960411972964275, |
| "grad_norm": 0.7006994780430169, |
| "learning_rate": 2.7545103092783505e-07, |
| "logits/chosen": -1.0391662120819092, |
| "logits/rejected": -1.0439056158065796, |
| "logps/chosen": -442.2250061035156, |
| "logps/rejected": -572.9000244140625, |
| "loss": 0.0318, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -4.493456840515137, |
| "rewards/margins": 16.2109375, |
| "rewards/rejected": -20.706249237060547, |
| "step": 2250 |
| }, |
| { |
| "epoch": 2.9089153524299967, |
| "grad_norm": 180.74744309920393, |
| "learning_rate": 2.7222938144329896e-07, |
| "logits/chosen": -0.9822998046875, |
| "logits/rejected": -0.9486083984375, |
| "logps/chosen": -439.6499938964844, |
| "logps/rejected": -573.0, |
| "loss": 0.0181, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -5.12890625, |
| "rewards/margins": 16.4921875, |
| "rewards/rejected": -21.623437881469727, |
| "step": 2260 |
| }, |
| { |
| "epoch": 2.9217895075635663, |
| "grad_norm": 32.30656704261848, |
| "learning_rate": 2.6900773195876287e-07, |
| "logits/chosen": -1.033935546875, |
| "logits/rejected": -1.022558569908142, |
| "logps/chosen": -424.67498779296875, |
| "logps/rejected": -589.4500122070312, |
| "loss": 0.0319, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -4.392675876617432, |
| "rewards/margins": 16.057811737060547, |
| "rewards/rejected": -20.462499618530273, |
| "step": 2270 |
| }, |
| { |
| "epoch": 2.9346636626971354, |
| "grad_norm": 0.5484870470342832, |
| "learning_rate": 2.657860824742268e-07, |
| "logits/chosen": -0.999804675579071, |
| "logits/rejected": -0.9871581792831421, |
| "logps/chosen": -435.17498779296875, |
| "logps/rejected": -564.5, |
| "loss": 0.0273, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -4.9228515625, |
| "rewards/margins": 14.823437690734863, |
| "rewards/rejected": -19.743749618530273, |
| "step": 2280 |
| }, |
| { |
| "epoch": 2.947537817830705, |
| "grad_norm": 0.06661369102436356, |
| "learning_rate": 2.6256443298969074e-07, |
| "logits/chosen": -1.0869140625, |
| "logits/rejected": -1.0365722179412842, |
| "logps/chosen": -414.7749938964844, |
| "logps/rejected": -537.1500244140625, |
| "loss": 0.0223, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -4.404394626617432, |
| "rewards/margins": 16.225780487060547, |
| "rewards/rejected": -20.629688262939453, |
| "step": 2290 |
| }, |
| { |
| "epoch": 2.960411972964274, |
| "grad_norm": 0.38801027736265253, |
| "learning_rate": 2.5934278350515465e-07, |
| "logits/chosen": -0.953857421875, |
| "logits/rejected": -0.930737316608429, |
| "logps/chosen": -417.5249938964844, |
| "logps/rejected": -546.25, |
| "loss": 0.0278, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -5.3330078125, |
| "rewards/margins": 15.092187881469727, |
| "rewards/rejected": -20.431249618530273, |
| "step": 2300 |
| }, |
| { |
| "epoch": 2.9732861280978433, |
| "grad_norm": 0.5569756223752637, |
| "learning_rate": 2.561211340206185e-07, |
| "logits/chosen": -0.974780261516571, |
| "logits/rejected": -0.959912121295929, |
| "logps/chosen": -425.5249938964844, |
| "logps/rejected": -586.7000122070312, |
| "loss": 0.0455, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -4.039258003234863, |
| "rewards/margins": 16.984375, |
| "rewards/rejected": -21.012500762939453, |
| "step": 2310 |
| }, |
| { |
| "epoch": 2.986160283231413, |
| "grad_norm": 0.1140239873803737, |
| "learning_rate": 2.5289948453608247e-07, |
| "logits/chosen": -0.954150378704071, |
| "logits/rejected": -0.9415038824081421, |
| "logps/chosen": -446.0, |
| "logps/rejected": -591.0, |
| "loss": 0.0339, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -5.755810737609863, |
| "rewards/margins": 16.087499618530273, |
| "rewards/rejected": -21.845312118530273, |
| "step": 2320 |
| }, |
| { |
| "epoch": 2.9990344383649825, |
| "grad_norm": 0.23176827452269116, |
| "learning_rate": 2.496778350515464e-07, |
| "logits/chosen": -0.892468273639679, |
| "logits/rejected": -0.8869384527206421, |
| "logps/chosen": -395.25, |
| "logps/rejected": -524.5499877929688, |
| "loss": 0.0355, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -4.820703029632568, |
| "rewards/margins": 17.042186737060547, |
| "rewards/rejected": -21.870311737060547, |
| "step": 2330 |
| }, |
| { |
| "epoch": 3.0115867396202125, |
| "grad_norm": 0.04030797300705242, |
| "learning_rate": 2.464561855670103e-07, |
| "logits/chosen": -0.9333683848381042, |
| "logits/rejected": -0.9138621687889099, |
| "logps/chosen": -412.23077392578125, |
| "logps/rejected": -552.2051391601562, |
| "loss": 0.0346, |
| "rewards/accuracies": 0.9487179517745972, |
| "rewards/chosen": -5.357221603393555, |
| "rewards/margins": 15.293269157409668, |
| "rewards/rejected": -20.644229888916016, |
| "step": 2340 |
| }, |
| { |
| "epoch": 3.0244608947537817, |
| "grad_norm": 0.011972067245460228, |
| "learning_rate": 2.432345360824742e-07, |
| "logits/chosen": -1.035009741783142, |
| "logits/rejected": -0.9994140863418579, |
| "logps/chosen": -425.4750061035156, |
| "logps/rejected": -578.2000122070312, |
| "loss": 0.0218, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -4.936822414398193, |
| "rewards/margins": 16.629688262939453, |
| "rewards/rejected": -21.5546875, |
| "step": 2350 |
| }, |
| { |
| "epoch": 3.0373350498873513, |
| "grad_norm": 0.014209079031048687, |
| "learning_rate": 2.4001288659793816e-07, |
| "logits/chosen": -0.955126941204071, |
| "logits/rejected": -0.945361316204071, |
| "logps/chosen": -409.75, |
| "logps/rejected": -589.1500244140625, |
| "loss": 0.0087, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -4.591943264007568, |
| "rewards/margins": 18.920312881469727, |
| "rewards/rejected": -23.517187118530273, |
| "step": 2360 |
| }, |
| { |
| "epoch": 3.0502092050209204, |
| "grad_norm": 0.003246395439313634, |
| "learning_rate": 2.3679123711340205e-07, |
| "logits/chosen": -0.972424328327179, |
| "logits/rejected": -0.954345703125, |
| "logps/chosen": -430.32501220703125, |
| "logps/rejected": -595.8499755859375, |
| "loss": 0.013, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -5.572167873382568, |
| "rewards/margins": 17.579687118530273, |
| "rewards/rejected": -23.153125762939453, |
| "step": 2370 |
| }, |
| { |
| "epoch": 3.06308336015449, |
| "grad_norm": 0.01783900566903516, |
| "learning_rate": 2.3356958762886598e-07, |
| "logits/chosen": -0.9012451171875, |
| "logits/rejected": -0.8792480230331421, |
| "logps/chosen": -434.2749938964844, |
| "logps/rejected": -566.9500122070312, |
| "loss": 0.0131, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -5.078955173492432, |
| "rewards/margins": 15.729687690734863, |
| "rewards/rejected": -20.803125381469727, |
| "step": 2380 |
| }, |
| { |
| "epoch": 3.075957515288059, |
| "grad_norm": 0.01685872055087948, |
| "learning_rate": 2.3034793814432987e-07, |
| "logits/chosen": -0.8941406011581421, |
| "logits/rejected": -0.877490222454071, |
| "logps/chosen": -404.9125061035156, |
| "logps/rejected": -555.625, |
| "loss": 0.0217, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -5.329394340515137, |
| "rewards/margins": 16.1328125, |
| "rewards/rejected": -21.4609375, |
| "step": 2390 |
| }, |
| { |
| "epoch": 3.0888316704216288, |
| "grad_norm": 0.004614662175876239, |
| "learning_rate": 2.271262886597938e-07, |
| "logits/chosen": -1.0093262195587158, |
| "logits/rejected": -1.0094726085662842, |
| "logps/chosen": -451.75, |
| "logps/rejected": -598.8499755859375, |
| "loss": 0.0216, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -4.370312690734863, |
| "rewards/margins": 17.09375, |
| "rewards/rejected": -21.471874237060547, |
| "step": 2400 |
| }, |
| { |
| "epoch": 3.101705825555198, |
| "grad_norm": 0.010560387649899727, |
| "learning_rate": 2.2390463917525774e-07, |
| "logits/chosen": -0.912158191204071, |
| "logits/rejected": -0.93896484375, |
| "logps/chosen": -415.3500061035156, |
| "logps/rejected": -563.9749755859375, |
| "loss": 0.0182, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -5.893652439117432, |
| "rewards/margins": 16.9609375, |
| "rewards/rejected": -22.845312118530273, |
| "step": 2410 |
| }, |
| { |
| "epoch": 3.1145799806887675, |
| "grad_norm": 0.009728145604973306, |
| "learning_rate": 2.2068298969072162e-07, |
| "logits/chosen": -0.8603515625, |
| "logits/rejected": -0.8588622808456421, |
| "logps/chosen": -397.51251220703125, |
| "logps/rejected": -570.9249877929688, |
| "loss": 0.0088, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -5.475195407867432, |
| "rewards/margins": 17.790624618530273, |
| "rewards/rejected": -23.262500762939453, |
| "step": 2420 |
| }, |
| { |
| "epoch": 3.1274541358223367, |
| "grad_norm": 0.022851496718264388, |
| "learning_rate": 2.1746134020618556e-07, |
| "logits/chosen": -0.90869140625, |
| "logits/rejected": -0.901171863079071, |
| "logps/chosen": -409.5, |
| "logps/rejected": -572.25, |
| "loss": 0.0303, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -6.165625095367432, |
| "rewards/margins": 16.734375, |
| "rewards/rejected": -22.893749237060547, |
| "step": 2430 |
| }, |
| { |
| "epoch": 3.140328290955906, |
| "grad_norm": 0.2577532223041415, |
| "learning_rate": 2.1423969072164947e-07, |
| "logits/chosen": -0.9032226800918579, |
| "logits/rejected": -0.899731457233429, |
| "logps/chosen": -430.29998779296875, |
| "logps/rejected": -578.6500244140625, |
| "loss": 0.0263, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -5.7548828125, |
| "rewards/margins": 17.462499618530273, |
| "rewards/rejected": -23.220312118530273, |
| "step": 2440 |
| }, |
| { |
| "epoch": 3.1532024460894754, |
| "grad_norm": 0.7364005790474251, |
| "learning_rate": 2.1101804123711338e-07, |
| "logits/chosen": -0.976318359375, |
| "logits/rejected": -0.982128918170929, |
| "logps/chosen": -452.0, |
| "logps/rejected": -611.3499755859375, |
| "loss": 0.0087, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -6.922656059265137, |
| "rewards/margins": 17.292186737060547, |
| "rewards/rejected": -24.209375381469727, |
| "step": 2450 |
| }, |
| { |
| "epoch": 3.1660766012230446, |
| "grad_norm": 0.2985041143166853, |
| "learning_rate": 2.0779639175257732e-07, |
| "logits/chosen": -0.9498962163925171, |
| "logits/rejected": -0.9401611089706421, |
| "logps/chosen": -484.8500061035156, |
| "logps/rejected": -589.2999877929688, |
| "loss": 0.035, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -5.613110542297363, |
| "rewards/margins": 16.129688262939453, |
| "rewards/rejected": -21.748437881469727, |
| "step": 2460 |
| }, |
| { |
| "epoch": 3.178950756356614, |
| "grad_norm": 0.2869817908499183, |
| "learning_rate": 2.0457474226804123e-07, |
| "logits/chosen": -0.969921886920929, |
| "logits/rejected": -0.9568847417831421, |
| "logps/chosen": -450.20001220703125, |
| "logps/rejected": -599.5750122070312, |
| "loss": 0.0131, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -6.140234470367432, |
| "rewards/margins": 17.448436737060547, |
| "rewards/rejected": -23.598438262939453, |
| "step": 2470 |
| }, |
| { |
| "epoch": 3.1918249114901833, |
| "grad_norm": 0.04088909996359556, |
| "learning_rate": 2.0135309278350516e-07, |
| "logits/chosen": -0.958251953125, |
| "logits/rejected": -0.948974609375, |
| "logps/chosen": -490.25, |
| "logps/rejected": -668.0, |
| "loss": 0.0225, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -5.629956245422363, |
| "rewards/margins": 17.813282012939453, |
| "rewards/rejected": -23.443750381469727, |
| "step": 2480 |
| }, |
| { |
| "epoch": 3.204699066623753, |
| "grad_norm": 0.08684549219199517, |
| "learning_rate": 1.9813144329896905e-07, |
| "logits/chosen": -0.9354492425918579, |
| "logits/rejected": -0.907482922077179, |
| "logps/chosen": -438.79998779296875, |
| "logps/rejected": -593.4000244140625, |
| "loss": 0.0217, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -6.191357612609863, |
| "rewards/margins": 17.426563262939453, |
| "rewards/rejected": -23.637500762939453, |
| "step": 2490 |
| }, |
| { |
| "epoch": 3.217573221757322, |
| "grad_norm": 0.01725372117673741, |
| "learning_rate": 1.9490979381443298e-07, |
| "logits/chosen": -0.9265502691268921, |
| "logits/rejected": -0.91583251953125, |
| "logps/chosen": -365.125, |
| "logps/rejected": -559.0499877929688, |
| "loss": 0.0216, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -5.943749904632568, |
| "rewards/margins": 17.767187118530273, |
| "rewards/rejected": -23.715625762939453, |
| "step": 2500 |
| }, |
| { |
| "epoch": 3.2304473768908917, |
| "grad_norm": 0.010133221834613732, |
| "learning_rate": 1.9168814432989692e-07, |
| "logits/chosen": -0.9111572504043579, |
| "logits/rejected": -0.9021972417831421, |
| "logps/chosen": -419.7749938964844, |
| "logps/rejected": -557.8499755859375, |
| "loss": 0.0217, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -6.076904296875, |
| "rewards/margins": 16.378124237060547, |
| "rewards/rejected": -22.446874618530273, |
| "step": 2510 |
| }, |
| { |
| "epoch": 3.243321532024461, |
| "grad_norm": 0.024756280732985112, |
| "learning_rate": 1.884664948453608e-07, |
| "logits/chosen": -0.906176745891571, |
| "logits/rejected": -0.9052368402481079, |
| "logps/chosen": -460.57501220703125, |
| "logps/rejected": -616.2000122070312, |
| "loss": 0.0174, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -6.07861328125, |
| "rewards/margins": 17.920312881469727, |
| "rewards/rejected": -23.996875762939453, |
| "step": 2520 |
| }, |
| { |
| "epoch": 3.2561956871580304, |
| "grad_norm": 0.10786257268593104, |
| "learning_rate": 1.8524484536082474e-07, |
| "logits/chosen": -0.9489501714706421, |
| "logits/rejected": -0.9321533441543579, |
| "logps/chosen": -427.0, |
| "logps/rejected": -575.25, |
| "loss": 0.0274, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -6.699999809265137, |
| "rewards/margins": 17.05078125, |
| "rewards/rejected": -23.767187118530273, |
| "step": 2530 |
| }, |
| { |
| "epoch": 3.2690698422915996, |
| "grad_norm": 0.026345085205565732, |
| "learning_rate": 1.8202319587628865e-07, |
| "logits/chosen": -0.996826171875, |
| "logits/rejected": -0.998339831829071, |
| "logps/chosen": -443.1000061035156, |
| "logps/rejected": -621.7999877929688, |
| "loss": 0.013, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -6.483300685882568, |
| "rewards/margins": 18.181249618530273, |
| "rewards/rejected": -24.65625, |
| "step": 2540 |
| }, |
| { |
| "epoch": 3.281943997425169, |
| "grad_norm": 0.04457572944694021, |
| "learning_rate": 1.7880154639175256e-07, |
| "logits/chosen": -0.869128406047821, |
| "logits/rejected": -0.8661864995956421, |
| "logps/chosen": -410.375, |
| "logps/rejected": -556.5499877929688, |
| "loss": 0.0433, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -6.444140434265137, |
| "rewards/margins": 17.700000762939453, |
| "rewards/rejected": -24.139062881469727, |
| "step": 2550 |
| }, |
| { |
| "epoch": 3.2948181525587383, |
| "grad_norm": 0.03949142107866748, |
| "learning_rate": 1.7557989690721647e-07, |
| "logits/chosen": -0.8691161870956421, |
| "logits/rejected": -0.840991199016571, |
| "logps/chosen": -453.04998779296875, |
| "logps/rejected": -596.4500122070312, |
| "loss": 0.042, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": -5.3740234375, |
| "rewards/margins": 16.5234375, |
| "rewards/rejected": -21.896875381469727, |
| "step": 2560 |
| }, |
| { |
| "epoch": 3.3076923076923075, |
| "grad_norm": 0.031361210073259906, |
| "learning_rate": 1.723582474226804e-07, |
| "logits/chosen": -0.9446777105331421, |
| "logits/rejected": -0.921069324016571, |
| "logps/chosen": -458.07501220703125, |
| "logps/rejected": -588.4500122070312, |
| "loss": 0.0132, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -6.862548828125, |
| "rewards/margins": 16.87890625, |
| "rewards/rejected": -23.737499237060547, |
| "step": 2570 |
| }, |
| { |
| "epoch": 3.320566462825877, |
| "grad_norm": 0.008891302161788853, |
| "learning_rate": 1.6913659793814432e-07, |
| "logits/chosen": -0.942626953125, |
| "logits/rejected": -0.9102783203125, |
| "logps/chosen": -403.2562561035156, |
| "logps/rejected": -579.3125, |
| "loss": 0.0347, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -5.142187595367432, |
| "rewards/margins": 18.44140625, |
| "rewards/rejected": -23.569530487060547, |
| "step": 2580 |
| }, |
| { |
| "epoch": 3.3334406179594462, |
| "grad_norm": 0.017404878791276196, |
| "learning_rate": 1.6591494845360823e-07, |
| "logits/chosen": -0.902905285358429, |
| "logits/rejected": -0.8969970941543579, |
| "logps/chosen": -416.6499938964844, |
| "logps/rejected": -574.6500244140625, |
| "loss": 0.0132, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -6.773388862609863, |
| "rewards/margins": 16.15234375, |
| "rewards/rejected": -22.926563262939453, |
| "step": 2590 |
| }, |
| { |
| "epoch": 3.346314773093016, |
| "grad_norm": 0.021856131216759354, |
| "learning_rate": 1.6269329896907217e-07, |
| "logits/chosen": -0.9424804449081421, |
| "logits/rejected": -0.927441418170929, |
| "logps/chosen": -418.5874938964844, |
| "logps/rejected": -579.5750122070312, |
| "loss": 0.0303, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -6.185742378234863, |
| "rewards/margins": 17.252344131469727, |
| "rewards/rejected": -23.4453125, |
| "step": 2600 |
| }, |
| { |
| "epoch": 3.359188928226585, |
| "grad_norm": 0.014925207618162201, |
| "learning_rate": 1.5947164948453608e-07, |
| "logits/chosen": -0.932934582233429, |
| "logits/rejected": -0.905834972858429, |
| "logps/chosen": -457.17498779296875, |
| "logps/rejected": -600.9500122070312, |
| "loss": 0.026, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -5.945703029632568, |
| "rewards/margins": 16.714061737060547, |
| "rewards/rejected": -22.6640625, |
| "step": 2610 |
| }, |
| { |
| "epoch": 3.3720630833601546, |
| "grad_norm": 0.005844375059446013, |
| "learning_rate": 1.5624999999999999e-07, |
| "logits/chosen": -0.8345702886581421, |
| "logits/rejected": -0.8366943597793579, |
| "logps/chosen": -393.5249938964844, |
| "logps/rejected": -576.4500122070312, |
| "loss": 0.0347, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -5.985547065734863, |
| "rewards/margins": 16.9765625, |
| "rewards/rejected": -22.96875, |
| "step": 2620 |
| }, |
| { |
| "epoch": 3.3849372384937237, |
| "grad_norm": 109.07022759608414, |
| "learning_rate": 1.5302835051546392e-07, |
| "logits/chosen": -0.9147682189941406, |
| "logits/rejected": -0.895800769329071, |
| "logps/chosen": -412.9750061035156, |
| "logps/rejected": -585.1500244140625, |
| "loss": 0.0378, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -5.943163871765137, |
| "rewards/margins": 16.678125381469727, |
| "rewards/rejected": -22.629688262939453, |
| "step": 2630 |
| }, |
| { |
| "epoch": 3.3978113936272933, |
| "grad_norm": 0.02354853531279238, |
| "learning_rate": 1.4980670103092783e-07, |
| "logits/chosen": -0.981213390827179, |
| "logits/rejected": -0.9670654535293579, |
| "logps/chosen": -484.3999938964844, |
| "logps/rejected": -592.5999755859375, |
| "loss": 0.0176, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -6.2783203125, |
| "rewards/margins": 16.71875, |
| "rewards/rejected": -23.001562118530273, |
| "step": 2640 |
| }, |
| { |
| "epoch": 3.4106855487608625, |
| "grad_norm": 0.023273211343261376, |
| "learning_rate": 1.4658505154639174e-07, |
| "logits/chosen": -0.9508301019668579, |
| "logits/rejected": -0.9560546875, |
| "logps/chosen": -414.2250061035156, |
| "logps/rejected": -592.4000244140625, |
| "loss": 0.0087, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -6.26123046875, |
| "rewards/margins": 18.1796875, |
| "rewards/rejected": -24.450000762939453, |
| "step": 2650 |
| }, |
| { |
| "epoch": 3.423559703894432, |
| "grad_norm": 0.029222841155265352, |
| "learning_rate": 1.4336340206185565e-07, |
| "logits/chosen": -1.0565917491912842, |
| "logits/rejected": -1.0272216796875, |
| "logps/chosen": -450.79998779296875, |
| "logps/rejected": -590.4000244140625, |
| "loss": 0.0216, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -5.90673828125, |
| "rewards/margins": 16.371875762939453, |
| "rewards/rejected": -22.2890625, |
| "step": 2660 |
| }, |
| { |
| "epoch": 3.4364338590280012, |
| "grad_norm": 0.014108471125617151, |
| "learning_rate": 1.401417525773196e-07, |
| "logits/chosen": -0.9363158941268921, |
| "logits/rejected": -0.9248901605606079, |
| "logps/chosen": -404.20001220703125, |
| "logps/rejected": -574.1500244140625, |
| "loss": 0.0389, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": -6.414563179016113, |
| "rewards/margins": 16.696874618530273, |
| "rewards/rejected": -23.107812881469727, |
| "step": 2670 |
| }, |
| { |
| "epoch": 3.449308014161571, |
| "grad_norm": 0.023177327589133483, |
| "learning_rate": 1.369201030927835e-07, |
| "logits/chosen": -0.9132324457168579, |
| "logits/rejected": -0.8884521722793579, |
| "logps/chosen": -407.0, |
| "logps/rejected": -574.5, |
| "loss": 0.0224, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -6.184960842132568, |
| "rewards/margins": 16.817188262939453, |
| "rewards/rejected": -23.001562118530273, |
| "step": 2680 |
| }, |
| { |
| "epoch": 3.46218216929514, |
| "grad_norm": 0.008481541035888551, |
| "learning_rate": 1.336984536082474e-07, |
| "logits/chosen": -0.9932616949081421, |
| "logits/rejected": -0.978710949420929, |
| "logps/chosen": -453.04998779296875, |
| "logps/rejected": -601.9500122070312, |
| "loss": 0.0049, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -5.75390625, |
| "rewards/margins": 18.299999237060547, |
| "rewards/rejected": -24.037500381469727, |
| "step": 2690 |
| }, |
| { |
| "epoch": 3.475056324428709, |
| "grad_norm": 0.26185590688201954, |
| "learning_rate": 1.3047680412371135e-07, |
| "logits/chosen": -0.9244629144668579, |
| "logits/rejected": -0.9140380620956421, |
| "logps/chosen": -403.98748779296875, |
| "logps/rejected": -566.2000122070312, |
| "loss": 0.0303, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -5.669335842132568, |
| "rewards/margins": 15.934374809265137, |
| "rewards/rejected": -21.596874237060547, |
| "step": 2700 |
| }, |
| { |
| "epoch": 3.4879304795622788, |
| "grad_norm": 0.13721565109158115, |
| "learning_rate": 1.2725515463917523e-07, |
| "logits/chosen": -0.8556152582168579, |
| "logits/rejected": -0.823779284954071, |
| "logps/chosen": -427.8999938964844, |
| "logps/rejected": -566.7000122070312, |
| "loss": 0.0392, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": -5.447851657867432, |
| "rewards/margins": 17.535938262939453, |
| "rewards/rejected": -22.975000381469727, |
| "step": 2710 |
| }, |
| { |
| "epoch": 3.5008046346958483, |
| "grad_norm": 0.0073649770109624054, |
| "learning_rate": 1.2403350515463917e-07, |
| "logits/chosen": -0.888256847858429, |
| "logits/rejected": -0.86865234375, |
| "logps/chosen": -408.2749938964844, |
| "logps/rejected": -548.5499877929688, |
| "loss": 0.048, |
| "rewards/accuracies": 0.9312499761581421, |
| "rewards/chosen": -5.692773342132568, |
| "rewards/margins": 17.471874237060547, |
| "rewards/rejected": -23.168750762939453, |
| "step": 2720 |
| }, |
| { |
| "epoch": 3.5136787898294175, |
| "grad_norm": 0.17569853974318173, |
| "learning_rate": 1.2081185567010308e-07, |
| "logits/chosen": -0.995800793170929, |
| "logits/rejected": -0.9750610589981079, |
| "logps/chosen": -477.54998779296875, |
| "logps/rejected": -588.5999755859375, |
| "loss": 0.0188, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -6.515722751617432, |
| "rewards/margins": 17.01171875, |
| "rewards/rejected": -23.518749237060547, |
| "step": 2730 |
| }, |
| { |
| "epoch": 3.5265529449629867, |
| "grad_norm": 0.02761008615338435, |
| "learning_rate": 1.17590206185567e-07, |
| "logits/chosen": -0.8585449457168579, |
| "logits/rejected": -0.8765014410018921, |
| "logps/chosen": -437.82501220703125, |
| "logps/rejected": -586.0999755859375, |
| "loss": 0.0353, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -6.2158203125, |
| "rewards/margins": 17.151561737060547, |
| "rewards/rejected": -23.360937118530273, |
| "step": 2740 |
| }, |
| { |
| "epoch": 3.5394271000965563, |
| "grad_norm": 11.66898424743255, |
| "learning_rate": 1.1436855670103092e-07, |
| "logits/chosen": -0.9114745855331421, |
| "logits/rejected": -0.9142822027206421, |
| "logps/chosen": -435.0249938964844, |
| "logps/rejected": -567.9749755859375, |
| "loss": 0.0276, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -6.442236423492432, |
| "rewards/margins": 17.442188262939453, |
| "rewards/rejected": -23.887500762939453, |
| "step": 2750 |
| }, |
| { |
| "epoch": 3.5523012552301254, |
| "grad_norm": 0.004880119483037049, |
| "learning_rate": 1.1114690721649483e-07, |
| "logits/chosen": -1.0186278820037842, |
| "logits/rejected": -1.0231444835662842, |
| "logps/chosen": -427.3500061035156, |
| "logps/rejected": -613.0499877929688, |
| "loss": 0.0259, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -7.323437690734863, |
| "rewards/margins": 18.084375381469727, |
| "rewards/rejected": -25.395313262939453, |
| "step": 2760 |
| }, |
| { |
| "epoch": 3.565175410363695, |
| "grad_norm": 0.07925246005961702, |
| "learning_rate": 1.0792525773195876e-07, |
| "logits/chosen": -0.949267566204071, |
| "logits/rejected": -0.93017578125, |
| "logps/chosen": -414.04998779296875, |
| "logps/rejected": -588.3499755859375, |
| "loss": 0.0432, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -7.015234470367432, |
| "rewards/margins": 17.198436737060547, |
| "rewards/rejected": -24.21875, |
| "step": 2770 |
| }, |
| { |
| "epoch": 3.578049565497264, |
| "grad_norm": 0.19368073236661892, |
| "learning_rate": 1.0470360824742268e-07, |
| "logits/chosen": -0.9741576910018921, |
| "logits/rejected": -0.9656127691268921, |
| "logps/chosen": -451.0375061035156, |
| "logps/rejected": -590.7999877929688, |
| "loss": 0.0259, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -6.281054496765137, |
| "rewards/margins": 16.860937118530273, |
| "rewards/rejected": -23.146875381469727, |
| "step": 2780 |
| }, |
| { |
| "epoch": 3.5909237206308338, |
| "grad_norm": 0.0023883308566828173, |
| "learning_rate": 1.0148195876288659e-07, |
| "logits/chosen": -0.9483276605606079, |
| "logits/rejected": -0.9399169683456421, |
| "logps/chosen": -424.0, |
| "logps/rejected": -585.25, |
| "loss": 0.026, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -6.641797065734863, |
| "rewards/margins": 17.5078125, |
| "rewards/rejected": -24.1640625, |
| "step": 2790 |
| }, |
| { |
| "epoch": 3.603797875764403, |
| "grad_norm": 0.06892660166670882, |
| "learning_rate": 9.826030927835051e-08, |
| "logits/chosen": -1.00048828125, |
| "logits/rejected": -0.9769287109375, |
| "logps/chosen": -482.375, |
| "logps/rejected": -592.6500244140625, |
| "loss": 0.0216, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -6.117089748382568, |
| "rewards/margins": 16.573436737060547, |
| "rewards/rejected": -22.678125381469727, |
| "step": 2800 |
| }, |
| { |
| "epoch": 3.6166720308979725, |
| "grad_norm": 0.05744424533947675, |
| "learning_rate": 9.503865979381443e-08, |
| "logits/chosen": -0.894519031047821, |
| "logits/rejected": -0.908447265625, |
| "logps/chosen": -415.67498779296875, |
| "logps/rejected": -574.2999877929688, |
| "loss": 0.0216, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -6.658593654632568, |
| "rewards/margins": 17.971874237060547, |
| "rewards/rejected": -24.634374618530273, |
| "step": 2810 |
| }, |
| { |
| "epoch": 3.6295461860315417, |
| "grad_norm": 0.19428536422265558, |
| "learning_rate": 9.181701030927835e-08, |
| "logits/chosen": -1.0341308116912842, |
| "logits/rejected": -1.0309569835662842, |
| "logps/chosen": -467.95001220703125, |
| "logps/rejected": -612.2000122070312, |
| "loss": 0.0389, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": -6.307226657867432, |
| "rewards/margins": 17.603124618530273, |
| "rewards/rejected": -23.903125762939453, |
| "step": 2820 |
| }, |
| { |
| "epoch": 3.642420341165111, |
| "grad_norm": 0.19115611112653982, |
| "learning_rate": 8.859536082474227e-08, |
| "logits/chosen": -0.9891357421875, |
| "logits/rejected": -0.9699951410293579, |
| "logps/chosen": -465.7250061035156, |
| "logps/rejected": -616.0499877929688, |
| "loss": 0.0389, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": -7.32843017578125, |
| "rewards/margins": 16.487499237060547, |
| "rewards/rejected": -23.821874618530273, |
| "step": 2830 |
| }, |
| { |
| "epoch": 3.6552944962986804, |
| "grad_norm": 0.0054378709746274664, |
| "learning_rate": 8.537371134020618e-08, |
| "logits/chosen": -0.9260498285293579, |
| "logits/rejected": -0.933642566204071, |
| "logps/chosen": -429.7749938964844, |
| "logps/rejected": -589.6749877929688, |
| "loss": 0.0391, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": -7.364355564117432, |
| "rewards/margins": 17.025781631469727, |
| "rewards/rejected": -24.393749237060547, |
| "step": 2840 |
| }, |
| { |
| "epoch": 3.66816865143225, |
| "grad_norm": 0.7156365283310772, |
| "learning_rate": 8.21520618556701e-08, |
| "logits/chosen": -0.91796875, |
| "logits/rejected": -0.901538074016571, |
| "logps/chosen": -438.5249938964844, |
| "logps/rejected": -601.7999877929688, |
| "loss": 0.0519, |
| "rewards/accuracies": 0.925000011920929, |
| "rewards/chosen": -6.447070121765137, |
| "rewards/margins": 17.762500762939453, |
| "rewards/rejected": -24.206249237060547, |
| "step": 2850 |
| }, |
| { |
| "epoch": 3.681042806565819, |
| "grad_norm": 0.03899053476143833, |
| "learning_rate": 7.893041237113402e-08, |
| "logits/chosen": -0.9479004144668579, |
| "logits/rejected": -0.9349365234375, |
| "logps/chosen": -460.54998779296875, |
| "logps/rejected": -621.7999877929688, |
| "loss": 0.0311, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -7.09765625, |
| "rewards/margins": 18.182811737060547, |
| "rewards/rejected": -25.298437118530273, |
| "step": 2860 |
| }, |
| { |
| "epoch": 3.6939169616993883, |
| "grad_norm": 0.41126203127443395, |
| "learning_rate": 7.570876288659793e-08, |
| "logits/chosen": -1.0304687023162842, |
| "logits/rejected": -1.018457055091858, |
| "logps/chosen": -480.1499938964844, |
| "logps/rejected": -617.9000244140625, |
| "loss": 0.009, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -6.525000095367432, |
| "rewards/margins": 18.549999237060547, |
| "rewards/rejected": -25.092187881469727, |
| "step": 2870 |
| }, |
| { |
| "epoch": 3.706791116832958, |
| "grad_norm": 0.4478569386073059, |
| "learning_rate": 7.248711340206186e-08, |
| "logits/chosen": -0.938159167766571, |
| "logits/rejected": -0.941943347454071, |
| "logps/chosen": -460.8500061035156, |
| "logps/rejected": -612.7000122070312, |
| "loss": 0.0346, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -7.170507907867432, |
| "rewards/margins": 16.8828125, |
| "rewards/rejected": -24.060937881469727, |
| "step": 2880 |
| }, |
| { |
| "epoch": 3.719665271966527, |
| "grad_norm": 0.01080667980438113, |
| "learning_rate": 6.926546391752577e-08, |
| "logits/chosen": -0.9442809820175171, |
| "logits/rejected": -0.935223400592804, |
| "logps/chosen": -422.2250061035156, |
| "logps/rejected": -603.4500122070312, |
| "loss": 0.0217, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -7.079492092132568, |
| "rewards/margins": 18.901561737060547, |
| "rewards/rejected": -25.981250762939453, |
| "step": 2890 |
| }, |
| { |
| "epoch": 3.7325394271000967, |
| "grad_norm": 0.03399816651851313, |
| "learning_rate": 6.604381443298968e-08, |
| "logits/chosen": -0.965527355670929, |
| "logits/rejected": -0.977587878704071, |
| "logps/chosen": -444.4750061035156, |
| "logps/rejected": -602.9500122070312, |
| "loss": 0.03, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -7.051367282867432, |
| "rewards/margins": 17.271875381469727, |
| "rewards/rejected": -24.328125, |
| "step": 2900 |
| }, |
| { |
| "epoch": 3.745413582233666, |
| "grad_norm": 0.04098255840783047, |
| "learning_rate": 6.28221649484536e-08, |
| "logits/chosen": -0.9448486566543579, |
| "logits/rejected": -0.95050048828125, |
| "logps/chosen": -414.125, |
| "logps/rejected": -563.75, |
| "loss": 0.0144, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -6.292578220367432, |
| "rewards/margins": 17.878124237060547, |
| "rewards/rejected": -24.176563262939453, |
| "step": 2910 |
| }, |
| { |
| "epoch": 3.7582877373672354, |
| "grad_norm": 0.10065251851869494, |
| "learning_rate": 5.960051546391753e-08, |
| "logits/chosen": -0.9204956293106079, |
| "logits/rejected": -0.922607421875, |
| "logps/chosen": -433.3500061035156, |
| "logps/rejected": -584.4500122070312, |
| "loss": 0.041, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -6.477929592132568, |
| "rewards/margins": 17.8828125, |
| "rewards/rejected": -24.348438262939453, |
| "step": 2920 |
| }, |
| { |
| "epoch": 3.7711618925008046, |
| "grad_norm": 0.00807285233362444, |
| "learning_rate": 5.637886597938144e-08, |
| "logits/chosen": -0.885009765625, |
| "logits/rejected": -0.8709716796875, |
| "logps/chosen": -433.4750061035156, |
| "logps/rejected": -583.4000244140625, |
| "loss": 0.0389, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": -6.649609565734863, |
| "rewards/margins": 17.6484375, |
| "rewards/rejected": -24.296875, |
| "step": 2930 |
| }, |
| { |
| "epoch": 3.784036047634374, |
| "grad_norm": 0.4957017627720933, |
| "learning_rate": 5.3157216494845357e-08, |
| "logits/chosen": -0.9778808355331421, |
| "logits/rejected": -0.9657226800918579, |
| "logps/chosen": -444.82501220703125, |
| "logps/rejected": -590.25, |
| "loss": 0.0173, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -6.065234184265137, |
| "rewards/margins": 17.642187118530273, |
| "rewards/rejected": -23.71875, |
| "step": 2940 |
| }, |
| { |
| "epoch": 3.7969102027679433, |
| "grad_norm": 0.055678910275804926, |
| "learning_rate": 4.993556701030928e-08, |
| "logits/chosen": -0.9195556640625, |
| "logits/rejected": -0.902905285358429, |
| "logps/chosen": -437.20001220703125, |
| "logps/rejected": -604.5, |
| "loss": 0.0451, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -6.924023628234863, |
| "rewards/margins": 16.248437881469727, |
| "rewards/rejected": -23.171875, |
| "step": 2950 |
| }, |
| { |
| "epoch": 3.8097843579015125, |
| "grad_norm": 0.009201975484736785, |
| "learning_rate": 4.67139175257732e-08, |
| "logits/chosen": -0.957104504108429, |
| "logits/rejected": -0.9645019769668579, |
| "logps/chosen": -465.8999938964844, |
| "logps/rejected": -607.7000122070312, |
| "loss": 0.0267, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -5.574267387390137, |
| "rewards/margins": 19.049999237060547, |
| "rewards/rejected": -24.634374618530273, |
| "step": 2960 |
| }, |
| { |
| "epoch": 3.822658513035082, |
| "grad_norm": 0.011481163313679986, |
| "learning_rate": 4.349226804123711e-08, |
| "logits/chosen": -0.9524902105331421, |
| "logits/rejected": -0.9425293207168579, |
| "logps/chosen": -419.1875, |
| "logps/rejected": -583.7999877929688, |
| "loss": 0.0218, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -6.461523532867432, |
| "rewards/margins": 17.15625, |
| "rewards/rejected": -23.610937118530273, |
| "step": 2970 |
| }, |
| { |
| "epoch": 3.8355326681686517, |
| "grad_norm": 0.448772967049888, |
| "learning_rate": 4.027061855670103e-08, |
| "logits/chosen": -0.8423095941543579, |
| "logits/rejected": -0.84716796875, |
| "logps/chosen": -422.95001220703125, |
| "logps/rejected": -553.5999755859375, |
| "loss": 0.0308, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -6.774609565734863, |
| "rewards/margins": 17.057811737060547, |
| "rewards/rejected": -23.846874237060547, |
| "step": 2980 |
| }, |
| { |
| "epoch": 3.848406823302221, |
| "grad_norm": 0.018685653504530834, |
| "learning_rate": 3.704896907216495e-08, |
| "logits/chosen": -0.8719238042831421, |
| "logits/rejected": -0.865161120891571, |
| "logps/chosen": -433.1499938964844, |
| "logps/rejected": -584.7000122070312, |
| "loss": 0.0401, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": -6.331408500671387, |
| "rewards/margins": 16.582813262939453, |
| "rewards/rejected": -22.91015625, |
| "step": 2990 |
| }, |
| { |
| "epoch": 3.86128097843579, |
| "grad_norm": 0.007926715342673765, |
| "learning_rate": 3.3827319587628864e-08, |
| "logits/chosen": -1.019750952720642, |
| "logits/rejected": -1.0137450695037842, |
| "logps/chosen": -481.2250061035156, |
| "logps/rejected": -622.2000122070312, |
| "loss": 0.0173, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -6.560217380523682, |
| "rewards/margins": 18.018749237060547, |
| "rewards/rejected": -24.575000762939453, |
| "step": 3000 |
| }, |
| { |
| "epoch": 3.8741551335693596, |
| "grad_norm": 0.0065490532563155885, |
| "learning_rate": 3.060567010309278e-08, |
| "logits/chosen": -0.9202514886856079, |
| "logits/rejected": -0.8963623046875, |
| "logps/chosen": -411.1000061035156, |
| "logps/rejected": -550.4000244140625, |
| "loss": 0.0267, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -7.885351657867432, |
| "rewards/margins": 16.58203125, |
| "rewards/rejected": -24.4609375, |
| "step": 3010 |
| }, |
| { |
| "epoch": 3.8870292887029287, |
| "grad_norm": 0.01802235010361078, |
| "learning_rate": 2.73840206185567e-08, |
| "logits/chosen": -1.0900390148162842, |
| "logits/rejected": -1.0775146484375, |
| "logps/chosen": -432.2250061035156, |
| "logps/rejected": -624.7999877929688, |
| "loss": 0.013, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -7.279541015625, |
| "rewards/margins": 19.043750762939453, |
| "rewards/rejected": -26.329687118530273, |
| "step": 3020 |
| }, |
| { |
| "epoch": 3.8999034438364983, |
| "grad_norm": 0.004374947890940602, |
| "learning_rate": 2.4162371134020617e-08, |
| "logits/chosen": -0.9076904058456421, |
| "logits/rejected": -0.9190429449081421, |
| "logps/chosen": -424.32501220703125, |
| "logps/rejected": -582.8499755859375, |
| "loss": 0.0173, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -6.466504096984863, |
| "rewards/margins": 17.926563262939453, |
| "rewards/rejected": -24.399999618530273, |
| "step": 3030 |
| }, |
| { |
| "epoch": 3.9127775989700675, |
| "grad_norm": 0.0060379466399444, |
| "learning_rate": 2.0940721649484534e-08, |
| "logits/chosen": -1.004003882408142, |
| "logits/rejected": -1.003271460533142, |
| "logps/chosen": -484.04998779296875, |
| "logps/rejected": -650.7999877929688, |
| "loss": 0.0306, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -6.380663871765137, |
| "rewards/margins": 18.453125, |
| "rewards/rejected": -24.842187881469727, |
| "step": 3040 |
| }, |
| { |
| "epoch": 3.925651754103637, |
| "grad_norm": 1.2095296649438, |
| "learning_rate": 1.771907216494845e-08, |
| "logits/chosen": -0.9027954339981079, |
| "logits/rejected": -0.8821441531181335, |
| "logps/chosen": -456.4375, |
| "logps/rejected": -596.0250244140625, |
| "loss": 0.026, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -6.953515529632568, |
| "rewards/margins": 18.3046875, |
| "rewards/rejected": -25.2578125, |
| "step": 3050 |
| }, |
| { |
| "epoch": 3.9385259092372062, |
| "grad_norm": 0.056068725721234, |
| "learning_rate": 1.4497422680412371e-08, |
| "logits/chosen": -0.9928649663925171, |
| "logits/rejected": -0.989697277545929, |
| "logps/chosen": -453.4750061035156, |
| "logps/rejected": -609.5499877929688, |
| "loss": 0.0309, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -7.050879001617432, |
| "rewards/margins": 16.801563262939453, |
| "rewards/rejected": -23.850000381469727, |
| "step": 3060 |
| }, |
| { |
| "epoch": 3.951400064370776, |
| "grad_norm": 0.28646859373193095, |
| "learning_rate": 1.1275773195876288e-08, |
| "logits/chosen": -1.015356421470642, |
| "logits/rejected": -1.011962890625, |
| "logps/chosen": -427.82501220703125, |
| "logps/rejected": -580.7000122070312, |
| "loss": 0.0173, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -6.504296779632568, |
| "rewards/margins": 17.173437118530273, |
| "rewards/rejected": -23.674999237060547, |
| "step": 3070 |
| }, |
| { |
| "epoch": 3.964274219504345, |
| "grad_norm": 0.23205040355122056, |
| "learning_rate": 8.054123711340205e-09, |
| "logits/chosen": -1.075097680091858, |
| "logits/rejected": -1.074121117591858, |
| "logps/chosen": -479.375, |
| "logps/rejected": -621.25, |
| "loss": 0.0087, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -7.373827934265137, |
| "rewards/margins": 17.626562118530273, |
| "rewards/rejected": -24.998437881469727, |
| "step": 3080 |
| }, |
| { |
| "epoch": 3.977148374637914, |
| "grad_norm": 0.03176959803882333, |
| "learning_rate": 4.832474226804124e-09, |
| "logits/chosen": -1.0000488758087158, |
| "logits/rejected": -0.9969726800918579, |
| "logps/chosen": -428.57501220703125, |
| "logps/rejected": -571.9500122070312, |
| "loss": 0.0173, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -6.996972560882568, |
| "rewards/margins": 17.692188262939453, |
| "rewards/rejected": -24.689062118530273, |
| "step": 3090 |
| }, |
| { |
| "epoch": 3.9900225297714838, |
| "grad_norm": 0.012454681894423131, |
| "learning_rate": 1.610824742268041e-09, |
| "logits/chosen": -1.0048828125, |
| "logits/rejected": -1.0068848133087158, |
| "logps/chosen": -428.17498779296875, |
| "logps/rejected": -616.25, |
| "loss": 0.0173, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -5.966845512390137, |
| "rewards/margins": 19.012500762939453, |
| "rewards/rejected": -24.996875762939453, |
| "step": 3100 |
| } |
| ], |
| "logging_steps": 10, |
| "max_steps": 3104, |
| "num_input_tokens_seen": 0, |
| "num_train_epochs": 4, |
| "save_steps": 500, |
| "stateful_callbacks": { |
| "TrainerControl": { |
| "args": { |
| "should_epoch_stop": false, |
| "should_evaluate": false, |
| "should_log": false, |
| "should_save": true, |
| "should_training_stop": true |
| }, |
| "attributes": {} |
| } |
| }, |
| "total_flos": 0.0, |
| "train_batch_size": 1, |
| "trial_name": null, |
| "trial_params": null |
| } |
|
|
