| { |
| "best_global_step": null, |
| "best_metric": null, |
| "best_model_checkpoint": null, |
| "epoch": 3.9977470228516254, |
| "eval_steps": 500, |
| "global_step": 6212, |
| "is_hyper_param_search": false, |
| "is_local_process_zero": true, |
| "is_world_process_zero": true, |
| "log_history": [ |
| { |
| "epoch": 0.006437077566784679, |
| "grad_norm": 120.16608464445075, |
| "learning_rate": 9.98551191242756e-07, |
| "logits/chosen": -0.5285888910293579, |
| "logits/rejected": -0.550219714641571, |
| "logps/chosen": -367.70001220703125, |
| "logps/rejected": -355.82501220703125, |
| "loss": 0.6736, |
| "rewards/accuracies": 0.3812499940395355, |
| "rewards/chosen": -0.07652588188648224, |
| "rewards/margins": 0.06703948974609375, |
| "rewards/rejected": -0.1435089111328125, |
| "step": 10 |
| }, |
| { |
| "epoch": 0.012874155133569359, |
| "grad_norm": 135.5669189775458, |
| "learning_rate": 9.96941403734707e-07, |
| "logits/chosen": -0.5475219488143921, |
| "logits/rejected": -0.556469738483429, |
| "logps/chosen": -368.625, |
| "logps/rejected": -344.0, |
| "loss": 0.6127, |
| "rewards/accuracies": 0.6499999761581421, |
| "rewards/chosen": -0.3770507872104645, |
| "rewards/margins": 0.3885040283203125, |
| "rewards/rejected": -0.7652343511581421, |
| "step": 20 |
| }, |
| { |
| "epoch": 0.01931123270035404, |
| "grad_norm": 137.23954540855013, |
| "learning_rate": 9.953316162266581e-07, |
| "logits/chosen": -0.4871459901332855, |
| "logits/rejected": -0.5256897211074829, |
| "logps/chosen": -347.4750061035156, |
| "logps/rejected": -340.875, |
| "loss": 0.5851, |
| "rewards/accuracies": 0.65625, |
| "rewards/chosen": -0.0062271119095385075, |
| "rewards/margins": 0.5819091796875, |
| "rewards/rejected": -0.5878967046737671, |
| "step": 30 |
| }, |
| { |
| "epoch": 0.025748310267138717, |
| "grad_norm": 162.8825460193566, |
| "learning_rate": 9.93721828718609e-07, |
| "logits/chosen": -0.540789783000946, |
| "logits/rejected": -0.5523315668106079, |
| "logps/chosen": -392.82501220703125, |
| "logps/rejected": -339.5874938964844, |
| "loss": 0.6844, |
| "rewards/accuracies": 0.5687500238418579, |
| "rewards/chosen": -0.02431030198931694, |
| "rewards/margins": 0.4530700743198395, |
| "rewards/rejected": -0.4765075743198395, |
| "step": 40 |
| }, |
| { |
| "epoch": 0.0321853878339234, |
| "grad_norm": 111.36198187326185, |
| "learning_rate": 9.9211204121056e-07, |
| "logits/chosen": -0.5651000738143921, |
| "logits/rejected": -0.560577392578125, |
| "logps/chosen": -364.45001220703125, |
| "logps/rejected": -334.11248779296875, |
| "loss": 0.5565, |
| "rewards/accuracies": 0.699999988079071, |
| "rewards/chosen": 0.3750457763671875, |
| "rewards/margins": 0.7763427495956421, |
| "rewards/rejected": -0.401620477437973, |
| "step": 50 |
| }, |
| { |
| "epoch": 0.03862246540070808, |
| "grad_norm": 99.12262240934288, |
| "learning_rate": 9.905022537025112e-07, |
| "logits/chosen": -0.4527038633823395, |
| "logits/rejected": -0.4679809510707855, |
| "logps/chosen": -359.25, |
| "logps/rejected": -340.92498779296875, |
| "loss": 0.6116, |
| "rewards/accuracies": 0.6312500238418579, |
| "rewards/chosen": 0.714129626750946, |
| "rewards/margins": 0.6304260492324829, |
| "rewards/rejected": 0.08298339694738388, |
| "step": 60 |
| }, |
| { |
| "epoch": 0.04505954296749276, |
| "grad_norm": 88.34508629839479, |
| "learning_rate": 9.888924661944624e-07, |
| "logits/chosen": -0.48187255859375, |
| "logits/rejected": -0.49010008573532104, |
| "logps/chosen": -346.2250061035156, |
| "logps/rejected": -339.4624938964844, |
| "loss": 0.5153, |
| "rewards/accuracies": 0.7437499761581421, |
| "rewards/chosen": 0.822558581829071, |
| "rewards/margins": 0.859326183795929, |
| "rewards/rejected": -0.03675537183880806, |
| "step": 70 |
| }, |
| { |
| "epoch": 0.051496620534277435, |
| "grad_norm": 111.01162050170656, |
| "learning_rate": 9.872826786864134e-07, |
| "logits/chosen": -0.545391857624054, |
| "logits/rejected": -0.537060558795929, |
| "logps/chosen": -408.15625, |
| "logps/rejected": -402.45001220703125, |
| "loss": 0.5767, |
| "rewards/accuracies": 0.731249988079071, |
| "rewards/chosen": 0.3031005859375, |
| "rewards/margins": 0.9102783203125, |
| "rewards/rejected": -0.6064819097518921, |
| "step": 80 |
| }, |
| { |
| "epoch": 0.057933698101062116, |
| "grad_norm": 99.33785963112945, |
| "learning_rate": 9.856728911783643e-07, |
| "logits/chosen": -0.57373046875, |
| "logits/rejected": -0.5748535394668579, |
| "logps/chosen": -393.45001220703125, |
| "logps/rejected": -360.95001220703125, |
| "loss": 0.5902, |
| "rewards/accuracies": 0.6499999761581421, |
| "rewards/chosen": -0.21026000380516052, |
| "rewards/margins": 0.7870117425918579, |
| "rewards/rejected": -0.997509777545929, |
| "step": 90 |
| }, |
| { |
| "epoch": 0.0643707756678468, |
| "grad_norm": 99.12075537368256, |
| "learning_rate": 9.840631036703153e-07, |
| "logits/chosen": -0.523236095905304, |
| "logits/rejected": -0.5228271484375, |
| "logps/chosen": -348.42498779296875, |
| "logps/rejected": -317.54998779296875, |
| "loss": 0.6495, |
| "rewards/accuracies": 0.59375, |
| "rewards/chosen": -0.176036074757576, |
| "rewards/margins": 0.4732727110385895, |
| "rewards/rejected": -0.6496826410293579, |
| "step": 100 |
| }, |
| { |
| "epoch": 0.07080785323463147, |
| "grad_norm": 115.52409219899631, |
| "learning_rate": 9.824533161622665e-07, |
| "logits/chosen": -0.4966674745082855, |
| "logits/rejected": -0.5160156488418579, |
| "logps/chosen": -362.0249938964844, |
| "logps/rejected": -325.75, |
| "loss": 0.6544, |
| "rewards/accuracies": 0.5874999761581421, |
| "rewards/chosen": 0.4094482362270355, |
| "rewards/margins": 0.36219483613967896, |
| "rewards/rejected": 0.0468292236328125, |
| "step": 110 |
| }, |
| { |
| "epoch": 0.07724493080141616, |
| "grad_norm": 108.91370235062881, |
| "learning_rate": 9.808435286542177e-07, |
| "logits/chosen": -0.533123791217804, |
| "logits/rejected": -0.561358630657196, |
| "logps/chosen": -368.6499938964844, |
| "logps/rejected": -338.875, |
| "loss": 0.5637, |
| "rewards/accuracies": 0.699999988079071, |
| "rewards/chosen": 0.9539794921875, |
| "rewards/margins": 0.5897582769393921, |
| "rewards/rejected": 0.3646179139614105, |
| "step": 120 |
| }, |
| { |
| "epoch": 0.08368200836820083, |
| "grad_norm": 86.74166714430766, |
| "learning_rate": 9.792337411461686e-07, |
| "logits/chosen": -0.45283204317092896, |
| "logits/rejected": -0.47824400663375854, |
| "logps/chosen": -350.3999938964844, |
| "logps/rejected": -337.0249938964844, |
| "loss": 0.647, |
| "rewards/accuracies": 0.5687500238418579, |
| "rewards/chosen": 0.527966320514679, |
| "rewards/margins": 0.5209716558456421, |
| "rewards/rejected": 0.0069335936568677425, |
| "step": 130 |
| }, |
| { |
| "epoch": 0.09011908593498552, |
| "grad_norm": 103.64877138648835, |
| "learning_rate": 9.776239536381196e-07, |
| "logits/chosen": -0.5349365472793579, |
| "logits/rejected": -0.564208984375, |
| "logps/chosen": -401.625, |
| "logps/rejected": -352.07501220703125, |
| "loss": 0.5847, |
| "rewards/accuracies": 0.6812499761581421, |
| "rewards/chosen": -0.16360779106616974, |
| "rewards/margins": 0.592480480670929, |
| "rewards/rejected": -0.7553344964981079, |
| "step": 140 |
| }, |
| { |
| "epoch": 0.0965561635017702, |
| "grad_norm": 116.65061603445305, |
| "learning_rate": 9.760141661300708e-07, |
| "logits/chosen": -0.4708251953125, |
| "logits/rejected": -0.4907592833042145, |
| "logps/chosen": -364.95001220703125, |
| "logps/rejected": -327.5249938964844, |
| "loss": 0.5924, |
| "rewards/accuracies": 0.65625, |
| "rewards/chosen": 0.10672607272863388, |
| "rewards/margins": 0.5644897222518921, |
| "rewards/rejected": -0.4580444395542145, |
| "step": 150 |
| }, |
| { |
| "epoch": 0.10299324106855487, |
| "grad_norm": 81.12322692751495, |
| "learning_rate": 9.744043786220218e-07, |
| "logits/chosen": -0.5120483636856079, |
| "logits/rejected": -0.5198394656181335, |
| "logps/chosen": -359.8500061035156, |
| "logps/rejected": -337.11248779296875, |
| "loss": 0.5912, |
| "rewards/accuracies": 0.6812499761581421, |
| "rewards/chosen": 0.36287230253219604, |
| "rewards/margins": 0.609118640422821, |
| "rewards/rejected": -0.24659423530101776, |
| "step": 160 |
| }, |
| { |
| "epoch": 0.10943031863533956, |
| "grad_norm": 74.31516061519605, |
| "learning_rate": 9.72794591113973e-07, |
| "logits/chosen": -0.5851074457168579, |
| "logits/rejected": -0.5960327386856079, |
| "logps/chosen": -374.1000061035156, |
| "logps/rejected": -360.8500061035156, |
| "loss": 0.5686, |
| "rewards/accuracies": 0.637499988079071, |
| "rewards/chosen": 0.30054932832717896, |
| "rewards/margins": 0.8222290277481079, |
| "rewards/rejected": -0.5218139886856079, |
| "step": 170 |
| }, |
| { |
| "epoch": 0.11586739620212423, |
| "grad_norm": 113.20782415215908, |
| "learning_rate": 9.71184803605924e-07, |
| "logits/chosen": -0.5738281011581421, |
| "logits/rejected": -0.579742431640625, |
| "logps/chosen": -387.3500061035156, |
| "logps/rejected": -373.2250061035156, |
| "loss": 0.6087, |
| "rewards/accuracies": 0.675000011920929, |
| "rewards/chosen": 0.24147339165210724, |
| "rewards/margins": 0.787463366985321, |
| "rewards/rejected": -0.5462890863418579, |
| "step": 180 |
| }, |
| { |
| "epoch": 0.12230447376890892, |
| "grad_norm": 130.1431697433103, |
| "learning_rate": 9.69575016097875e-07, |
| "logits/chosen": -0.583453357219696, |
| "logits/rejected": -0.5979553461074829, |
| "logps/chosen": -372.29998779296875, |
| "logps/rejected": -363.6000061035156, |
| "loss": 0.6054, |
| "rewards/accuracies": 0.6625000238418579, |
| "rewards/chosen": 0.41649168729782104, |
| "rewards/margins": 1.0996825695037842, |
| "rewards/rejected": -0.683605968952179, |
| "step": 190 |
| }, |
| { |
| "epoch": 0.1287415513356936, |
| "grad_norm": 91.70486267908973, |
| "learning_rate": 9.67965228589826e-07, |
| "logits/chosen": -0.5898193120956421, |
| "logits/rejected": -0.587890625, |
| "logps/chosen": -408.375, |
| "logps/rejected": -347.3999938964844, |
| "loss": 0.5409, |
| "rewards/accuracies": 0.731249988079071, |
| "rewards/chosen": 0.648663341999054, |
| "rewards/margins": 1.29571533203125, |
| "rewards/rejected": -0.64654541015625, |
| "step": 200 |
| }, |
| { |
| "epoch": 0.13517862890247828, |
| "grad_norm": 130.11983466346078, |
| "learning_rate": 9.663554410817772e-07, |
| "logits/chosen": -0.4863647520542145, |
| "logits/rejected": -0.539746105670929, |
| "logps/chosen": -347.0375061035156, |
| "logps/rejected": -338.32501220703125, |
| "loss": 0.6107, |
| "rewards/accuracies": 0.643750011920929, |
| "rewards/chosen": 0.397308349609375, |
| "rewards/margins": 0.7805420160293579, |
| "rewards/rejected": -0.38372802734375, |
| "step": 210 |
| }, |
| { |
| "epoch": 0.14161570646926294, |
| "grad_norm": 75.88348717635424, |
| "learning_rate": 9.647456535737282e-07, |
| "logits/chosen": -0.4582275450229645, |
| "logits/rejected": -0.49046629667282104, |
| "logps/chosen": -374.8500061035156, |
| "logps/rejected": -359.04998779296875, |
| "loss": 0.5886, |
| "rewards/accuracies": 0.6625000238418579, |
| "rewards/chosen": 0.744860827922821, |
| "rewards/margins": 0.8912719488143921, |
| "rewards/rejected": -0.14583739638328552, |
| "step": 220 |
| }, |
| { |
| "epoch": 0.14805278403604763, |
| "grad_norm": 72.21523489539429, |
| "learning_rate": 9.631358660656794e-07, |
| "logits/chosen": -0.4630493223667145, |
| "logits/rejected": -0.4919677674770355, |
| "logps/chosen": -342.1875, |
| "logps/rejected": -331.42498779296875, |
| "loss": 0.6393, |
| "rewards/accuracies": 0.5874999761581421, |
| "rewards/chosen": 0.988513171672821, |
| "rewards/margins": 0.6120818853378296, |
| "rewards/rejected": 0.3760314881801605, |
| "step": 230 |
| }, |
| { |
| "epoch": 0.15448986160283232, |
| "grad_norm": 96.90032768184547, |
| "learning_rate": 9.615260785576303e-07, |
| "logits/chosen": -0.5086669921875, |
| "logits/rejected": -0.520275890827179, |
| "logps/chosen": -369.5375061035156, |
| "logps/rejected": -331.95001220703125, |
| "loss": 0.6711, |
| "rewards/accuracies": 0.606249988079071, |
| "rewards/chosen": 0.80767822265625, |
| "rewards/margins": 0.4611572325229645, |
| "rewards/rejected": 0.3460754454135895, |
| "step": 240 |
| }, |
| { |
| "epoch": 0.160926939169617, |
| "grad_norm": 117.88627280563217, |
| "learning_rate": 9.599162910495813e-07, |
| "logits/chosen": -0.578723132610321, |
| "logits/rejected": -0.607922375202179, |
| "logps/chosen": -410.42498779296875, |
| "logps/rejected": -369.25, |
| "loss": 0.6423, |
| "rewards/accuracies": 0.6499999761581421, |
| "rewards/chosen": -0.02398681640625, |
| "rewards/margins": 0.622692883014679, |
| "rewards/rejected": -0.646209716796875, |
| "step": 250 |
| }, |
| { |
| "epoch": 0.16736401673640167, |
| "grad_norm": 114.49562778791112, |
| "learning_rate": 9.583065035415325e-07, |
| "logits/chosen": -0.546374499797821, |
| "logits/rejected": -0.5821288824081421, |
| "logps/chosen": -358.5625, |
| "logps/rejected": -334.8125, |
| "loss": 0.6349, |
| "rewards/accuracies": 0.643750011920929, |
| "rewards/chosen": -0.6829162836074829, |
| "rewards/margins": 0.758496105670929, |
| "rewards/rejected": -1.4420654773712158, |
| "step": 260 |
| }, |
| { |
| "epoch": 0.17380109430318635, |
| "grad_norm": 252.1668555560546, |
| "learning_rate": 9.566967160334835e-07, |
| "logits/chosen": -0.496002197265625, |
| "logits/rejected": -0.529162585735321, |
| "logps/chosen": -341.23748779296875, |
| "logps/rejected": -344.5687561035156, |
| "loss": 0.6088, |
| "rewards/accuracies": 0.668749988079071, |
| "rewards/chosen": -0.3066039979457855, |
| "rewards/margins": 0.724609375, |
| "rewards/rejected": -1.030542016029358, |
| "step": 270 |
| }, |
| { |
| "epoch": 0.18023817186997104, |
| "grad_norm": 102.98363396913689, |
| "learning_rate": 9.550869285254346e-07, |
| "logits/chosen": -0.546069324016571, |
| "logits/rejected": -0.558239758014679, |
| "logps/chosen": -370.57501220703125, |
| "logps/rejected": -368.75, |
| "loss": 0.5405, |
| "rewards/accuracies": 0.7124999761581421, |
| "rewards/chosen": 0.058502197265625, |
| "rewards/margins": 0.809436023235321, |
| "rewards/rejected": -0.751208484172821, |
| "step": 280 |
| }, |
| { |
| "epoch": 0.1866752494367557, |
| "grad_norm": 91.00041151449058, |
| "learning_rate": 9.534771410173856e-07, |
| "logits/chosen": -0.577075183391571, |
| "logits/rejected": -0.5903075933456421, |
| "logps/chosen": -375.0, |
| "logps/rejected": -360.98748779296875, |
| "loss": 0.6413, |
| "rewards/accuracies": 0.637499988079071, |
| "rewards/chosen": 0.1882476806640625, |
| "rewards/margins": 0.7995971441268921, |
| "rewards/rejected": -0.612011730670929, |
| "step": 290 |
| }, |
| { |
| "epoch": 0.1931123270035404, |
| "grad_norm": 102.0109893516658, |
| "learning_rate": 9.518673535093368e-07, |
| "logits/chosen": -0.4876770079135895, |
| "logits/rejected": -0.489013671875, |
| "logps/chosen": -374.79998779296875, |
| "logps/rejected": -336.17498779296875, |
| "loss": 0.618, |
| "rewards/accuracies": 0.6499999761581421, |
| "rewards/chosen": 0.07204589992761612, |
| "rewards/margins": 0.842114269733429, |
| "rewards/rejected": -0.770495593547821, |
| "step": 300 |
| }, |
| { |
| "epoch": 0.19954940457032508, |
| "grad_norm": 76.86741916960312, |
| "learning_rate": 9.502575660012879e-07, |
| "logits/chosen": -0.5677276849746704, |
| "logits/rejected": -0.5572754144668579, |
| "logps/chosen": -359.3999938964844, |
| "logps/rejected": -367.92498779296875, |
| "loss": 0.6237, |
| "rewards/accuracies": 0.6312500238418579, |
| "rewards/chosen": 0.3591064512729645, |
| "rewards/margins": 0.6754516363143921, |
| "rewards/rejected": -0.3162597715854645, |
| "step": 310 |
| }, |
| { |
| "epoch": 0.20598648213710974, |
| "grad_norm": 108.9030110579976, |
| "learning_rate": 9.486477784932388e-07, |
| "logits/chosen": -0.6708739995956421, |
| "logits/rejected": -0.655590832233429, |
| "logps/chosen": -376.125, |
| "logps/rejected": -373.375, |
| "loss": 0.5166, |
| "rewards/accuracies": 0.7250000238418579, |
| "rewards/chosen": 0.2652526795864105, |
| "rewards/margins": 1.1403319835662842, |
| "rewards/rejected": -0.8749328851699829, |
| "step": 320 |
| }, |
| { |
| "epoch": 0.21242355970389443, |
| "grad_norm": 107.07521441276505, |
| "learning_rate": 9.470379909851899e-07, |
| "logits/chosen": -0.5230712890625, |
| "logits/rejected": -0.551342785358429, |
| "logps/chosen": -373.09375, |
| "logps/rejected": -354.7875061035156, |
| "loss": 0.6214, |
| "rewards/accuracies": 0.675000011920929, |
| "rewards/chosen": -0.1680450439453125, |
| "rewards/margins": 0.9344116449356079, |
| "rewards/rejected": -1.102380394935608, |
| "step": 330 |
| }, |
| { |
| "epoch": 0.21886063727067911, |
| "grad_norm": 80.28388865404655, |
| "learning_rate": 9.454282034771411e-07, |
| "logits/chosen": -0.6387573480606079, |
| "logits/rejected": -0.661419689655304, |
| "logps/chosen": -365.1499938964844, |
| "logps/rejected": -351.125, |
| "loss": 0.5976, |
| "rewards/accuracies": 0.668749988079071, |
| "rewards/chosen": -0.03964843600988388, |
| "rewards/margins": 0.987072765827179, |
| "rewards/rejected": -1.0261566638946533, |
| "step": 340 |
| }, |
| { |
| "epoch": 0.2252977148374638, |
| "grad_norm": 66.301132685081, |
| "learning_rate": 9.43818415969092e-07, |
| "logits/chosen": -0.5279541015625, |
| "logits/rejected": -0.5782226324081421, |
| "logps/chosen": -373.9750061035156, |
| "logps/rejected": -359.0249938964844, |
| "loss": 0.5829, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": 0.40840452909469604, |
| "rewards/margins": 0.9593750238418579, |
| "rewards/rejected": -0.552032470703125, |
| "step": 350 |
| }, |
| { |
| "epoch": 0.23173479240424846, |
| "grad_norm": 133.60786340460766, |
| "learning_rate": 9.422086284610431e-07, |
| "logits/chosen": -0.4904952943325043, |
| "logits/rejected": -0.4936676025390625, |
| "logps/chosen": -351.54998779296875, |
| "logps/rejected": -352.1875, |
| "loss": 0.6358, |
| "rewards/accuracies": 0.625, |
| "rewards/chosen": 0.3689941465854645, |
| "rewards/margins": 0.801318347454071, |
| "rewards/rejected": -0.43192750215530396, |
| "step": 360 |
| }, |
| { |
| "epoch": 0.23817186997103315, |
| "grad_norm": 146.73254711353752, |
| "learning_rate": 9.405988409529941e-07, |
| "logits/chosen": -0.4536499083042145, |
| "logits/rejected": -0.48334044218063354, |
| "logps/chosen": -394.20001220703125, |
| "logps/rejected": -344.3999938964844, |
| "loss": 0.622, |
| "rewards/accuracies": 0.6499999761581421, |
| "rewards/chosen": 0.44520872831344604, |
| "rewards/margins": 0.8761352300643921, |
| "rewards/rejected": -0.43049317598342896, |
| "step": 370 |
| }, |
| { |
| "epoch": 0.24460894753781784, |
| "grad_norm": 108.12091497250533, |
| "learning_rate": 9.389890534449453e-07, |
| "logits/chosen": -0.48052978515625, |
| "logits/rejected": -0.48167723417282104, |
| "logps/chosen": -388.875, |
| "logps/rejected": -367.875, |
| "loss": 0.5671, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": 0.10392455756664276, |
| "rewards/margins": 0.9621337652206421, |
| "rewards/rejected": -0.858563244342804, |
| "step": 380 |
| }, |
| { |
| "epoch": 0.2510460251046025, |
| "grad_norm": 85.66165714869449, |
| "learning_rate": 9.373792659368963e-07, |
| "logits/chosen": -0.504321277141571, |
| "logits/rejected": -0.52301025390625, |
| "logps/chosen": -362.92498779296875, |
| "logps/rejected": -352.42498779296875, |
| "loss": 0.5526, |
| "rewards/accuracies": 0.6812499761581421, |
| "rewards/chosen": -0.561572253704071, |
| "rewards/margins": 1.221276879310608, |
| "rewards/rejected": -1.7831909656524658, |
| "step": 390 |
| }, |
| { |
| "epoch": 0.2574831026713872, |
| "grad_norm": 136.73196581533503, |
| "learning_rate": 9.357694784288473e-07, |
| "logits/chosen": -0.49071043729782104, |
| "logits/rejected": -0.51934814453125, |
| "logps/chosen": -366.36248779296875, |
| "logps/rejected": -363.25, |
| "loss": 0.6429, |
| "rewards/accuracies": 0.637499988079071, |
| "rewards/chosen": -0.3850997984409332, |
| "rewards/margins": 1.0957520008087158, |
| "rewards/rejected": -1.481896996498108, |
| "step": 400 |
| }, |
| { |
| "epoch": 0.2639201802381719, |
| "grad_norm": 62.68096976935343, |
| "learning_rate": 9.341596909207984e-07, |
| "logits/chosen": -0.4698120057582855, |
| "logits/rejected": -0.49330443143844604, |
| "logps/chosen": -373.75, |
| "logps/rejected": -357.20001220703125, |
| "loss": 0.554, |
| "rewards/accuracies": 0.706250011920929, |
| "rewards/chosen": -0.1954345703125, |
| "rewards/margins": 0.96142578125, |
| "rewards/rejected": -1.1562988758087158, |
| "step": 410 |
| }, |
| { |
| "epoch": 0.27035725780495656, |
| "grad_norm": 117.58664324623555, |
| "learning_rate": 9.325499034127496e-07, |
| "logits/chosen": -0.5307983160018921, |
| "logits/rejected": -0.5357711911201477, |
| "logps/chosen": -402.75, |
| "logps/rejected": -393.67498779296875, |
| "loss": 0.5572, |
| "rewards/accuracies": 0.706250011920929, |
| "rewards/chosen": -0.15812988579273224, |
| "rewards/margins": 1.0307495594024658, |
| "rewards/rejected": -1.1893126964569092, |
| "step": 420 |
| }, |
| { |
| "epoch": 0.27679433537174125, |
| "grad_norm": 93.75687305331235, |
| "learning_rate": 9.309401159047005e-07, |
| "logits/chosen": -0.5401611328125, |
| "logits/rejected": -0.578295886516571, |
| "logps/chosen": -353.67498779296875, |
| "logps/rejected": -350.29998779296875, |
| "loss": 0.697, |
| "rewards/accuracies": 0.643750011920929, |
| "rewards/chosen": -0.4753173887729645, |
| "rewards/margins": 0.94769287109375, |
| "rewards/rejected": -1.422509789466858, |
| "step": 430 |
| }, |
| { |
| "epoch": 0.2832314129385259, |
| "grad_norm": 95.5374818544099, |
| "learning_rate": 9.293303283966516e-07, |
| "logits/chosen": -0.619433581829071, |
| "logits/rejected": -0.6208251714706421, |
| "logps/chosen": -394.98748779296875, |
| "logps/rejected": -389.11248779296875, |
| "loss": 0.6189, |
| "rewards/accuracies": 0.6187499761581421, |
| "rewards/chosen": -0.571759045124054, |
| "rewards/margins": 0.7771240472793579, |
| "rewards/rejected": -1.349853515625, |
| "step": 440 |
| }, |
| { |
| "epoch": 0.28966849050531057, |
| "grad_norm": 125.84423019681405, |
| "learning_rate": 9.277205408886027e-07, |
| "logits/chosen": -0.53460693359375, |
| "logits/rejected": -0.530902087688446, |
| "logps/chosen": -368.8999938964844, |
| "logps/rejected": -367.3500061035156, |
| "loss": 0.627, |
| "rewards/accuracies": 0.7124999761581421, |
| "rewards/chosen": -0.482666015625, |
| "rewards/margins": 1.0159423351287842, |
| "rewards/rejected": -1.497259497642517, |
| "step": 450 |
| }, |
| { |
| "epoch": 0.29610556807209526, |
| "grad_norm": 119.6694420609397, |
| "learning_rate": 9.261107533805537e-07, |
| "logits/chosen": -0.5695999264717102, |
| "logits/rejected": -0.574688732624054, |
| "logps/chosen": -392.45001220703125, |
| "logps/rejected": -376.875, |
| "loss": 0.6691, |
| "rewards/accuracies": 0.612500011920929, |
| "rewards/chosen": -0.489309698343277, |
| "rewards/margins": 0.785595715045929, |
| "rewards/rejected": -1.275183081626892, |
| "step": 460 |
| }, |
| { |
| "epoch": 0.30254264563887995, |
| "grad_norm": 109.69479101002719, |
| "learning_rate": 9.245009658725048e-07, |
| "logits/chosen": -0.5947021245956421, |
| "logits/rejected": -0.5928710699081421, |
| "logps/chosen": -404.1625061035156, |
| "logps/rejected": -355.2749938964844, |
| "loss": 0.6002, |
| "rewards/accuracies": 0.65625, |
| "rewards/chosen": -0.16807861626148224, |
| "rewards/margins": 0.9239562749862671, |
| "rewards/rejected": -1.091485619544983, |
| "step": 470 |
| }, |
| { |
| "epoch": 0.30897972320566464, |
| "grad_norm": 114.73014097944892, |
| "learning_rate": 9.228911783644559e-07, |
| "logits/chosen": -0.572558581829071, |
| "logits/rejected": -0.602734386920929, |
| "logps/chosen": -368.2749938964844, |
| "logps/rejected": -358.1499938964844, |
| "loss": 0.5264, |
| "rewards/accuracies": 0.731249988079071, |
| "rewards/chosen": 0.10758056491613388, |
| "rewards/margins": 1.0086791515350342, |
| "rewards/rejected": -0.9011077880859375, |
| "step": 480 |
| }, |
| { |
| "epoch": 0.3154168007724493, |
| "grad_norm": 127.76637945406556, |
| "learning_rate": 9.212813908564069e-07, |
| "logits/chosen": -0.54168701171875, |
| "logits/rejected": -0.570635974407196, |
| "logps/chosen": -363.2875061035156, |
| "logps/rejected": -343.17498779296875, |
| "loss": 0.6542, |
| "rewards/accuracies": 0.643750011920929, |
| "rewards/chosen": 0.015155029483139515, |
| "rewards/margins": 0.724200427532196, |
| "rewards/rejected": -0.708392322063446, |
| "step": 490 |
| }, |
| { |
| "epoch": 0.321853878339234, |
| "grad_norm": 92.4760176391222, |
| "learning_rate": 9.19671603348358e-07, |
| "logits/chosen": -0.5834716558456421, |
| "logits/rejected": -0.608386218547821, |
| "logps/chosen": -370.13751220703125, |
| "logps/rejected": -359.7749938964844, |
| "loss": 0.6958, |
| "rewards/accuracies": 0.643750011920929, |
| "rewards/chosen": -0.02927246131002903, |
| "rewards/margins": 0.681835949420929, |
| "rewards/rejected": -0.7109962701797485, |
| "step": 500 |
| }, |
| { |
| "epoch": 0.32829095590601864, |
| "grad_norm": 120.62329927967555, |
| "learning_rate": 9.18061815840309e-07, |
| "logits/chosen": -0.5348449945449829, |
| "logits/rejected": -0.5554443597793579, |
| "logps/chosen": -365.8999938964844, |
| "logps/rejected": -356.29998779296875, |
| "loss": 0.5751, |
| "rewards/accuracies": 0.675000011920929, |
| "rewards/chosen": 0.2884887754917145, |
| "rewards/margins": 0.861279308795929, |
| "rewards/rejected": -0.5736328363418579, |
| "step": 510 |
| }, |
| { |
| "epoch": 0.33472803347280333, |
| "grad_norm": 130.9763223613383, |
| "learning_rate": 9.164520283322601e-07, |
| "logits/chosen": -0.451071172952652, |
| "logits/rejected": -0.48179322481155396, |
| "logps/chosen": -363.6000061035156, |
| "logps/rejected": -347.92498779296875, |
| "loss": 0.6538, |
| "rewards/accuracies": 0.6312500238418579, |
| "rewards/chosen": 0.3598876893520355, |
| "rewards/margins": 0.71923828125, |
| "rewards/rejected": -0.359466552734375, |
| "step": 520 |
| }, |
| { |
| "epoch": 0.341165111039588, |
| "grad_norm": 109.62659240266296, |
| "learning_rate": 9.148422408242112e-07, |
| "logits/chosen": -0.4910888671875, |
| "logits/rejected": -0.519116222858429, |
| "logps/chosen": -358.5, |
| "logps/rejected": -364.45001220703125, |
| "loss": 0.6195, |
| "rewards/accuracies": 0.6000000238418579, |
| "rewards/chosen": 0.5476440191268921, |
| "rewards/margins": 0.73748779296875, |
| "rewards/rejected": -0.19072571396827698, |
| "step": 530 |
| }, |
| { |
| "epoch": 0.3476021886063727, |
| "grad_norm": 109.68298542748705, |
| "learning_rate": 9.132324533161622e-07, |
| "logits/chosen": -0.4764648377895355, |
| "logits/rejected": -0.502703845500946, |
| "logps/chosen": -336.875, |
| "logps/rejected": -318.25, |
| "loss": 0.64, |
| "rewards/accuracies": 0.643750011920929, |
| "rewards/chosen": 0.613238513469696, |
| "rewards/margins": 0.6423705816268921, |
| "rewards/rejected": -0.02930908277630806, |
| "step": 540 |
| }, |
| { |
| "epoch": 0.3540392661731574, |
| "grad_norm": 105.51467874045339, |
| "learning_rate": 9.116226658081133e-07, |
| "logits/chosen": -0.5284057855606079, |
| "logits/rejected": -0.5428222417831421, |
| "logps/chosen": -361.125, |
| "logps/rejected": -332.3500061035156, |
| "loss": 0.5463, |
| "rewards/accuracies": 0.7124999761581421, |
| "rewards/chosen": 0.4479126036167145, |
| "rewards/margins": 1.0054442882537842, |
| "rewards/rejected": -0.558178722858429, |
| "step": 550 |
| }, |
| { |
| "epoch": 0.3604763437399421, |
| "grad_norm": 99.67713382177797, |
| "learning_rate": 9.100128783000644e-07, |
| "logits/chosen": -0.5566619634628296, |
| "logits/rejected": -0.5879882574081421, |
| "logps/chosen": -348.375, |
| "logps/rejected": -369.625, |
| "loss": 0.5016, |
| "rewards/accuracies": 0.699999988079071, |
| "rewards/chosen": 0.610461413860321, |
| "rewards/margins": 1.308984398841858, |
| "rewards/rejected": -0.698779284954071, |
| "step": 560 |
| }, |
| { |
| "epoch": 0.3669134213067268, |
| "grad_norm": 131.25070143478592, |
| "learning_rate": 9.084030907920153e-07, |
| "logits/chosen": -0.5046325922012329, |
| "logits/rejected": -0.545117199420929, |
| "logps/chosen": -364.75, |
| "logps/rejected": -372.5249938964844, |
| "loss": 0.6631, |
| "rewards/accuracies": 0.6937500238418579, |
| "rewards/chosen": 0.550830066204071, |
| "rewards/margins": 0.9921020269393921, |
| "rewards/rejected": -0.4401001036167145, |
| "step": 570 |
| }, |
| { |
| "epoch": 0.3733504988735114, |
| "grad_norm": 109.14671818372386, |
| "learning_rate": 9.067933032839665e-07, |
| "logits/chosen": -0.4894042909145355, |
| "logits/rejected": -0.4809204041957855, |
| "logps/chosen": -340.4624938964844, |
| "logps/rejected": -360.8500061035156, |
| "loss": 0.6645, |
| "rewards/accuracies": 0.65625, |
| "rewards/chosen": 0.671185314655304, |
| "rewards/margins": 0.7091064453125, |
| "rewards/rejected": -0.03690185397863388, |
| "step": 580 |
| }, |
| { |
| "epoch": 0.3797875764402961, |
| "grad_norm": 102.87805305261952, |
| "learning_rate": 9.051835157759176e-07, |
| "logits/chosen": -0.6076415777206421, |
| "logits/rejected": -0.63128662109375, |
| "logps/chosen": -394.92498779296875, |
| "logps/rejected": -383.4375, |
| "loss": 0.581, |
| "rewards/accuracies": 0.668749988079071, |
| "rewards/chosen": 0.48107606172561646, |
| "rewards/margins": 1.0627930164337158, |
| "rewards/rejected": -0.581591784954071, |
| "step": 590 |
| }, |
| { |
| "epoch": 0.3862246540070808, |
| "grad_norm": 101.2501221806809, |
| "learning_rate": 9.035737282678686e-07, |
| "logits/chosen": -0.606689453125, |
| "logits/rejected": -0.617114245891571, |
| "logps/chosen": -325.23748779296875, |
| "logps/rejected": -309.0874938964844, |
| "loss": 0.599, |
| "rewards/accuracies": 0.65625, |
| "rewards/chosen": 0.31675416231155396, |
| "rewards/margins": 0.9888916015625, |
| "rewards/rejected": -0.671948254108429, |
| "step": 600 |
| }, |
| { |
| "epoch": 0.39266173157386547, |
| "grad_norm": 112.02032310278221, |
| "learning_rate": 9.019639407598196e-07, |
| "logits/chosen": -0.5888427495956421, |
| "logits/rejected": -0.611279308795929, |
| "logps/chosen": -370.3500061035156, |
| "logps/rejected": -363.8500061035156, |
| "loss": 0.5333, |
| "rewards/accuracies": 0.7124999761581421, |
| "rewards/chosen": 0.5602661371231079, |
| "rewards/margins": 1.0866515636444092, |
| "rewards/rejected": -0.5262695550918579, |
| "step": 610 |
| }, |
| { |
| "epoch": 0.39909880914065016, |
| "grad_norm": 116.1836878867331, |
| "learning_rate": 9.003541532517708e-07, |
| "logits/chosen": -0.561572253704071, |
| "logits/rejected": -0.577441394329071, |
| "logps/chosen": -362.29998779296875, |
| "logps/rejected": -352.7250061035156, |
| "loss": 0.6513, |
| "rewards/accuracies": 0.668749988079071, |
| "rewards/chosen": 0.3332366943359375, |
| "rewards/margins": 0.7449706792831421, |
| "rewards/rejected": -0.4120422303676605, |
| "step": 620 |
| }, |
| { |
| "epoch": 0.40553588670743485, |
| "grad_norm": 119.84221150272923, |
| "learning_rate": 8.987443657437218e-07, |
| "logits/chosen": -0.551226794719696, |
| "logits/rejected": -0.5623534917831421, |
| "logps/chosen": -386.70001220703125, |
| "logps/rejected": -373.07501220703125, |
| "loss": 0.6302, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": -0.28680419921875, |
| "rewards/margins": 0.9903320074081421, |
| "rewards/rejected": -1.2776367664337158, |
| "step": 630 |
| }, |
| { |
| "epoch": 0.4119729642742195, |
| "grad_norm": 119.20355705473106, |
| "learning_rate": 8.971345782356729e-07, |
| "logits/chosen": -0.5877685546875, |
| "logits/rejected": -0.6196533441543579, |
| "logps/chosen": -386.4750061035156, |
| "logps/rejected": -392.2250061035156, |
| "loss": 0.618, |
| "rewards/accuracies": 0.65625, |
| "rewards/chosen": -0.45903319120407104, |
| "rewards/margins": 0.8375488519668579, |
| "rewards/rejected": -1.2969481945037842, |
| "step": 640 |
| }, |
| { |
| "epoch": 0.41841004184100417, |
| "grad_norm": 114.39134560983841, |
| "learning_rate": 8.955247907276239e-07, |
| "logits/chosen": -0.4747314453125, |
| "logits/rejected": -0.520581066608429, |
| "logps/chosen": -361.45001220703125, |
| "logps/rejected": -340.42498779296875, |
| "loss": 0.6872, |
| "rewards/accuracies": 0.59375, |
| "rewards/chosen": -0.09818115085363388, |
| "rewards/margins": 0.6923888921737671, |
| "rewards/rejected": -0.791015625, |
| "step": 650 |
| }, |
| { |
| "epoch": 0.42484711940778885, |
| "grad_norm": 107.98675915515811, |
| "learning_rate": 8.93915003219575e-07, |
| "logits/chosen": -0.539172351360321, |
| "logits/rejected": -0.5562499761581421, |
| "logps/chosen": -381.54998779296875, |
| "logps/rejected": -356.75, |
| "loss": 0.551, |
| "rewards/accuracies": 0.6937500238418579, |
| "rewards/chosen": 0.00932922400534153, |
| "rewards/margins": 1.1506836414337158, |
| "rewards/rejected": -1.142364501953125, |
| "step": 660 |
| }, |
| { |
| "epoch": 0.43128419697457354, |
| "grad_norm": 103.2010559979854, |
| "learning_rate": 8.923052157115261e-07, |
| "logits/chosen": -0.528704822063446, |
| "logits/rejected": -0.556774914264679, |
| "logps/chosen": -388.6000061035156, |
| "logps/rejected": -368.70001220703125, |
| "loss": 0.5941, |
| "rewards/accuracies": 0.6812499761581421, |
| "rewards/chosen": -0.13016967475414276, |
| "rewards/margins": 1.0416138172149658, |
| "rewards/rejected": -1.172509789466858, |
| "step": 670 |
| }, |
| { |
| "epoch": 0.43772127454135823, |
| "grad_norm": 123.08466573292604, |
| "learning_rate": 8.90695428203477e-07, |
| "logits/chosen": -0.5683318972587585, |
| "logits/rejected": -0.575610339641571, |
| "logps/chosen": -399.875, |
| "logps/rejected": -361.45001220703125, |
| "loss": 0.5778, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": -0.6572631597518921, |
| "rewards/margins": 1.0535399913787842, |
| "rewards/rejected": -1.7103271484375, |
| "step": 680 |
| }, |
| { |
| "epoch": 0.4441583521081429, |
| "grad_norm": 83.98215014096604, |
| "learning_rate": 8.890856406954281e-07, |
| "logits/chosen": -0.523547351360321, |
| "logits/rejected": -0.5286010503768921, |
| "logps/chosen": -384.7749938964844, |
| "logps/rejected": -357.57501220703125, |
| "loss": 0.5256, |
| "rewards/accuracies": 0.7437499761581421, |
| "rewards/chosen": -0.02643432654440403, |
| "rewards/margins": 1.292199730873108, |
| "rewards/rejected": -1.3194580078125, |
| "step": 690 |
| }, |
| { |
| "epoch": 0.4505954296749276, |
| "grad_norm": 104.87203994034158, |
| "learning_rate": 8.874758531873793e-07, |
| "logits/chosen": -0.538464367389679, |
| "logits/rejected": -0.5656677484512329, |
| "logps/chosen": -364.5625, |
| "logps/rejected": -369.8999938964844, |
| "loss": 0.4687, |
| "rewards/accuracies": 0.6937500238418579, |
| "rewards/chosen": 0.04945068433880806, |
| "rewards/margins": 1.432836890220642, |
| "rewards/rejected": -1.3836181163787842, |
| "step": 700 |
| }, |
| { |
| "epoch": 0.45703250724171224, |
| "grad_norm": 57.76406203268148, |
| "learning_rate": 8.858660656793303e-07, |
| "logits/chosen": -0.512127697467804, |
| "logits/rejected": -0.5491943359375, |
| "logps/chosen": -405.0, |
| "logps/rejected": -395.3374938964844, |
| "loss": 0.5798, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": 0.303070068359375, |
| "rewards/margins": 1.1065673828125, |
| "rewards/rejected": -0.8034912347793579, |
| "step": 710 |
| }, |
| { |
| "epoch": 0.4634695848084969, |
| "grad_norm": 69.89352328895056, |
| "learning_rate": 8.842562781712813e-07, |
| "logits/chosen": -0.4332519471645355, |
| "logits/rejected": -0.45560914278030396, |
| "logps/chosen": -350.20001220703125, |
| "logps/rejected": -314.125, |
| "loss": 0.6324, |
| "rewards/accuracies": 0.6499999761581421, |
| "rewards/chosen": 0.3115478456020355, |
| "rewards/margins": 0.9028564691543579, |
| "rewards/rejected": -0.591540515422821, |
| "step": 720 |
| }, |
| { |
| "epoch": 0.4699066623752816, |
| "grad_norm": 83.67294747788176, |
| "learning_rate": 8.826464906632324e-07, |
| "logits/chosen": -0.51568603515625, |
| "logits/rejected": -0.5252624750137329, |
| "logps/chosen": -381.92498779296875, |
| "logps/rejected": -349.32501220703125, |
| "loss": 0.6184, |
| "rewards/accuracies": 0.6499999761581421, |
| "rewards/chosen": 0.4420166015625, |
| "rewards/margins": 0.99859619140625, |
| "rewards/rejected": -0.556780993938446, |
| "step": 730 |
| }, |
| { |
| "epoch": 0.4763437399420663, |
| "grad_norm": 104.60759299733223, |
| "learning_rate": 8.810367031551835e-07, |
| "logits/chosen": -0.42892152070999146, |
| "logits/rejected": -0.4578002989292145, |
| "logps/chosen": -409.29998779296875, |
| "logps/rejected": -351.3500061035156, |
| "loss": 0.5794, |
| "rewards/accuracies": 0.6812499761581421, |
| "rewards/chosen": 0.088287353515625, |
| "rewards/margins": 1.06494140625, |
| "rewards/rejected": -0.9759521484375, |
| "step": 740 |
| }, |
| { |
| "epoch": 0.482780817508851, |
| "grad_norm": 70.65265994679216, |
| "learning_rate": 8.794269156471346e-07, |
| "logits/chosen": -0.4439941346645355, |
| "logits/rejected": -0.502819836139679, |
| "logps/chosen": -358.51251220703125, |
| "logps/rejected": -343.7250061035156, |
| "loss": 0.603, |
| "rewards/accuracies": 0.71875, |
| "rewards/chosen": 0.317587286233902, |
| "rewards/margins": 1.092810034751892, |
| "rewards/rejected": -0.774609386920929, |
| "step": 750 |
| }, |
| { |
| "epoch": 0.4892178950756357, |
| "grad_norm": 99.48553520919884, |
| "learning_rate": 8.778171281390856e-07, |
| "logits/chosen": -0.454559326171875, |
| "logits/rejected": -0.46809083223342896, |
| "logps/chosen": -387.20001220703125, |
| "logps/rejected": -361.875, |
| "loss": 0.591, |
| "rewards/accuracies": 0.643750011920929, |
| "rewards/chosen": 0.2822631895542145, |
| "rewards/margins": 0.992480456829071, |
| "rewards/rejected": -0.710858166217804, |
| "step": 760 |
| }, |
| { |
| "epoch": 0.49565497264242037, |
| "grad_norm": 107.63172335214213, |
| "learning_rate": 8.762073406310366e-07, |
| "logits/chosen": -0.5004516839981079, |
| "logits/rejected": -0.5167480707168579, |
| "logps/chosen": -349.20001220703125, |
| "logps/rejected": -351.1499938964844, |
| "loss": 0.6649, |
| "rewards/accuracies": 0.6312500238418579, |
| "rewards/chosen": -0.29923707246780396, |
| "rewards/margins": 0.953686535358429, |
| "rewards/rejected": -1.253198266029358, |
| "step": 770 |
| }, |
| { |
| "epoch": 0.502092050209205, |
| "grad_norm": 112.79451532298022, |
| "learning_rate": 8.745975531229878e-07, |
| "logits/chosen": -0.50775146484375, |
| "logits/rejected": -0.5383545160293579, |
| "logps/chosen": -372.2875061035156, |
| "logps/rejected": -374.7749938964844, |
| "loss": 0.6127, |
| "rewards/accuracies": 0.706250011920929, |
| "rewards/chosen": -0.543988049030304, |
| "rewards/margins": 1.2046020030975342, |
| "rewards/rejected": -1.748632788658142, |
| "step": 780 |
| }, |
| { |
| "epoch": 0.5085291277759897, |
| "grad_norm": 92.84930514639605, |
| "learning_rate": 8.729877656149389e-07, |
| "logits/chosen": -0.511303722858429, |
| "logits/rejected": -0.52484130859375, |
| "logps/chosen": -356.45001220703125, |
| "logps/rejected": -347.0249938964844, |
| "loss": 0.5365, |
| "rewards/accuracies": 0.706250011920929, |
| "rewards/chosen": -0.19889526069164276, |
| "rewards/margins": 1.256262183189392, |
| "rewards/rejected": -1.454187035560608, |
| "step": 790 |
| }, |
| { |
| "epoch": 0.5149662053427744, |
| "grad_norm": 118.08576455579058, |
| "learning_rate": 8.713779781068898e-07, |
| "logits/chosen": -0.526232898235321, |
| "logits/rejected": -0.544995129108429, |
| "logps/chosen": -344.3374938964844, |
| "logps/rejected": -359.29998779296875, |
| "loss": 0.567, |
| "rewards/accuracies": 0.6812499761581421, |
| "rewards/chosen": -0.2644104063510895, |
| "rewards/margins": 1.2300536632537842, |
| "rewards/rejected": -1.4940307140350342, |
| "step": 800 |
| }, |
| { |
| "epoch": 0.5214032829095591, |
| "grad_norm": 149.5051687226972, |
| "learning_rate": 8.697681905988409e-07, |
| "logits/chosen": -0.533679187297821, |
| "logits/rejected": -0.546582043170929, |
| "logps/chosen": -416.1000061035156, |
| "logps/rejected": -381.8500061035156, |
| "loss": 0.6592, |
| "rewards/accuracies": 0.6499999761581421, |
| "rewards/chosen": -0.562670886516571, |
| "rewards/margins": 0.9259033203125, |
| "rewards/rejected": -1.48779296875, |
| "step": 810 |
| }, |
| { |
| "epoch": 0.5278403604763438, |
| "grad_norm": 116.30993199121478, |
| "learning_rate": 8.681584030907921e-07, |
| "logits/chosen": -0.548510730266571, |
| "logits/rejected": -0.5396972894668579, |
| "logps/chosen": -415.82501220703125, |
| "logps/rejected": -355.17498779296875, |
| "loss": 0.604, |
| "rewards/accuracies": 0.612500011920929, |
| "rewards/chosen": -0.35222166776657104, |
| "rewards/margins": 0.830737292766571, |
| "rewards/rejected": -1.182348608970642, |
| "step": 820 |
| }, |
| { |
| "epoch": 0.5342774380431284, |
| "grad_norm": 88.88351113445877, |
| "learning_rate": 8.66548615582743e-07, |
| "logits/chosen": -0.41846925020217896, |
| "logits/rejected": -0.44677734375, |
| "logps/chosen": -355.38751220703125, |
| "logps/rejected": -348.2749938964844, |
| "loss": 0.7307, |
| "rewards/accuracies": 0.606249988079071, |
| "rewards/chosen": -0.31745606660842896, |
| "rewards/margins": 0.7986205816268921, |
| "rewards/rejected": -1.1161620616912842, |
| "step": 830 |
| }, |
| { |
| "epoch": 0.5407145156099131, |
| "grad_norm": 141.95886167076992, |
| "learning_rate": 8.649388280746941e-07, |
| "logits/chosen": -0.4906982481479645, |
| "logits/rejected": -0.5586181879043579, |
| "logps/chosen": -411.54998779296875, |
| "logps/rejected": -367.82501220703125, |
| "loss": 0.6348, |
| "rewards/accuracies": 0.6625000238418579, |
| "rewards/chosen": 0.28123778104782104, |
| "rewards/margins": 1.0207641124725342, |
| "rewards/rejected": -0.7385803461074829, |
| "step": 840 |
| }, |
| { |
| "epoch": 0.5471515931766978, |
| "grad_norm": 119.67858559796636, |
| "learning_rate": 8.633290405666451e-07, |
| "logits/chosen": -0.53289794921875, |
| "logits/rejected": -0.526135265827179, |
| "logps/chosen": -361.0, |
| "logps/rejected": -353.1000061035156, |
| "loss": 0.5787, |
| "rewards/accuracies": 0.699999988079071, |
| "rewards/chosen": -0.18852539360523224, |
| "rewards/margins": 1.079687476158142, |
| "rewards/rejected": -1.2688720226287842, |
| "step": 850 |
| }, |
| { |
| "epoch": 0.5535886707434825, |
| "grad_norm": 106.12475270901847, |
| "learning_rate": 8.617192530585963e-07, |
| "logits/chosen": -0.5385986566543579, |
| "logits/rejected": -0.542651355266571, |
| "logps/chosen": -427.5375061035156, |
| "logps/rejected": -417.3999938964844, |
| "loss": 0.6631, |
| "rewards/accuracies": 0.699999988079071, |
| "rewards/chosen": -0.42586058378219604, |
| "rewards/margins": 1.089013695716858, |
| "rewards/rejected": -1.515173316001892, |
| "step": 860 |
| }, |
| { |
| "epoch": 0.5600257483102672, |
| "grad_norm": 98.81134073733207, |
| "learning_rate": 8.601094655505473e-07, |
| "logits/chosen": -0.5274902582168579, |
| "logits/rejected": -0.522229015827179, |
| "logps/chosen": -392.29998779296875, |
| "logps/rejected": -394.5249938964844, |
| "loss": 0.613, |
| "rewards/accuracies": 0.668749988079071, |
| "rewards/chosen": -0.3875488340854645, |
| "rewards/margins": 1.0703125, |
| "rewards/rejected": -1.4576904773712158, |
| "step": 870 |
| }, |
| { |
| "epoch": 0.5664628258770518, |
| "grad_norm": 120.13198690851658, |
| "learning_rate": 8.584996780424983e-07, |
| "logits/chosen": -0.55718994140625, |
| "logits/rejected": -0.553845226764679, |
| "logps/chosen": -369.82501220703125, |
| "logps/rejected": -343.07501220703125, |
| "loss": 0.639, |
| "rewards/accuracies": 0.5874999761581421, |
| "rewards/chosen": -0.23508301377296448, |
| "rewards/margins": 0.7032836675643921, |
| "rewards/rejected": -0.939038097858429, |
| "step": 880 |
| }, |
| { |
| "epoch": 0.5728999034438365, |
| "grad_norm": 62.94101570205876, |
| "learning_rate": 8.568898905344494e-07, |
| "logits/chosen": -0.4923339784145355, |
| "logits/rejected": -0.5071777105331421, |
| "logps/chosen": -350.5874938964844, |
| "logps/rejected": -328.25, |
| "loss": 0.602, |
| "rewards/accuracies": 0.6499999761581421, |
| "rewards/chosen": 0.03252258151769638, |
| "rewards/margins": 0.9979614019393921, |
| "rewards/rejected": -0.9653381109237671, |
| "step": 890 |
| }, |
| { |
| "epoch": 0.5793369810106211, |
| "grad_norm": 112.45942278359949, |
| "learning_rate": 8.552801030264006e-07, |
| "logits/chosen": -0.5096191167831421, |
| "logits/rejected": -0.4962158203125, |
| "logps/chosen": -376.375, |
| "logps/rejected": -356.29998779296875, |
| "loss": 0.6255, |
| "rewards/accuracies": 0.65625, |
| "rewards/chosen": -0.581451416015625, |
| "rewards/margins": 0.798583984375, |
| "rewards/rejected": -1.37945556640625, |
| "step": 900 |
| }, |
| { |
| "epoch": 0.5857740585774058, |
| "grad_norm": 108.90325211362334, |
| "learning_rate": 8.536703155183515e-07, |
| "logits/chosen": -0.552380383014679, |
| "logits/rejected": -0.5654449462890625, |
| "logps/chosen": -417.51251220703125, |
| "logps/rejected": -411.625, |
| "loss": 0.54, |
| "rewards/accuracies": 0.6812499761581421, |
| "rewards/chosen": -0.811279296875, |
| "rewards/margins": 1.20068359375, |
| "rewards/rejected": -2.0130858421325684, |
| "step": 910 |
| }, |
| { |
| "epoch": 0.5922111361441905, |
| "grad_norm": 98.0959184069916, |
| "learning_rate": 8.520605280103026e-07, |
| "logits/chosen": -0.5830078125, |
| "logits/rejected": -0.6092773675918579, |
| "logps/chosen": -353.6625061035156, |
| "logps/rejected": -364.67498779296875, |
| "loss": 0.6542, |
| "rewards/accuracies": 0.612500011920929, |
| "rewards/chosen": -0.9505707025527954, |
| "rewards/margins": 0.778289794921875, |
| "rewards/rejected": -1.729894995689392, |
| "step": 920 |
| }, |
| { |
| "epoch": 0.5986482137109752, |
| "grad_norm": 99.99889212589564, |
| "learning_rate": 8.504507405022537e-07, |
| "logits/chosen": -0.47906494140625, |
| "logits/rejected": -0.4897399842739105, |
| "logps/chosen": -379.625, |
| "logps/rejected": -377.2250061035156, |
| "loss": 0.6211, |
| "rewards/accuracies": 0.6812499761581421, |
| "rewards/chosen": -0.747698962688446, |
| "rewards/margins": 1.053491234779358, |
| "rewards/rejected": -1.801123023033142, |
| "step": 930 |
| }, |
| { |
| "epoch": 0.6050852912777599, |
| "grad_norm": 89.85940157465599, |
| "learning_rate": 8.488409529942047e-07, |
| "logits/chosen": -0.526043713092804, |
| "logits/rejected": -0.553393542766571, |
| "logps/chosen": -363.2250061035156, |
| "logps/rejected": -347.8999938964844, |
| "loss": 0.5999, |
| "rewards/accuracies": 0.643750011920929, |
| "rewards/chosen": -0.332254022359848, |
| "rewards/margins": 1.034509301185608, |
| "rewards/rejected": -1.367010474205017, |
| "step": 940 |
| }, |
| { |
| "epoch": 0.6115223688445446, |
| "grad_norm": 70.14280663264151, |
| "learning_rate": 8.472311654861558e-07, |
| "logits/chosen": -0.6128768920898438, |
| "logits/rejected": -0.6567413210868835, |
| "logps/chosen": -391.5, |
| "logps/rejected": -375.8500061035156, |
| "loss": 0.5634, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": -0.06875000149011612, |
| "rewards/margins": 1.0836913585662842, |
| "rewards/rejected": -1.152441382408142, |
| "step": 950 |
| }, |
| { |
| "epoch": 0.6179594464113293, |
| "grad_norm": 120.47345616509051, |
| "learning_rate": 8.456213779781069e-07, |
| "logits/chosen": -0.5240844488143921, |
| "logits/rejected": -0.5285888910293579, |
| "logps/chosen": -332.4125061035156, |
| "logps/rejected": -343.63751220703125, |
| "loss": 0.6299, |
| "rewards/accuracies": 0.6625000238418579, |
| "rewards/chosen": -0.15395812690258026, |
| "rewards/margins": 1.132959008216858, |
| "rewards/rejected": -1.2863037586212158, |
| "step": 960 |
| }, |
| { |
| "epoch": 0.624396523978114, |
| "grad_norm": 92.26766562500985, |
| "learning_rate": 8.440115904700579e-07, |
| "logits/chosen": -0.578857421875, |
| "logits/rejected": -0.598950207233429, |
| "logps/chosen": -400.1000061035156, |
| "logps/rejected": -380.45001220703125, |
| "loss": 0.6269, |
| "rewards/accuracies": 0.6499999761581421, |
| "rewards/chosen": 0.084320068359375, |
| "rewards/margins": 1.0315673351287842, |
| "rewards/rejected": -0.947705090045929, |
| "step": 970 |
| }, |
| { |
| "epoch": 0.6308336015448986, |
| "grad_norm": 85.69917047655541, |
| "learning_rate": 8.42401802962009e-07, |
| "logits/chosen": -0.6307373046875, |
| "logits/rejected": -0.6424804925918579, |
| "logps/chosen": -399.7749938964844, |
| "logps/rejected": -380.07501220703125, |
| "loss": 0.5451, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": -0.20753173530101776, |
| "rewards/margins": 1.097631812095642, |
| "rewards/rejected": -1.3062012195587158, |
| "step": 980 |
| }, |
| { |
| "epoch": 0.6372706791116833, |
| "grad_norm": 99.91368279665215, |
| "learning_rate": 8.4079201545396e-07, |
| "logits/chosen": -0.53326416015625, |
| "logits/rejected": -0.5157104730606079, |
| "logps/chosen": -370.75, |
| "logps/rejected": -344.92498779296875, |
| "loss": 0.6636, |
| "rewards/accuracies": 0.668749988079071, |
| "rewards/chosen": -0.07122802734375, |
| "rewards/margins": 0.891162097454071, |
| "rewards/rejected": -0.9619506597518921, |
| "step": 990 |
| }, |
| { |
| "epoch": 0.643707756678468, |
| "grad_norm": 111.09429678604513, |
| "learning_rate": 8.391822279459111e-07, |
| "logits/chosen": -0.5557830929756165, |
| "logits/rejected": -0.578845202922821, |
| "logps/chosen": -386.07501220703125, |
| "logps/rejected": -370.2250061035156, |
| "loss": 0.5953, |
| "rewards/accuracies": 0.706250011920929, |
| "rewards/chosen": -0.06235351413488388, |
| "rewards/margins": 1.00732421875, |
| "rewards/rejected": -1.069116234779358, |
| "step": 1000 |
| }, |
| { |
| "epoch": 0.6501448342452526, |
| "grad_norm": 108.22353551131003, |
| "learning_rate": 8.375724404378622e-07, |
| "logits/chosen": -0.5925537347793579, |
| "logits/rejected": -0.598681628704071, |
| "logps/chosen": -346.75, |
| "logps/rejected": -322.8999938964844, |
| "loss": 0.6265, |
| "rewards/accuracies": 0.643750011920929, |
| "rewards/chosen": -0.20738525688648224, |
| "rewards/margins": 0.9044555425643921, |
| "rewards/rejected": -1.1110718250274658, |
| "step": 1010 |
| }, |
| { |
| "epoch": 0.6565819118120373, |
| "grad_norm": 84.63983202639838, |
| "learning_rate": 8.359626529298132e-07, |
| "logits/chosen": -0.501538097858429, |
| "logits/rejected": -0.47550660371780396, |
| "logps/chosen": -374.3500061035156, |
| "logps/rejected": -349.63751220703125, |
| "loss": 0.5603, |
| "rewards/accuracies": 0.6625000238418579, |
| "rewards/chosen": -0.20573119819164276, |
| "rewards/margins": 0.952099621295929, |
| "rewards/rejected": -1.15771484375, |
| "step": 1020 |
| }, |
| { |
| "epoch": 0.663018989378822, |
| "grad_norm": 94.59002794744202, |
| "learning_rate": 8.343528654217643e-07, |
| "logits/chosen": -0.45048826932907104, |
| "logits/rejected": -0.44732666015625, |
| "logps/chosen": -379.25, |
| "logps/rejected": -379.625, |
| "loss": 0.6549, |
| "rewards/accuracies": 0.625, |
| "rewards/chosen": -0.4843383729457855, |
| "rewards/margins": 0.846728503704071, |
| "rewards/rejected": -1.330957055091858, |
| "step": 1030 |
| }, |
| { |
| "epoch": 0.6694560669456067, |
| "grad_norm": 128.4884473761372, |
| "learning_rate": 8.327430779137154e-07, |
| "logits/chosen": -0.5034545660018921, |
| "logits/rejected": -0.524914562702179, |
| "logps/chosen": -385.2749938964844, |
| "logps/rejected": -350.29998779296875, |
| "loss": 0.5829, |
| "rewards/accuracies": 0.6937500238418579, |
| "rewards/chosen": -0.407562255859375, |
| "rewards/margins": 1.02783203125, |
| "rewards/rejected": -1.435888648033142, |
| "step": 1040 |
| }, |
| { |
| "epoch": 0.6758931445123914, |
| "grad_norm": 107.25803443764735, |
| "learning_rate": 8.311332904056663e-07, |
| "logits/chosen": -0.46173095703125, |
| "logits/rejected": -0.5187743902206421, |
| "logps/chosen": -372.2749938964844, |
| "logps/rejected": -341.04998779296875, |
| "loss": 0.6023, |
| "rewards/accuracies": 0.6812499761581421, |
| "rewards/chosen": -0.3631958067417145, |
| "rewards/margins": 1.004797339439392, |
| "rewards/rejected": -1.368017554283142, |
| "step": 1050 |
| }, |
| { |
| "epoch": 0.682330222079176, |
| "grad_norm": 126.80827854824096, |
| "learning_rate": 8.295235028976175e-07, |
| "logits/chosen": -0.47755128145217896, |
| "logits/rejected": -0.509960949420929, |
| "logps/chosen": -350.42498779296875, |
| "logps/rejected": -357.9375, |
| "loss": 0.6958, |
| "rewards/accuracies": 0.6625000238418579, |
| "rewards/chosen": -0.19310760498046875, |
| "rewards/margins": 0.7228027582168579, |
| "rewards/rejected": -0.9160064458847046, |
| "step": 1060 |
| }, |
| { |
| "epoch": 0.6887672996459607, |
| "grad_norm": 81.10373132613064, |
| "learning_rate": 8.279137153895686e-07, |
| "logits/chosen": -0.42036741971969604, |
| "logits/rejected": -0.4339141845703125, |
| "logps/chosen": -385.1499938964844, |
| "logps/rejected": -373.45001220703125, |
| "loss": 0.6191, |
| "rewards/accuracies": 0.643750011920929, |
| "rewards/chosen": -0.156494140625, |
| "rewards/margins": 0.8669067621231079, |
| "rewards/rejected": -1.024011254310608, |
| "step": 1070 |
| }, |
| { |
| "epoch": 0.6952043772127454, |
| "grad_norm": 54.410337891839184, |
| "learning_rate": 8.263039278815196e-07, |
| "logits/chosen": -0.47700196504592896, |
| "logits/rejected": -0.5096435546875, |
| "logps/chosen": -364.7749938964844, |
| "logps/rejected": -334.29998779296875, |
| "loss": 0.4926, |
| "rewards/accuracies": 0.7437499761581421, |
| "rewards/chosen": 0.12125243991613388, |
| "rewards/margins": 1.276452660560608, |
| "rewards/rejected": -1.155175805091858, |
| "step": 1080 |
| }, |
| { |
| "epoch": 0.7016414547795301, |
| "grad_norm": 103.66262035258725, |
| "learning_rate": 8.246941403734706e-07, |
| "logits/chosen": -0.45051270723342896, |
| "logits/rejected": -0.4598144590854645, |
| "logps/chosen": -361.4750061035156, |
| "logps/rejected": -396.45001220703125, |
| "loss": 0.6205, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": -0.45460206270217896, |
| "rewards/margins": 1.1227295398712158, |
| "rewards/rejected": -1.5778076648712158, |
| "step": 1090 |
| }, |
| { |
| "epoch": 0.7080785323463148, |
| "grad_norm": 110.31202907587142, |
| "learning_rate": 8.230843528654218e-07, |
| "logits/chosen": -0.36857908964157104, |
| "logits/rejected": -0.4070068299770355, |
| "logps/chosen": -347.8500061035156, |
| "logps/rejected": -325.8999938964844, |
| "loss": 0.6092, |
| "rewards/accuracies": 0.637499988079071, |
| "rewards/chosen": -0.662280261516571, |
| "rewards/margins": 1.011010766029358, |
| "rewards/rejected": -1.6741943359375, |
| "step": 1100 |
| }, |
| { |
| "epoch": 0.7145156099130995, |
| "grad_norm": 113.5610108186264, |
| "learning_rate": 8.214745653573728e-07, |
| "logits/chosen": -0.45977783203125, |
| "logits/rejected": -0.4886474609375, |
| "logps/chosen": -422.75, |
| "logps/rejected": -394.92498779296875, |
| "loss": 0.5894, |
| "rewards/accuracies": 0.6625000238418579, |
| "rewards/chosen": -0.7705932855606079, |
| "rewards/margins": 1.0309569835662842, |
| "rewards/rejected": -1.8014647960662842, |
| "step": 1110 |
| }, |
| { |
| "epoch": 0.7209526874798842, |
| "grad_norm": 123.78507873787207, |
| "learning_rate": 8.198647778493239e-07, |
| "logits/chosen": -0.48577880859375, |
| "logits/rejected": -0.5226074457168579, |
| "logps/chosen": -393.75, |
| "logps/rejected": -369.07501220703125, |
| "loss": 0.5765, |
| "rewards/accuracies": 0.6625000238418579, |
| "rewards/chosen": -0.38194578886032104, |
| "rewards/margins": 1.10162353515625, |
| "rewards/rejected": -1.484594702720642, |
| "step": 1120 |
| }, |
| { |
| "epoch": 0.7273897650466689, |
| "grad_norm": 127.99687246016022, |
| "learning_rate": 8.182549903412748e-07, |
| "logits/chosen": -0.4909118711948395, |
| "logits/rejected": -0.4707275331020355, |
| "logps/chosen": -353.5249938964844, |
| "logps/rejected": -350.3500061035156, |
| "loss": 0.6522, |
| "rewards/accuracies": 0.637499988079071, |
| "rewards/chosen": -0.3532470762729645, |
| "rewards/margins": 0.9039306640625, |
| "rewards/rejected": -1.257135033607483, |
| "step": 1130 |
| }, |
| { |
| "epoch": 0.7338268426134535, |
| "grad_norm": 100.74894669506055, |
| "learning_rate": 8.16645202833226e-07, |
| "logits/chosen": -0.4802612364292145, |
| "logits/rejected": -0.5090576410293579, |
| "logps/chosen": -360.29998779296875, |
| "logps/rejected": -378.67498779296875, |
| "loss": 0.6234, |
| "rewards/accuracies": 0.6937500238418579, |
| "rewards/chosen": -0.32078856229782104, |
| "rewards/margins": 1.0515868663787842, |
| "rewards/rejected": -1.372583031654358, |
| "step": 1140 |
| }, |
| { |
| "epoch": 0.7402639201802381, |
| "grad_norm": 124.88195018416995, |
| "learning_rate": 8.150354153251771e-07, |
| "logits/chosen": -0.46788328886032104, |
| "logits/rejected": -0.4967285096645355, |
| "logps/chosen": -366.98748779296875, |
| "logps/rejected": -367.8999938964844, |
| "loss": 0.6054, |
| "rewards/accuracies": 0.65625, |
| "rewards/chosen": 0.23341064155101776, |
| "rewards/margins": 1.050317406654358, |
| "rewards/rejected": -0.817883312702179, |
| "step": 1150 |
| }, |
| { |
| "epoch": 0.7467009977470228, |
| "grad_norm": 115.48073006138456, |
| "learning_rate": 8.13425627817128e-07, |
| "logits/chosen": -0.48234254121780396, |
| "logits/rejected": -0.49266356229782104, |
| "logps/chosen": -372.6499938964844, |
| "logps/rejected": -340.3999938964844, |
| "loss": 0.538, |
| "rewards/accuracies": 0.71875, |
| "rewards/chosen": 0.525585949420929, |
| "rewards/margins": 1.1814696788787842, |
| "rewards/rejected": -0.655688464641571, |
| "step": 1160 |
| }, |
| { |
| "epoch": 0.7531380753138075, |
| "grad_norm": 85.01864055690064, |
| "learning_rate": 8.118158403090791e-07, |
| "logits/chosen": -0.3582519590854645, |
| "logits/rejected": -0.3791137635707855, |
| "logps/chosen": -382.1000061035156, |
| "logps/rejected": -345.29998779296875, |
| "loss": 0.6241, |
| "rewards/accuracies": 0.6812499761581421, |
| "rewards/chosen": 0.05527343600988388, |
| "rewards/margins": 1.0609130859375, |
| "rewards/rejected": -1.0050170421600342, |
| "step": 1170 |
| }, |
| { |
| "epoch": 0.7595751528805922, |
| "grad_norm": 114.9441718662302, |
| "learning_rate": 8.102060528010303e-07, |
| "logits/chosen": -0.46600341796875, |
| "logits/rejected": -0.4788574278354645, |
| "logps/chosen": -399.51251220703125, |
| "logps/rejected": -397.3500061035156, |
| "loss": 0.5901, |
| "rewards/accuracies": 0.699999988079071, |
| "rewards/chosen": -0.261474609375, |
| "rewards/margins": 1.3340332508087158, |
| "rewards/rejected": -1.596215844154358, |
| "step": 1180 |
| }, |
| { |
| "epoch": 0.7660122304473769, |
| "grad_norm": 108.13993143362673, |
| "learning_rate": 8.085962652929813e-07, |
| "logits/chosen": -0.45205992460250854, |
| "logits/rejected": -0.4634765684604645, |
| "logps/chosen": -386.98748779296875, |
| "logps/rejected": -358.625, |
| "loss": 0.6667, |
| "rewards/accuracies": 0.699999988079071, |
| "rewards/chosen": -0.49949342012405396, |
| "rewards/margins": 1.317773461341858, |
| "rewards/rejected": -1.817724585533142, |
| "step": 1190 |
| }, |
| { |
| "epoch": 0.7724493080141616, |
| "grad_norm": 147.08462386802262, |
| "learning_rate": 8.069864777849323e-07, |
| "logits/chosen": -0.49937134981155396, |
| "logits/rejected": -0.505419909954071, |
| "logps/chosen": -369.0625, |
| "logps/rejected": -382.4750061035156, |
| "loss": 0.7564, |
| "rewards/accuracies": 0.6187499761581421, |
| "rewards/chosen": -0.2877746522426605, |
| "rewards/margins": 0.7278076410293579, |
| "rewards/rejected": -1.015435814857483, |
| "step": 1200 |
| }, |
| { |
| "epoch": 0.7788863855809462, |
| "grad_norm": 115.55623695277468, |
| "learning_rate": 8.053766902768834e-07, |
| "logits/chosen": -0.4474243223667145, |
| "logits/rejected": -0.46574705839157104, |
| "logps/chosen": -355.86248779296875, |
| "logps/rejected": -345.07501220703125, |
| "loss": 0.569, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": 0.0020507811568677425, |
| "rewards/margins": 0.920788586139679, |
| "rewards/rejected": -0.917816162109375, |
| "step": 1210 |
| }, |
| { |
| "epoch": 0.7853234631477309, |
| "grad_norm": 98.71710653719403, |
| "learning_rate": 8.037669027688345e-07, |
| "logits/chosen": -0.484619140625, |
| "logits/rejected": -0.47932130098342896, |
| "logps/chosen": -372.7250061035156, |
| "logps/rejected": -371.20001220703125, |
| "loss": 0.5893, |
| "rewards/accuracies": 0.637499988079071, |
| "rewards/chosen": -0.21046753227710724, |
| "rewards/margins": 0.9305664300918579, |
| "rewards/rejected": -1.142114281654358, |
| "step": 1220 |
| }, |
| { |
| "epoch": 0.7917605407145156, |
| "grad_norm": 116.58967696349914, |
| "learning_rate": 8.021571152607856e-07, |
| "logits/chosen": -0.41608887910842896, |
| "logits/rejected": -0.4575439393520355, |
| "logps/chosen": -374.63751220703125, |
| "logps/rejected": -356.92498779296875, |
| "loss": 0.6804, |
| "rewards/accuracies": 0.6312500238418579, |
| "rewards/chosen": -0.17164306342601776, |
| "rewards/margins": 0.873791515827179, |
| "rewards/rejected": -1.045166015625, |
| "step": 1230 |
| }, |
| { |
| "epoch": 0.7981976182813003, |
| "grad_norm": 89.90212458778099, |
| "learning_rate": 8.005473277527366e-07, |
| "logits/chosen": -0.47612303495407104, |
| "logits/rejected": -0.46992188692092896, |
| "logps/chosen": -382.98748779296875, |
| "logps/rejected": -376.42498779296875, |
| "loss": 0.6204, |
| "rewards/accuracies": 0.699999988079071, |
| "rewards/chosen": -0.11334228515625, |
| "rewards/margins": 1.0497925281524658, |
| "rewards/rejected": -1.1633727550506592, |
| "step": 1240 |
| }, |
| { |
| "epoch": 0.804634695848085, |
| "grad_norm": 113.94650452614209, |
| "learning_rate": 7.989375402446876e-07, |
| "logits/chosen": -0.4763244688510895, |
| "logits/rejected": -0.49517822265625, |
| "logps/chosen": -347.23748779296875, |
| "logps/rejected": -325.0249938964844, |
| "loss": 0.623, |
| "rewards/accuracies": 0.6812499761581421, |
| "rewards/chosen": -0.11918945610523224, |
| "rewards/margins": 0.8921142816543579, |
| "rewards/rejected": -1.0122039318084717, |
| "step": 1250 |
| }, |
| { |
| "epoch": 0.8110717734148697, |
| "grad_norm": 98.14429996042087, |
| "learning_rate": 7.973277527366388e-07, |
| "logits/chosen": -0.4635772705078125, |
| "logits/rejected": -0.4755493104457855, |
| "logps/chosen": -373.70001220703125, |
| "logps/rejected": -357.25, |
| "loss": 0.6548, |
| "rewards/accuracies": 0.65625, |
| "rewards/chosen": -0.548840343952179, |
| "rewards/margins": 0.88629150390625, |
| "rewards/rejected": -1.435449242591858, |
| "step": 1260 |
| }, |
| { |
| "epoch": 0.8175088509816544, |
| "grad_norm": 97.0864968634607, |
| "learning_rate": 7.957179652285899e-07, |
| "logits/chosen": -0.525805652141571, |
| "logits/rejected": -0.510266125202179, |
| "logps/chosen": -354.6499938964844, |
| "logps/rejected": -351.95001220703125, |
| "loss": 0.5806, |
| "rewards/accuracies": 0.6937500238418579, |
| "rewards/chosen": -0.37434083223342896, |
| "rewards/margins": 1.059057593345642, |
| "rewards/rejected": -1.4339721202850342, |
| "step": 1270 |
| }, |
| { |
| "epoch": 0.823945928548439, |
| "grad_norm": 81.72733031375124, |
| "learning_rate": 7.941081777205408e-07, |
| "logits/chosen": -0.47092896699905396, |
| "logits/rejected": -0.47998350858688354, |
| "logps/chosen": -387.8374938964844, |
| "logps/rejected": -374.79998779296875, |
| "loss": 0.5909, |
| "rewards/accuracies": 0.668749988079071, |
| "rewards/chosen": -0.31041258573532104, |
| "rewards/margins": 1.3404357433319092, |
| "rewards/rejected": -1.651647925376892, |
| "step": 1280 |
| }, |
| { |
| "epoch": 0.8303830061152236, |
| "grad_norm": 61.78251081973873, |
| "learning_rate": 7.924983902124919e-07, |
| "logits/chosen": -0.3810485899448395, |
| "logits/rejected": -0.41566163301467896, |
| "logps/chosen": -376.1000061035156, |
| "logps/rejected": -358.54998779296875, |
| "loss": 0.5402, |
| "rewards/accuracies": 0.699999988079071, |
| "rewards/chosen": -0.06690673530101776, |
| "rewards/margins": 1.2426269054412842, |
| "rewards/rejected": -1.310052514076233, |
| "step": 1290 |
| }, |
| { |
| "epoch": 0.8368200836820083, |
| "grad_norm": 109.83631710755957, |
| "learning_rate": 7.908886027044431e-07, |
| "logits/chosen": -0.42792969942092896, |
| "logits/rejected": -0.45692139863967896, |
| "logps/chosen": -384.875, |
| "logps/rejected": -376.32501220703125, |
| "loss": 0.6567, |
| "rewards/accuracies": 0.637499988079071, |
| "rewards/chosen": -0.03769531100988388, |
| "rewards/margins": 1.0447266101837158, |
| "rewards/rejected": -1.082800269126892, |
| "step": 1300 |
| }, |
| { |
| "epoch": 0.843257161248793, |
| "grad_norm": 91.84896650553513, |
| "learning_rate": 7.89278815196394e-07, |
| "logits/chosen": -0.36790770292282104, |
| "logits/rejected": -0.3922485411167145, |
| "logps/chosen": -359.7749938964844, |
| "logps/rejected": -365.0249938964844, |
| "loss": 0.5082, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -0.12186889350414276, |
| "rewards/margins": 1.2075378894805908, |
| "rewards/rejected": -1.328283667564392, |
| "step": 1310 |
| }, |
| { |
| "epoch": 0.8496942388155777, |
| "grad_norm": 123.08939858015702, |
| "learning_rate": 7.876690276883451e-07, |
| "logits/chosen": -0.4756835997104645, |
| "logits/rejected": -0.5056854486465454, |
| "logps/chosen": -392.625, |
| "logps/rejected": -380.5249938964844, |
| "loss": 0.6656, |
| "rewards/accuracies": 0.668749988079071, |
| "rewards/chosen": -0.29423826932907104, |
| "rewards/margins": 0.930346667766571, |
| "rewards/rejected": -1.225061058998108, |
| "step": 1320 |
| }, |
| { |
| "epoch": 0.8561313163823624, |
| "grad_norm": 89.38220487310218, |
| "learning_rate": 7.860592401802961e-07, |
| "logits/chosen": -0.5153290033340454, |
| "logits/rejected": -0.5448852777481079, |
| "logps/chosen": -406.92498779296875, |
| "logps/rejected": -377.625, |
| "loss": 0.6152, |
| "rewards/accuracies": 0.6625000238418579, |
| "rewards/chosen": -0.2519470155239105, |
| "rewards/margins": 0.8592529296875, |
| "rewards/rejected": -1.1110961437225342, |
| "step": 1330 |
| }, |
| { |
| "epoch": 0.8625683939491471, |
| "grad_norm": 138.69932522724545, |
| "learning_rate": 7.844494526722473e-07, |
| "logits/chosen": -0.43743896484375, |
| "logits/rejected": -0.45673829317092896, |
| "logps/chosen": -381.0375061035156, |
| "logps/rejected": -372.75, |
| "loss": 0.6838, |
| "rewards/accuracies": 0.612500011920929, |
| "rewards/chosen": -0.09254150092601776, |
| "rewards/margins": 0.789379894733429, |
| "rewards/rejected": -0.882275402545929, |
| "step": 1340 |
| }, |
| { |
| "epoch": 0.8690054715159318, |
| "grad_norm": 101.36114737984545, |
| "learning_rate": 7.828396651641983e-07, |
| "logits/chosen": -0.5053650140762329, |
| "logits/rejected": -0.511737048625946, |
| "logps/chosen": -386.0874938964844, |
| "logps/rejected": -340.79376220703125, |
| "loss": 0.6082, |
| "rewards/accuracies": 0.637499988079071, |
| "rewards/chosen": 0.2793136537075043, |
| "rewards/margins": 1.0363891124725342, |
| "rewards/rejected": -0.7570556402206421, |
| "step": 1350 |
| }, |
| { |
| "epoch": 0.8754425490827165, |
| "grad_norm": 56.94113918190888, |
| "learning_rate": 7.812298776561493e-07, |
| "logits/chosen": -0.45692139863967896, |
| "logits/rejected": -0.47791749238967896, |
| "logps/chosen": -400.7250061035156, |
| "logps/rejected": -383.375, |
| "loss": 0.5721, |
| "rewards/accuracies": 0.71875, |
| "rewards/chosen": 0.06065673753619194, |
| "rewards/margins": 1.01776123046875, |
| "rewards/rejected": -0.9569336175918579, |
| "step": 1360 |
| }, |
| { |
| "epoch": 0.8818796266495011, |
| "grad_norm": 87.21788738477099, |
| "learning_rate": 7.796200901481004e-07, |
| "logits/chosen": -0.43765562772750854, |
| "logits/rejected": -0.4749755859375, |
| "logps/chosen": -340.1000061035156, |
| "logps/rejected": -339.7250061035156, |
| "loss": 0.7079, |
| "rewards/accuracies": 0.625, |
| "rewards/chosen": -0.2620178163051605, |
| "rewards/margins": 0.7551513910293579, |
| "rewards/rejected": -1.0172851085662842, |
| "step": 1370 |
| }, |
| { |
| "epoch": 0.8883167042162858, |
| "grad_norm": 116.26399959827697, |
| "learning_rate": 7.780103026400516e-07, |
| "logits/chosen": -0.457510381937027, |
| "logits/rejected": -0.48211669921875, |
| "logps/chosen": -376.32501220703125, |
| "logps/rejected": -366.95001220703125, |
| "loss": 0.7661, |
| "rewards/accuracies": 0.581250011920929, |
| "rewards/chosen": -0.46033018827438354, |
| "rewards/margins": 0.6608642339706421, |
| "rewards/rejected": -1.121118187904358, |
| "step": 1380 |
| }, |
| { |
| "epoch": 0.8947537817830705, |
| "grad_norm": 70.21186430940142, |
| "learning_rate": 7.764005151320025e-07, |
| "logits/chosen": -0.47960203886032104, |
| "logits/rejected": -0.517138659954071, |
| "logps/chosen": -344.79998779296875, |
| "logps/rejected": -343.3999938964844, |
| "loss": 0.4856, |
| "rewards/accuracies": 0.7124999761581421, |
| "rewards/chosen": -0.19971923530101776, |
| "rewards/margins": 1.256591796875, |
| "rewards/rejected": -1.4572632312774658, |
| "step": 1390 |
| }, |
| { |
| "epoch": 0.9011908593498552, |
| "grad_norm": 112.5823484060804, |
| "learning_rate": 7.747907276239536e-07, |
| "logits/chosen": -0.500781238079071, |
| "logits/rejected": -0.5128418207168579, |
| "logps/chosen": -392.375, |
| "logps/rejected": -373.1000061035156, |
| "loss": 0.5976, |
| "rewards/accuracies": 0.699999988079071, |
| "rewards/chosen": -0.542877197265625, |
| "rewards/margins": 1.1980102062225342, |
| "rewards/rejected": -1.740942358970642, |
| "step": 1400 |
| }, |
| { |
| "epoch": 0.9076279369166399, |
| "grad_norm": 108.00708309053341, |
| "learning_rate": 7.731809401159047e-07, |
| "logits/chosen": -0.44998472929000854, |
| "logits/rejected": -0.4686279296875, |
| "logps/chosen": -384.8374938964844, |
| "logps/rejected": -384.95001220703125, |
| "loss": 0.5577, |
| "rewards/accuracies": 0.6625000238418579, |
| "rewards/chosen": -0.6136230230331421, |
| "rewards/margins": 1.4021484851837158, |
| "rewards/rejected": -2.0157883167266846, |
| "step": 1410 |
| }, |
| { |
| "epoch": 0.9140650144834245, |
| "grad_norm": 75.63651943592622, |
| "learning_rate": 7.715711526078557e-07, |
| "logits/chosen": -0.4461120665073395, |
| "logits/rejected": -0.46266478300094604, |
| "logps/chosen": -357.75, |
| "logps/rejected": -340.36248779296875, |
| "loss": 0.6517, |
| "rewards/accuracies": 0.6812499761581421, |
| "rewards/chosen": -0.07795409858226776, |
| "rewards/margins": 1.0103881359100342, |
| "rewards/rejected": -1.087805151939392, |
| "step": 1420 |
| }, |
| { |
| "epoch": 0.9205020920502092, |
| "grad_norm": 82.65401725045969, |
| "learning_rate": 7.699613650998068e-07, |
| "logits/chosen": -0.4739746153354645, |
| "logits/rejected": -0.5115722417831421, |
| "logps/chosen": -378.875, |
| "logps/rejected": -375.20001220703125, |
| "loss": 0.6256, |
| "rewards/accuracies": 0.6625000238418579, |
| "rewards/chosen": 0.321298211812973, |
| "rewards/margins": 0.9541260004043579, |
| "rewards/rejected": -0.6323302984237671, |
| "step": 1430 |
| }, |
| { |
| "epoch": 0.9269391696169939, |
| "grad_norm": 112.68383111556054, |
| "learning_rate": 7.683515775917579e-07, |
| "logits/chosen": -0.5020507574081421, |
| "logits/rejected": -0.510693371295929, |
| "logps/chosen": -360.61248779296875, |
| "logps/rejected": -361.67498779296875, |
| "loss": 0.6268, |
| "rewards/accuracies": 0.699999988079071, |
| "rewards/chosen": 0.434814453125, |
| "rewards/margins": 0.9604736566543579, |
| "rewards/rejected": -0.5249465703964233, |
| "step": 1440 |
| }, |
| { |
| "epoch": 0.9333762471837785, |
| "grad_norm": 121.03410484944297, |
| "learning_rate": 7.667417900837089e-07, |
| "logits/chosen": -0.44097900390625, |
| "logits/rejected": -0.45458984375, |
| "logps/chosen": -374.5625, |
| "logps/rejected": -340.5249938964844, |
| "loss": 0.6559, |
| "rewards/accuracies": 0.6625000238418579, |
| "rewards/chosen": 0.13691405951976776, |
| "rewards/margins": 1.165869116783142, |
| "rewards/rejected": -1.029052734375, |
| "step": 1450 |
| }, |
| { |
| "epoch": 0.9398133247505632, |
| "grad_norm": 99.05584268644827, |
| "learning_rate": 7.6513200257566e-07, |
| "logits/chosen": -0.5095580816268921, |
| "logits/rejected": -0.5365966558456421, |
| "logps/chosen": -380.42498779296875, |
| "logps/rejected": -349.1499938964844, |
| "loss": 0.59, |
| "rewards/accuracies": 0.71875, |
| "rewards/chosen": 0.10870361328125, |
| "rewards/margins": 1.219384789466858, |
| "rewards/rejected": -1.110510230064392, |
| "step": 1460 |
| }, |
| { |
| "epoch": 0.9462504023173479, |
| "grad_norm": 130.78950082280264, |
| "learning_rate": 7.63522215067611e-07, |
| "logits/chosen": -0.45726317167282104, |
| "logits/rejected": -0.4390853941440582, |
| "logps/chosen": -384.7124938964844, |
| "logps/rejected": -380.2749938964844, |
| "loss": 0.7047, |
| "rewards/accuracies": 0.6625000238418579, |
| "rewards/chosen": -0.13157348334789276, |
| "rewards/margins": 0.8795715570449829, |
| "rewards/rejected": -1.01123046875, |
| "step": 1470 |
| }, |
| { |
| "epoch": 0.9526874798841326, |
| "grad_norm": 100.85870827827803, |
| "learning_rate": 7.619124275595621e-07, |
| "logits/chosen": -0.4605956971645355, |
| "logits/rejected": -0.5074218511581421, |
| "logps/chosen": -406.5249938964844, |
| "logps/rejected": -398.3500061035156, |
| "loss": 0.5323, |
| "rewards/accuracies": 0.6937500238418579, |
| "rewards/chosen": 0.14664916694164276, |
| "rewards/margins": 1.08514404296875, |
| "rewards/rejected": -0.939013659954071, |
| "step": 1480 |
| }, |
| { |
| "epoch": 0.9591245574509173, |
| "grad_norm": 99.51512777249033, |
| "learning_rate": 7.603026400515131e-07, |
| "logits/chosen": -0.519457995891571, |
| "logits/rejected": -0.5551391839981079, |
| "logps/chosen": -412.1499938964844, |
| "logps/rejected": -398.42498779296875, |
| "loss": 0.5587, |
| "rewards/accuracies": 0.699999988079071, |
| "rewards/chosen": -0.12549439072608948, |
| "rewards/margins": 1.107519507408142, |
| "rewards/rejected": -1.232666015625, |
| "step": 1490 |
| }, |
| { |
| "epoch": 0.965561635017702, |
| "grad_norm": 118.0541567845769, |
| "learning_rate": 7.586928525434642e-07, |
| "logits/chosen": -0.43101805448532104, |
| "logits/rejected": -0.40216064453125, |
| "logps/chosen": -390.5249938964844, |
| "logps/rejected": -370.0, |
| "loss": 0.6822, |
| "rewards/accuracies": 0.59375, |
| "rewards/chosen": -0.705487072467804, |
| "rewards/margins": 1.0155029296875, |
| "rewards/rejected": -1.7212402820587158, |
| "step": 1500 |
| }, |
| { |
| "epoch": 0.9719987125844867, |
| "grad_norm": 98.22598118527986, |
| "learning_rate": 7.570830650354153e-07, |
| "logits/chosen": -0.46629029512405396, |
| "logits/rejected": -0.47856444120407104, |
| "logps/chosen": -374.36248779296875, |
| "logps/rejected": -351.8999938964844, |
| "loss": 0.5244, |
| "rewards/accuracies": 0.7562500238418579, |
| "rewards/chosen": -0.17139282822608948, |
| "rewards/margins": 1.374121069908142, |
| "rewards/rejected": -1.545922875404358, |
| "step": 1510 |
| }, |
| { |
| "epoch": 0.9784357901512714, |
| "grad_norm": 117.92656420297726, |
| "learning_rate": 7.554732775273664e-07, |
| "logits/chosen": -0.486328125, |
| "logits/rejected": -0.4902099668979645, |
| "logps/chosen": -353.3500061035156, |
| "logps/rejected": -372.4750061035156, |
| "loss": 0.6914, |
| "rewards/accuracies": 0.675000011920929, |
| "rewards/chosen": 0.07288818061351776, |
| "rewards/margins": 0.8248291015625, |
| "rewards/rejected": -0.752392590045929, |
| "step": 1520 |
| }, |
| { |
| "epoch": 0.984872867718056, |
| "grad_norm": 103.7352972935465, |
| "learning_rate": 7.538634900193173e-07, |
| "logits/chosen": -0.44585877656936646, |
| "logits/rejected": -0.4797119200229645, |
| "logps/chosen": -364.51251220703125, |
| "logps/rejected": -334.07501220703125, |
| "loss": 0.5836, |
| "rewards/accuracies": 0.706250011920929, |
| "rewards/chosen": 0.17331238090991974, |
| "rewards/margins": 1.103173851966858, |
| "rewards/rejected": -0.929003894329071, |
| "step": 1530 |
| }, |
| { |
| "epoch": 0.9913099452848407, |
| "grad_norm": 92.0120508509047, |
| "learning_rate": 7.522537025112685e-07, |
| "logits/chosen": -0.432577520608902, |
| "logits/rejected": -0.44459229707717896, |
| "logps/chosen": -374.3999938964844, |
| "logps/rejected": -374.625, |
| "loss": 0.5247, |
| "rewards/accuracies": 0.7250000238418579, |
| "rewards/chosen": 0.04023437574505806, |
| "rewards/margins": 1.244116187095642, |
| "rewards/rejected": -1.2042725086212158, |
| "step": 1540 |
| }, |
| { |
| "epoch": 0.9977470228516253, |
| "grad_norm": 113.60262425936246, |
| "learning_rate": 7.506439150032196e-07, |
| "logits/chosen": -0.44282227754592896, |
| "logits/rejected": -0.4719482362270355, |
| "logps/chosen": -399.1499938964844, |
| "logps/rejected": -388.2749938964844, |
| "loss": 0.5858, |
| "rewards/accuracies": 0.6499999761581421, |
| "rewards/chosen": -0.48509520292282104, |
| "rewards/margins": 1.2941710948944092, |
| "rewards/rejected": -1.7789428234100342, |
| "step": 1550 |
| }, |
| { |
| "epoch": 1.0038622465400708, |
| "grad_norm": 28.701645450043788, |
| "learning_rate": 7.490341274951706e-07, |
| "logits/chosen": -0.5120528340339661, |
| "logits/rejected": -0.5163445472717285, |
| "logps/chosen": -368.6842041015625, |
| "logps/rejected": -373.47369384765625, |
| "loss": 0.2804, |
| "rewards/accuracies": 0.8223684430122375, |
| "rewards/chosen": 0.9294819235801697, |
| "rewards/margins": 4.5301833152771, |
| "rewards/rejected": -3.5959086418151855, |
| "step": 1560 |
| }, |
| { |
| "epoch": 1.0102993241068554, |
| "grad_norm": 60.32074871596137, |
| "learning_rate": 7.474243399871216e-07, |
| "logits/chosen": -0.4825439453125, |
| "logits/rejected": -0.52020263671875, |
| "logps/chosen": -371.92498779296875, |
| "logps/rejected": -425.45001220703125, |
| "loss": 0.0487, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": 2.0789122581481934, |
| "rewards/margins": 7.958593845367432, |
| "rewards/rejected": -5.882421970367432, |
| "step": 1570 |
| }, |
| { |
| "epoch": 1.0167364016736402, |
| "grad_norm": 71.52396938669607, |
| "learning_rate": 7.458145524790728e-07, |
| "logits/chosen": -0.6259857416152954, |
| "logits/rejected": -0.646166980266571, |
| "logps/chosen": -352.3999938964844, |
| "logps/rejected": -411.70001220703125, |
| "loss": 0.0354, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": 1.0871460437774658, |
| "rewards/margins": 8.010156631469727, |
| "rewards/rejected": -6.9296875, |
| "step": 1580 |
| }, |
| { |
| "epoch": 1.0231734792404248, |
| "grad_norm": 9.724575140553721, |
| "learning_rate": 7.442047649710238e-07, |
| "logits/chosen": -0.687255859375, |
| "logits/rejected": -0.711743175983429, |
| "logps/chosen": -335.79998779296875, |
| "logps/rejected": -400.25, |
| "loss": 0.052, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -0.412109375, |
| "rewards/margins": 8.798828125, |
| "rewards/rejected": -9.207422256469727, |
| "step": 1590 |
| }, |
| { |
| "epoch": 1.0296105568072096, |
| "grad_norm": 42.3472252623567, |
| "learning_rate": 7.425949774629749e-07, |
| "logits/chosen": -0.783618152141571, |
| "logits/rejected": -0.7865371704101562, |
| "logps/chosen": -406.625, |
| "logps/rejected": -462.3999938964844, |
| "loss": 0.0677, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -0.48292237520217896, |
| "rewards/margins": 10.419140815734863, |
| "rewards/rejected": -10.904687881469727, |
| "step": 1600 |
| }, |
| { |
| "epoch": 1.0360476343739942, |
| "grad_norm": 44.590992711597345, |
| "learning_rate": 7.409851899549258e-07, |
| "logits/chosen": -0.7693115472793579, |
| "logits/rejected": -0.7798522710800171, |
| "logps/chosen": -367.6499938964844, |
| "logps/rejected": -441.5625, |
| "loss": 0.0596, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -0.7735351324081421, |
| "rewards/margins": 9.705469131469727, |
| "rewards/rejected": -10.483593940734863, |
| "step": 1610 |
| }, |
| { |
| "epoch": 1.042484711940779, |
| "grad_norm": 12.122547274948735, |
| "learning_rate": 7.39375402446877e-07, |
| "logits/chosen": -0.7435302734375, |
| "logits/rejected": -0.745318591594696, |
| "logps/chosen": -353.125, |
| "logps/rejected": -426.17498779296875, |
| "loss": 0.0561, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -0.11784668266773224, |
| "rewards/margins": 9.178125381469727, |
| "rewards/rejected": -9.298437118530273, |
| "step": 1620 |
| }, |
| { |
| "epoch": 1.0489217895075635, |
| "grad_norm": 4.7060888899695135, |
| "learning_rate": 7.377656149388281e-07, |
| "logits/chosen": -0.799609363079071, |
| "logits/rejected": -0.780871570110321, |
| "logps/chosen": -393.625, |
| "logps/rejected": -469.5249938964844, |
| "loss": 0.0357, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -0.3866333067417145, |
| "rewards/margins": 9.78515625, |
| "rewards/rejected": -10.168749809265137, |
| "step": 1630 |
| }, |
| { |
| "epoch": 1.0553588670743483, |
| "grad_norm": 19.42538895152891, |
| "learning_rate": 7.36155827430779e-07, |
| "logits/chosen": -0.8111816644668579, |
| "logits/rejected": -0.814648449420929, |
| "logps/chosen": -377.04998779296875, |
| "logps/rejected": -452.11248779296875, |
| "loss": 0.0331, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -0.6400299072265625, |
| "rewards/margins": 9.115625381469727, |
| "rewards/rejected": -9.759374618530273, |
| "step": 1640 |
| }, |
| { |
| "epoch": 1.061795944641133, |
| "grad_norm": 1.1011234013173983, |
| "learning_rate": 7.345460399227301e-07, |
| "logits/chosen": -0.773486316204071, |
| "logits/rejected": -0.772021472454071, |
| "logps/chosen": -405.7749938964844, |
| "logps/rejected": -478.2749938964844, |
| "loss": 0.0311, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -0.60272216796875, |
| "rewards/margins": 10.611719131469727, |
| "rewards/rejected": -11.21875, |
| "step": 1650 |
| }, |
| { |
| "epoch": 1.0682330222079175, |
| "grad_norm": 26.178562770306236, |
| "learning_rate": 7.329362524146813e-07, |
| "logits/chosen": -0.712841808795929, |
| "logits/rejected": -0.7065674066543579, |
| "logps/chosen": -353.73748779296875, |
| "logps/rejected": -436.2749938964844, |
| "loss": 0.0538, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -0.7074829339981079, |
| "rewards/margins": 8.717187881469727, |
| "rewards/rejected": -9.428906440734863, |
| "step": 1660 |
| }, |
| { |
| "epoch": 1.0746700997747023, |
| "grad_norm": 1.209371501717102, |
| "learning_rate": 7.313264649066323e-07, |
| "logits/chosen": -0.7484130859375, |
| "logits/rejected": -0.774340808391571, |
| "logps/chosen": -378.5625, |
| "logps/rejected": -436.29998779296875, |
| "loss": 0.0313, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 0.3489746153354645, |
| "rewards/margins": 8.931640625, |
| "rewards/rejected": -8.578906059265137, |
| "step": 1670 |
| }, |
| { |
| "epoch": 1.0811071773414869, |
| "grad_norm": 5.4457811328971815, |
| "learning_rate": 7.297166773985833e-07, |
| "logits/chosen": -0.726611316204071, |
| "logits/rejected": -0.738110363483429, |
| "logps/chosen": -353.79998779296875, |
| "logps/rejected": -463.6499938964844, |
| "loss": 0.0396, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -0.09562988579273224, |
| "rewards/margins": 10.096094131469727, |
| "rewards/rejected": -10.197656631469727, |
| "step": 1680 |
| }, |
| { |
| "epoch": 1.0875442549082717, |
| "grad_norm": 78.32697539506492, |
| "learning_rate": 7.281068898905344e-07, |
| "logits/chosen": -0.871044933795929, |
| "logits/rejected": -0.8653320074081421, |
| "logps/chosen": -401.8500061035156, |
| "logps/rejected": -505.54998779296875, |
| "loss": 0.0434, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -1.7566101551055908, |
| "rewards/margins": 10.932031631469727, |
| "rewards/rejected": -12.690625190734863, |
| "step": 1690 |
| }, |
| { |
| "epoch": 1.0939813324750562, |
| "grad_norm": 30.05286319924308, |
| "learning_rate": 7.264971023824855e-07, |
| "logits/chosen": -0.869873046875, |
| "logits/rejected": -0.87158203125, |
| "logps/chosen": -365.3374938964844, |
| "logps/rejected": -441.5, |
| "loss": 0.0598, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -2.2253174781799316, |
| "rewards/margins": 10.394922256469727, |
| "rewards/rejected": -12.62890625, |
| "step": 1700 |
| }, |
| { |
| "epoch": 1.100418410041841, |
| "grad_norm": 14.131277242386574, |
| "learning_rate": 7.248873148744366e-07, |
| "logits/chosen": -0.841992199420929, |
| "logits/rejected": -0.883532702922821, |
| "logps/chosen": -409.67498779296875, |
| "logps/rejected": -448.1000061035156, |
| "loss": 0.0498, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -2.0384764671325684, |
| "rewards/margins": 9.805468559265137, |
| "rewards/rejected": -11.846094131469727, |
| "step": 1710 |
| }, |
| { |
| "epoch": 1.1068554876086256, |
| "grad_norm": 7.132116530747616, |
| "learning_rate": 7.232775273663876e-07, |
| "logits/chosen": -0.8307861089706421, |
| "logits/rejected": -0.877148449420929, |
| "logps/chosen": -413.26251220703125, |
| "logps/rejected": -495.6499938964844, |
| "loss": 0.0771, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -0.5871948003768921, |
| "rewards/margins": 11.547656059265137, |
| "rewards/rejected": -12.12109375, |
| "step": 1720 |
| }, |
| { |
| "epoch": 1.1132925651754104, |
| "grad_norm": 0.47230522302878464, |
| "learning_rate": 7.216677398583386e-07, |
| "logits/chosen": -0.8125, |
| "logits/rejected": -0.8301757574081421, |
| "logps/chosen": -352.57501220703125, |
| "logps/rejected": -443.875, |
| "loss": 0.0518, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -0.527294933795929, |
| "rewards/margins": 10.287500381469727, |
| "rewards/rejected": -10.814844131469727, |
| "step": 1730 |
| }, |
| { |
| "epoch": 1.119729642742195, |
| "grad_norm": 37.48581782587474, |
| "learning_rate": 7.200579523502898e-07, |
| "logits/chosen": -0.687573254108429, |
| "logits/rejected": -0.688122570514679, |
| "logps/chosen": -382.375, |
| "logps/rejected": -495.54998779296875, |
| "loss": 0.0648, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.15927734971046448, |
| "rewards/margins": 11.528905868530273, |
| "rewards/rejected": -11.6953125, |
| "step": 1740 |
| }, |
| { |
| "epoch": 1.1261667203089798, |
| "grad_norm": 41.26402499082609, |
| "learning_rate": 7.184481648422408e-07, |
| "logits/chosen": -0.878955066204071, |
| "logits/rejected": -0.896044909954071, |
| "logps/chosen": -364.6000061035156, |
| "logps/rejected": -447.7250061035156, |
| "loss": 0.0548, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -0.19716796278953552, |
| "rewards/margins": 10.053515434265137, |
| "rewards/rejected": -10.253125190734863, |
| "step": 1750 |
| }, |
| { |
| "epoch": 1.1326037978757644, |
| "grad_norm": 3.6822102341100567, |
| "learning_rate": 7.168383773341918e-07, |
| "logits/chosen": -0.8634277582168579, |
| "logits/rejected": -0.873974621295929, |
| "logps/chosen": -392.20001220703125, |
| "logps/rejected": -458.20001220703125, |
| "loss": 0.0257, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -0.2974609434604645, |
| "rewards/margins": 10.384374618530273, |
| "rewards/rejected": -10.675000190734863, |
| "step": 1760 |
| }, |
| { |
| "epoch": 1.1390408754425492, |
| "grad_norm": 4.0760843716589426, |
| "learning_rate": 7.152285898261429e-07, |
| "logits/chosen": -0.8376098871231079, |
| "logits/rejected": -0.8130859136581421, |
| "logps/chosen": -376.8999938964844, |
| "logps/rejected": -478.8999938964844, |
| "loss": 0.0287, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -1.418371558189392, |
| "rewards/margins": 11.157031059265137, |
| "rewards/rejected": -12.575780868530273, |
| "step": 1770 |
| }, |
| { |
| "epoch": 1.1454779530093337, |
| "grad_norm": 17.081009996286852, |
| "learning_rate": 7.136188023180941e-07, |
| "logits/chosen": -0.81988525390625, |
| "logits/rejected": -0.8177734613418579, |
| "logps/chosen": -406.5, |
| "logps/rejected": -492.4750061035156, |
| "loss": 0.0404, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -2.221630811691284, |
| "rewards/margins": 11.858593940734863, |
| "rewards/rejected": -14.083593368530273, |
| "step": 1780 |
| }, |
| { |
| "epoch": 1.1519150305761183, |
| "grad_norm": 16.670165572478858, |
| "learning_rate": 7.12009014810045e-07, |
| "logits/chosen": -0.878173828125, |
| "logits/rejected": -0.886474609375, |
| "logps/chosen": -406.82501220703125, |
| "logps/rejected": -475.7250061035156, |
| "loss": 0.0203, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -2.0299072265625, |
| "rewards/margins": 11.866796493530273, |
| "rewards/rejected": -13.893750190734863, |
| "step": 1790 |
| }, |
| { |
| "epoch": 1.1583521081429031, |
| "grad_norm": 14.789491416020818, |
| "learning_rate": 7.103992273019961e-07, |
| "logits/chosen": -0.789306640625, |
| "logits/rejected": -0.776171863079071, |
| "logps/chosen": -414.42498779296875, |
| "logps/rejected": -475.95001220703125, |
| "loss": 0.034, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -1.1190063953399658, |
| "rewards/margins": 10.380859375, |
| "rewards/rejected": -11.503125190734863, |
| "step": 1800 |
| }, |
| { |
| "epoch": 1.164789185709688, |
| "grad_norm": 13.586311507009619, |
| "learning_rate": 7.087894397939471e-07, |
| "logits/chosen": -0.8396972417831421, |
| "logits/rejected": -0.8243652582168579, |
| "logps/chosen": -399.6499938964844, |
| "logps/rejected": -458.79998779296875, |
| "loss": 0.0374, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -1.6400878429412842, |
| "rewards/margins": 10.001562118530273, |
| "rewards/rejected": -11.642187118530273, |
| "step": 1810 |
| }, |
| { |
| "epoch": 1.1712262632764725, |
| "grad_norm": 55.57830649329583, |
| "learning_rate": 7.071796522858983e-07, |
| "logits/chosen": -0.861035168170929, |
| "logits/rejected": -0.846386730670929, |
| "logps/chosen": -402.2250061035156, |
| "logps/rejected": -496.1000061035156, |
| "loss": 0.0441, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -1.8700439929962158, |
| "rewards/margins": 11.387499809265137, |
| "rewards/rejected": -13.250781059265137, |
| "step": 1820 |
| }, |
| { |
| "epoch": 1.177663340843257, |
| "grad_norm": 18.828474480482623, |
| "learning_rate": 7.055698647778493e-07, |
| "logits/chosen": -0.851696789264679, |
| "logits/rejected": -0.8456054925918579, |
| "logps/chosen": -372.8999938964844, |
| "logps/rejected": -439.6000061035156, |
| "loss": 0.066, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -3.3226075172424316, |
| "rewards/margins": 10.33984375, |
| "rewards/rejected": -13.662500381469727, |
| "step": 1830 |
| }, |
| { |
| "epoch": 1.1841004184100419, |
| "grad_norm": 0.9764059185254257, |
| "learning_rate": 7.039600772698003e-07, |
| "logits/chosen": -0.82470703125, |
| "logits/rejected": -0.806689441204071, |
| "logps/chosen": -388.01251220703125, |
| "logps/rejected": -486.29998779296875, |
| "loss": 0.0703, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -2.5329222679138184, |
| "rewards/margins": 10.471094131469727, |
| "rewards/rejected": -13.009374618530273, |
| "step": 1840 |
| }, |
| { |
| "epoch": 1.1905374959768265, |
| "grad_norm": 3.7355014099628647, |
| "learning_rate": 7.023502897617514e-07, |
| "logits/chosen": -0.7955566644668579, |
| "logits/rejected": -0.8161865472793579, |
| "logps/chosen": -420.95001220703125, |
| "logps/rejected": -477.20001220703125, |
| "loss": 0.0414, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -1.4476196765899658, |
| "rewards/margins": 10.553515434265137, |
| "rewards/rejected": -12.003125190734863, |
| "step": 1850 |
| }, |
| { |
| "epoch": 1.1969745735436113, |
| "grad_norm": 23.271976132825916, |
| "learning_rate": 7.007405022537025e-07, |
| "logits/chosen": -0.7967590093612671, |
| "logits/rejected": -0.8336181640625, |
| "logps/chosen": -372.2749938964844, |
| "logps/rejected": -485.0249938964844, |
| "loss": 0.0653, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -3.135058641433716, |
| "rewards/margins": 11.382031440734863, |
| "rewards/rejected": -14.517969131469727, |
| "step": 1860 |
| }, |
| { |
| "epoch": 1.2034116511103958, |
| "grad_norm": 65.18811151029988, |
| "learning_rate": 6.991307147456535e-07, |
| "logits/chosen": -0.8287597894668579, |
| "logits/rejected": -0.823779284954071, |
| "logps/chosen": -387.79998779296875, |
| "logps/rejected": -462.4750061035156, |
| "loss": 0.035, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -2.011474609375, |
| "rewards/margins": 11.100781440734863, |
| "rewards/rejected": -13.106249809265137, |
| "step": 1870 |
| }, |
| { |
| "epoch": 1.2098487286771806, |
| "grad_norm": 10.973771958166278, |
| "learning_rate": 6.975209272376046e-07, |
| "logits/chosen": -0.7869873046875, |
| "logits/rejected": -0.7986816167831421, |
| "logps/chosen": -360.5249938964844, |
| "logps/rejected": -439.20001220703125, |
| "loss": 0.038, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -1.812768578529358, |
| "rewards/margins": 10.350781440734863, |
| "rewards/rejected": -12.167187690734863, |
| "step": 1880 |
| }, |
| { |
| "epoch": 1.2162858062439652, |
| "grad_norm": 0.1795200411379023, |
| "learning_rate": 6.959111397295557e-07, |
| "logits/chosen": -0.8089843988418579, |
| "logits/rejected": -0.7916015386581421, |
| "logps/chosen": -327.3500061035156, |
| "logps/rejected": -447.54998779296875, |
| "loss": 0.0326, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -0.839111328125, |
| "rewards/margins": 10.127344131469727, |
| "rewards/rejected": -10.97265625, |
| "step": 1890 |
| }, |
| { |
| "epoch": 1.22272288381075, |
| "grad_norm": 6.31069489977895, |
| "learning_rate": 6.943013522215067e-07, |
| "logits/chosen": -0.7904297113418579, |
| "logits/rejected": -0.789257824420929, |
| "logps/chosen": -380.79998779296875, |
| "logps/rejected": -457.5, |
| "loss": 0.0564, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 0.0034423829056322575, |
| "rewards/margins": 10.767187118530273, |
| "rewards/rejected": -10.768750190734863, |
| "step": 1900 |
| }, |
| { |
| "epoch": 1.2291599613775346, |
| "grad_norm": 1.890302975072155, |
| "learning_rate": 6.926915647134578e-07, |
| "logits/chosen": -0.8453521728515625, |
| "logits/rejected": -0.8412109613418579, |
| "logps/chosen": -400.70001220703125, |
| "logps/rejected": -479.8500061035156, |
| "loss": 0.0241, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -0.521453857421875, |
| "rewards/margins": 10.838281631469727, |
| "rewards/rejected": -11.364062309265137, |
| "step": 1910 |
| }, |
| { |
| "epoch": 1.2355970389443192, |
| "grad_norm": 5.2209876862192015, |
| "learning_rate": 6.910817772054089e-07, |
| "logits/chosen": -0.8232421875, |
| "logits/rejected": -0.831591784954071, |
| "logps/chosen": -363.8500061035156, |
| "logps/rejected": -451.125, |
| "loss": 0.0463, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -1.1357421875, |
| "rewards/margins": 10.284375190734863, |
| "rewards/rejected": -11.41796875, |
| "step": 1920 |
| }, |
| { |
| "epoch": 1.242034116511104, |
| "grad_norm": 8.784840887843162, |
| "learning_rate": 6.894719896973599e-07, |
| "logits/chosen": -0.854248046875, |
| "logits/rejected": -0.857421875, |
| "logps/chosen": -385.2250061035156, |
| "logps/rejected": -471.3999938964844, |
| "loss": 0.0269, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -0.4132934510707855, |
| "rewards/margins": 10.916406631469727, |
| "rewards/rejected": -11.321093559265137, |
| "step": 1930 |
| }, |
| { |
| "epoch": 1.2484711940778888, |
| "grad_norm": 11.499308969293939, |
| "learning_rate": 6.87862202189311e-07, |
| "logits/chosen": -0.8663085699081421, |
| "logits/rejected": -0.8819824457168579, |
| "logps/chosen": -367.20001220703125, |
| "logps/rejected": -482.4750061035156, |
| "loss": 0.0393, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -0.7445312738418579, |
| "rewards/margins": 10.687891006469727, |
| "rewards/rejected": -11.436718940734863, |
| "step": 1940 |
| }, |
| { |
| "epoch": 1.2549082716446733, |
| "grad_norm": 5.476246704655694, |
| "learning_rate": 6.86252414681262e-07, |
| "logits/chosen": -0.785839855670929, |
| "logits/rejected": -0.802783191204071, |
| "logps/chosen": -393.25, |
| "logps/rejected": -481.1499938964844, |
| "loss": 0.07, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": -1.304406762123108, |
| "rewards/margins": 10.49609375, |
| "rewards/rejected": -11.80078125, |
| "step": 1950 |
| }, |
| { |
| "epoch": 1.261345349211458, |
| "grad_norm": 1.8068433654986311, |
| "learning_rate": 6.846426271732131e-07, |
| "logits/chosen": -0.8782958984375, |
| "logits/rejected": -0.878369152545929, |
| "logps/chosen": -366.95001220703125, |
| "logps/rejected": -481.4750061035156, |
| "loss": 0.0281, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -1.360253930091858, |
| "rewards/margins": 11.563281059265137, |
| "rewards/rejected": -12.9296875, |
| "step": 1960 |
| }, |
| { |
| "epoch": 1.2677824267782427, |
| "grad_norm": 1.5992649831411958, |
| "learning_rate": 6.830328396651641e-07, |
| "logits/chosen": -0.916015625, |
| "logits/rejected": -0.910888671875, |
| "logps/chosen": -351.2749938964844, |
| "logps/rejected": -440.92498779296875, |
| "loss": 0.0428, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -1.484375, |
| "rewards/margins": 10.620312690734863, |
| "rewards/rejected": -12.095312118530273, |
| "step": 1970 |
| }, |
| { |
| "epoch": 1.2742195043450273, |
| "grad_norm": 11.932348554376217, |
| "learning_rate": 6.814230521571152e-07, |
| "logits/chosen": -0.7955566644668579, |
| "logits/rejected": -0.812182605266571, |
| "logps/chosen": -405.0249938964844, |
| "logps/rejected": -501.5, |
| "loss": 0.0258, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -1.031854271888733, |
| "rewards/margins": 11.592187881469727, |
| "rewards/rejected": -12.624218940734863, |
| "step": 1980 |
| }, |
| { |
| "epoch": 1.280656581911812, |
| "grad_norm": 0.7284651571104193, |
| "learning_rate": 6.798132646490663e-07, |
| "logits/chosen": -0.687243640422821, |
| "logits/rejected": -0.7085205316543579, |
| "logps/chosen": -370.9125061035156, |
| "logps/rejected": -477.25, |
| "loss": 0.0594, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -1.3655884265899658, |
| "rewards/margins": 11.599218368530273, |
| "rewards/rejected": -12.961718559265137, |
| "step": 1990 |
| }, |
| { |
| "epoch": 1.2870936594785967, |
| "grad_norm": 0.1726195121834098, |
| "learning_rate": 6.782034771410174e-07, |
| "logits/chosen": -0.886474609375, |
| "logits/rejected": -0.9327148199081421, |
| "logps/chosen": -374.79998779296875, |
| "logps/rejected": -464.75, |
| "loss": 0.0157, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -1.9635009765625, |
| "rewards/margins": 10.838281631469727, |
| "rewards/rejected": -12.809374809265137, |
| "step": 2000 |
| }, |
| { |
| "epoch": 1.2935307370453815, |
| "grad_norm": 4.855214989312065, |
| "learning_rate": 6.765936896329683e-07, |
| "logits/chosen": -0.7554687261581421, |
| "logits/rejected": -0.762524425983429, |
| "logps/chosen": -353.7749938964844, |
| "logps/rejected": -453.79998779296875, |
| "loss": 0.0237, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -2.166912794113159, |
| "rewards/margins": 10.567187309265137, |
| "rewards/rejected": -12.739062309265137, |
| "step": 2010 |
| }, |
| { |
| "epoch": 1.299967814612166, |
| "grad_norm": 3.805160643591438, |
| "learning_rate": 6.749839021249195e-07, |
| "logits/chosen": -0.774462878704071, |
| "logits/rejected": -0.8001464605331421, |
| "logps/chosen": -431.25, |
| "logps/rejected": -509.375, |
| "loss": 0.0564, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -2.2074341773986816, |
| "rewards/margins": 11.928125381469727, |
| "rewards/rejected": -14.134374618530273, |
| "step": 2020 |
| }, |
| { |
| "epoch": 1.3064048921789508, |
| "grad_norm": 12.782556298742978, |
| "learning_rate": 6.733741146168706e-07, |
| "logits/chosen": -0.6968749761581421, |
| "logits/rejected": -0.719860851764679, |
| "logps/chosen": -363.04998779296875, |
| "logps/rejected": -473.6499938964844, |
| "loss": 0.03, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -1.2151978015899658, |
| "rewards/margins": 12.274218559265137, |
| "rewards/rejected": -13.489062309265137, |
| "step": 2030 |
| }, |
| { |
| "epoch": 1.3128419697457354, |
| "grad_norm": 90.75357097442223, |
| "learning_rate": 6.717643271088216e-07, |
| "logits/chosen": -0.7369629144668579, |
| "logits/rejected": -0.760327160358429, |
| "logps/chosen": -395.13751220703125, |
| "logps/rejected": -462.79998779296875, |
| "loss": 0.0882, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -2.4952149391174316, |
| "rewards/margins": 10.8828125, |
| "rewards/rejected": -13.3671875, |
| "step": 2040 |
| }, |
| { |
| "epoch": 1.31927904731252, |
| "grad_norm": 0.5115947981333193, |
| "learning_rate": 6.701545396007726e-07, |
| "logits/chosen": -0.807690441608429, |
| "logits/rejected": -0.8475097417831421, |
| "logps/chosen": -365.8999938964844, |
| "logps/rejected": -443.20001220703125, |
| "loss": 0.0529, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -2.2901368141174316, |
| "rewards/margins": 10.971875190734863, |
| "rewards/rejected": -13.258593559265137, |
| "step": 2050 |
| }, |
| { |
| "epoch": 1.3257161248793048, |
| "grad_norm": 0.18940957690475052, |
| "learning_rate": 6.685447520927238e-07, |
| "logits/chosen": -0.775683581829071, |
| "logits/rejected": -0.818310558795929, |
| "logps/chosen": -360.25, |
| "logps/rejected": -440.8999938964844, |
| "loss": 0.0328, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -1.39141845703125, |
| "rewards/margins": 10.676562309265137, |
| "rewards/rejected": -12.060155868530273, |
| "step": 2060 |
| }, |
| { |
| "epoch": 1.3321532024460896, |
| "grad_norm": 6.818009967898228, |
| "learning_rate": 6.669349645846748e-07, |
| "logits/chosen": -0.794970691204071, |
| "logits/rejected": -0.8131347894668579, |
| "logps/chosen": -396.70001220703125, |
| "logps/rejected": -451.1499938964844, |
| "loss": 0.036, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -0.8651977777481079, |
| "rewards/margins": 10.630468368530273, |
| "rewards/rejected": -11.493749618530273, |
| "step": 2070 |
| }, |
| { |
| "epoch": 1.3385902800128742, |
| "grad_norm": 20.477227570556575, |
| "learning_rate": 6.653251770766258e-07, |
| "logits/chosen": -0.772412121295929, |
| "logits/rejected": -0.787646472454071, |
| "logps/chosen": -381.73748779296875, |
| "logps/rejected": -452.6499938964844, |
| "loss": 0.046, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -0.25780028104782104, |
| "rewards/margins": 10.538281440734863, |
| "rewards/rejected": -10.794530868530273, |
| "step": 2080 |
| }, |
| { |
| "epoch": 1.3450273575796587, |
| "grad_norm": 1.8958489445333633, |
| "learning_rate": 6.637153895685768e-07, |
| "logits/chosen": -0.758129894733429, |
| "logits/rejected": -0.7566162347793579, |
| "logps/chosen": -376.875, |
| "logps/rejected": -473.07501220703125, |
| "loss": 0.0434, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -0.41459959745407104, |
| "rewards/margins": 10.59375, |
| "rewards/rejected": -11.004687309265137, |
| "step": 2090 |
| }, |
| { |
| "epoch": 1.3514644351464435, |
| "grad_norm": 15.797070454042899, |
| "learning_rate": 6.62105602060528e-07, |
| "logits/chosen": -0.7794433832168579, |
| "logits/rejected": -0.765332043170929, |
| "logps/chosen": -375.7749938964844, |
| "logps/rejected": -448.20001220703125, |
| "loss": 0.0274, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -0.5860840082168579, |
| "rewards/margins": 10.074609756469727, |
| "rewards/rejected": -10.665234565734863, |
| "step": 2100 |
| }, |
| { |
| "epoch": 1.3579015127132281, |
| "grad_norm": 17.274563010286307, |
| "learning_rate": 6.604958145524791e-07, |
| "logits/chosen": -0.7362338900566101, |
| "logits/rejected": -0.7653137445449829, |
| "logps/chosen": -395.25, |
| "logps/rejected": -454.4750061035156, |
| "loss": 0.0504, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -1.1345703601837158, |
| "rewards/margins": 9.571093559265137, |
| "rewards/rejected": -10.703906059265137, |
| "step": 2110 |
| }, |
| { |
| "epoch": 1.364338590280013, |
| "grad_norm": 34.1120810421438, |
| "learning_rate": 6.5888602704443e-07, |
| "logits/chosen": -0.7216430902481079, |
| "logits/rejected": -0.7294555902481079, |
| "logps/chosen": -366.0375061035156, |
| "logps/rejected": -448.3999938964844, |
| "loss": 0.04, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": 0.01531982421875, |
| "rewards/margins": 10.12109375, |
| "rewards/rejected": -10.103906631469727, |
| "step": 2120 |
| }, |
| { |
| "epoch": 1.3707756678467975, |
| "grad_norm": 35.51436144182908, |
| "learning_rate": 6.572762395363811e-07, |
| "logits/chosen": -0.7086426019668579, |
| "logits/rejected": -0.7312988042831421, |
| "logps/chosen": -378.67498779296875, |
| "logps/rejected": -473.5375061035156, |
| "loss": 0.0406, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": 0.3042968809604645, |
| "rewards/margins": 10.831250190734863, |
| "rewards/rejected": -10.525781631469727, |
| "step": 2130 |
| }, |
| { |
| "epoch": 1.3772127454135823, |
| "grad_norm": 7.163516950663505, |
| "learning_rate": 6.556664520283323e-07, |
| "logits/chosen": -0.7219482660293579, |
| "logits/rejected": -0.7713867425918579, |
| "logps/chosen": -375.95001220703125, |
| "logps/rejected": -441.20001220703125, |
| "loss": 0.0507, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -0.01722412183880806, |
| "rewards/margins": 10.337499618530273, |
| "rewards/rejected": -10.358593940734863, |
| "step": 2140 |
| }, |
| { |
| "epoch": 1.3836498229803669, |
| "grad_norm": 5.920479235836306, |
| "learning_rate": 6.540566645202833e-07, |
| "logits/chosen": -0.726318359375, |
| "logits/rejected": -0.734570324420929, |
| "logps/chosen": -371.0, |
| "logps/rejected": -441.1000061035156, |
| "loss": 0.0288, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -0.1243896484375, |
| "rewards/margins": 10.610937118530273, |
| "rewards/rejected": -10.733593940734863, |
| "step": 2150 |
| }, |
| { |
| "epoch": 1.3900869005471517, |
| "grad_norm": 8.789538295836175, |
| "learning_rate": 6.524468770122343e-07, |
| "logits/chosen": -0.649243175983429, |
| "logits/rejected": -0.679980456829071, |
| "logps/chosen": -353.42498779296875, |
| "logps/rejected": -464.07501220703125, |
| "loss": 0.0424, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -0.13173827528953552, |
| "rewards/margins": 11.746874809265137, |
| "rewards/rejected": -11.876562118530273, |
| "step": 2160 |
| }, |
| { |
| "epoch": 1.3965239781139362, |
| "grad_norm": 10.025462859350501, |
| "learning_rate": 6.508370895041854e-07, |
| "logits/chosen": -0.73486328125, |
| "logits/rejected": -0.7211669683456421, |
| "logps/chosen": -390.8500061035156, |
| "logps/rejected": -462.9750061035156, |
| "loss": 0.0531, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -1.454126000404358, |
| "rewards/margins": 11.36328125, |
| "rewards/rejected": -12.815625190734863, |
| "step": 2170 |
| }, |
| { |
| "epoch": 1.4029610556807208, |
| "grad_norm": 11.342140558916348, |
| "learning_rate": 6.492273019961365e-07, |
| "logits/chosen": -0.7691650390625, |
| "logits/rejected": -0.7724243402481079, |
| "logps/chosen": -381.4750061035156, |
| "logps/rejected": -460.6499938964844, |
| "loss": 0.0549, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -2.3446898460388184, |
| "rewards/margins": 10.916406631469727, |
| "rewards/rejected": -13.258593559265137, |
| "step": 2180 |
| }, |
| { |
| "epoch": 1.4093981332475056, |
| "grad_norm": 0.7950552155099938, |
| "learning_rate": 6.476175144880875e-07, |
| "logits/chosen": -0.7981933355331421, |
| "logits/rejected": -0.8179931640625, |
| "logps/chosen": -403.0249938964844, |
| "logps/rejected": -469.625, |
| "loss": 0.0397, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -1.8957030773162842, |
| "rewards/margins": 11.323827743530273, |
| "rewards/rejected": -13.215624809265137, |
| "step": 2190 |
| }, |
| { |
| "epoch": 1.4158352108142904, |
| "grad_norm": 20.523500260808675, |
| "learning_rate": 6.460077269800386e-07, |
| "logits/chosen": -0.7896484136581421, |
| "logits/rejected": -0.8050537109375, |
| "logps/chosen": -353.57501220703125, |
| "logps/rejected": -488.6499938964844, |
| "loss": 0.0333, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -0.691693127155304, |
| "rewards/margins": 11.971094131469727, |
| "rewards/rejected": -12.664843559265137, |
| "step": 2200 |
| }, |
| { |
| "epoch": 1.422272288381075, |
| "grad_norm": 4.165148973749865, |
| "learning_rate": 6.443979394719896e-07, |
| "logits/chosen": -0.7835693359375, |
| "logits/rejected": -0.8207031488418579, |
| "logps/chosen": -404.6499938964844, |
| "logps/rejected": -483.82501220703125, |
| "loss": 0.0284, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -1.223669409751892, |
| "rewards/margins": 11.237500190734863, |
| "rewards/rejected": -12.465624809265137, |
| "step": 2210 |
| }, |
| { |
| "epoch": 1.4287093659478596, |
| "grad_norm": 5.180688078431531, |
| "learning_rate": 6.427881519639408e-07, |
| "logits/chosen": -0.684216320514679, |
| "logits/rejected": -0.7363525629043579, |
| "logps/chosen": -403.5249938964844, |
| "logps/rejected": -505.1000061035156, |
| "loss": 0.0265, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -1.9525146484375, |
| "rewards/margins": 12.81640625, |
| "rewards/rejected": -14.774999618530273, |
| "step": 2220 |
| }, |
| { |
| "epoch": 1.4351464435146444, |
| "grad_norm": 14.212893761212056, |
| "learning_rate": 6.411783644558918e-07, |
| "logits/chosen": -0.8042968511581421, |
| "logits/rejected": -0.794909656047821, |
| "logps/chosen": -403.8999938964844, |
| "logps/rejected": -481.2250061035156, |
| "loss": 0.0624, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -1.993322730064392, |
| "rewards/margins": 11.608593940734863, |
| "rewards/rejected": -13.594141006469727, |
| "step": 2230 |
| }, |
| { |
| "epoch": 1.441583521081429, |
| "grad_norm": 6.871037130676326, |
| "learning_rate": 6.395685769478428e-07, |
| "logits/chosen": -0.700634777545929, |
| "logits/rejected": -0.715405285358429, |
| "logps/chosen": -352.38751220703125, |
| "logps/rejected": -474.79998779296875, |
| "loss": 0.0557, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -0.9448608160018921, |
| "rewards/margins": 10.33984375, |
| "rewards/rejected": -11.283594131469727, |
| "step": 2240 |
| }, |
| { |
| "epoch": 1.4480205986482138, |
| "grad_norm": 0.18746076155577607, |
| "learning_rate": 6.379587894397939e-07, |
| "logits/chosen": -0.752368152141571, |
| "logits/rejected": -0.7897704839706421, |
| "logps/chosen": -397.29998779296875, |
| "logps/rejected": -457.3500061035156, |
| "loss": 0.0314, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -1.0015380382537842, |
| "rewards/margins": 10.325780868530273, |
| "rewards/rejected": -11.323437690734863, |
| "step": 2250 |
| }, |
| { |
| "epoch": 1.4544576762149983, |
| "grad_norm": 2.5507421968165973, |
| "learning_rate": 6.363490019317451e-07, |
| "logits/chosen": -0.742602527141571, |
| "logits/rejected": -0.7437683343887329, |
| "logps/chosen": -383.07501220703125, |
| "logps/rejected": -486.375, |
| "loss": 0.0506, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -1.1386229991912842, |
| "rewards/margins": 12.111719131469727, |
| "rewards/rejected": -13.255468368530273, |
| "step": 2260 |
| }, |
| { |
| "epoch": 1.4608947537817831, |
| "grad_norm": 2.2763823295013963, |
| "learning_rate": 6.34739214423696e-07, |
| "logits/chosen": -0.7447265386581421, |
| "logits/rejected": -0.748046875, |
| "logps/chosen": -410.625, |
| "logps/rejected": -481.70001220703125, |
| "loss": 0.0478, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -1.0862915515899658, |
| "rewards/margins": 10.287500381469727, |
| "rewards/rejected": -11.372655868530273, |
| "step": 2270 |
| }, |
| { |
| "epoch": 1.4673318313485677, |
| "grad_norm": 0.4260998042645073, |
| "learning_rate": 6.331294269156471e-07, |
| "logits/chosen": -0.845996081829071, |
| "logits/rejected": -0.8616698980331421, |
| "logps/chosen": -393.95001220703125, |
| "logps/rejected": -484.67498779296875, |
| "loss": 0.0348, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -1.0209228992462158, |
| "rewards/margins": 11.013280868530273, |
| "rewards/rejected": -12.045312881469727, |
| "step": 2280 |
| }, |
| { |
| "epoch": 1.4737689089153525, |
| "grad_norm": 123.17291450113709, |
| "learning_rate": 6.315196394075981e-07, |
| "logits/chosen": -0.721881091594696, |
| "logits/rejected": -0.744799792766571, |
| "logps/chosen": -368.5874938964844, |
| "logps/rejected": -469.8999938964844, |
| "loss": 0.0443, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -1.0027587413787842, |
| "rewards/margins": 10.260156631469727, |
| "rewards/rejected": -11.270703315734863, |
| "step": 2290 |
| }, |
| { |
| "epoch": 1.480205986482137, |
| "grad_norm": 17.53638862046089, |
| "learning_rate": 6.299098518995493e-07, |
| "logits/chosen": -0.7756592035293579, |
| "logits/rejected": -0.777020275592804, |
| "logps/chosen": -386.3999938964844, |
| "logps/rejected": -475.2749938964844, |
| "loss": 0.0551, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -1.0754516124725342, |
| "rewards/margins": 10.524999618530273, |
| "rewards/rejected": -11.599218368530273, |
| "step": 2300 |
| }, |
| { |
| "epoch": 1.4866430640489217, |
| "grad_norm": 40.43102058317517, |
| "learning_rate": 6.283000643915003e-07, |
| "logits/chosen": -0.7683349847793579, |
| "logits/rejected": -0.8065429925918579, |
| "logps/chosen": -340.1499938964844, |
| "logps/rejected": -432.79998779296875, |
| "loss": 0.0536, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -0.523364245891571, |
| "rewards/margins": 10.595312118530273, |
| "rewards/rejected": -11.111719131469727, |
| "step": 2310 |
| }, |
| { |
| "epoch": 1.4930801416157065, |
| "grad_norm": 0.9213782840626744, |
| "learning_rate": 6.266902768834513e-07, |
| "logits/chosen": -0.7571777105331421, |
| "logits/rejected": -0.7656280398368835, |
| "logps/chosen": -394.17498779296875, |
| "logps/rejected": -493.625, |
| "loss": 0.0195, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -0.618359386920929, |
| "rewards/margins": 11.130468368530273, |
| "rewards/rejected": -11.751562118530273, |
| "step": 2320 |
| }, |
| { |
| "epoch": 1.4995172191824913, |
| "grad_norm": 22.104939943938994, |
| "learning_rate": 6.250804893754024e-07, |
| "logits/chosen": -0.784497082233429, |
| "logits/rejected": -0.773754894733429, |
| "logps/chosen": -378.79998779296875, |
| "logps/rejected": -431.25, |
| "loss": 0.0639, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -0.8636840581893921, |
| "rewards/margins": 10.059765815734863, |
| "rewards/rejected": -10.924219131469727, |
| "step": 2330 |
| }, |
| { |
| "epoch": 1.5059542967492758, |
| "grad_norm": 1.6915245555680156, |
| "learning_rate": 6.234707018673535e-07, |
| "logits/chosen": -0.834716796875, |
| "logits/rejected": -0.8809570074081421, |
| "logps/chosen": -370.54998779296875, |
| "logps/rejected": -439.8500061035156, |
| "loss": 0.052, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -1.103784203529358, |
| "rewards/margins": 10.172656059265137, |
| "rewards/rejected": -11.285937309265137, |
| "step": 2340 |
| }, |
| { |
| "epoch": 1.5123913743160604, |
| "grad_norm": 17.859220853952976, |
| "learning_rate": 6.218609143593045e-07, |
| "logits/chosen": -0.827197253704071, |
| "logits/rejected": -0.8333495855331421, |
| "logps/chosen": -385.57501220703125, |
| "logps/rejected": -463.6000061035156, |
| "loss": 0.0792, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": -1.35723876953125, |
| "rewards/margins": 10.078125, |
| "rewards/rejected": -11.44140625, |
| "step": 2350 |
| }, |
| { |
| "epoch": 1.5188284518828452, |
| "grad_norm": 0.2607261691968034, |
| "learning_rate": 6.202511268512556e-07, |
| "logits/chosen": -0.760913074016571, |
| "logits/rejected": -0.776318371295929, |
| "logps/chosen": -389.17498779296875, |
| "logps/rejected": -465.5249938964844, |
| "loss": 0.0401, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -1.0231444835662842, |
| "rewards/margins": 9.904296875, |
| "rewards/rejected": -10.931640625, |
| "step": 2360 |
| }, |
| { |
| "epoch": 1.52526552944963, |
| "grad_norm": 15.593710610614462, |
| "learning_rate": 6.186413393432067e-07, |
| "logits/chosen": -0.8382812738418579, |
| "logits/rejected": -0.856738269329071, |
| "logps/chosen": -407.2250061035156, |
| "logps/rejected": -464.125, |
| "loss": 0.0363, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -1.1426513195037842, |
| "rewards/margins": 10.119531631469727, |
| "rewards/rejected": -11.264062881469727, |
| "step": 2370 |
| }, |
| { |
| "epoch": 1.5317026070164146, |
| "grad_norm": 7.034407955917834, |
| "learning_rate": 6.170315518351577e-07, |
| "logits/chosen": -0.7542724609375, |
| "logits/rejected": -0.766064465045929, |
| "logps/chosen": -378.70001220703125, |
| "logps/rejected": -481.57501220703125, |
| "loss": 0.0612, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -1.373419165611267, |
| "rewards/margins": 10.766406059265137, |
| "rewards/rejected": -12.145312309265137, |
| "step": 2380 |
| }, |
| { |
| "epoch": 1.5381396845831992, |
| "grad_norm": 6.582834920072221, |
| "learning_rate": 6.154217643271088e-07, |
| "logits/chosen": -0.723925769329071, |
| "logits/rejected": -0.734448254108429, |
| "logps/chosen": -404.125, |
| "logps/rejected": -502.75, |
| "loss": 0.0557, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -1.978906273841858, |
| "rewards/margins": 10.76171875, |
| "rewards/rejected": -12.744531631469727, |
| "step": 2390 |
| }, |
| { |
| "epoch": 1.544576762149984, |
| "grad_norm": 0.4422958879108554, |
| "learning_rate": 6.138119768190599e-07, |
| "logits/chosen": -0.8257080316543579, |
| "logits/rejected": -0.862231433391571, |
| "logps/chosen": -370.45001220703125, |
| "logps/rejected": -459.9750061035156, |
| "loss": 0.047, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -2.163891553878784, |
| "rewards/margins": 11.420312881469727, |
| "rewards/rejected": -13.585156440734863, |
| "step": 2400 |
| }, |
| { |
| "epoch": 1.5510138397167685, |
| "grad_norm": 23.621743283386632, |
| "learning_rate": 6.122021893110108e-07, |
| "logits/chosen": -0.7707275152206421, |
| "logits/rejected": -0.782275378704071, |
| "logps/chosen": -374.75, |
| "logps/rejected": -467.54998779296875, |
| "loss": 0.0745, |
| "rewards/accuracies": 0.9312499761581421, |
| "rewards/chosen": -1.935156226158142, |
| "rewards/margins": 10.9296875, |
| "rewards/rejected": -12.871874809265137, |
| "step": 2410 |
| }, |
| { |
| "epoch": 1.5574509172835533, |
| "grad_norm": 174.52576579494084, |
| "learning_rate": 6.10592401802962e-07, |
| "logits/chosen": -0.825268566608429, |
| "logits/rejected": -0.830151379108429, |
| "logps/chosen": -368.95001220703125, |
| "logps/rejected": -481.8500061035156, |
| "loss": 0.1028, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -1.627960205078125, |
| "rewards/margins": 10.973047256469727, |
| "rewards/rejected": -12.61328125, |
| "step": 2420 |
| }, |
| { |
| "epoch": 1.563887994850338, |
| "grad_norm": 6.353329449967956, |
| "learning_rate": 6.08982614294913e-07, |
| "logits/chosen": -0.8233642578125, |
| "logits/rejected": -0.8514159917831421, |
| "logps/chosen": -370.625, |
| "logps/rejected": -465.70001220703125, |
| "loss": 0.0247, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -1.6372802257537842, |
| "rewards/margins": 10.8671875, |
| "rewards/rejected": -12.513280868530273, |
| "step": 2430 |
| }, |
| { |
| "epoch": 1.5703250724171225, |
| "grad_norm": 42.632038258106874, |
| "learning_rate": 6.073728267868641e-07, |
| "logits/chosen": -0.7555176019668579, |
| "logits/rejected": -0.775775134563446, |
| "logps/chosen": -403.6499938964844, |
| "logps/rejected": -472.1499938964844, |
| "loss": 0.0496, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -0.56494140625, |
| "rewards/margins": 10.884374618530273, |
| "rewards/rejected": -11.444531440734863, |
| "step": 2440 |
| }, |
| { |
| "epoch": 1.5767621499839073, |
| "grad_norm": 4.234950034854178, |
| "learning_rate": 6.057630392788152e-07, |
| "logits/chosen": -0.7583984136581421, |
| "logits/rejected": -0.771484375, |
| "logps/chosen": -392.73748779296875, |
| "logps/rejected": -478.0249938964844, |
| "loss": 0.0933, |
| "rewards/accuracies": 0.9312499761581421, |
| "rewards/chosen": -1.113104224205017, |
| "rewards/margins": 10.796093940734863, |
| "rewards/rejected": -11.916406631469727, |
| "step": 2450 |
| }, |
| { |
| "epoch": 1.583199227550692, |
| "grad_norm": 3.9163310479581566, |
| "learning_rate": 6.041532517707662e-07, |
| "logits/chosen": -0.8863281011581421, |
| "logits/rejected": -0.895947277545929, |
| "logps/chosen": -395.3500061035156, |
| "logps/rejected": -483.54998779296875, |
| "loss": 0.0371, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -1.408911108970642, |
| "rewards/margins": 11.666406631469727, |
| "rewards/rejected": -13.063281059265137, |
| "step": 2460 |
| }, |
| { |
| "epoch": 1.5896363051174767, |
| "grad_norm": 62.02613513340239, |
| "learning_rate": 6.025434642627173e-07, |
| "logits/chosen": -0.8659423589706421, |
| "logits/rejected": -0.879638671875, |
| "logps/chosen": -402.1499938964844, |
| "logps/rejected": -507.375, |
| "loss": 0.039, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -1.2672119140625, |
| "rewards/margins": 11.375781059265137, |
| "rewards/rejected": -12.642969131469727, |
| "step": 2470 |
| }, |
| { |
| "epoch": 1.5960733826842612, |
| "grad_norm": 3.3628306955247598, |
| "learning_rate": 6.009336767546684e-07, |
| "logits/chosen": -0.891796886920929, |
| "logits/rejected": -0.920849621295929, |
| "logps/chosen": -405.875, |
| "logps/rejected": -493.67498779296875, |
| "loss": 0.0499, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -0.6985229253768921, |
| "rewards/margins": 11.516406059265137, |
| "rewards/rejected": -12.217968940734863, |
| "step": 2480 |
| }, |
| { |
| "epoch": 1.602510460251046, |
| "grad_norm": 6.944832508288456, |
| "learning_rate": 5.993238892466194e-07, |
| "logits/chosen": -0.8244873285293579, |
| "logits/rejected": -0.838452160358429, |
| "logps/chosen": -371.3999938964844, |
| "logps/rejected": -458.82501220703125, |
| "loss": 0.0703, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -0.8302001953125, |
| "rewards/margins": 10.130468368530273, |
| "rewards/rejected": -10.963281631469727, |
| "step": 2490 |
| }, |
| { |
| "epoch": 1.6089475378178308, |
| "grad_norm": 16.431404128160846, |
| "learning_rate": 5.977141017385705e-07, |
| "logits/chosen": -1.0031249523162842, |
| "logits/rejected": -1.000390648841858, |
| "logps/chosen": -377.54998779296875, |
| "logps/rejected": -472.5, |
| "loss": 0.0395, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -0.730297863483429, |
| "rewards/margins": 10.501562118530273, |
| "rewards/rejected": -11.234375, |
| "step": 2500 |
| }, |
| { |
| "epoch": 1.6153846153846154, |
| "grad_norm": 0.9146958952965157, |
| "learning_rate": 5.961043142305216e-07, |
| "logits/chosen": -0.8604491949081421, |
| "logits/rejected": -0.87451171875, |
| "logps/chosen": -382.51251220703125, |
| "logps/rejected": -439.375, |
| "loss": 0.052, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -0.5503753423690796, |
| "rewards/margins": 9.967187881469727, |
| "rewards/rejected": -10.515625, |
| "step": 2510 |
| }, |
| { |
| "epoch": 1.6218216929514, |
| "grad_norm": 1.2281780912032512, |
| "learning_rate": 5.944945267224725e-07, |
| "logits/chosen": -0.8853515386581421, |
| "logits/rejected": -0.891796886920929, |
| "logps/chosen": -390.8500061035156, |
| "logps/rejected": -468.1499938964844, |
| "loss": 0.04, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -0.706024169921875, |
| "rewards/margins": 11.03125, |
| "rewards/rejected": -11.733593940734863, |
| "step": 2520 |
| }, |
| { |
| "epoch": 1.6282587705181848, |
| "grad_norm": 0.9436650570997343, |
| "learning_rate": 5.928847392144237e-07, |
| "logits/chosen": -0.8895508050918579, |
| "logits/rejected": -0.8848632574081421, |
| "logps/chosen": -392.96875, |
| "logps/rejected": -470.4750061035156, |
| "loss": 0.0295, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -0.9778808355331421, |
| "rewards/margins": 10.90625, |
| "rewards/rejected": -11.883593559265137, |
| "step": 2530 |
| }, |
| { |
| "epoch": 1.6346958480849694, |
| "grad_norm": 0.2775703409910018, |
| "learning_rate": 5.912749517063748e-07, |
| "logits/chosen": -0.940356433391571, |
| "logits/rejected": -0.9769531488418579, |
| "logps/chosen": -396.0, |
| "logps/rejected": -452.0249938964844, |
| "loss": 0.0129, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -1.201135277748108, |
| "rewards/margins": 11.169530868530273, |
| "rewards/rejected": -12.374218940734863, |
| "step": 2540 |
| }, |
| { |
| "epoch": 1.6411329256517542, |
| "grad_norm": 31.914598423294287, |
| "learning_rate": 5.896651641983258e-07, |
| "logits/chosen": -1.011376976966858, |
| "logits/rejected": -1.037695288658142, |
| "logps/chosen": -381.79998779296875, |
| "logps/rejected": -435.6499938964844, |
| "loss": 0.0321, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -1.6757323741912842, |
| "rewards/margins": 10.783594131469727, |
| "rewards/rejected": -12.4609375, |
| "step": 2550 |
| }, |
| { |
| "epoch": 1.6475700032185387, |
| "grad_norm": 72.59157572433088, |
| "learning_rate": 5.880553766902768e-07, |
| "logits/chosen": -0.977001965045929, |
| "logits/rejected": -0.993701159954071, |
| "logps/chosen": -403.92498779296875, |
| "logps/rejected": -496.8999938964844, |
| "loss": 0.0418, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -1.3961181640625, |
| "rewards/margins": 11.057031631469727, |
| "rewards/rejected": -12.449999809265137, |
| "step": 2560 |
| }, |
| { |
| "epoch": 1.6540070807853233, |
| "grad_norm": 14.184682824537003, |
| "learning_rate": 5.86445589182228e-07, |
| "logits/chosen": -0.903369128704071, |
| "logits/rejected": -0.904467761516571, |
| "logps/chosen": -392.32501220703125, |
| "logps/rejected": -475.57501220703125, |
| "loss": 0.0652, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -1.341552734375, |
| "rewards/margins": 12.142969131469727, |
| "rewards/rejected": -13.478124618530273, |
| "step": 2570 |
| }, |
| { |
| "epoch": 1.6604441583521081, |
| "grad_norm": 5.25218753944226, |
| "learning_rate": 5.84835801674179e-07, |
| "logits/chosen": -0.955322265625, |
| "logits/rejected": -0.977343738079071, |
| "logps/chosen": -375.13751220703125, |
| "logps/rejected": -466.1499938964844, |
| "loss": 0.0314, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -1.451562523841858, |
| "rewards/margins": 11.546093940734863, |
| "rewards/rejected": -12.996874809265137, |
| "step": 2580 |
| }, |
| { |
| "epoch": 1.666881235918893, |
| "grad_norm": 0.25832510910108303, |
| "learning_rate": 5.832260141661301e-07, |
| "logits/chosen": -0.9334472417831421, |
| "logits/rejected": -0.955920398235321, |
| "logps/chosen": -417.75, |
| "logps/rejected": -496.07501220703125, |
| "loss": 0.0242, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -1.4179565906524658, |
| "rewards/margins": 11.3984375, |
| "rewards/rejected": -12.81640625, |
| "step": 2590 |
| }, |
| { |
| "epoch": 1.6733183134856775, |
| "grad_norm": 8.01847196423977, |
| "learning_rate": 5.81616226658081e-07, |
| "logits/chosen": -0.925707995891571, |
| "logits/rejected": -0.940673828125, |
| "logps/chosen": -397.67498779296875, |
| "logps/rejected": -461.79998779296875, |
| "loss": 0.0398, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -1.860571265220642, |
| "rewards/margins": 11.041406631469727, |
| "rewards/rejected": -12.903905868530273, |
| "step": 2600 |
| }, |
| { |
| "epoch": 1.679755391052462, |
| "grad_norm": 8.250100146015557, |
| "learning_rate": 5.800064391500322e-07, |
| "logits/chosen": -0.935986340045929, |
| "logits/rejected": -0.957763671875, |
| "logps/chosen": -415.67498779296875, |
| "logps/rejected": -511.6000061035156, |
| "loss": 0.1153, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -2.994311571121216, |
| "rewards/margins": 11.036718368530273, |
| "rewards/rejected": -14.035937309265137, |
| "step": 2610 |
| }, |
| { |
| "epoch": 1.6861924686192469, |
| "grad_norm": 13.134105792412328, |
| "learning_rate": 5.783966516419833e-07, |
| "logits/chosen": -0.983447253704071, |
| "logits/rejected": -0.974560558795929, |
| "logps/chosen": -386.82501220703125, |
| "logps/rejected": -476.75, |
| "loss": 0.0335, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -2.349414110183716, |
| "rewards/margins": 11.560155868530273, |
| "rewards/rejected": -13.899999618530273, |
| "step": 2620 |
| }, |
| { |
| "epoch": 1.6926295461860317, |
| "grad_norm": 116.36227986157657, |
| "learning_rate": 5.767868641339343e-07, |
| "logits/chosen": -0.8326416015625, |
| "logits/rejected": -0.829663097858429, |
| "logps/chosen": -353.1499938964844, |
| "logps/rejected": -462.7749938964844, |
| "loss": 0.0935, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -2.1159729957580566, |
| "rewards/margins": 11.43359375, |
| "rewards/rejected": -13.545312881469727, |
| "step": 2630 |
| }, |
| { |
| "epoch": 1.6990666237528163, |
| "grad_norm": 4.5619674703943085, |
| "learning_rate": 5.751770766258853e-07, |
| "logits/chosen": -0.8194214105606079, |
| "logits/rejected": -0.84722900390625, |
| "logps/chosen": -422.07501220703125, |
| "logps/rejected": -498.375, |
| "loss": 0.0954, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": -1.7252624034881592, |
| "rewards/margins": 10.96875, |
| "rewards/rejected": -12.690625190734863, |
| "step": 2640 |
| }, |
| { |
| "epoch": 1.7055037013196008, |
| "grad_norm": 3.006491121324236, |
| "learning_rate": 5.735672891178365e-07, |
| "logits/chosen": -0.9326721429824829, |
| "logits/rejected": -0.929638683795929, |
| "logps/chosen": -418.79998779296875, |
| "logps/rejected": -483.8999938964844, |
| "loss": 0.0323, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -2.345507860183716, |
| "rewards/margins": 11.989062309265137, |
| "rewards/rejected": -14.334375381469727, |
| "step": 2650 |
| }, |
| { |
| "epoch": 1.7119407788863856, |
| "grad_norm": 4.13341575346933, |
| "learning_rate": 5.719575016097875e-07, |
| "logits/chosen": -0.828857421875, |
| "logits/rejected": -0.8162841796875, |
| "logps/chosen": -356.2250061035156, |
| "logps/rejected": -526.7000122070312, |
| "loss": 0.045, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -3.123608350753784, |
| "rewards/margins": 13.339062690734863, |
| "rewards/rejected": -16.462499618530273, |
| "step": 2660 |
| }, |
| { |
| "epoch": 1.7183778564531704, |
| "grad_norm": 22.117373309483987, |
| "learning_rate": 5.703477141017385e-07, |
| "logits/chosen": -0.844775378704071, |
| "logits/rejected": -0.8448730707168579, |
| "logps/chosen": -404.32501220703125, |
| "logps/rejected": -502.25, |
| "loss": 0.0476, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -3.886608839035034, |
| "rewards/margins": 12.126562118530273, |
| "rewards/rejected": -16.014062881469727, |
| "step": 2670 |
| }, |
| { |
| "epoch": 1.724814934019955, |
| "grad_norm": 22.82723612005747, |
| "learning_rate": 5.687379265936896e-07, |
| "logits/chosen": -0.9837402105331421, |
| "logits/rejected": -1.027099609375, |
| "logps/chosen": -404.8999938964844, |
| "logps/rejected": -503.8999938964844, |
| "loss": 0.0874, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -5.081250190734863, |
| "rewards/margins": 12.1015625, |
| "rewards/rejected": -17.173437118530273, |
| "step": 2680 |
| }, |
| { |
| "epoch": 1.7312520115867396, |
| "grad_norm": 10.704380536262521, |
| "learning_rate": 5.671281390856407e-07, |
| "logits/chosen": -0.9532226324081421, |
| "logits/rejected": -0.9735351800918579, |
| "logps/chosen": -422.7749938964844, |
| "logps/rejected": -521.2999877929688, |
| "loss": 0.0372, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -4.213476657867432, |
| "rewards/margins": 12.93359375, |
| "rewards/rejected": -17.1484375, |
| "step": 2690 |
| }, |
| { |
| "epoch": 1.7376890891535242, |
| "grad_norm": 2.73829439018618, |
| "learning_rate": 5.655183515775918e-07, |
| "logits/chosen": -0.94189453125, |
| "logits/rejected": -0.97607421875, |
| "logps/chosen": -440.29998779296875, |
| "logps/rejected": -522.8499755859375, |
| "loss": 0.0201, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -2.729840040206909, |
| "rewards/margins": 12.442187309265137, |
| "rewards/rejected": -15.1640625, |
| "step": 2700 |
| }, |
| { |
| "epoch": 1.744126166720309, |
| "grad_norm": 36.282664030023824, |
| "learning_rate": 5.639085640695428e-07, |
| "logits/chosen": -0.8478027582168579, |
| "logits/rejected": -0.8516601324081421, |
| "logps/chosen": -416.2250061035156, |
| "logps/rejected": -503.7749938964844, |
| "loss": 0.151, |
| "rewards/accuracies": 0.925000011920929, |
| "rewards/chosen": -2.7637696266174316, |
| "rewards/margins": 11.846875190734863, |
| "rewards/rejected": -14.606249809265137, |
| "step": 2710 |
| }, |
| { |
| "epoch": 1.7505632442870938, |
| "grad_norm": 2.1283637906112407, |
| "learning_rate": 5.622987765614938e-07, |
| "logits/chosen": -0.9247070550918579, |
| "logits/rejected": -0.9312499761581421, |
| "logps/chosen": -409.7749938964844, |
| "logps/rejected": -511.5, |
| "loss": 0.0602, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -2.046765089035034, |
| "rewards/margins": 12.314062118530273, |
| "rewards/rejected": -14.3515625, |
| "step": 2720 |
| }, |
| { |
| "epoch": 1.7570003218538783, |
| "grad_norm": 134.5608375076651, |
| "learning_rate": 5.60688989053445e-07, |
| "logits/chosen": -0.8810790777206421, |
| "logits/rejected": -0.922985851764679, |
| "logps/chosen": -412.5, |
| "logps/rejected": -479.42498779296875, |
| "loss": 0.083, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -2.50537109375, |
| "rewards/margins": 11.076562881469727, |
| "rewards/rejected": -13.574999809265137, |
| "step": 2730 |
| }, |
| { |
| "epoch": 1.763437399420663, |
| "grad_norm": 44.62636520944944, |
| "learning_rate": 5.590792015453961e-07, |
| "logits/chosen": -0.8690429925918579, |
| "logits/rejected": -0.862597644329071, |
| "logps/chosen": -439.42498779296875, |
| "logps/rejected": -518.0999755859375, |
| "loss": 0.0385, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -2.424877882003784, |
| "rewards/margins": 12.205469131469727, |
| "rewards/rejected": -14.629687309265137, |
| "step": 2740 |
| }, |
| { |
| "epoch": 1.7698744769874477, |
| "grad_norm": 11.000258418597038, |
| "learning_rate": 5.57469414037347e-07, |
| "logits/chosen": -0.8440917730331421, |
| "logits/rejected": -0.877392590045929, |
| "logps/chosen": -431.0, |
| "logps/rejected": -490.1000061035156, |
| "loss": 0.0655, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -2.1365966796875, |
| "rewards/margins": 11.603124618530273, |
| "rewards/rejected": -13.740625381469727, |
| "step": 2750 |
| }, |
| { |
| "epoch": 1.7763115545542325, |
| "grad_norm": 24.728950785009513, |
| "learning_rate": 5.558596265292981e-07, |
| "logits/chosen": -0.793408215045929, |
| "logits/rejected": -0.82470703125, |
| "logps/chosen": -384.2250061035156, |
| "logps/rejected": -455.75, |
| "loss": 0.0423, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -1.9411132335662842, |
| "rewards/margins": 11.362500190734863, |
| "rewards/rejected": -13.303125381469727, |
| "step": 2760 |
| }, |
| { |
| "epoch": 1.782748632121017, |
| "grad_norm": 18.185794833969, |
| "learning_rate": 5.542498390212492e-07, |
| "logits/chosen": -0.8042999505996704, |
| "logits/rejected": -0.7938781976699829, |
| "logps/chosen": -376.0, |
| "logps/rejected": -467.70001220703125, |
| "loss": 0.0949, |
| "rewards/accuracies": 0.9312499761581421, |
| "rewards/chosen": -1.829443335533142, |
| "rewards/margins": 10.978906631469727, |
| "rewards/rejected": -12.80078125, |
| "step": 2770 |
| }, |
| { |
| "epoch": 1.7891857096878017, |
| "grad_norm": 36.77834630320014, |
| "learning_rate": 5.526400515132002e-07, |
| "logits/chosen": -0.898144543170929, |
| "logits/rejected": -0.8753417730331421, |
| "logps/chosen": -413.8999938964844, |
| "logps/rejected": -524.0999755859375, |
| "loss": 0.0878, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -2.239550828933716, |
| "rewards/margins": 11.630468368530273, |
| "rewards/rejected": -13.878125190734863, |
| "step": 2780 |
| }, |
| { |
| "epoch": 1.7956227872545865, |
| "grad_norm": 50.535377745071834, |
| "learning_rate": 5.510302640051513e-07, |
| "logits/chosen": -0.871777355670929, |
| "logits/rejected": -0.909472644329071, |
| "logps/chosen": -396.1000061035156, |
| "logps/rejected": -475.0, |
| "loss": 0.037, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -1.981176733970642, |
| "rewards/margins": 11.844531059265137, |
| "rewards/rejected": -13.824999809265137, |
| "step": 2790 |
| }, |
| { |
| "epoch": 1.8020598648213713, |
| "grad_norm": 187.81436429316912, |
| "learning_rate": 5.494204764971023e-07, |
| "logits/chosen": -0.8958984613418579, |
| "logits/rejected": -0.9166015386581421, |
| "logps/chosen": -413.04998779296875, |
| "logps/rejected": -503.125, |
| "loss": 0.0652, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -2.3756103515625, |
| "rewards/margins": 11.768750190734863, |
| "rewards/rejected": -14.146875381469727, |
| "step": 2800 |
| }, |
| { |
| "epoch": 1.8084969423881558, |
| "grad_norm": 2.46028333934951, |
| "learning_rate": 5.478106889890535e-07, |
| "logits/chosen": -0.9292968511581421, |
| "logits/rejected": -0.9300781488418579, |
| "logps/chosen": -367.375, |
| "logps/rejected": -495.1000061035156, |
| "loss": 0.0362, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -1.463037133216858, |
| "rewards/margins": 12.038281440734863, |
| "rewards/rejected": -13.5078125, |
| "step": 2810 |
| }, |
| { |
| "epoch": 1.8149340199549404, |
| "grad_norm": 231.08328656997156, |
| "learning_rate": 5.462009014810045e-07, |
| "logits/chosen": -0.952929675579071, |
| "logits/rejected": -0.947460949420929, |
| "logps/chosen": -433.07501220703125, |
| "logps/rejected": -511.25, |
| "loss": 0.0422, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -1.267065405845642, |
| "rewards/margins": 13.215624809265137, |
| "rewards/rejected": -14.4765625, |
| "step": 2820 |
| }, |
| { |
| "epoch": 1.821371097521725, |
| "grad_norm": 20.163208476823606, |
| "learning_rate": 5.445911139729555e-07, |
| "logits/chosen": -0.944793701171875, |
| "logits/rejected": -0.9900878667831421, |
| "logps/chosen": -394.07501220703125, |
| "logps/rejected": -501.0, |
| "loss": 0.0365, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -2.1129150390625, |
| "rewards/margins": 12.45703125, |
| "rewards/rejected": -14.5625, |
| "step": 2830 |
| }, |
| { |
| "epoch": 1.8278081750885098, |
| "grad_norm": 144.33944891843873, |
| "learning_rate": 5.429813264649066e-07, |
| "logits/chosen": -0.917529284954071, |
| "logits/rejected": -0.927441418170929, |
| "logps/chosen": -393.42498779296875, |
| "logps/rejected": -512.9500122070312, |
| "loss": 0.0597, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -2.2644286155700684, |
| "rewards/margins": 12.955469131469727, |
| "rewards/rejected": -15.217187881469727, |
| "step": 2840 |
| }, |
| { |
| "epoch": 1.8342452526552946, |
| "grad_norm": 21.552588183350537, |
| "learning_rate": 5.413715389568578e-07, |
| "logits/chosen": -0.94049072265625, |
| "logits/rejected": -0.9557861089706421, |
| "logps/chosen": -371.3374938964844, |
| "logps/rejected": -473.95001220703125, |
| "loss": 0.0413, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -1.8835632801055908, |
| "rewards/margins": 12.410937309265137, |
| "rewards/rejected": -14.28515625, |
| "step": 2850 |
| }, |
| { |
| "epoch": 1.8406823302220792, |
| "grad_norm": 0.33607759158894557, |
| "learning_rate": 5.397617514488087e-07, |
| "logits/chosen": -1.03466796875, |
| "logits/rejected": -1.0185546875, |
| "logps/chosen": -397.7749938964844, |
| "logps/rejected": -503.3500061035156, |
| "loss": 0.1239, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -2.576586961746216, |
| "rewards/margins": 11.790624618530273, |
| "rewards/rejected": -14.370312690734863, |
| "step": 2860 |
| }, |
| { |
| "epoch": 1.8471194077888637, |
| "grad_norm": 7.301426102324497, |
| "learning_rate": 5.381519639407598e-07, |
| "logits/chosen": -0.9674316644668579, |
| "logits/rejected": -0.9830077886581421, |
| "logps/chosen": -387.2250061035156, |
| "logps/rejected": -525.6500244140625, |
| "loss": 0.0404, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -3.5799803733825684, |
| "rewards/margins": 12.387499809265137, |
| "rewards/rejected": -15.962499618530273, |
| "step": 2870 |
| }, |
| { |
| "epoch": 1.8535564853556485, |
| "grad_norm": 8.160121310799461, |
| "learning_rate": 5.365421764327109e-07, |
| "logits/chosen": -1.011450171470642, |
| "logits/rejected": -1.022363305091858, |
| "logps/chosen": -412.375, |
| "logps/rejected": -507.95001220703125, |
| "loss": 0.1486, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -3.354541063308716, |
| "rewards/margins": 11.653124809265137, |
| "rewards/rejected": -15.006250381469727, |
| "step": 2880 |
| }, |
| { |
| "epoch": 1.8599935629224333, |
| "grad_norm": 5.827434791236949, |
| "learning_rate": 5.34932388924662e-07, |
| "logits/chosen": -0.955517590045929, |
| "logits/rejected": -0.963061511516571, |
| "logps/chosen": -429.04998779296875, |
| "logps/rejected": -512.7125244140625, |
| "loss": 0.1118, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": -3.6991209983825684, |
| "rewards/margins": 12.186718940734863, |
| "rewards/rejected": -15.900781631469727, |
| "step": 2890 |
| }, |
| { |
| "epoch": 1.866430640489218, |
| "grad_norm": 4.154458697921168, |
| "learning_rate": 5.33322601416613e-07, |
| "logits/chosen": -0.95458984375, |
| "logits/rejected": -0.9620605707168579, |
| "logps/chosen": -408.5249938964844, |
| "logps/rejected": -510.5, |
| "loss": 0.056, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -2.6628174781799316, |
| "rewards/margins": 11.815625190734863, |
| "rewards/rejected": -14.484375, |
| "step": 2900 |
| }, |
| { |
| "epoch": 1.8728677180560025, |
| "grad_norm": 25.750618944342897, |
| "learning_rate": 5.31712813908564e-07, |
| "logits/chosen": -0.978076159954071, |
| "logits/rejected": -0.968994140625, |
| "logps/chosen": -376.70001220703125, |
| "logps/rejected": -513.2000122070312, |
| "loss": 0.0537, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -2.340625047683716, |
| "rewards/margins": 11.696874618530273, |
| "rewards/rejected": -14.035937309265137, |
| "step": 2910 |
| }, |
| { |
| "epoch": 1.8793047956227873, |
| "grad_norm": 34.482062477676116, |
| "learning_rate": 5.301030264005151e-07, |
| "logits/chosen": -0.887988269329071, |
| "logits/rejected": -0.9131835699081421, |
| "logps/chosen": -393.8999938964844, |
| "logps/rejected": -460.1499938964844, |
| "loss": 0.034, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -1.0007812976837158, |
| "rewards/margins": 11.300000190734863, |
| "rewards/rejected": -12.290624618530273, |
| "step": 2920 |
| }, |
| { |
| "epoch": 1.885741873189572, |
| "grad_norm": 0.6449608149453013, |
| "learning_rate": 5.284932388924662e-07, |
| "logits/chosen": -0.908447265625, |
| "logits/rejected": -0.928955078125, |
| "logps/chosen": -366.4624938964844, |
| "logps/rejected": -453.07501220703125, |
| "loss": 0.0462, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -1.143225073814392, |
| "rewards/margins": 11.33984375, |
| "rewards/rejected": -12.48046875, |
| "step": 2930 |
| }, |
| { |
| "epoch": 1.8921789507563567, |
| "grad_norm": 10.761664351455913, |
| "learning_rate": 5.268834513844172e-07, |
| "logits/chosen": -0.9051758050918579, |
| "logits/rejected": -0.942919909954071, |
| "logps/chosen": -378.25, |
| "logps/rejected": -482.6000061035156, |
| "loss": 0.0644, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -1.2742919921875, |
| "rewards/margins": 10.936327934265137, |
| "rewards/rejected": -12.215624809265137, |
| "step": 2940 |
| }, |
| { |
| "epoch": 1.8986160283231412, |
| "grad_norm": 91.23334834373519, |
| "learning_rate": 5.252736638763683e-07, |
| "logits/chosen": -0.938720703125, |
| "logits/rejected": -0.9503418207168579, |
| "logps/chosen": -413.625, |
| "logps/rejected": -475.70001220703125, |
| "loss": 0.0455, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -0.732348620891571, |
| "rewards/margins": 11.340624809265137, |
| "rewards/rejected": -12.078125, |
| "step": 2950 |
| }, |
| { |
| "epoch": 1.9050531058899258, |
| "grad_norm": 12.915735407500012, |
| "learning_rate": 5.236638763683194e-07, |
| "logits/chosen": -0.85382080078125, |
| "logits/rejected": -0.871350109577179, |
| "logps/chosen": -366.95001220703125, |
| "logps/rejected": -493.75, |
| "loss": 0.0473, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -0.8136841058731079, |
| "rewards/margins": 11.69140625, |
| "rewards/rejected": -12.508593559265137, |
| "step": 2960 |
| }, |
| { |
| "epoch": 1.9114901834567106, |
| "grad_norm": 227.12309414417433, |
| "learning_rate": 5.220540888602704e-07, |
| "logits/chosen": -0.9459472894668579, |
| "logits/rejected": -0.935351550579071, |
| "logps/chosen": -412.7749938964844, |
| "logps/rejected": -502.0, |
| "loss": 0.0448, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -0.43291014432907104, |
| "rewards/margins": 11.698437690734863, |
| "rewards/rejected": -12.13671875, |
| "step": 2970 |
| }, |
| { |
| "epoch": 1.9179272610234954, |
| "grad_norm": 0.196421800385388, |
| "learning_rate": 5.204443013522215e-07, |
| "logits/chosen": -0.9383789300918579, |
| "logits/rejected": -0.953320324420929, |
| "logps/chosen": -379.6000061035156, |
| "logps/rejected": -474.6499938964844, |
| "loss": 0.0569, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -1.2946655750274658, |
| "rewards/margins": 11.255078315734863, |
| "rewards/rejected": -12.55078125, |
| "step": 2980 |
| }, |
| { |
| "epoch": 1.92436433859028, |
| "grad_norm": 1.346626934480126, |
| "learning_rate": 5.188345138441726e-07, |
| "logits/chosen": -0.924121081829071, |
| "logits/rejected": -0.9498046636581421, |
| "logps/chosen": -392.54998779296875, |
| "logps/rejected": -464.4750061035156, |
| "loss": 0.0398, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -0.9569946527481079, |
| "rewards/margins": 11.33203125, |
| "rewards/rejected": -12.289843559265137, |
| "step": 2990 |
| }, |
| { |
| "epoch": 1.9308014161570646, |
| "grad_norm": 8.375416288346715, |
| "learning_rate": 5.172247263361235e-07, |
| "logits/chosen": -1.010498046875, |
| "logits/rejected": -1.031152367591858, |
| "logps/chosen": -442.7250061035156, |
| "logps/rejected": -524.5999755859375, |
| "loss": 0.031, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -1.4194824695587158, |
| "rewards/margins": 12.033594131469727, |
| "rewards/rejected": -13.460156440734863, |
| "step": 3000 |
| }, |
| { |
| "epoch": 1.9372384937238494, |
| "grad_norm": 12.910753832152915, |
| "learning_rate": 5.156149388280747e-07, |
| "logits/chosen": -0.9345703125, |
| "logits/rejected": -0.956250011920929, |
| "logps/chosen": -392.625, |
| "logps/rejected": -460.6499938964844, |
| "loss": 0.0398, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -2.1406006813049316, |
| "rewards/margins": 12.096094131469727, |
| "rewards/rejected": -14.235937118530273, |
| "step": 3010 |
| }, |
| { |
| "epoch": 1.9436755712906342, |
| "grad_norm": 1.0305047534231448, |
| "learning_rate": 5.140051513200258e-07, |
| "logits/chosen": -0.94677734375, |
| "logits/rejected": -0.9766601324081421, |
| "logps/chosen": -381.4750061035156, |
| "logps/rejected": -468.8500061035156, |
| "loss": 0.0257, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -1.2906372547149658, |
| "rewards/margins": 11.268750190734863, |
| "rewards/rejected": -12.560155868530273, |
| "step": 3020 |
| }, |
| { |
| "epoch": 1.9501126488574188, |
| "grad_norm": 0.36209350367976334, |
| "learning_rate": 5.123953638119768e-07, |
| "logits/chosen": -1.013085961341858, |
| "logits/rejected": -1.033300757408142, |
| "logps/chosen": -421.5249938964844, |
| "logps/rejected": -502.75, |
| "loss": 0.0626, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -1.6013672351837158, |
| "rewards/margins": 11.564844131469727, |
| "rewards/rejected": -13.159375190734863, |
| "step": 3030 |
| }, |
| { |
| "epoch": 1.9565497264242033, |
| "grad_norm": 11.479780504467316, |
| "learning_rate": 5.107855763039278e-07, |
| "logits/chosen": -1.0580322742462158, |
| "logits/rejected": -1.06884765625, |
| "logps/chosen": -393.7749938964844, |
| "logps/rejected": -477.07501220703125, |
| "loss": 0.0234, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -1.379449486732483, |
| "rewards/margins": 11.125781059265137, |
| "rewards/rejected": -12.494531631469727, |
| "step": 3040 |
| }, |
| { |
| "epoch": 1.9629868039909881, |
| "grad_norm": 18.87941406876472, |
| "learning_rate": 5.09175788795879e-07, |
| "logits/chosen": -0.984814465045929, |
| "logits/rejected": -1.028466820716858, |
| "logps/chosen": -381.54998779296875, |
| "logps/rejected": -484.4750061035156, |
| "loss": 0.0733, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -1.355200171470642, |
| "rewards/margins": 12.080469131469727, |
| "rewards/rejected": -13.435937881469727, |
| "step": 3050 |
| }, |
| { |
| "epoch": 1.969423881557773, |
| "grad_norm": 46.197107715130024, |
| "learning_rate": 5.0756600128783e-07, |
| "logits/chosen": -1.0607421398162842, |
| "logits/rejected": -1.0695312023162842, |
| "logps/chosen": -407.5249938964844, |
| "logps/rejected": -496.1000061035156, |
| "loss": 0.0459, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -2.9796385765075684, |
| "rewards/margins": 12.356249809265137, |
| "rewards/rejected": -15.340624809265137, |
| "step": 3060 |
| }, |
| { |
| "epoch": 1.9758609591245575, |
| "grad_norm": 2.624768811992857, |
| "learning_rate": 5.059562137797811e-07, |
| "logits/chosen": -1.1554687023162842, |
| "logits/rejected": -1.15771484375, |
| "logps/chosen": -403.79998779296875, |
| "logps/rejected": -472.04998779296875, |
| "loss": 0.0206, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -3.441601514816284, |
| "rewards/margins": 12.533594131469727, |
| "rewards/rejected": -15.979687690734863, |
| "step": 3070 |
| }, |
| { |
| "epoch": 1.982298036691342, |
| "grad_norm": 1.3093063165499863, |
| "learning_rate": 5.04346426271732e-07, |
| "logits/chosen": -1.0871093273162842, |
| "logits/rejected": -1.099609375, |
| "logps/chosen": -447.0249938964844, |
| "logps/rejected": -526.1500244140625, |
| "loss": 0.0173, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -3.60986328125, |
| "rewards/margins": 13.119531631469727, |
| "rewards/rejected": -16.731250762939453, |
| "step": 3080 |
| }, |
| { |
| "epoch": 1.9887351142581267, |
| "grad_norm": 0.12450097571649546, |
| "learning_rate": 5.027366387636832e-07, |
| "logits/chosen": -1.007080078125, |
| "logits/rejected": -1.0055663585662842, |
| "logps/chosen": -383.2749938964844, |
| "logps/rejected": -498.3500061035156, |
| "loss": 0.0646, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -4.04296875, |
| "rewards/margins": 12.362500190734863, |
| "rewards/rejected": -16.412500381469727, |
| "step": 3090 |
| }, |
| { |
| "epoch": 1.9951721918249115, |
| "grad_norm": 1.3344752910311428, |
| "learning_rate": 5.011268512556343e-07, |
| "logits/chosen": -1.055419921875, |
| "logits/rejected": -1.0906250476837158, |
| "logps/chosen": -408.9750061035156, |
| "logps/rejected": -504.7250061035156, |
| "loss": 0.0326, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -3.272106885910034, |
| "rewards/margins": 12.796093940734863, |
| "rewards/rejected": -16.071094512939453, |
| "step": 3100 |
| }, |
| { |
| "epoch": 2.001287415513357, |
| "grad_norm": 0.1077285501528772, |
| "learning_rate": 4.995170637475852e-07, |
| "logits/chosen": -1.0288857221603394, |
| "logits/rejected": -1.0637335777282715, |
| "logps/chosen": -362.5789489746094, |
| "logps/rejected": -500.15789794921875, |
| "loss": 0.0212, |
| "rewards/accuracies": 0.9934210777282715, |
| "rewards/chosen": -3.1022307872772217, |
| "rewards/margins": 13.444901466369629, |
| "rewards/rejected": -16.542762756347656, |
| "step": 3110 |
| }, |
| { |
| "epoch": 2.0077244930801417, |
| "grad_norm": 0.09265850851102228, |
| "learning_rate": 4.979072762395364e-07, |
| "logits/chosen": -1.086035132408142, |
| "logits/rejected": -1.103613257408142, |
| "logps/chosen": -406.2749938964844, |
| "logps/rejected": -530.4000244140625, |
| "loss": 0.0227, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -1.9188964366912842, |
| "rewards/margins": 15.073437690734863, |
| "rewards/rejected": -16.998437881469727, |
| "step": 3120 |
| }, |
| { |
| "epoch": 2.0141615706469262, |
| "grad_norm": 3.06968476113232, |
| "learning_rate": 4.962974887314874e-07, |
| "logits/chosen": -1.0173828601837158, |
| "logits/rejected": -1.0482666492462158, |
| "logps/chosen": -374.79998779296875, |
| "logps/rejected": -495.29998779296875, |
| "loss": 0.0458, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -2.3733763694763184, |
| "rewards/margins": 15.339062690734863, |
| "rewards/rejected": -17.7109375, |
| "step": 3130 |
| }, |
| { |
| "epoch": 2.020598648213711, |
| "grad_norm": 6.148575080941695, |
| "learning_rate": 4.946877012234385e-07, |
| "logits/chosen": -1.033300757408142, |
| "logits/rejected": -1.0410645008087158, |
| "logps/chosen": -366.11248779296875, |
| "logps/rejected": -507.20001220703125, |
| "loss": 0.0245, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -2.705615282058716, |
| "rewards/margins": 14.09765625, |
| "rewards/rejected": -16.806249618530273, |
| "step": 3140 |
| }, |
| { |
| "epoch": 2.027035725780496, |
| "grad_norm": 3.966203474050368, |
| "learning_rate": 4.930779137153895e-07, |
| "logits/chosen": -0.929516613483429, |
| "logits/rejected": -0.9665282964706421, |
| "logps/chosen": -374.4750061035156, |
| "logps/rejected": -498.2749938964844, |
| "loss": 0.0152, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -2.3712525367736816, |
| "rewards/margins": 14.732030868530273, |
| "rewards/rejected": -17.109375, |
| "step": 3150 |
| }, |
| { |
| "epoch": 2.0334728033472804, |
| "grad_norm": 0.1594348793038751, |
| "learning_rate": 4.914681262073406e-07, |
| "logits/chosen": -1.0707519054412842, |
| "logits/rejected": -1.0604248046875, |
| "logps/chosen": -389.5, |
| "logps/rejected": -529.5, |
| "loss": 0.0136, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -2.8618407249450684, |
| "rewards/margins": 15.59375, |
| "rewards/rejected": -18.470312118530273, |
| "step": 3160 |
| }, |
| { |
| "epoch": 2.039909880914065, |
| "grad_norm": 0.014532573399047911, |
| "learning_rate": 4.898583386992917e-07, |
| "logits/chosen": -1.038476586341858, |
| "logits/rejected": -1.043554663658142, |
| "logps/chosen": -366.57501220703125, |
| "logps/rejected": -519.25, |
| "loss": 0.0406, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -2.771923780441284, |
| "rewards/margins": 15.8203125, |
| "rewards/rejected": -18.603124618530273, |
| "step": 3170 |
| }, |
| { |
| "epoch": 2.0463469584808496, |
| "grad_norm": 0.7015845235202134, |
| "learning_rate": 4.882485511912428e-07, |
| "logits/chosen": -1.0251953601837158, |
| "logits/rejected": -1.0403563976287842, |
| "logps/chosen": -433.5375061035156, |
| "logps/rejected": -579.0, |
| "loss": 0.0147, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -3.81591796875, |
| "rewards/margins": 16.274999618530273, |
| "rewards/rejected": -20.09375, |
| "step": 3180 |
| }, |
| { |
| "epoch": 2.052784036047634, |
| "grad_norm": 0.11338404332547947, |
| "learning_rate": 4.866387636831938e-07, |
| "logits/chosen": -1.0687987804412842, |
| "logits/rejected": -1.0724608898162842, |
| "logps/chosen": -381.3999938964844, |
| "logps/rejected": -527.0499877929688, |
| "loss": 0.0854, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -4.091454982757568, |
| "rewards/margins": 15.41796875, |
| "rewards/rejected": -19.514062881469727, |
| "step": 3190 |
| }, |
| { |
| "epoch": 2.059221113614419, |
| "grad_norm": 0.07307065029187323, |
| "learning_rate": 4.850289761751449e-07, |
| "logits/chosen": -1.0838134288787842, |
| "logits/rejected": -1.1128661632537842, |
| "logps/chosen": -390.20001220703125, |
| "logps/rejected": -526.5499877929688, |
| "loss": 0.0145, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -3.879467725753784, |
| "rewards/margins": 15.40625, |
| "rewards/rejected": -19.292186737060547, |
| "step": 3200 |
| }, |
| { |
| "epoch": 2.0656581911812038, |
| "grad_norm": 0.4252136551579939, |
| "learning_rate": 4.834191886670959e-07, |
| "logits/chosen": -1.02734375, |
| "logits/rejected": -1.0543944835662842, |
| "logps/chosen": -434.95001220703125, |
| "logps/rejected": -551.9500122070312, |
| "loss": 0.0057, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -3.806933641433716, |
| "rewards/margins": 15.7109375, |
| "rewards/rejected": -19.514062881469727, |
| "step": 3210 |
| }, |
| { |
| "epoch": 2.0720952687479883, |
| "grad_norm": 0.22065842973577401, |
| "learning_rate": 4.818094011590471e-07, |
| "logits/chosen": -1.009033203125, |
| "logits/rejected": -1.04412841796875, |
| "logps/chosen": -389.54998779296875, |
| "logps/rejected": -543.875, |
| "loss": 0.0228, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -3.7743163108825684, |
| "rewards/margins": 16.923437118530273, |
| "rewards/rejected": -20.692188262939453, |
| "step": 3220 |
| }, |
| { |
| "epoch": 2.078532346314773, |
| "grad_norm": 3.366783353939681, |
| "learning_rate": 4.80199613650998e-07, |
| "logits/chosen": -1.0969727039337158, |
| "logits/rejected": -1.1025390625, |
| "logps/chosen": -449.0249938964844, |
| "logps/rejected": -566.5499877929688, |
| "loss": 0.0137, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -3.6866211891174316, |
| "rewards/margins": 16.260936737060547, |
| "rewards/rejected": -19.940624237060547, |
| "step": 3230 |
| }, |
| { |
| "epoch": 2.084969423881558, |
| "grad_norm": 1.3351611741915037, |
| "learning_rate": 4.785898261429491e-07, |
| "logits/chosen": -1.1100585460662842, |
| "logits/rejected": -1.1037108898162842, |
| "logps/chosen": -400.375, |
| "logps/rejected": -533.0999755859375, |
| "loss": 0.0096, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -4.844922065734863, |
| "rewards/margins": 15.753125190734863, |
| "rewards/rejected": -20.606250762939453, |
| "step": 3240 |
| }, |
| { |
| "epoch": 2.0914065014483425, |
| "grad_norm": 0.671875093679778, |
| "learning_rate": 4.769800386349002e-07, |
| "logits/chosen": -1.1155273914337158, |
| "logits/rejected": -1.1248047351837158, |
| "logps/chosen": -423.29998779296875, |
| "logps/rejected": -563.4500122070312, |
| "loss": 0.0046, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -4.465918064117432, |
| "rewards/margins": 15.721875190734863, |
| "rewards/rejected": -20.184375762939453, |
| "step": 3250 |
| }, |
| { |
| "epoch": 2.097843579015127, |
| "grad_norm": 12.140898346072861, |
| "learning_rate": 4.7537025112685124e-07, |
| "logits/chosen": -1.0279541015625, |
| "logits/rejected": -1.0253509283065796, |
| "logps/chosen": -385.79998779296875, |
| "logps/rejected": -513.7249755859375, |
| "loss": 0.0232, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -3.7276854515075684, |
| "rewards/margins": 14.610937118530273, |
| "rewards/rejected": -18.337499618530273, |
| "step": 3260 |
| }, |
| { |
| "epoch": 2.1042806565819117, |
| "grad_norm": 2.623763638066555, |
| "learning_rate": 4.7376046361880226e-07, |
| "logits/chosen": -1.017480492591858, |
| "logits/rejected": -1.009667992591858, |
| "logps/chosen": -428.0, |
| "logps/rejected": -552.2249755859375, |
| "loss": 0.0279, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -3.2283692359924316, |
| "rewards/margins": 16.339061737060547, |
| "rewards/rejected": -19.5703125, |
| "step": 3270 |
| }, |
| { |
| "epoch": 2.1107177341486967, |
| "grad_norm": 0.4497581563597138, |
| "learning_rate": 4.721506761107534e-07, |
| "logits/chosen": -1.0849609375, |
| "logits/rejected": -1.1357421875, |
| "logps/chosen": -423.75, |
| "logps/rejected": -557.0, |
| "loss": 0.0014, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -3.2232909202575684, |
| "rewards/margins": 16.860937118530273, |
| "rewards/rejected": -20.0859375, |
| "step": 3280 |
| }, |
| { |
| "epoch": 2.1171548117154813, |
| "grad_norm": 1.3339774174292212, |
| "learning_rate": 4.705408886027044e-07, |
| "logits/chosen": -0.9937744140625, |
| "logits/rejected": -0.982617199420929, |
| "logps/chosen": -368.25, |
| "logps/rejected": -534.9000244140625, |
| "loss": 0.0594, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": -3.929394483566284, |
| "rewards/margins": 16.626562118530273, |
| "rewards/rejected": -20.557811737060547, |
| "step": 3290 |
| }, |
| { |
| "epoch": 2.123591889282266, |
| "grad_norm": 0.0051571726531345384, |
| "learning_rate": 4.689311010946555e-07, |
| "logits/chosen": -1.063818335533142, |
| "logits/rejected": -1.061132788658142, |
| "logps/chosen": -383.9750061035156, |
| "logps/rejected": -543.3499755859375, |
| "loss": 0.0181, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -3.5445799827575684, |
| "rewards/margins": 16.131250381469727, |
| "rewards/rejected": -19.682811737060547, |
| "step": 3300 |
| }, |
| { |
| "epoch": 2.1300289668490504, |
| "grad_norm": 0.059582082026372385, |
| "learning_rate": 4.6732131358660656e-07, |
| "logits/chosen": -0.9920898675918579, |
| "logits/rejected": -1.022851586341858, |
| "logps/chosen": -393.1499938964844, |
| "logps/rejected": -518.4000244140625, |
| "loss": 0.0238, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -3.5086913108825684, |
| "rewards/margins": 14.434374809265137, |
| "rewards/rejected": -17.932811737060547, |
| "step": 3310 |
| }, |
| { |
| "epoch": 2.136466044415835, |
| "grad_norm": 0.05020904902019842, |
| "learning_rate": 4.6571152607855763e-07, |
| "logits/chosen": -0.925341784954071, |
| "logits/rejected": -0.948437511920929, |
| "logps/chosen": -405.0, |
| "logps/rejected": -542.6500244140625, |
| "loss": 0.025, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -1.8140380382537842, |
| "rewards/margins": 16.903125762939453, |
| "rewards/rejected": -18.720312118530273, |
| "step": 3320 |
| }, |
| { |
| "epoch": 2.14290312198262, |
| "grad_norm": 0.9021303309972339, |
| "learning_rate": 4.6410173857050865e-07, |
| "logits/chosen": -1.002783179283142, |
| "logits/rejected": -0.99658203125, |
| "logps/chosen": -420.1000061035156, |
| "logps/rejected": -559.9500122070312, |
| "loss": 0.0169, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -4.581494331359863, |
| "rewards/margins": 15.056249618530273, |
| "rewards/rejected": -19.6328125, |
| "step": 3330 |
| }, |
| { |
| "epoch": 2.1493401995494046, |
| "grad_norm": 0.3791121350351295, |
| "learning_rate": 4.624919510624598e-07, |
| "logits/chosen": -1.060644507408142, |
| "logits/rejected": -1.098242163658142, |
| "logps/chosen": -393.42498779296875, |
| "logps/rejected": -543.2999877929688, |
| "loss": 0.0144, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -3.459033250808716, |
| "rewards/margins": 16.667186737060547, |
| "rewards/rejected": -20.120311737060547, |
| "step": 3340 |
| }, |
| { |
| "epoch": 2.155777277116189, |
| "grad_norm": 0.045148022238358086, |
| "learning_rate": 4.608821635544108e-07, |
| "logits/chosen": -0.9743896722793579, |
| "logits/rejected": -1.025634765625, |
| "logps/chosen": -433.61248779296875, |
| "logps/rejected": -542.5499877929688, |
| "loss": 0.0166, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -2.9928221702575684, |
| "rewards/margins": 16.131250381469727, |
| "rewards/rejected": -19.1328125, |
| "step": 3350 |
| }, |
| { |
| "epoch": 2.1622143546829737, |
| "grad_norm": 0.7861734963114936, |
| "learning_rate": 4.5927237604636187e-07, |
| "logits/chosen": -0.998339831829071, |
| "logits/rejected": -1.03173828125, |
| "logps/chosen": -432.07501220703125, |
| "logps/rejected": -545.2999877929688, |
| "loss": 0.0151, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -3.386035203933716, |
| "rewards/margins": 15.671875, |
| "rewards/rejected": -19.059375762939453, |
| "step": 3360 |
| }, |
| { |
| "epoch": 2.1686514322497588, |
| "grad_norm": 0.25553750983202145, |
| "learning_rate": 4.576625885383129e-07, |
| "logits/chosen": -0.948168933391571, |
| "logits/rejected": -0.9716796875, |
| "logps/chosen": -445.82501220703125, |
| "logps/rejected": -577.1500244140625, |
| "loss": 0.0047, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -3.176953077316284, |
| "rewards/margins": 16.501562118530273, |
| "rewards/rejected": -19.682811737060547, |
| "step": 3370 |
| }, |
| { |
| "epoch": 2.1750885098165433, |
| "grad_norm": 0.29127717157768923, |
| "learning_rate": 4.56052801030264e-07, |
| "logits/chosen": -1.062585473060608, |
| "logits/rejected": -1.0586426258087158, |
| "logps/chosen": -405.9750061035156, |
| "logps/rejected": -540.9249877929688, |
| "loss": 0.0088, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -3.174121141433716, |
| "rewards/margins": 15.868749618530273, |
| "rewards/rejected": -19.057811737060547, |
| "step": 3380 |
| }, |
| { |
| "epoch": 2.181525587383328, |
| "grad_norm": 0.10089211561220655, |
| "learning_rate": 4.5444301352221504e-07, |
| "logits/chosen": -1.073828101158142, |
| "logits/rejected": -1.073632836341858, |
| "logps/chosen": -457.9750061035156, |
| "logps/rejected": -594.4000244140625, |
| "loss": 0.0132, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -4.7056884765625, |
| "rewards/margins": 15.8515625, |
| "rewards/rejected": -20.556249618530273, |
| "step": 3390 |
| }, |
| { |
| "epoch": 2.1879626649501125, |
| "grad_norm": 0.0006958724241457409, |
| "learning_rate": 4.528332260141661e-07, |
| "logits/chosen": -1.022985816001892, |
| "logits/rejected": -1.043212890625, |
| "logps/chosen": -411.5, |
| "logps/rejected": -548.3250122070312, |
| "loss": 0.0101, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -4.134961128234863, |
| "rewards/margins": 16.254688262939453, |
| "rewards/rejected": -20.389062881469727, |
| "step": 3400 |
| }, |
| { |
| "epoch": 2.1943997425168975, |
| "grad_norm": 14.135716304337999, |
| "learning_rate": 4.512234385061172e-07, |
| "logits/chosen": -0.994384765625, |
| "logits/rejected": -1.0382568836212158, |
| "logps/chosen": -464.2250061035156, |
| "logps/rejected": -574.7000122070312, |
| "loss": 0.0188, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -4.747363090515137, |
| "rewards/margins": 16.634374618530273, |
| "rewards/rejected": -21.364063262939453, |
| "step": 3410 |
| }, |
| { |
| "epoch": 2.200836820083682, |
| "grad_norm": 0.31310709472957954, |
| "learning_rate": 4.4961365099806826e-07, |
| "logits/chosen": -1.076269507408142, |
| "logits/rejected": -1.083398461341858, |
| "logps/chosen": -407.2250061035156, |
| "logps/rejected": -520.4500122070312, |
| "loss": 0.0011, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -3.8101563453674316, |
| "rewards/margins": 17.2265625, |
| "rewards/rejected": -21.035938262939453, |
| "step": 3420 |
| }, |
| { |
| "epoch": 2.2072738976504667, |
| "grad_norm": 0.008193914417783646, |
| "learning_rate": 4.480038634900193e-07, |
| "logits/chosen": -0.981396496295929, |
| "logits/rejected": -0.9964355230331421, |
| "logps/chosen": -392.75, |
| "logps/rejected": -537.625, |
| "loss": 0.0184, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -4.210669040679932, |
| "rewards/margins": 16.301563262939453, |
| "rewards/rejected": -20.514842987060547, |
| "step": 3430 |
| }, |
| { |
| "epoch": 2.2137109752172512, |
| "grad_norm": 0.5881690669947562, |
| "learning_rate": 4.4639407598197035e-07, |
| "logits/chosen": -1.028222680091858, |
| "logits/rejected": -1.021386742591858, |
| "logps/chosen": -438.875, |
| "logps/rejected": -579.25, |
| "loss": 0.0069, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -4.130139350891113, |
| "rewards/margins": 16.584375381469727, |
| "rewards/rejected": -20.701562881469727, |
| "step": 3440 |
| }, |
| { |
| "epoch": 2.220148052784036, |
| "grad_norm": 1.8005519369536847, |
| "learning_rate": 4.447842884739214e-07, |
| "logits/chosen": -0.9946044683456421, |
| "logits/rejected": -1.0242187976837158, |
| "logps/chosen": -400.92498779296875, |
| "logps/rejected": -537.0499877929688, |
| "loss": 0.0187, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -4.555712699890137, |
| "rewards/margins": 16.268749237060547, |
| "rewards/rejected": -20.821874618530273, |
| "step": 3450 |
| }, |
| { |
| "epoch": 2.226585130350821, |
| "grad_norm": 0.03380963376951087, |
| "learning_rate": 4.431745009658725e-07, |
| "logits/chosen": -0.999707043170929, |
| "logits/rejected": -1.041748046875, |
| "logps/chosen": -408.6000061035156, |
| "logps/rejected": -551.0750122070312, |
| "loss": 0.0045, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -4.134863376617432, |
| "rewards/margins": 16.989063262939453, |
| "rewards/rejected": -21.118749618530273, |
| "step": 3460 |
| }, |
| { |
| "epoch": 2.2330222079176054, |
| "grad_norm": 0.033513908815007994, |
| "learning_rate": 4.415647134578235e-07, |
| "logits/chosen": -1.010839819908142, |
| "logits/rejected": -1.040429711341858, |
| "logps/chosen": -424.95001220703125, |
| "logps/rejected": -571.0, |
| "loss": 0.0094, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -4.512225151062012, |
| "rewards/margins": 17.228124618530273, |
| "rewards/rejected": -21.732812881469727, |
| "step": 3470 |
| }, |
| { |
| "epoch": 2.23945928548439, |
| "grad_norm": 0.2740000677105209, |
| "learning_rate": 4.3995492594977465e-07, |
| "logits/chosen": -0.97900390625, |
| "logits/rejected": -0.9859619140625, |
| "logps/chosen": -396.0249938964844, |
| "logps/rejected": -535.1749877929688, |
| "loss": 0.0206, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -5.21337890625, |
| "rewards/margins": 15.759374618530273, |
| "rewards/rejected": -20.957813262939453, |
| "step": 3480 |
| }, |
| { |
| "epoch": 2.2458963630511746, |
| "grad_norm": 0.15223632079148156, |
| "learning_rate": 4.3834513844172567e-07, |
| "logits/chosen": -1.014062523841858, |
| "logits/rejected": -1.0511353015899658, |
| "logps/chosen": -442.7250061035156, |
| "logps/rejected": -571.4749755859375, |
| "loss": 0.0295, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -5.354589939117432, |
| "rewards/margins": 15.748437881469727, |
| "rewards/rejected": -21.092187881469727, |
| "step": 3490 |
| }, |
| { |
| "epoch": 2.2523334406179596, |
| "grad_norm": 0.13728561648140786, |
| "learning_rate": 4.3673535093367674e-07, |
| "logits/chosen": -1.0329101085662842, |
| "logits/rejected": -1.036962866783142, |
| "logps/chosen": -382.4750061035156, |
| "logps/rejected": -534.5, |
| "loss": 0.005, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -5.963476657867432, |
| "rewards/margins": 16.146875381469727, |
| "rewards/rejected": -22.104686737060547, |
| "step": 3500 |
| }, |
| { |
| "epoch": 2.258770518184744, |
| "grad_norm": 10.709637400608113, |
| "learning_rate": 4.3512556342562776e-07, |
| "logits/chosen": -1.0541503429412842, |
| "logits/rejected": -1.050878882408142, |
| "logps/chosen": -443.9375, |
| "logps/rejected": -584.8499755859375, |
| "loss": 0.017, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -5.727734565734863, |
| "rewards/margins": 17.151561737060547, |
| "rewards/rejected": -22.8671875, |
| "step": 3510 |
| }, |
| { |
| "epoch": 2.2652075957515287, |
| "grad_norm": 0.7174296831368239, |
| "learning_rate": 4.335157759175789e-07, |
| "logits/chosen": -1.0232422351837158, |
| "logits/rejected": -1.032568335533142, |
| "logps/chosen": -406.875, |
| "logps/rejected": -583.6500244140625, |
| "loss": 0.0365, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -5.3134765625, |
| "rewards/margins": 17.565624237060547, |
| "rewards/rejected": -22.865625381469727, |
| "step": 3520 |
| }, |
| { |
| "epoch": 2.2716446733183133, |
| "grad_norm": 0.6260293290211867, |
| "learning_rate": 4.319059884095299e-07, |
| "logits/chosen": -1.082421898841858, |
| "logits/rejected": -1.1144530773162842, |
| "logps/chosen": -413.70001220703125, |
| "logps/rejected": -555.6500244140625, |
| "loss": 0.0217, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -5.38232421875, |
| "rewards/margins": 16.2734375, |
| "rewards/rejected": -21.674999237060547, |
| "step": 3530 |
| }, |
| { |
| "epoch": 2.2780817508850983, |
| "grad_norm": 1.703213885575144, |
| "learning_rate": 4.30296200901481e-07, |
| "logits/chosen": -0.9996337890625, |
| "logits/rejected": -1.014892578125, |
| "logps/chosen": -439.04998779296875, |
| "logps/rejected": -576.0750122070312, |
| "loss": 0.0138, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -4.720019340515137, |
| "rewards/margins": 16.753124237060547, |
| "rewards/rejected": -21.4765625, |
| "step": 3540 |
| }, |
| { |
| "epoch": 2.284518828451883, |
| "grad_norm": 2.632339736606739, |
| "learning_rate": 4.2868641339343205e-07, |
| "logits/chosen": -1.0037109851837158, |
| "logits/rejected": -1.0343017578125, |
| "logps/chosen": -425.125, |
| "logps/rejected": -536.75, |
| "loss": 0.0159, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -4.025781154632568, |
| "rewards/margins": 16.020313262939453, |
| "rewards/rejected": -20.042186737060547, |
| "step": 3550 |
| }, |
| { |
| "epoch": 2.2909559060186675, |
| "grad_norm": 0.20391483750374004, |
| "learning_rate": 4.2707662588538313e-07, |
| "logits/chosen": -1.028173804283142, |
| "logits/rejected": -1.041967749595642, |
| "logps/chosen": -420.92498779296875, |
| "logps/rejected": -558.2000122070312, |
| "loss": 0.0198, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -4.070458889007568, |
| "rewards/margins": 15.535937309265137, |
| "rewards/rejected": -19.610937118530273, |
| "step": 3560 |
| }, |
| { |
| "epoch": 2.297392983585452, |
| "grad_norm": 0.22150018868741062, |
| "learning_rate": 4.2546683837733415e-07, |
| "logits/chosen": -1.0495116710662842, |
| "logits/rejected": -1.0852539539337158, |
| "logps/chosen": -424.1499938964844, |
| "logps/rejected": -548.6500244140625, |
| "loss": 0.0221, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -3.4621949195861816, |
| "rewards/margins": 16.325000762939453, |
| "rewards/rejected": -19.792186737060547, |
| "step": 3570 |
| }, |
| { |
| "epoch": 2.3038300611522367, |
| "grad_norm": 0.09769741421519501, |
| "learning_rate": 4.238570508692853e-07, |
| "logits/chosen": -1.0507323741912842, |
| "logits/rejected": -1.040673851966858, |
| "logps/chosen": -417.2749938964844, |
| "logps/rejected": -568.0499877929688, |
| "loss": 0.0132, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -4.569872856140137, |
| "rewards/margins": 16.069530487060547, |
| "rewards/rejected": -20.625, |
| "step": 3580 |
| }, |
| { |
| "epoch": 2.3102671387190217, |
| "grad_norm": 0.3834551065281765, |
| "learning_rate": 4.222472633612363e-07, |
| "logits/chosen": -1.0048736333847046, |
| "logits/rejected": -1.031103491783142, |
| "logps/chosen": -425.29998779296875, |
| "logps/rejected": -565.8499755859375, |
| "loss": 0.0092, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -3.7234864234924316, |
| "rewards/margins": 16.870311737060547, |
| "rewards/rejected": -20.598438262939453, |
| "step": 3590 |
| }, |
| { |
| "epoch": 2.3167042162858063, |
| "grad_norm": 1.9144721824557707, |
| "learning_rate": 4.2063747585318737e-07, |
| "logits/chosen": -1.0072753429412842, |
| "logits/rejected": -1.0269043445587158, |
| "logps/chosen": -381.07501220703125, |
| "logps/rejected": -536.6500244140625, |
| "loss": 0.005, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -4.131249904632568, |
| "rewards/margins": 16.9375, |
| "rewards/rejected": -21.082813262939453, |
| "step": 3600 |
| }, |
| { |
| "epoch": 2.323141293852591, |
| "grad_norm": 36.214552892647944, |
| "learning_rate": 4.190276883451384e-07, |
| "logits/chosen": -1.02734375, |
| "logits/rejected": -1.035888671875, |
| "logps/chosen": -441.3374938964844, |
| "logps/rejected": -592.0499877929688, |
| "loss": 0.0068, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -4.705712795257568, |
| "rewards/margins": 17.370311737060547, |
| "rewards/rejected": -22.082813262939453, |
| "step": 3610 |
| }, |
| { |
| "epoch": 2.329578371419376, |
| "grad_norm": 0.047373315431620606, |
| "learning_rate": 4.174179008370895e-07, |
| "logits/chosen": -1.1228516101837158, |
| "logits/rejected": -1.125585913658142, |
| "logps/chosen": -416.7749938964844, |
| "logps/rejected": -573.9000244140625, |
| "loss": 0.0159, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -4.840966701507568, |
| "rewards/margins": 16.671875, |
| "rewards/rejected": -21.510936737060547, |
| "step": 3620 |
| }, |
| { |
| "epoch": 2.3360154489861604, |
| "grad_norm": 0.5143692709122827, |
| "learning_rate": 4.1580811332904054e-07, |
| "logits/chosen": -1.047265648841858, |
| "logits/rejected": -1.072656273841858, |
| "logps/chosen": -401.3999938964844, |
| "logps/rejected": -503.2250061035156, |
| "loss": 0.0226, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -4.970654487609863, |
| "rewards/margins": 15.118749618530273, |
| "rewards/rejected": -20.075000762939453, |
| "step": 3630 |
| }, |
| { |
| "epoch": 2.342452526552945, |
| "grad_norm": 0.13432767152874814, |
| "learning_rate": 4.141983258209916e-07, |
| "logits/chosen": -1.0247802734375, |
| "logits/rejected": -1.006445288658142, |
| "logps/chosen": -397.3125, |
| "logps/rejected": -522.9000244140625, |
| "loss": 0.0371, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -3.4481444358825684, |
| "rewards/margins": 15.620312690734863, |
| "rewards/rejected": -19.079687118530273, |
| "step": 3640 |
| }, |
| { |
| "epoch": 2.3488896041197296, |
| "grad_norm": 0.34195251874543514, |
| "learning_rate": 4.125885383129427e-07, |
| "logits/chosen": -0.9477783441543579, |
| "logits/rejected": -0.9656982421875, |
| "logps/chosen": -404.51251220703125, |
| "logps/rejected": -527.25, |
| "loss": 0.0182, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -4.066552639007568, |
| "rewards/margins": 15.949999809265137, |
| "rewards/rejected": -20.014062881469727, |
| "step": 3650 |
| }, |
| { |
| "epoch": 2.355326681686514, |
| "grad_norm": 2.526519532504119, |
| "learning_rate": 4.1097875080489376e-07, |
| "logits/chosen": -1.093359351158142, |
| "logits/rejected": -1.110498070716858, |
| "logps/chosen": -408.98748779296875, |
| "logps/rejected": -553.0499877929688, |
| "loss": 0.0089, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -4.961523532867432, |
| "rewards/margins": 16.207813262939453, |
| "rewards/rejected": -21.168750762939453, |
| "step": 3660 |
| }, |
| { |
| "epoch": 2.361763759253299, |
| "grad_norm": 0.06085996531440162, |
| "learning_rate": 4.093689632968448e-07, |
| "logits/chosen": -1.125634789466858, |
| "logits/rejected": -1.1251952648162842, |
| "logps/chosen": -434.2250061035156, |
| "logps/rejected": -596.0499877929688, |
| "loss": 0.005, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -4.989697456359863, |
| "rewards/margins": 17.685937881469727, |
| "rewards/rejected": -22.674999237060547, |
| "step": 3670 |
| }, |
| { |
| "epoch": 2.3682008368200838, |
| "grad_norm": 1.0978779971202857, |
| "learning_rate": 4.0775917578879585e-07, |
| "logits/chosen": -1.00048828125, |
| "logits/rejected": -1.0075194835662842, |
| "logps/chosen": -383.17498779296875, |
| "logps/rejected": -517.9500122070312, |
| "loss": 0.0307, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -4.374267578125, |
| "rewards/margins": 15.589062690734863, |
| "rewards/rejected": -19.954687118530273, |
| "step": 3680 |
| }, |
| { |
| "epoch": 2.3746379143868683, |
| "grad_norm": 0.004285537440066397, |
| "learning_rate": 4.061493882807469e-07, |
| "logits/chosen": -0.9914916753768921, |
| "logits/rejected": -1.009521484375, |
| "logps/chosen": -420.8374938964844, |
| "logps/rejected": -559.1500244140625, |
| "loss": 0.0128, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -5.005566596984863, |
| "rewards/margins": 17.623437881469727, |
| "rewards/rejected": -22.639062881469727, |
| "step": 3690 |
| }, |
| { |
| "epoch": 2.381074991953653, |
| "grad_norm": 0.0753736597073573, |
| "learning_rate": 4.04539600772698e-07, |
| "logits/chosen": -0.992919921875, |
| "logits/rejected": -1.01214599609375, |
| "logps/chosen": -429.8500061035156, |
| "logps/rejected": -572.2000122070312, |
| "loss": 0.0165, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -4.004248142242432, |
| "rewards/margins": 17.706249237060547, |
| "rewards/rejected": -21.701562881469727, |
| "step": 3700 |
| }, |
| { |
| "epoch": 2.3875120695204375, |
| "grad_norm": 12.902251613669751, |
| "learning_rate": 4.02929813264649e-07, |
| "logits/chosen": -1.042822241783142, |
| "logits/rejected": -1.052832007408142, |
| "logps/chosen": -420.6000061035156, |
| "logps/rejected": -559.1500244140625, |
| "loss": 0.0239, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -5.941210746765137, |
| "rewards/margins": 15.0078125, |
| "rewards/rejected": -20.9375, |
| "step": 3710 |
| }, |
| { |
| "epoch": 2.3939491470872225, |
| "grad_norm": 4.529963145208864, |
| "learning_rate": 4.0132002575660014e-07, |
| "logits/chosen": -0.9861816167831421, |
| "logits/rejected": -1.039648413658142, |
| "logps/chosen": -417.32501220703125, |
| "logps/rejected": -582.5499877929688, |
| "loss": 0.0046, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -4.405077934265137, |
| "rewards/margins": 17.334375381469727, |
| "rewards/rejected": -21.7265625, |
| "step": 3720 |
| }, |
| { |
| "epoch": 2.400386224654007, |
| "grad_norm": 0.9629898122683673, |
| "learning_rate": 3.9971023824855116e-07, |
| "logits/chosen": -1.044580101966858, |
| "logits/rejected": -1.0702636241912842, |
| "logps/chosen": -394.17498779296875, |
| "logps/rejected": -539.5999755859375, |
| "loss": 0.0088, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -3.7095704078674316, |
| "rewards/margins": 17.260936737060547, |
| "rewards/rejected": -20.965625762939453, |
| "step": 3730 |
| }, |
| { |
| "epoch": 2.4068233022207917, |
| "grad_norm": 0.20698119594926714, |
| "learning_rate": 3.9810045074050224e-07, |
| "logits/chosen": -1.0583007335662842, |
| "logits/rejected": -1.0828125476837158, |
| "logps/chosen": -473.20001220703125, |
| "logps/rejected": -547.3499755859375, |
| "loss": 0.0101, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -3.494311571121216, |
| "rewards/margins": 16.2109375, |
| "rewards/rejected": -19.709375381469727, |
| "step": 3740 |
| }, |
| { |
| "epoch": 2.4132603797875767, |
| "grad_norm": 0.013493617041070298, |
| "learning_rate": 3.9649066323245326e-07, |
| "logits/chosen": -0.940869152545929, |
| "logits/rejected": -0.9414306879043579, |
| "logps/chosen": -391.5, |
| "logps/rejected": -520.6749877929688, |
| "loss": 0.0205, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -3.1650633811950684, |
| "rewards/margins": 16.353124618530273, |
| "rewards/rejected": -19.526561737060547, |
| "step": 3750 |
| }, |
| { |
| "epoch": 2.4196974573543613, |
| "grad_norm": 0.013139894005101382, |
| "learning_rate": 3.948808757244044e-07, |
| "logits/chosen": -0.9778076410293579, |
| "logits/rejected": -0.998120129108429, |
| "logps/chosen": -427.25, |
| "logps/rejected": -550.0499877929688, |
| "loss": 0.0311, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -3.592041015625, |
| "rewards/margins": 15.556249618530273, |
| "rewards/rejected": -19.140625, |
| "step": 3760 |
| }, |
| { |
| "epoch": 2.426134534921146, |
| "grad_norm": 7.327332077965629, |
| "learning_rate": 3.932710882163554e-07, |
| "logits/chosen": -1.0476562976837158, |
| "logits/rejected": -1.07373046875, |
| "logps/chosen": -432.92498779296875, |
| "logps/rejected": -563.9500122070312, |
| "loss": 0.0118, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -3.391796827316284, |
| "rewards/margins": 17.470312118530273, |
| "rewards/rejected": -20.870311737060547, |
| "step": 3770 |
| }, |
| { |
| "epoch": 2.4325716124879304, |
| "grad_norm": 0.1810775822374288, |
| "learning_rate": 3.916613007083065e-07, |
| "logits/chosen": -1.052490234375, |
| "logits/rejected": -1.092626929283142, |
| "logps/chosen": -379.625, |
| "logps/rejected": -523.9500122070312, |
| "loss": 0.0189, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -3.958789110183716, |
| "rewards/margins": 16.422657012939453, |
| "rewards/rejected": -20.373437881469727, |
| "step": 3780 |
| }, |
| { |
| "epoch": 2.439008690054715, |
| "grad_norm": 0.23327394750489913, |
| "learning_rate": 3.9005151320025755e-07, |
| "logits/chosen": -1.0048339366912842, |
| "logits/rejected": -1.0261719226837158, |
| "logps/chosen": -396.11248779296875, |
| "logps/rejected": -539.2000122070312, |
| "loss": 0.0096, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -3.868115186691284, |
| "rewards/margins": 16.415624618530273, |
| "rewards/rejected": -20.299999237060547, |
| "step": 3790 |
| }, |
| { |
| "epoch": 2.4454457676215, |
| "grad_norm": 0.02558078884725733, |
| "learning_rate": 3.884417256922086e-07, |
| "logits/chosen": -0.972582995891571, |
| "logits/rejected": -0.9895995855331421, |
| "logps/chosen": -400.0249938964844, |
| "logps/rejected": -556.4000244140625, |
| "loss": 0.0014, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -3.6697998046875, |
| "rewards/margins": 16.653125762939453, |
| "rewards/rejected": -20.310937881469727, |
| "step": 3800 |
| }, |
| { |
| "epoch": 2.4518828451882846, |
| "grad_norm": 0.05959869141952533, |
| "learning_rate": 3.8683193818415965e-07, |
| "logits/chosen": -0.9869140386581421, |
| "logits/rejected": -0.977246105670929, |
| "logps/chosen": -407.04998779296875, |
| "logps/rejected": -573.4000244140625, |
| "loss": 0.0133, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -4.500256538391113, |
| "rewards/margins": 15.440625190734863, |
| "rewards/rejected": -19.932811737060547, |
| "step": 3810 |
| }, |
| { |
| "epoch": 2.458319922755069, |
| "grad_norm": 0.08842132782631504, |
| "learning_rate": 3.8522215067611077e-07, |
| "logits/chosen": -1.037939429283142, |
| "logits/rejected": -1.053808569908142, |
| "logps/chosen": -423.07501220703125, |
| "logps/rejected": -584.8499755859375, |
| "loss": 0.0145, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -4.993945121765137, |
| "rewards/margins": 17.1484375, |
| "rewards/rejected": -22.149999618530273, |
| "step": 3820 |
| }, |
| { |
| "epoch": 2.4647570003218537, |
| "grad_norm": 30.947385597831875, |
| "learning_rate": 3.836123631680618e-07, |
| "logits/chosen": -1.028906226158142, |
| "logits/rejected": -1.0302002429962158, |
| "logps/chosen": -372.0, |
| "logps/rejected": -546.8250122070312, |
| "loss": 0.0212, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -4.22998046875, |
| "rewards/margins": 16.628124237060547, |
| "rewards/rejected": -20.865625381469727, |
| "step": 3830 |
| }, |
| { |
| "epoch": 2.4711940778886383, |
| "grad_norm": 0.05211605695940069, |
| "learning_rate": 3.8200257566001287e-07, |
| "logits/chosen": -1.055566430091858, |
| "logits/rejected": -1.0647461414337158, |
| "logps/chosen": -409.04998779296875, |
| "logps/rejected": -544.0499877929688, |
| "loss": 0.0048, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -3.8119139671325684, |
| "rewards/margins": 18.073436737060547, |
| "rewards/rejected": -21.878124237060547, |
| "step": 3840 |
| }, |
| { |
| "epoch": 2.4776311554554233, |
| "grad_norm": 9.213720198102742, |
| "learning_rate": 3.803927881519639e-07, |
| "logits/chosen": -1.0117919445037842, |
| "logits/rejected": -1.0427124500274658, |
| "logps/chosen": -433.5249938964844, |
| "logps/rejected": -587.8499755859375, |
| "loss": 0.0304, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -4.554394721984863, |
| "rewards/margins": 16.564062118530273, |
| "rewards/rejected": -21.112499237060547, |
| "step": 3850 |
| }, |
| { |
| "epoch": 2.484068233022208, |
| "grad_norm": 0.24431172158453063, |
| "learning_rate": 3.78783000643915e-07, |
| "logits/chosen": -1.0658690929412842, |
| "logits/rejected": -1.073632836341858, |
| "logps/chosen": -367.375, |
| "logps/rejected": -512.0999755859375, |
| "loss": 0.0288, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -3.274951219558716, |
| "rewards/margins": 15.74609375, |
| "rewards/rejected": -19.03125, |
| "step": 3860 |
| }, |
| { |
| "epoch": 2.4905053105889925, |
| "grad_norm": 71.25388858956305, |
| "learning_rate": 3.7717321313586603e-07, |
| "logits/chosen": -1.061425805091858, |
| "logits/rejected": -1.0839354991912842, |
| "logps/chosen": -455.8374938964844, |
| "logps/rejected": -562.2000122070312, |
| "loss": 0.0104, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -4.843505859375, |
| "rewards/margins": 16.064062118530273, |
| "rewards/rejected": -20.912500381469727, |
| "step": 3870 |
| }, |
| { |
| "epoch": 2.4969423881557775, |
| "grad_norm": 1.8350700859136448, |
| "learning_rate": 3.755634256278171e-07, |
| "logits/chosen": -0.9950927495956421, |
| "logits/rejected": -1.032861351966858, |
| "logps/chosen": -421.9750061035156, |
| "logps/rejected": -555.4000244140625, |
| "loss": 0.0139, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -3.4852051734924316, |
| "rewards/margins": 16.821874618530273, |
| "rewards/rejected": -20.3125, |
| "step": 3880 |
| }, |
| { |
| "epoch": 2.503379465722562, |
| "grad_norm": 5.050750304416249, |
| "learning_rate": 3.739536381197682e-07, |
| "logits/chosen": -1.1130859851837158, |
| "logits/rejected": -1.1386229991912842, |
| "logps/chosen": -406.8500061035156, |
| "logps/rejected": -538.0499877929688, |
| "loss": 0.0127, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -5.451952934265137, |
| "rewards/margins": 16.712499618530273, |
| "rewards/rejected": -22.167186737060547, |
| "step": 3890 |
| }, |
| { |
| "epoch": 2.5098165432893467, |
| "grad_norm": 5.355053391341112, |
| "learning_rate": 3.7234385061171925e-07, |
| "logits/chosen": -0.9847412109375, |
| "logits/rejected": -1.0299804210662842, |
| "logps/chosen": -412.7250061035156, |
| "logps/rejected": -550.6500244140625, |
| "loss": 0.017, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -4.985693454742432, |
| "rewards/margins": 18.120311737060547, |
| "rewards/rejected": -23.103124618530273, |
| "step": 3900 |
| }, |
| { |
| "epoch": 2.5162536208561312, |
| "grad_norm": 0.0193165295031974, |
| "learning_rate": 3.707340631036703e-07, |
| "logits/chosen": -1.0419189929962158, |
| "logits/rejected": -1.063818335533142, |
| "logps/chosen": -406.9624938964844, |
| "logps/rejected": -541.1500244140625, |
| "loss": 0.0132, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -5.443945407867432, |
| "rewards/margins": 17.647655487060547, |
| "rewards/rejected": -23.082813262939453, |
| "step": 3910 |
| }, |
| { |
| "epoch": 2.522690698422916, |
| "grad_norm": 1.277209930482019, |
| "learning_rate": 3.6912427559562135e-07, |
| "logits/chosen": -1.0840332508087158, |
| "logits/rejected": -1.095361351966858, |
| "logps/chosen": -430.375, |
| "logps/rejected": -571.4000244140625, |
| "loss": 0.0064, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -5.085156440734863, |
| "rewards/margins": 16.314062118530273, |
| "rewards/rejected": -21.404687881469727, |
| "step": 3920 |
| }, |
| { |
| "epoch": 2.529127775989701, |
| "grad_norm": 14.484108215530707, |
| "learning_rate": 3.675144880875724e-07, |
| "logits/chosen": -1.103124976158142, |
| "logits/rejected": -1.1331055164337158, |
| "logps/chosen": -451.17498779296875, |
| "logps/rejected": -567.4000244140625, |
| "loss": 0.0092, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -4.98681640625, |
| "rewards/margins": 17.245311737060547, |
| "rewards/rejected": -22.225000381469727, |
| "step": 3930 |
| }, |
| { |
| "epoch": 2.5355648535564854, |
| "grad_norm": 0.08431049236655122, |
| "learning_rate": 3.659047005795235e-07, |
| "logits/chosen": -1.0732421875, |
| "logits/rejected": -1.0867431163787842, |
| "logps/chosen": -447.67498779296875, |
| "logps/rejected": -586.1500244140625, |
| "loss": 0.0087, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -4.617505073547363, |
| "rewards/margins": 17.545312881469727, |
| "rewards/rejected": -22.178125381469727, |
| "step": 3940 |
| }, |
| { |
| "epoch": 2.54200193112327, |
| "grad_norm": 0.001819636969881986, |
| "learning_rate": 3.642949130714745e-07, |
| "logits/chosen": -1.0378296375274658, |
| "logits/rejected": -1.06689453125, |
| "logps/chosen": -378.20001220703125, |
| "logps/rejected": -521.8499755859375, |
| "loss": 0.0131, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -4.829297065734863, |
| "rewards/margins": 16.543750762939453, |
| "rewards/rejected": -21.359375, |
| "step": 3950 |
| }, |
| { |
| "epoch": 2.5484390086900546, |
| "grad_norm": 9.099365929589844, |
| "learning_rate": 3.6268512556342564e-07, |
| "logits/chosen": -1.011962890625, |
| "logits/rejected": -1.035668969154358, |
| "logps/chosen": -410.3500061035156, |
| "logps/rejected": -571.75, |
| "loss": 0.0182, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -4.998046875, |
| "rewards/margins": 17.016407012939453, |
| "rewards/rejected": -22.017187118530273, |
| "step": 3960 |
| }, |
| { |
| "epoch": 2.554876086256839, |
| "grad_norm": 0.0459123345549701, |
| "learning_rate": 3.6107533805537666e-07, |
| "logits/chosen": -1.1208984851837158, |
| "logits/rejected": -1.1435546875, |
| "logps/chosen": -439.8999938964844, |
| "logps/rejected": -591.9500122070312, |
| "loss": 0.0131, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -4.621191501617432, |
| "rewards/margins": 18.442188262939453, |
| "rewards/rejected": -23.067188262939453, |
| "step": 3970 |
| }, |
| { |
| "epoch": 2.561313163823624, |
| "grad_norm": 1.9349974401487566, |
| "learning_rate": 3.5946555054732774e-07, |
| "logits/chosen": -1.0195801258087158, |
| "logits/rejected": -1.0270507335662842, |
| "logps/chosen": -426.95001220703125, |
| "logps/rejected": -560.0, |
| "loss": 0.0157, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -5.405713081359863, |
| "rewards/margins": 17.421875, |
| "rewards/rejected": -22.818750381469727, |
| "step": 3980 |
| }, |
| { |
| "epoch": 2.5677502413904087, |
| "grad_norm": 0.4891014195678684, |
| "learning_rate": 3.5785576303927876e-07, |
| "logits/chosen": -1.0330078601837158, |
| "logits/rejected": -1.0408203601837158, |
| "logps/chosen": -429.7749938964844, |
| "logps/rejected": -601.8499755859375, |
| "loss": 0.0099, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -5.31982421875, |
| "rewards/margins": 17.131250381469727, |
| "rewards/rejected": -22.443750381469727, |
| "step": 3990 |
| }, |
| { |
| "epoch": 2.5741873189571933, |
| "grad_norm": 1.5627512684455094, |
| "learning_rate": 3.562459755312299e-07, |
| "logits/chosen": -1.0496826171875, |
| "logits/rejected": -1.080175757408142, |
| "logps/chosen": -415.54998779296875, |
| "logps/rejected": -572.1500244140625, |
| "loss": 0.0004, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -5.251953125, |
| "rewards/margins": 17.899999618530273, |
| "rewards/rejected": -23.146875381469727, |
| "step": 4000 |
| }, |
| { |
| "epoch": 2.5806243965239783, |
| "grad_norm": 0.08739818731065581, |
| "learning_rate": 3.546361880231809e-07, |
| "logits/chosen": -1.082617163658142, |
| "logits/rejected": -1.087744116783142, |
| "logps/chosen": -473.4750061035156, |
| "logps/rejected": -611.75, |
| "loss": 0.0101, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -6.598681449890137, |
| "rewards/margins": 17.556249618530273, |
| "rewards/rejected": -24.149999618530273, |
| "step": 4010 |
| }, |
| { |
| "epoch": 2.587061474090763, |
| "grad_norm": 0.3471564851888695, |
| "learning_rate": 3.53026400515132e-07, |
| "logits/chosen": -1.062597632408142, |
| "logits/rejected": -1.064550757408142, |
| "logps/chosen": -414.20001220703125, |
| "logps/rejected": -553.9000244140625, |
| "loss": 0.0095, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -5.682898044586182, |
| "rewards/margins": 16.796875, |
| "rewards/rejected": -22.484375, |
| "step": 4020 |
| }, |
| { |
| "epoch": 2.5934985516575475, |
| "grad_norm": 0.014063778025355996, |
| "learning_rate": 3.5141661300708305e-07, |
| "logits/chosen": -1.0272216796875, |
| "logits/rejected": -1.03466796875, |
| "logps/chosen": -370.1875, |
| "logps/rejected": -538.1500244140625, |
| "loss": 0.0206, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -4.160009860992432, |
| "rewards/margins": 17.0546875, |
| "rewards/rejected": -21.209375381469727, |
| "step": 4030 |
| }, |
| { |
| "epoch": 2.599935629224332, |
| "grad_norm": 192.87824451729486, |
| "learning_rate": 3.498068254990341e-07, |
| "logits/chosen": -1.0537598133087158, |
| "logits/rejected": -1.0952637195587158, |
| "logps/chosen": -421.1499938964844, |
| "logps/rejected": -559.3499755859375, |
| "loss": 0.0109, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -4.37060546875, |
| "rewards/margins": 16.501562118530273, |
| "rewards/rejected": -20.875, |
| "step": 4040 |
| }, |
| { |
| "epoch": 2.6063727067911167, |
| "grad_norm": 0.08960042129229139, |
| "learning_rate": 3.4819703799098514e-07, |
| "logits/chosen": -1.115136742591858, |
| "logits/rejected": -1.1196777820587158, |
| "logps/chosen": -430.95001220703125, |
| "logps/rejected": -589.5999755859375, |
| "loss": 0.0109, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -5.207763671875, |
| "rewards/margins": 16.924999237060547, |
| "rewards/rejected": -22.129688262939453, |
| "step": 4050 |
| }, |
| { |
| "epoch": 2.6128097843579017, |
| "grad_norm": 0.20842717683750414, |
| "learning_rate": 3.4658725048293627e-07, |
| "logits/chosen": -1.0168945789337158, |
| "logits/rejected": -1.046289086341858, |
| "logps/chosen": -443.2250061035156, |
| "logps/rejected": -565.0999755859375, |
| "loss": 0.013, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -4.522119045257568, |
| "rewards/margins": 16.446874618530273, |
| "rewards/rejected": -20.96875, |
| "step": 4060 |
| }, |
| { |
| "epoch": 2.6192468619246863, |
| "grad_norm": 0.25064627796735495, |
| "learning_rate": 3.449774629748873e-07, |
| "logits/chosen": -1.0129883289337158, |
| "logits/rejected": -1.024804711341858, |
| "logps/chosen": -379.375, |
| "logps/rejected": -528.3499755859375, |
| "loss": 0.0263, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -3.7671875953674316, |
| "rewards/margins": 16.249217987060547, |
| "rewards/rejected": -20.014062881469727, |
| "step": 4070 |
| }, |
| { |
| "epoch": 2.625683939491471, |
| "grad_norm": 0.08946523961309825, |
| "learning_rate": 3.4336767546683836e-07, |
| "logits/chosen": -1.0355956554412842, |
| "logits/rejected": -1.0556151866912842, |
| "logps/chosen": -433.6499938964844, |
| "logps/rejected": -589.7000122070312, |
| "loss": 0.0141, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -5.182519435882568, |
| "rewards/margins": 15.75, |
| "rewards/rejected": -20.9296875, |
| "step": 4080 |
| }, |
| { |
| "epoch": 2.6321210170582554, |
| "grad_norm": 0.009005467084888181, |
| "learning_rate": 3.417578879587894e-07, |
| "logits/chosen": -1.0455322265625, |
| "logits/rejected": -1.050805687904358, |
| "logps/chosen": -472.875, |
| "logps/rejected": -595.9500122070312, |
| "loss": 0.0027, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -4.389208793640137, |
| "rewards/margins": 17.415624618530273, |
| "rewards/rejected": -21.810937881469727, |
| "step": 4090 |
| }, |
| { |
| "epoch": 2.63855809462504, |
| "grad_norm": 0.03113362210332774, |
| "learning_rate": 3.401481004507405e-07, |
| "logits/chosen": -1.061279296875, |
| "logits/rejected": -1.085205078125, |
| "logps/chosen": -399.92498779296875, |
| "logps/rejected": -553.8499755859375, |
| "loss": 0.0112, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -4.193945407867432, |
| "rewards/margins": 16.964061737060547, |
| "rewards/rejected": -21.159374237060547, |
| "step": 4100 |
| }, |
| { |
| "epoch": 2.644995172191825, |
| "grad_norm": 0.37326898026603983, |
| "learning_rate": 3.3853831294269153e-07, |
| "logits/chosen": -1.027929663658142, |
| "logits/rejected": -1.041015625, |
| "logps/chosen": -418.70001220703125, |
| "logps/rejected": -548.0, |
| "loss": 0.0094, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -4.916943550109863, |
| "rewards/margins": 16.417186737060547, |
| "rewards/rejected": -21.334375381469727, |
| "step": 4110 |
| }, |
| { |
| "epoch": 2.6514322497586096, |
| "grad_norm": 7.897992961962735, |
| "learning_rate": 3.369285254346426e-07, |
| "logits/chosen": -0.9991455078125, |
| "logits/rejected": -1.0133056640625, |
| "logps/chosen": -395.0249938964844, |
| "logps/rejected": -551.0499877929688, |
| "loss": 0.0139, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -3.975878953933716, |
| "rewards/margins": 17.159374237060547, |
| "rewards/rejected": -21.131250381469727, |
| "step": 4120 |
| }, |
| { |
| "epoch": 2.657869327325394, |
| "grad_norm": 0.07509522718083096, |
| "learning_rate": 3.353187379265937e-07, |
| "logits/chosen": -1.073095679283142, |
| "logits/rejected": -1.070459008216858, |
| "logps/chosen": -396.95001220703125, |
| "logps/rejected": -549.5, |
| "loss": 0.0196, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -4.819872856140137, |
| "rewards/margins": 17.485937118530273, |
| "rewards/rejected": -22.306249618530273, |
| "step": 4130 |
| }, |
| { |
| "epoch": 2.664306404892179, |
| "grad_norm": 0.5200990215076599, |
| "learning_rate": 3.3370895041854475e-07, |
| "logits/chosen": -1.039453148841858, |
| "logits/rejected": -1.0915038585662842, |
| "logps/chosen": -453.75, |
| "logps/rejected": -574.2999877929688, |
| "loss": 0.0225, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -4.282422065734863, |
| "rewards/margins": 15.921875, |
| "rewards/rejected": -20.217187881469727, |
| "step": 4140 |
| }, |
| { |
| "epoch": 2.6707434824589638, |
| "grad_norm": 0.25225569399148123, |
| "learning_rate": 3.3209916291049577e-07, |
| "logits/chosen": -1.0086181163787842, |
| "logits/rejected": -1.0000488758087158, |
| "logps/chosen": -372.42498779296875, |
| "logps/rejected": -509.29998779296875, |
| "loss": 0.0469, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -4.168701171875, |
| "rewards/margins": 14.98828125, |
| "rewards/rejected": -19.173437118530273, |
| "step": 4150 |
| }, |
| { |
| "epoch": 2.6771805600257483, |
| "grad_norm": 0.08673995336101631, |
| "learning_rate": 3.3048937540244685e-07, |
| "logits/chosen": -1.0426757335662842, |
| "logits/rejected": -1.098046898841858, |
| "logps/chosen": -460.25, |
| "logps/rejected": -575.3499755859375, |
| "loss": 0.0059, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -3.84619140625, |
| "rewards/margins": 16.293750762939453, |
| "rewards/rejected": -20.1328125, |
| "step": 4160 |
| }, |
| { |
| "epoch": 2.683617637592533, |
| "grad_norm": 13.353206676041454, |
| "learning_rate": 3.288795878943979e-07, |
| "logits/chosen": -1.021875023841858, |
| "logits/rejected": -1.0466797351837158, |
| "logps/chosen": -418.9375, |
| "logps/rejected": -592.9000244140625, |
| "loss": 0.005, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -3.449902296066284, |
| "rewards/margins": 17.214061737060547, |
| "rewards/rejected": -20.654687881469727, |
| "step": 4170 |
| }, |
| { |
| "epoch": 2.6900547151593175, |
| "grad_norm": 0.09935680046220585, |
| "learning_rate": 3.27269800386349e-07, |
| "logits/chosen": -0.990771472454071, |
| "logits/rejected": -1.0391356945037842, |
| "logps/chosen": -396.1499938964844, |
| "logps/rejected": -495.29998779296875, |
| "loss": 0.0223, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -3.1318116188049316, |
| "rewards/margins": 15.576562881469727, |
| "rewards/rejected": -18.704687118530273, |
| "step": 4180 |
| }, |
| { |
| "epoch": 2.6964917927261025, |
| "grad_norm": 2.336769551114649, |
| "learning_rate": 3.256600128783e-07, |
| "logits/chosen": -1.0742676258087158, |
| "logits/rejected": -1.09033203125, |
| "logps/chosen": -416.04998779296875, |
| "logps/rejected": -533.1749877929688, |
| "loss": 0.0215, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -3.793750047683716, |
| "rewards/margins": 14.542187690734863, |
| "rewards/rejected": -18.3359375, |
| "step": 4190 |
| }, |
| { |
| "epoch": 2.702928870292887, |
| "grad_norm": 1.6019642715844329, |
| "learning_rate": 3.2405022537025114e-07, |
| "logits/chosen": -1.0363037586212158, |
| "logits/rejected": -1.076416015625, |
| "logps/chosen": -403.375, |
| "logps/rejected": -547.3499755859375, |
| "loss": 0.011, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -3.212695360183716, |
| "rewards/margins": 15.584375381469727, |
| "rewards/rejected": -18.792186737060547, |
| "step": 4200 |
| }, |
| { |
| "epoch": 2.7093659478596717, |
| "grad_norm": 2.140234939910703, |
| "learning_rate": 3.2244043786220216e-07, |
| "logits/chosen": -1.002539038658142, |
| "logits/rejected": -1.024999976158142, |
| "logps/chosen": -464.86248779296875, |
| "logps/rejected": -558.5, |
| "loss": 0.0084, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -2.7227540016174316, |
| "rewards/margins": 14.568750381469727, |
| "rewards/rejected": -17.287500381469727, |
| "step": 4210 |
| }, |
| { |
| "epoch": 2.7158030254264562, |
| "grad_norm": 11.833136190216157, |
| "learning_rate": 3.2083065035415323e-07, |
| "logits/chosen": -0.994433581829071, |
| "logits/rejected": -1.042333960533142, |
| "logps/chosen": -406.4624938964844, |
| "logps/rejected": -500.2749938964844, |
| "loss": 0.0137, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -2.553906202316284, |
| "rewards/margins": 15.292187690734863, |
| "rewards/rejected": -17.853124618530273, |
| "step": 4220 |
| }, |
| { |
| "epoch": 2.722240102993241, |
| "grad_norm": 0.20643996913297855, |
| "learning_rate": 3.1922086284610425e-07, |
| "logits/chosen": -0.989501953125, |
| "logits/rejected": -1.02734375, |
| "logps/chosen": -407.3500061035156, |
| "logps/rejected": -531.7000122070312, |
| "loss": 0.0192, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -3.6068358421325684, |
| "rewards/margins": 14.211718559265137, |
| "rewards/rejected": -17.814062118530273, |
| "step": 4230 |
| }, |
| { |
| "epoch": 2.728677180560026, |
| "grad_norm": 0.05435070454747595, |
| "learning_rate": 3.176110753380554e-07, |
| "logits/chosen": -0.961718738079071, |
| "logits/rejected": -0.965283215045929, |
| "logps/chosen": -399.92498779296875, |
| "logps/rejected": -552.5, |
| "loss": 0.014, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -3.569287061691284, |
| "rewards/margins": 16.609375, |
| "rewards/rejected": -20.1796875, |
| "step": 4240 |
| }, |
| { |
| "epoch": 2.7351142581268104, |
| "grad_norm": 18.859101722505248, |
| "learning_rate": 3.160012878300064e-07, |
| "logits/chosen": -0.983837902545929, |
| "logits/rejected": -0.985034167766571, |
| "logps/chosen": -400.67498779296875, |
| "logps/rejected": -515.5, |
| "loss": 0.0057, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -3.3128905296325684, |
| "rewards/margins": 15.3515625, |
| "rewards/rejected": -18.6640625, |
| "step": 4250 |
| }, |
| { |
| "epoch": 2.741551335693595, |
| "grad_norm": 0.19188923573777983, |
| "learning_rate": 3.143915003219575e-07, |
| "logits/chosen": -1.030126929283142, |
| "logits/rejected": -1.056542992591858, |
| "logps/chosen": -402.17498779296875, |
| "logps/rejected": -529.5999755859375, |
| "loss": 0.0133, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -2.809277296066284, |
| "rewards/margins": 16.501562118530273, |
| "rewards/rejected": -19.318750381469727, |
| "step": 4260 |
| }, |
| { |
| "epoch": 2.74798841326038, |
| "grad_norm": 0.30196685444916815, |
| "learning_rate": 3.1278171281390855e-07, |
| "logits/chosen": -0.997265636920929, |
| "logits/rejected": -1.048925757408142, |
| "logps/chosen": -402.3999938964844, |
| "logps/rejected": -580.0, |
| "loss": 0.0344, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -3.306884765625, |
| "rewards/margins": 17.953125, |
| "rewards/rejected": -21.268749237060547, |
| "step": 4270 |
| }, |
| { |
| "epoch": 2.7544254908271646, |
| "grad_norm": 0.10959414872036556, |
| "learning_rate": 3.111719253058596e-07, |
| "logits/chosen": -1.031945824623108, |
| "logits/rejected": -1.056616187095642, |
| "logps/chosen": -427.4624938964844, |
| "logps/rejected": -535.9000244140625, |
| "loss": 0.0092, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -4.003564357757568, |
| "rewards/margins": 16.3671875, |
| "rewards/rejected": -20.381250381469727, |
| "step": 4280 |
| }, |
| { |
| "epoch": 2.760862568393949, |
| "grad_norm": 1.1898548530459272, |
| "learning_rate": 3.0956213779781064e-07, |
| "logits/chosen": -0.935986340045929, |
| "logits/rejected": -0.9755859375, |
| "logps/chosen": -395.4750061035156, |
| "logps/rejected": -536.5, |
| "loss": 0.0402, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -4.188818454742432, |
| "rewards/margins": 15.578906059265137, |
| "rewards/rejected": -19.765625, |
| "step": 4290 |
| }, |
| { |
| "epoch": 2.7672996459607337, |
| "grad_norm": 0.0349341778282771, |
| "learning_rate": 3.0795235028976177e-07, |
| "logits/chosen": -1.085205078125, |
| "logits/rejected": -1.1107909679412842, |
| "logps/chosen": -442.7250061035156, |
| "logps/rejected": -600.7000122070312, |
| "loss": 0.0097, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -3.6946043968200684, |
| "rewards/margins": 17.035938262939453, |
| "rewards/rejected": -20.739063262939453, |
| "step": 4300 |
| }, |
| { |
| "epoch": 2.7737367235275183, |
| "grad_norm": 0.022601355069047334, |
| "learning_rate": 3.063425627817128e-07, |
| "logits/chosen": -1.023046851158142, |
| "logits/rejected": -0.9951171875, |
| "logps/chosen": -410.1000061035156, |
| "logps/rejected": -555.0, |
| "loss": 0.0253, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -3.625805616378784, |
| "rewards/margins": 16.006250381469727, |
| "rewards/rejected": -19.629688262939453, |
| "step": 4310 |
| }, |
| { |
| "epoch": 2.7801738010943033, |
| "grad_norm": 0.4877006768131671, |
| "learning_rate": 3.0473277527366386e-07, |
| "logits/chosen": -1.080957055091858, |
| "logits/rejected": -1.0974609851837158, |
| "logps/chosen": -389.0249938964844, |
| "logps/rejected": -538.25, |
| "loss": 0.0089, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -3.261798143386841, |
| "rewards/margins": 16.251562118530273, |
| "rewards/rejected": -19.510936737060547, |
| "step": 4320 |
| }, |
| { |
| "epoch": 2.786610878661088, |
| "grad_norm": 0.03266239825168514, |
| "learning_rate": 3.031229877656149e-07, |
| "logits/chosen": -1.054443359375, |
| "logits/rejected": -1.0895507335662842, |
| "logps/chosen": -429.67498779296875, |
| "logps/rejected": -539.75, |
| "loss": 0.0094, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -4.355712890625, |
| "rewards/margins": 15.524999618530273, |
| "rewards/rejected": -19.889062881469727, |
| "step": 4330 |
| }, |
| { |
| "epoch": 2.7930479562278725, |
| "grad_norm": 3.742203170068705, |
| "learning_rate": 3.01513200257566e-07, |
| "logits/chosen": -1.047509789466858, |
| "logits/rejected": -1.061767578125, |
| "logps/chosen": -392.1499938964844, |
| "logps/rejected": -558.5750122070312, |
| "loss": 0.0141, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -3.4574217796325684, |
| "rewards/margins": 16.379688262939453, |
| "rewards/rejected": -19.839061737060547, |
| "step": 4340 |
| }, |
| { |
| "epoch": 2.799485033794657, |
| "grad_norm": 1.1471591775678358, |
| "learning_rate": 2.9990341274951703e-07, |
| "logits/chosen": -1.0753600597381592, |
| "logits/rejected": -1.1260833740234375, |
| "logps/chosen": -400.875, |
| "logps/rejected": -516.4000244140625, |
| "loss": 0.0219, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -3.001171827316284, |
| "rewards/margins": 15.435937881469727, |
| "rewards/rejected": -18.435937881469727, |
| "step": 4350 |
| }, |
| { |
| "epoch": 2.8059221113614417, |
| "grad_norm": 0.16781605556701026, |
| "learning_rate": 2.982936252414681e-07, |
| "logits/chosen": -1.0686767101287842, |
| "logits/rejected": -1.083740234375, |
| "logps/chosen": -399.0, |
| "logps/rejected": -552.8499755859375, |
| "loss": 0.0274, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -3.817089796066284, |
| "rewards/margins": 15.465624809265137, |
| "rewards/rejected": -19.284374237060547, |
| "step": 4360 |
| }, |
| { |
| "epoch": 2.8123591889282267, |
| "grad_norm": 1.9416319834531985, |
| "learning_rate": 2.966838377334192e-07, |
| "logits/chosen": -1.076025366783142, |
| "logits/rejected": -1.080078125, |
| "logps/chosen": -406.5249938964844, |
| "logps/rejected": -547.0499877929688, |
| "loss": 0.022, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -2.145703077316284, |
| "rewards/margins": 16.671875, |
| "rewards/rejected": -18.817188262939453, |
| "step": 4370 |
| }, |
| { |
| "epoch": 2.8187962664950112, |
| "grad_norm": 5.762013159949488, |
| "learning_rate": 2.9507405022537025e-07, |
| "logits/chosen": -1.105371117591858, |
| "logits/rejected": -1.121679663658142, |
| "logps/chosen": -432.67498779296875, |
| "logps/rejected": -557.6500244140625, |
| "loss": 0.0176, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -3.159130811691284, |
| "rewards/margins": 16.162500381469727, |
| "rewards/rejected": -19.323436737060547, |
| "step": 4380 |
| }, |
| { |
| "epoch": 2.825233344061796, |
| "grad_norm": 0.9620630047817021, |
| "learning_rate": 2.9346426271732127e-07, |
| "logits/chosen": -1.098535180091858, |
| "logits/rejected": -1.11181640625, |
| "logps/chosen": -383.42498779296875, |
| "logps/rejected": -552.375, |
| "loss": 0.014, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -3.7589354515075684, |
| "rewards/margins": 16.293750762939453, |
| "rewards/rejected": -20.051563262939453, |
| "step": 4390 |
| }, |
| { |
| "epoch": 2.831670421628581, |
| "grad_norm": 0.06912227797655064, |
| "learning_rate": 2.9185447520927234e-07, |
| "logits/chosen": -1.1637694835662842, |
| "logits/rejected": -1.1516602039337158, |
| "logps/chosen": -392.45001220703125, |
| "logps/rejected": -568.0999755859375, |
| "loss": 0.012, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -3.525561571121216, |
| "rewards/margins": 16.5859375, |
| "rewards/rejected": -20.115625381469727, |
| "step": 4400 |
| }, |
| { |
| "epoch": 2.8381074991953654, |
| "grad_norm": 0.23191462260970216, |
| "learning_rate": 2.902446877012234e-07, |
| "logits/chosen": -1.08929443359375, |
| "logits/rejected": -1.105126976966858, |
| "logps/chosen": -389.67498779296875, |
| "logps/rejected": -540.0499877929688, |
| "loss": 0.0138, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -2.943164110183716, |
| "rewards/margins": 15.912500381469727, |
| "rewards/rejected": -18.854686737060547, |
| "step": 4410 |
| }, |
| { |
| "epoch": 2.84454457676215, |
| "grad_norm": 0.22733204817053218, |
| "learning_rate": 2.886349001931745e-07, |
| "logits/chosen": -1.051367163658142, |
| "logits/rejected": -1.085546851158142, |
| "logps/chosen": -441.1000061035156, |
| "logps/rejected": -564.75, |
| "loss": 0.0176, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -2.992919921875, |
| "rewards/margins": 14.8984375, |
| "rewards/rejected": -17.885936737060547, |
| "step": 4420 |
| }, |
| { |
| "epoch": 2.8509816543289346, |
| "grad_norm": 0.35599187838621965, |
| "learning_rate": 2.870251126851255e-07, |
| "logits/chosen": -1.1135742664337158, |
| "logits/rejected": -1.129785180091858, |
| "logps/chosen": -402.0249938964844, |
| "logps/rejected": -500.67498779296875, |
| "loss": 0.0251, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -2.205517530441284, |
| "rewards/margins": 15.65625, |
| "rewards/rejected": -17.860937118530273, |
| "step": 4430 |
| }, |
| { |
| "epoch": 2.857418731895719, |
| "grad_norm": 0.5999684700633137, |
| "learning_rate": 2.8541532517707664e-07, |
| "logits/chosen": -1.177343726158142, |
| "logits/rejected": -1.2083008289337158, |
| "logps/chosen": -423.8999938964844, |
| "logps/rejected": -576.5999755859375, |
| "loss": 0.0089, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -2.318981885910034, |
| "rewards/margins": 16.435937881469727, |
| "rewards/rejected": -18.759374618530273, |
| "step": 4440 |
| }, |
| { |
| "epoch": 2.863855809462504, |
| "grad_norm": 0.41307829335787627, |
| "learning_rate": 2.8380553766902766e-07, |
| "logits/chosen": -1.1286132335662842, |
| "logits/rejected": -1.140625, |
| "logps/chosen": -418.79998779296875, |
| "logps/rejected": -560.75, |
| "loss": 0.0131, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -2.341552734375, |
| "rewards/margins": 16.698436737060547, |
| "rewards/rejected": -19.03125, |
| "step": 4450 |
| }, |
| { |
| "epoch": 2.8702928870292888, |
| "grad_norm": 0.06329392464426446, |
| "learning_rate": 2.8219575016097873e-07, |
| "logits/chosen": -1.0729491710662842, |
| "logits/rejected": -1.0994141101837158, |
| "logps/chosen": -405.7749938964844, |
| "logps/rejected": -532.4500122070312, |
| "loss": 0.0218, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -2.7913575172424316, |
| "rewards/margins": 15.798437118530273, |
| "rewards/rejected": -18.600000381469727, |
| "step": 4460 |
| }, |
| { |
| "epoch": 2.8767299645960733, |
| "grad_norm": 0.5929940057123108, |
| "learning_rate": 2.8058596265292975e-07, |
| "logits/chosen": -1.12939453125, |
| "logits/rejected": -1.158447265625, |
| "logps/chosen": -425.6000061035156, |
| "logps/rejected": -554.8499755859375, |
| "loss": 0.009, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -2.7857298851013184, |
| "rewards/margins": 16.956249237060547, |
| "rewards/rejected": -19.728124618530273, |
| "step": 4470 |
| }, |
| { |
| "epoch": 2.883167042162858, |
| "grad_norm": 10.781761622532624, |
| "learning_rate": 2.789761751448809e-07, |
| "logits/chosen": -1.1557128429412842, |
| "logits/rejected": -1.17626953125, |
| "logps/chosen": -387.67498779296875, |
| "logps/rejected": -530.3499755859375, |
| "loss": 0.0136, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -3.205273389816284, |
| "rewards/margins": 15.399999618530273, |
| "rewards/rejected": -18.606250762939453, |
| "step": 4480 |
| }, |
| { |
| "epoch": 2.8896041197296425, |
| "grad_norm": 5.678623060839604, |
| "learning_rate": 2.773663876368319e-07, |
| "logits/chosen": -1.122949242591858, |
| "logits/rejected": -1.1169922351837158, |
| "logps/chosen": -412.875, |
| "logps/rejected": -559.7000122070312, |
| "loss": 0.0155, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -4.047656059265137, |
| "rewards/margins": 15.893750190734863, |
| "rewards/rejected": -19.956249237060547, |
| "step": 4490 |
| }, |
| { |
| "epoch": 2.8960411972964275, |
| "grad_norm": 0.12843353367463994, |
| "learning_rate": 2.7575660012878297e-07, |
| "logits/chosen": -1.0998046398162842, |
| "logits/rejected": -1.0850098133087158, |
| "logps/chosen": -403.29998779296875, |
| "logps/rejected": -578.4000244140625, |
| "loss": 0.0046, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -4.0653076171875, |
| "rewards/margins": 15.801562309265137, |
| "rewards/rejected": -19.857812881469727, |
| "step": 4500 |
| }, |
| { |
| "epoch": 2.902478274863212, |
| "grad_norm": 0.18293130358900542, |
| "learning_rate": 2.7414681262073405e-07, |
| "logits/chosen": -1.146093726158142, |
| "logits/rejected": -1.1618163585662842, |
| "logps/chosen": -414.625, |
| "logps/rejected": -538.0999755859375, |
| "loss": 0.0088, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -3.832763671875, |
| "rewards/margins": 16.057811737060547, |
| "rewards/rejected": -19.892187118530273, |
| "step": 4510 |
| }, |
| { |
| "epoch": 2.9089153524299967, |
| "grad_norm": 15.929814025771526, |
| "learning_rate": 2.725370251126851e-07, |
| "logits/chosen": -1.20361328125, |
| "logits/rejected": -1.2047851085662842, |
| "logps/chosen": -441.79998779296875, |
| "logps/rejected": -596.75, |
| "loss": 0.0151, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -3.934033155441284, |
| "rewards/margins": 17.510936737060547, |
| "rewards/rejected": -21.459375381469727, |
| "step": 4520 |
| }, |
| { |
| "epoch": 2.9153524299967817, |
| "grad_norm": 26.74681492633243, |
| "learning_rate": 2.7092723760463614e-07, |
| "logits/chosen": -1.161865234375, |
| "logits/rejected": -1.198339819908142, |
| "logps/chosen": -409.6000061035156, |
| "logps/rejected": -548.0999755859375, |
| "loss": 0.0184, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -3.522265672683716, |
| "rewards/margins": 16.2265625, |
| "rewards/rejected": -19.748437881469727, |
| "step": 4530 |
| }, |
| { |
| "epoch": 2.9217895075635663, |
| "grad_norm": 0.18148041344029991, |
| "learning_rate": 2.6931745009658727e-07, |
| "logits/chosen": -1.149511694908142, |
| "logits/rejected": -1.160742163658142, |
| "logps/chosen": -404.625, |
| "logps/rejected": -598.25, |
| "loss": 0.0044, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -4.6748046875, |
| "rewards/margins": 17.774999618530273, |
| "rewards/rejected": -22.448436737060547, |
| "step": 4540 |
| }, |
| { |
| "epoch": 2.928226585130351, |
| "grad_norm": 0.09689001421228971, |
| "learning_rate": 2.677076625885383e-07, |
| "logits/chosen": -1.1152832508087158, |
| "logits/rejected": -1.1346313953399658, |
| "logps/chosen": -404.6875, |
| "logps/rejected": -555.875, |
| "loss": 0.026, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -4.319580078125, |
| "rewards/margins": 17.51171875, |
| "rewards/rejected": -21.8203125, |
| "step": 4550 |
| }, |
| { |
| "epoch": 2.9346636626971354, |
| "grad_norm": 0.8349717835904119, |
| "learning_rate": 2.6609787508048936e-07, |
| "logits/chosen": -1.2187011241912842, |
| "logits/rejected": -1.26025390625, |
| "logps/chosen": -419.57501220703125, |
| "logps/rejected": -553.0, |
| "loss": 0.0051, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -4.408154487609863, |
| "rewards/margins": 16.829687118530273, |
| "rewards/rejected": -21.245311737060547, |
| "step": 4560 |
| }, |
| { |
| "epoch": 2.94110074026392, |
| "grad_norm": 0.0018520052569766796, |
| "learning_rate": 2.644880875724404e-07, |
| "logits/chosen": -1.1864745616912842, |
| "logits/rejected": -1.1776854991912842, |
| "logps/chosen": -429.125, |
| "logps/rejected": -587.4500122070312, |
| "loss": 0.0065, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -5.62286376953125, |
| "rewards/margins": 17.575000762939453, |
| "rewards/rejected": -23.203125, |
| "step": 4570 |
| }, |
| { |
| "epoch": 2.947537817830705, |
| "grad_norm": 0.9827245912908983, |
| "learning_rate": 2.628783000643915e-07, |
| "logits/chosen": -1.075048804283142, |
| "logits/rejected": -1.100341796875, |
| "logps/chosen": -450.88751220703125, |
| "logps/rejected": -579.8499755859375, |
| "loss": 0.0218, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -5.349609375, |
| "rewards/margins": 17.246875762939453, |
| "rewards/rejected": -22.5859375, |
| "step": 4580 |
| }, |
| { |
| "epoch": 2.9539748953974896, |
| "grad_norm": 2.808445023918683, |
| "learning_rate": 2.6126851255634253e-07, |
| "logits/chosen": -1.074462890625, |
| "logits/rejected": -1.122167944908142, |
| "logps/chosen": -404.38751220703125, |
| "logps/rejected": -577.3499755859375, |
| "loss": 0.0465, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -5.451562404632568, |
| "rewards/margins": 16.517187118530273, |
| "rewards/rejected": -21.970312118530273, |
| "step": 4590 |
| }, |
| { |
| "epoch": 2.960411972964274, |
| "grad_norm": 0.31073133311477225, |
| "learning_rate": 2.596587250482936e-07, |
| "logits/chosen": -1.086157202720642, |
| "logits/rejected": -1.089868187904358, |
| "logps/chosen": -421.4624938964844, |
| "logps/rejected": -573.3499755859375, |
| "loss": 0.0195, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -5.573144435882568, |
| "rewards/margins": 17.239843368530273, |
| "rewards/rejected": -22.795312881469727, |
| "step": 4600 |
| }, |
| { |
| "epoch": 2.9668490505310587, |
| "grad_norm": 0.068889903289002, |
| "learning_rate": 2.580489375402447e-07, |
| "logits/chosen": -1.0750000476837158, |
| "logits/rejected": -1.1067383289337158, |
| "logps/chosen": -408.54998779296875, |
| "logps/rejected": -580.625, |
| "loss": 0.0148, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -4.872363090515137, |
| "rewards/margins": 17.399999618530273, |
| "rewards/rejected": -22.2890625, |
| "step": 4610 |
| }, |
| { |
| "epoch": 2.9732861280978433, |
| "grad_norm": 0.6319961977568703, |
| "learning_rate": 2.5643915003219575e-07, |
| "logits/chosen": -1.137109398841858, |
| "logits/rejected": -1.150292992591858, |
| "logps/chosen": -435.17498779296875, |
| "logps/rejected": -535.0, |
| "loss": 0.0127, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -4.769067287445068, |
| "rewards/margins": 15.90625, |
| "rewards/rejected": -20.671875, |
| "step": 4620 |
| }, |
| { |
| "epoch": 2.9797232056646283, |
| "grad_norm": 0.008954633914854768, |
| "learning_rate": 2.5482936252414677e-07, |
| "logits/chosen": -1.117285132408142, |
| "logits/rejected": -1.121679663658142, |
| "logps/chosen": -465.875, |
| "logps/rejected": -593.4000244140625, |
| "loss": 0.0262, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -4.642480373382568, |
| "rewards/margins": 16.438282012939453, |
| "rewards/rejected": -21.082813262939453, |
| "step": 4630 |
| }, |
| { |
| "epoch": 2.986160283231413, |
| "grad_norm": 0.2715041175189798, |
| "learning_rate": 2.5321957501609784e-07, |
| "logits/chosen": -1.084082007408142, |
| "logits/rejected": -1.119775414466858, |
| "logps/chosen": -480.92498779296875, |
| "logps/rejected": -611.3499755859375, |
| "loss": 0.0071, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -4.862011909484863, |
| "rewards/margins": 17.3046875, |
| "rewards/rejected": -22.178125381469727, |
| "step": 4640 |
| }, |
| { |
| "epoch": 2.9925973607981975, |
| "grad_norm": 0.07398779198819364, |
| "learning_rate": 2.516097875080489e-07, |
| "logits/chosen": -1.11474609375, |
| "logits/rejected": -1.156152367591858, |
| "logps/chosen": -449.57501220703125, |
| "logps/rejected": -600.5999755859375, |
| "loss": 0.0117, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -4.337744235992432, |
| "rewards/margins": 17.264062881469727, |
| "rewards/rejected": -21.604686737060547, |
| "step": 4650 |
| }, |
| { |
| "epoch": 2.9990344383649825, |
| "grad_norm": 0.6036221444805209, |
| "learning_rate": 2.5e-07, |
| "logits/chosen": -1.118505835533142, |
| "logits/rejected": -1.121191382408142, |
| "logps/chosen": -417.1000061035156, |
| "logps/rejected": -584.4000244140625, |
| "loss": 0.0179, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -3.984081983566284, |
| "rewards/margins": 17.8046875, |
| "rewards/rejected": -21.784374237060547, |
| "step": 4660 |
| }, |
| { |
| "epoch": 3.005149662053428, |
| "grad_norm": 0.02049975193867326, |
| "learning_rate": 2.4839021249195106e-07, |
| "logits/chosen": -1.1468955278396606, |
| "logits/rejected": -1.1493626832962036, |
| "logps/chosen": -438.1052551269531, |
| "logps/rejected": -590.6842041015625, |
| "loss": 0.0043, |
| "rewards/accuracies": 0.9934210777282715, |
| "rewards/chosen": -5.279964923858643, |
| "rewards/margins": 18.324012756347656, |
| "rewards/rejected": -23.601974487304688, |
| "step": 4670 |
| }, |
| { |
| "epoch": 3.0115867396202125, |
| "grad_norm": 0.007171380019719538, |
| "learning_rate": 2.4678042498390214e-07, |
| "logits/chosen": -1.125341773033142, |
| "logits/rejected": -1.152099609375, |
| "logps/chosen": -459.7749938964844, |
| "logps/rejected": -600.5499877929688, |
| "loss": 0.0015, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -5.583300590515137, |
| "rewards/margins": 17.621875762939453, |
| "rewards/rejected": -23.206249237060547, |
| "step": 4680 |
| }, |
| { |
| "epoch": 3.018023817186997, |
| "grad_norm": 0.015194056176862642, |
| "learning_rate": 2.4517063747585316e-07, |
| "logits/chosen": -1.083105444908142, |
| "logits/rejected": -1.109155297279358, |
| "logps/chosen": -415.1000061035156, |
| "logps/rejected": -594.9500122070312, |
| "loss": 0.0173, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -4.552294731140137, |
| "rewards/margins": 19.375, |
| "rewards/rejected": -23.924999237060547, |
| "step": 4690 |
| }, |
| { |
| "epoch": 3.0244608947537817, |
| "grad_norm": 4.468715166239343, |
| "learning_rate": 2.4356084996780423e-07, |
| "logits/chosen": -1.2006347179412842, |
| "logits/rejected": -1.192480444908142, |
| "logps/chosen": -440.1000061035156, |
| "logps/rejected": -595.4500122070312, |
| "loss": 0.0173, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -5.480175971984863, |
| "rewards/margins": 18.094532012939453, |
| "rewards/rejected": -23.568750381469727, |
| "step": 4700 |
| }, |
| { |
| "epoch": 3.0308979723205667, |
| "grad_norm": 0.021122244530159647, |
| "learning_rate": 2.419510624597553e-07, |
| "logits/chosen": -1.069067358970642, |
| "logits/rejected": -1.050115942955017, |
| "logps/chosen": -445.5, |
| "logps/rejected": -602.0499877929688, |
| "loss": 0.0092, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -5.22955322265625, |
| "rewards/margins": 17.5, |
| "rewards/rejected": -22.737499237060547, |
| "step": 4710 |
| }, |
| { |
| "epoch": 3.0373350498873513, |
| "grad_norm": 0.03307507714663866, |
| "learning_rate": 2.403412749517064e-07, |
| "logits/chosen": -1.173242211341858, |
| "logits/rejected": -1.1818358898162842, |
| "logps/chosen": -433.1000061035156, |
| "logps/rejected": -591.5999755859375, |
| "loss": 0.0044, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -4.977221488952637, |
| "rewards/margins": 18.759374618530273, |
| "rewards/rejected": -23.740625381469727, |
| "step": 4720 |
| }, |
| { |
| "epoch": 3.043772127454136, |
| "grad_norm": 0.02934355373683881, |
| "learning_rate": 2.3873148744365745e-07, |
| "logits/chosen": -1.133935570716858, |
| "logits/rejected": -1.1240723133087158, |
| "logps/chosen": -412.6499938964844, |
| "logps/rejected": -562.75, |
| "loss": 0.013, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -5.184765815734863, |
| "rewards/margins": 18.248437881469727, |
| "rewards/rejected": -23.424999237060547, |
| "step": 4730 |
| }, |
| { |
| "epoch": 3.0502092050209204, |
| "grad_norm": 0.019452870159428832, |
| "learning_rate": 2.371216999356085e-07, |
| "logits/chosen": -1.0964844226837158, |
| "logits/rejected": -1.0937011241912842, |
| "logps/chosen": -428.2749938964844, |
| "logps/rejected": -587.7999877929688, |
| "loss": 0.026, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -6.134753227233887, |
| "rewards/margins": 17.532032012939453, |
| "rewards/rejected": -23.657812118530273, |
| "step": 4740 |
| }, |
| { |
| "epoch": 3.056646282587705, |
| "grad_norm": 0.01026781399221231, |
| "learning_rate": 2.3551191242755954e-07, |
| "logits/chosen": -1.1647460460662842, |
| "logits/rejected": -1.160058617591858, |
| "logps/chosen": -442.70001220703125, |
| "logps/rejected": -598.5499877929688, |
| "loss": 0.0043, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -5.059496879577637, |
| "rewards/margins": 20.454687118530273, |
| "rewards/rejected": -25.529687881469727, |
| "step": 4750 |
| }, |
| { |
| "epoch": 3.06308336015449, |
| "grad_norm": 0.28593764965232715, |
| "learning_rate": 2.3390212491951062e-07, |
| "logits/chosen": -1.155908226966858, |
| "logits/rejected": -1.1824219226837158, |
| "logps/chosen": -442.8500061035156, |
| "logps/rejected": -572.7000122070312, |
| "loss": 0.0224, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -5.127831935882568, |
| "rewards/margins": 17.984375, |
| "rewards/rejected": -23.118749618530273, |
| "step": 4760 |
| }, |
| { |
| "epoch": 3.0695204377212746, |
| "grad_norm": 0.003290033796275054, |
| "learning_rate": 2.3229233741146166e-07, |
| "logits/chosen": -1.203222632408142, |
| "logits/rejected": -1.217187523841858, |
| "logps/chosen": -454.8500061035156, |
| "logps/rejected": -630.7000122070312, |
| "loss": 0.0021, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -5.734375, |
| "rewards/margins": 19.578125, |
| "rewards/rejected": -25.309375762939453, |
| "step": 4770 |
| }, |
| { |
| "epoch": 3.075957515288059, |
| "grad_norm": 0.0017905173138558442, |
| "learning_rate": 2.3068254990341274e-07, |
| "logits/chosen": -1.1566894054412842, |
| "logits/rejected": -1.195703148841858, |
| "logps/chosen": -481.125, |
| "logps/rejected": -612.9000244140625, |
| "loss": 0.0043, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -7.490429878234863, |
| "rewards/margins": 18.299999237060547, |
| "rewards/rejected": -25.787500381469727, |
| "step": 4780 |
| }, |
| { |
| "epoch": 3.0823945928548437, |
| "grad_norm": 1.331154357210038, |
| "learning_rate": 2.290727623953638e-07, |
| "logits/chosen": -1.2042968273162842, |
| "logits/rejected": -1.212011694908142, |
| "logps/chosen": -475.0249938964844, |
| "logps/rejected": -620.4500122070312, |
| "loss": 0.0133, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -7.7841796875, |
| "rewards/margins": 19.067188262939453, |
| "rewards/rejected": -26.856250762939453, |
| "step": 4790 |
| }, |
| { |
| "epoch": 3.0888316704216288, |
| "grad_norm": 0.008913213065396297, |
| "learning_rate": 2.2746297488731486e-07, |
| "logits/chosen": -1.195410132408142, |
| "logits/rejected": -1.2082030773162842, |
| "logps/chosen": -421.79998779296875, |
| "logps/rejected": -622.4000244140625, |
| "loss": 0.0043, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -6.528954982757568, |
| "rewards/margins": 19.818750381469727, |
| "rewards/rejected": -26.353124618530273, |
| "step": 4800 |
| }, |
| { |
| "epoch": 3.0952687479884133, |
| "grad_norm": 0.1029343972347026, |
| "learning_rate": 2.2585318737926593e-07, |
| "logits/chosen": -1.1962890625, |
| "logits/rejected": -1.1948730945587158, |
| "logps/chosen": -438.8999938964844, |
| "logps/rejected": -587.0499877929688, |
| "loss": 0.0029, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -7.034863471984863, |
| "rewards/margins": 19.465625762939453, |
| "rewards/rejected": -26.515625, |
| "step": 4810 |
| }, |
| { |
| "epoch": 3.101705825555198, |
| "grad_norm": 0.00040497685294049686, |
| "learning_rate": 2.2424339987121698e-07, |
| "logits/chosen": -1.1691405773162842, |
| "logits/rejected": -1.1959960460662842, |
| "logps/chosen": -445.5249938964844, |
| "logps/rejected": -604.0499877929688, |
| "loss": 0.0, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -7.252734184265137, |
| "rewards/margins": 19.690624237060547, |
| "rewards/rejected": -26.931249618530273, |
| "step": 4820 |
| }, |
| { |
| "epoch": 3.1081429031219825, |
| "grad_norm": 0.15227806520684364, |
| "learning_rate": 2.2263361236316805e-07, |
| "logits/chosen": -1.1714355945587158, |
| "logits/rejected": -1.162451148033142, |
| "logps/chosen": -425.70001220703125, |
| "logps/rejected": -615.7000122070312, |
| "loss": 0.0054, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -7.599999904632568, |
| "rewards/margins": 19.184375762939453, |
| "rewards/rejected": -26.774999618530273, |
| "step": 4830 |
| }, |
| { |
| "epoch": 3.1145799806887675, |
| "grad_norm": 0.023311318939842125, |
| "learning_rate": 2.210238248551191e-07, |
| "logits/chosen": -1.174707055091858, |
| "logits/rejected": -1.1965820789337158, |
| "logps/chosen": -453.04998779296875, |
| "logps/rejected": -608.7999877929688, |
| "loss": 0.0132, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -7.903124809265137, |
| "rewards/margins": 18.704687118530273, |
| "rewards/rejected": -26.618749618530273, |
| "step": 4840 |
| }, |
| { |
| "epoch": 3.121017058255552, |
| "grad_norm": 10.786296882917435, |
| "learning_rate": 2.1941403734707017e-07, |
| "logits/chosen": -1.1326172351837158, |
| "logits/rejected": -1.1515624523162842, |
| "logps/chosen": -422.5249938964844, |
| "logps/rejected": -609.1500244140625, |
| "loss": 0.0135, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -7.811327934265137, |
| "rewards/margins": 18.329687118530273, |
| "rewards/rejected": -26.146875381469727, |
| "step": 4850 |
| }, |
| { |
| "epoch": 3.1274541358223367, |
| "grad_norm": 0.01329847930022698, |
| "learning_rate": 2.1780424983902125e-07, |
| "logits/chosen": -1.146520972251892, |
| "logits/rejected": -1.1954345703125, |
| "logps/chosen": -444.375, |
| "logps/rejected": -611.2999877929688, |
| "loss": 0.013, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -6.67236328125, |
| "rewards/margins": 18.916406631469727, |
| "rewards/rejected": -25.596874237060547, |
| "step": 4860 |
| }, |
| { |
| "epoch": 3.1338912133891212, |
| "grad_norm": 0.04550344760940479, |
| "learning_rate": 2.161944623309723e-07, |
| "logits/chosen": -1.194433569908142, |
| "logits/rejected": -1.233496069908142, |
| "logps/chosen": -418.92498779296875, |
| "logps/rejected": -581.0, |
| "loss": 0.0054, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -7.391015529632568, |
| "rewards/margins": 18.495311737060547, |
| "rewards/rejected": -25.868749618530273, |
| "step": 4870 |
| }, |
| { |
| "epoch": 3.140328290955906, |
| "grad_norm": 0.11635747842049714, |
| "learning_rate": 2.1458467482292337e-07, |
| "logits/chosen": -1.123266577720642, |
| "logits/rejected": -1.149194359779358, |
| "logps/chosen": -495.25, |
| "logps/rejected": -631.2000122070312, |
| "loss": 0.0056, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -6.9462890625, |
| "rewards/margins": 19.1015625, |
| "rewards/rejected": -26.037500381469727, |
| "step": 4880 |
| }, |
| { |
| "epoch": 3.146765368522691, |
| "grad_norm": 0.0015098347770240576, |
| "learning_rate": 2.129748873148744e-07, |
| "logits/chosen": -1.099462866783142, |
| "logits/rejected": -1.12841796875, |
| "logps/chosen": -447.04998779296875, |
| "logps/rejected": -605.0499877929688, |
| "loss": 0.0045, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -6.196093559265137, |
| "rewards/margins": 19.821874618530273, |
| "rewards/rejected": -26.018749237060547, |
| "step": 4890 |
| }, |
| { |
| "epoch": 3.1532024460894754, |
| "grad_norm": 0.06132780820289997, |
| "learning_rate": 2.1136509980682549e-07, |
| "logits/chosen": -1.181860327720642, |
| "logits/rejected": -1.214599609375, |
| "logps/chosen": -458.0249938964844, |
| "logps/rejected": -618.4000244140625, |
| "loss": 0.0045, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -6.970703125, |
| "rewards/margins": 18.651561737060547, |
| "rewards/rejected": -25.634374618530273, |
| "step": 4900 |
| }, |
| { |
| "epoch": 3.15963952365626, |
| "grad_norm": 0.015670466658027772, |
| "learning_rate": 2.0975531229877656e-07, |
| "logits/chosen": -1.186669945716858, |
| "logits/rejected": -1.179956078529358, |
| "logps/chosen": -451.1499938964844, |
| "logps/rejected": -623.6500244140625, |
| "loss": 0.0173, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -6.311132907867432, |
| "rewards/margins": 18.420312881469727, |
| "rewards/rejected": -24.734375, |
| "step": 4910 |
| }, |
| { |
| "epoch": 3.1660766012230446, |
| "grad_norm": 0.7860096166411907, |
| "learning_rate": 2.081455247907276e-07, |
| "logits/chosen": -1.139672875404358, |
| "logits/rejected": -1.1612060070037842, |
| "logps/chosen": -434.07501220703125, |
| "logps/rejected": -601.9000244140625, |
| "loss": 0.0087, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -5.775292873382568, |
| "rewards/margins": 18.990625381469727, |
| "rewards/rejected": -24.756250381469727, |
| "step": 4920 |
| }, |
| { |
| "epoch": 3.1725136787898296, |
| "grad_norm": 0.009808484495203735, |
| "learning_rate": 2.0653573728267868e-07, |
| "logits/chosen": -1.0760498046875, |
| "logits/rejected": -1.0888671875, |
| "logps/chosen": -405.92498779296875, |
| "logps/rejected": -592.4500122070312, |
| "loss": 0.0256, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -7.089453220367432, |
| "rewards/margins": 18.778125762939453, |
| "rewards/rejected": -25.860937118530273, |
| "step": 4930 |
| }, |
| { |
| "epoch": 3.178950756356614, |
| "grad_norm": 0.0016510487300226008, |
| "learning_rate": 2.0492594977462973e-07, |
| "logits/chosen": -1.156835913658142, |
| "logits/rejected": -1.186132788658142, |
| "logps/chosen": -437.20001220703125, |
| "logps/rejected": -580.5, |
| "loss": 0.0087, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -5.806250095367432, |
| "rewards/margins": 18.1875, |
| "rewards/rejected": -24.0234375, |
| "step": 4940 |
| }, |
| { |
| "epoch": 3.1853878339233987, |
| "grad_norm": 0.04680967562897145, |
| "learning_rate": 2.033161622665808e-07, |
| "logits/chosen": -1.114355444908142, |
| "logits/rejected": -1.157324194908142, |
| "logps/chosen": -447.20001220703125, |
| "logps/rejected": -595.4000244140625, |
| "loss": 0.0044, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -5.834228515625, |
| "rewards/margins": 16.8828125, |
| "rewards/rejected": -22.717187881469727, |
| "step": 4950 |
| }, |
| { |
| "epoch": 3.1918249114901833, |
| "grad_norm": 0.03766884792215408, |
| "learning_rate": 2.0170637475853185e-07, |
| "logits/chosen": -1.107934594154358, |
| "logits/rejected": -1.1571044921875, |
| "logps/chosen": -463.6000061035156, |
| "logps/rejected": -588.5999755859375, |
| "loss": 0.0087, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -5.200463771820068, |
| "rewards/margins": 18.962499618530273, |
| "rewards/rejected": -24.15625, |
| "step": 4960 |
| }, |
| { |
| "epoch": 3.1982619890569683, |
| "grad_norm": 0.032789307952724236, |
| "learning_rate": 2.0009658725048292e-07, |
| "logits/chosen": -1.143945336341858, |
| "logits/rejected": -1.173437476158142, |
| "logps/chosen": -460.0, |
| "logps/rejected": -598.0499877929688, |
| "loss": 0.013, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -5.598474025726318, |
| "rewards/margins": 18.3203125, |
| "rewards/rejected": -23.903125762939453, |
| "step": 4970 |
| }, |
| { |
| "epoch": 3.204699066623753, |
| "grad_norm": 0.014658227990306128, |
| "learning_rate": 1.98486799742434e-07, |
| "logits/chosen": -1.2394530773162842, |
| "logits/rejected": -1.252832055091858, |
| "logps/chosen": -451.29998779296875, |
| "logps/rejected": -618.7999877929688, |
| "loss": 0.0088, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -6.079833984375, |
| "rewards/margins": 19.456249237060547, |
| "rewards/rejected": -25.534374237060547, |
| "step": 4980 |
| }, |
| { |
| "epoch": 3.2111361441905375, |
| "grad_norm": 0.046097116270495335, |
| "learning_rate": 1.9687701223438504e-07, |
| "logits/chosen": -1.2252929210662842, |
| "logits/rejected": -1.2561523914337158, |
| "logps/chosen": -442.57501220703125, |
| "logps/rejected": -594.75, |
| "loss": 0.013, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -6.322241306304932, |
| "rewards/margins": 19.120311737060547, |
| "rewards/rejected": -25.435937881469727, |
| "step": 4990 |
| }, |
| { |
| "epoch": 3.217573221757322, |
| "grad_norm": 0.09069092869768736, |
| "learning_rate": 1.9526722472633612e-07, |
| "logits/chosen": -1.132568359375, |
| "logits/rejected": -1.156494140625, |
| "logps/chosen": -425.82501220703125, |
| "logps/rejected": -578.7000122070312, |
| "loss": 0.0174, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -6.106738090515137, |
| "rewards/margins": 18.545312881469727, |
| "rewards/rejected": -24.645313262939453, |
| "step": 5000 |
| }, |
| { |
| "epoch": 3.2240102993241067, |
| "grad_norm": 0.01590234525948529, |
| "learning_rate": 1.9365743721828716e-07, |
| "logits/chosen": -1.1720702648162842, |
| "logits/rejected": -1.216406226158142, |
| "logps/chosen": -410.375, |
| "logps/rejected": -574.5999755859375, |
| "loss": 0.0216, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -6.863867282867432, |
| "rewards/margins": 18.245311737060547, |
| "rewards/rejected": -25.118749618530273, |
| "step": 5010 |
| }, |
| { |
| "epoch": 3.2304473768908917, |
| "grad_norm": 0.005519486507844155, |
| "learning_rate": 1.9204764971023824e-07, |
| "logits/chosen": -1.161962866783142, |
| "logits/rejected": -1.197119116783142, |
| "logps/chosen": -420.7250061035156, |
| "logps/rejected": -583.4000244140625, |
| "loss": 0.0044, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -6.3515625, |
| "rewards/margins": 19.045312881469727, |
| "rewards/rejected": -25.3828125, |
| "step": 5020 |
| }, |
| { |
| "epoch": 3.2368844544576763, |
| "grad_norm": 0.004607457293818073, |
| "learning_rate": 1.904378622021893e-07, |
| "logits/chosen": -1.165283203125, |
| "logits/rejected": -1.167138695716858, |
| "logps/chosen": -445.92498779296875, |
| "logps/rejected": -614.1500244140625, |
| "loss": 0.0173, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -6.177538871765137, |
| "rewards/margins": 19.678125381469727, |
| "rewards/rejected": -25.84375, |
| "step": 5030 |
| }, |
| { |
| "epoch": 3.243321532024461, |
| "grad_norm": 0.7275701284809951, |
| "learning_rate": 1.8882807469414036e-07, |
| "logits/chosen": -1.1663086414337158, |
| "logits/rejected": -1.1945312023162842, |
| "logps/chosen": -407.82501220703125, |
| "logps/rejected": -566.5499877929688, |
| "loss": 0.0113, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -6.144921779632568, |
| "rewards/margins": 18.651561737060547, |
| "rewards/rejected": -24.7734375, |
| "step": 5040 |
| }, |
| { |
| "epoch": 3.2497586095912454, |
| "grad_norm": 0.00925264942928273, |
| "learning_rate": 1.8721828718609143e-07, |
| "logits/chosen": -1.0818359851837158, |
| "logits/rejected": -1.1124999523162842, |
| "logps/chosen": -422.625, |
| "logps/rejected": -599.2000122070312, |
| "loss": 0.0216, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -5.3525390625, |
| "rewards/margins": 18.346874237060547, |
| "rewards/rejected": -23.692188262939453, |
| "step": 5050 |
| }, |
| { |
| "epoch": 3.2561956871580304, |
| "grad_norm": 0.018748740799270636, |
| "learning_rate": 1.8560849967804248e-07, |
| "logits/chosen": -1.139746069908142, |
| "logits/rejected": -1.134374976158142, |
| "logps/chosen": -432.95001220703125, |
| "logps/rejected": -592.5, |
| "loss": 0.0263, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -6.676465034484863, |
| "rewards/margins": 17.557811737060547, |
| "rewards/rejected": -24.228124618530273, |
| "step": 5060 |
| }, |
| { |
| "epoch": 3.262632764724815, |
| "grad_norm": 0.009384866899820094, |
| "learning_rate": 1.8399871216999355e-07, |
| "logits/chosen": -1.19677734375, |
| "logits/rejected": -1.204199194908142, |
| "logps/chosen": -433.67498779296875, |
| "logps/rejected": -604.8499755859375, |
| "loss": 0.0173, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -5.350488185882568, |
| "rewards/margins": 18.135936737060547, |
| "rewards/rejected": -23.475000381469727, |
| "step": 5070 |
| }, |
| { |
| "epoch": 3.2690698422915996, |
| "grad_norm": 0.02147237650112711, |
| "learning_rate": 1.823889246619446e-07, |
| "logits/chosen": -1.2500488758087158, |
| "logits/rejected": -1.2795898914337158, |
| "logps/chosen": -481.875, |
| "logps/rejected": -612.6500244140625, |
| "loss": 0.0044, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -7.387597560882568, |
| "rewards/margins": 17.568750381469727, |
| "rewards/rejected": -24.957813262939453, |
| "step": 5080 |
| }, |
| { |
| "epoch": 3.275506919858384, |
| "grad_norm": 0.02187166539536812, |
| "learning_rate": 1.8077913715389567e-07, |
| "logits/chosen": -1.118310570716858, |
| "logits/rejected": -1.1315186023712158, |
| "logps/chosen": -412.5, |
| "logps/rejected": -595.2999877929688, |
| "loss": 0.0303, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -5.707812309265137, |
| "rewards/margins": 18.108592987060547, |
| "rewards/rejected": -23.81640625, |
| "step": 5090 |
| }, |
| { |
| "epoch": 3.281943997425169, |
| "grad_norm": 0.13063587935498464, |
| "learning_rate": 1.7916934964584674e-07, |
| "logits/chosen": -1.181549072265625, |
| "logits/rejected": -1.2327148914337158, |
| "logps/chosen": -423.9750061035156, |
| "logps/rejected": -615.6500244140625, |
| "loss": 0.013, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -7.080273628234863, |
| "rewards/margins": 18.559375762939453, |
| "rewards/rejected": -25.653125762939453, |
| "step": 5100 |
| }, |
| { |
| "epoch": 3.2883810749919538, |
| "grad_norm": 0.005783772596752793, |
| "learning_rate": 1.775595621377978e-07, |
| "logits/chosen": -1.120336890220642, |
| "logits/rejected": -1.1363036632537842, |
| "logps/chosen": -441.20001220703125, |
| "logps/rejected": -580.0499877929688, |
| "loss": 0.0087, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -5.895410060882568, |
| "rewards/margins": 18.892187118530273, |
| "rewards/rejected": -24.774999618530273, |
| "step": 5110 |
| }, |
| { |
| "epoch": 3.2948181525587383, |
| "grad_norm": 0.009963166845550958, |
| "learning_rate": 1.7594977462974886e-07, |
| "logits/chosen": -1.1458008289337158, |
| "logits/rejected": -1.1650879383087158, |
| "logps/chosen": -396.3500061035156, |
| "logps/rejected": -568.3499755859375, |
| "loss": 0.013, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -6.274755954742432, |
| "rewards/margins": 18.643749237060547, |
| "rewards/rejected": -24.907812118530273, |
| "step": 5120 |
| }, |
| { |
| "epoch": 3.301255230125523, |
| "grad_norm": 0.06936182148963972, |
| "learning_rate": 1.743399871216999e-07, |
| "logits/chosen": -1.218896508216858, |
| "logits/rejected": -1.230566382408142, |
| "logps/chosen": -452.2749938964844, |
| "logps/rejected": -606.5999755859375, |
| "loss": 0.0101, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -5.982812404632568, |
| "rewards/margins": 17.821874618530273, |
| "rewards/rejected": -23.8203125, |
| "step": 5130 |
| }, |
| { |
| "epoch": 3.3076923076923075, |
| "grad_norm": 0.004261491928143392, |
| "learning_rate": 1.7273019961365098e-07, |
| "logits/chosen": -1.1884276866912842, |
| "logits/rejected": -1.212499976158142, |
| "logps/chosen": -437.8999938964844, |
| "logps/rejected": -612.7999877929688, |
| "loss": 0.0043, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -5.523461818695068, |
| "rewards/margins": 20.407812118530273, |
| "rewards/rejected": -25.935937881469727, |
| "step": 5140 |
| }, |
| { |
| "epoch": 3.3141293852590925, |
| "grad_norm": 0.013833502764908275, |
| "learning_rate": 1.7112041210560206e-07, |
| "logits/chosen": -1.161230444908142, |
| "logits/rejected": -1.194726586341858, |
| "logps/chosen": -460.1000061035156, |
| "logps/rejected": -585.75, |
| "loss": 0.013, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -6.311327934265137, |
| "rewards/margins": 17.731250762939453, |
| "rewards/rejected": -24.037500381469727, |
| "step": 5150 |
| }, |
| { |
| "epoch": 3.320566462825877, |
| "grad_norm": 0.02845853102131737, |
| "learning_rate": 1.695106245975531e-07, |
| "logits/chosen": -1.135888695716858, |
| "logits/rejected": -1.14990234375, |
| "logps/chosen": -443.6000061035156, |
| "logps/rejected": -636.25, |
| "loss": 0.0043, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -5.395410060882568, |
| "rewards/margins": 19.323436737060547, |
| "rewards/rejected": -24.731250762939453, |
| "step": 5160 |
| }, |
| { |
| "epoch": 3.3270035403926617, |
| "grad_norm": 0.0038233653605419843, |
| "learning_rate": 1.6790083708950418e-07, |
| "logits/chosen": -1.1759765148162842, |
| "logits/rejected": -1.20068359375, |
| "logps/chosen": -446.75, |
| "logps/rejected": -607.7000122070312, |
| "loss": 0.013, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -6.467577934265137, |
| "rewards/margins": 19.192188262939453, |
| "rewards/rejected": -25.649999618530273, |
| "step": 5170 |
| }, |
| { |
| "epoch": 3.3334406179594462, |
| "grad_norm": 1.268747881939313, |
| "learning_rate": 1.6629104958145523e-07, |
| "logits/chosen": -1.114404320716858, |
| "logits/rejected": -1.1251952648162842, |
| "logps/chosen": -414.70001220703125, |
| "logps/rejected": -584.0499877929688, |
| "loss": 0.0091, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -6.07421875, |
| "rewards/margins": 19.090625762939453, |
| "rewards/rejected": -25.174999237060547, |
| "step": 5180 |
| }, |
| { |
| "epoch": 3.3398776955262313, |
| "grad_norm": 0.11625811094142115, |
| "learning_rate": 1.646812620734063e-07, |
| "logits/chosen": -1.169042944908142, |
| "logits/rejected": -1.19970703125, |
| "logps/chosen": -440.625, |
| "logps/rejected": -621.0499877929688, |
| "loss": 0.013, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -6.413671970367432, |
| "rewards/margins": 18.1328125, |
| "rewards/rejected": -24.542186737060547, |
| "step": 5190 |
| }, |
| { |
| "epoch": 3.346314773093016, |
| "grad_norm": 0.01457558814733387, |
| "learning_rate": 1.6307147456535735e-07, |
| "logits/chosen": -1.1357421875, |
| "logits/rejected": -1.1613280773162842, |
| "logps/chosen": -445.125, |
| "logps/rejected": -589.5, |
| "loss": 0.0046, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -6.311450004577637, |
| "rewards/margins": 18.700000762939453, |
| "rewards/rejected": -25.009374618530273, |
| "step": 5200 |
| }, |
| { |
| "epoch": 3.3527518506598004, |
| "grad_norm": 0.0318603012676317, |
| "learning_rate": 1.6146168705730842e-07, |
| "logits/chosen": -1.173583984375, |
| "logits/rejected": -1.1763184070587158, |
| "logps/chosen": -449.125, |
| "logps/rejected": -632.0499877929688, |
| "loss": 0.0201, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -6.589745998382568, |
| "rewards/margins": 18.8046875, |
| "rewards/rejected": -25.396875381469727, |
| "step": 5210 |
| }, |
| { |
| "epoch": 3.359188928226585, |
| "grad_norm": 0.037256736610292676, |
| "learning_rate": 1.598518995492595e-07, |
| "logits/chosen": -1.100341796875, |
| "logits/rejected": -1.1125977039337158, |
| "logps/chosen": -424.2250061035156, |
| "logps/rejected": -582.9000244140625, |
| "loss": 0.026, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -6.180517673492432, |
| "rewards/margins": 18.310937881469727, |
| "rewards/rejected": -24.490625381469727, |
| "step": 5220 |
| }, |
| { |
| "epoch": 3.36562600579337, |
| "grad_norm": 0.10211936783870917, |
| "learning_rate": 1.5824211204121054e-07, |
| "logits/chosen": -1.0896484851837158, |
| "logits/rejected": -1.1443359851837158, |
| "logps/chosen": -449.375, |
| "logps/rejected": -602.0499877929688, |
| "loss": 0.0177, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -6.273779392242432, |
| "rewards/margins": 17.615625381469727, |
| "rewards/rejected": -23.881250381469727, |
| "step": 5230 |
| }, |
| { |
| "epoch": 3.3720630833601546, |
| "grad_norm": 0.01454528726990812, |
| "learning_rate": 1.566323245331616e-07, |
| "logits/chosen": -1.1654052734375, |
| "logits/rejected": -1.2077147960662842, |
| "logps/chosen": -443.2250061035156, |
| "logps/rejected": -612.1500244140625, |
| "loss": 0.0173, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -6.409081935882568, |
| "rewards/margins": 19.645313262939453, |
| "rewards/rejected": -26.056249618530273, |
| "step": 5240 |
| }, |
| { |
| "epoch": 3.378500160926939, |
| "grad_norm": 0.9637410547971565, |
| "learning_rate": 1.5502253702511266e-07, |
| "logits/chosen": -1.1218750476837158, |
| "logits/rejected": -1.118749976158142, |
| "logps/chosen": -419.0874938964844, |
| "logps/rejected": -581.5999755859375, |
| "loss": 0.0174, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -6.750048637390137, |
| "rewards/margins": 17.667186737060547, |
| "rewards/rejected": -24.440624237060547, |
| "step": 5250 |
| }, |
| { |
| "epoch": 3.3849372384937237, |
| "grad_norm": 0.03149535057004577, |
| "learning_rate": 1.5341274951706373e-07, |
| "logits/chosen": -1.111108422279358, |
| "logits/rejected": -1.144677758216858, |
| "logps/chosen": -461.375, |
| "logps/rejected": -606.0999755859375, |
| "loss": 0.0216, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -6.674023628234863, |
| "rewards/margins": 17.876562118530273, |
| "rewards/rejected": -24.568750381469727, |
| "step": 5260 |
| }, |
| { |
| "epoch": 3.3913743160605083, |
| "grad_norm": 0.010542321081653615, |
| "learning_rate": 1.518029620090148e-07, |
| "logits/chosen": -1.197851538658142, |
| "logits/rejected": -1.2239258289337158, |
| "logps/chosen": -411.45001220703125, |
| "logps/rejected": -562.6500244140625, |
| "loss": 0.0107, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -5.745898246765137, |
| "rewards/margins": 19.893749237060547, |
| "rewards/rejected": -25.65625, |
| "step": 5270 |
| }, |
| { |
| "epoch": 3.3978113936272933, |
| "grad_norm": 0.009674870957905443, |
| "learning_rate": 1.5019317450096585e-07, |
| "logits/chosen": -1.1496093273162842, |
| "logits/rejected": -1.172216773033142, |
| "logps/chosen": -393.3374938964844, |
| "logps/rejected": -585.4500122070312, |
| "loss": 0.0153, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -5.639721870422363, |
| "rewards/margins": 18.8828125, |
| "rewards/rejected": -24.532812118530273, |
| "step": 5280 |
| }, |
| { |
| "epoch": 3.404248471194078, |
| "grad_norm": 0.03660607691936547, |
| "learning_rate": 1.4858338699291693e-07, |
| "logits/chosen": -1.12353515625, |
| "logits/rejected": -1.0983397960662842, |
| "logps/chosen": -398.54998779296875, |
| "logps/rejected": -549.8499755859375, |
| "loss": 0.0135, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -5.693066596984863, |
| "rewards/margins": 18.256250381469727, |
| "rewards/rejected": -23.946874618530273, |
| "step": 5290 |
| }, |
| { |
| "epoch": 3.4106855487608625, |
| "grad_norm": 0.0018207283503531464, |
| "learning_rate": 1.4697359948486797e-07, |
| "logits/chosen": -1.1821777820587158, |
| "logits/rejected": -1.2270019054412842, |
| "logps/chosen": -451.57501220703125, |
| "logps/rejected": -590.9500122070312, |
| "loss": 0.0045, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -5.689355373382568, |
| "rewards/margins": 20.020313262939453, |
| "rewards/rejected": -25.712499618530273, |
| "step": 5300 |
| }, |
| { |
| "epoch": 3.417122626327647, |
| "grad_norm": 0.013680348716149777, |
| "learning_rate": 1.4536381197681905e-07, |
| "logits/chosen": -1.1581542491912842, |
| "logits/rejected": -1.185937523841858, |
| "logps/chosen": -468.75, |
| "logps/rejected": -631.25, |
| "loss": 0.0002, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -6.91015625, |
| "rewards/margins": 17.759374618530273, |
| "rewards/rejected": -24.668750762939453, |
| "step": 5310 |
| }, |
| { |
| "epoch": 3.423559703894432, |
| "grad_norm": 0.9264470620519801, |
| "learning_rate": 1.437540244687701e-07, |
| "logits/chosen": -1.13616943359375, |
| "logits/rejected": -1.133520483970642, |
| "logps/chosen": -420.3999938964844, |
| "logps/rejected": -594.5999755859375, |
| "loss": 0.013, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -6.451952934265137, |
| "rewards/margins": 19.596874237060547, |
| "rewards/rejected": -26.049999237060547, |
| "step": 5320 |
| }, |
| { |
| "epoch": 3.4299967814612167, |
| "grad_norm": 0.0014284653419422593, |
| "learning_rate": 1.4214423696072117e-07, |
| "logits/chosen": -1.182226538658142, |
| "logits/rejected": -1.187597632408142, |
| "logps/chosen": -430.0, |
| "logps/rejected": -622.7000122070312, |
| "loss": 0.0051, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -5.946875095367432, |
| "rewards/margins": 20.178125381469727, |
| "rewards/rejected": -26.125, |
| "step": 5330 |
| }, |
| { |
| "epoch": 3.4364338590280012, |
| "grad_norm": 0.016144057920363487, |
| "learning_rate": 1.4053444945267224e-07, |
| "logits/chosen": -1.2000000476837158, |
| "logits/rejected": -1.2268555164337158, |
| "logps/chosen": -426.4750061035156, |
| "logps/rejected": -583.9000244140625, |
| "loss": 0.013, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -5.616894721984863, |
| "rewards/margins": 18.295312881469727, |
| "rewards/rejected": -23.901561737060547, |
| "step": 5340 |
| }, |
| { |
| "epoch": 3.442870936594786, |
| "grad_norm": 0.002661825772172003, |
| "learning_rate": 1.389246619446233e-07, |
| "logits/chosen": -1.1572754383087158, |
| "logits/rejected": -1.1572754383087158, |
| "logps/chosen": -408.67498779296875, |
| "logps/rejected": -589.7999877929688, |
| "loss": 0.0177, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -6.3740234375, |
| "rewards/margins": 18.325000762939453, |
| "rewards/rejected": -24.712499618530273, |
| "step": 5350 |
| }, |
| { |
| "epoch": 3.449308014161571, |
| "grad_norm": 0.17931528592870347, |
| "learning_rate": 1.3731487443657436e-07, |
| "logits/chosen": -1.0957763195037842, |
| "logits/rejected": -1.1067383289337158, |
| "logps/chosen": -425.45001220703125, |
| "logps/rejected": -586.2000122070312, |
| "loss": 0.026, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -6.767773628234863, |
| "rewards/margins": 17.924999237060547, |
| "rewards/rejected": -24.690624237060547, |
| "step": 5360 |
| }, |
| { |
| "epoch": 3.4557450917283554, |
| "grad_norm": 0.00752747415982439, |
| "learning_rate": 1.357050869285254e-07, |
| "logits/chosen": -1.0899658203125, |
| "logits/rejected": -1.118408203125, |
| "logps/chosen": -446.88751220703125, |
| "logps/rejected": -618.4000244140625, |
| "loss": 0.013, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -6.139916896820068, |
| "rewards/margins": 19.3515625, |
| "rewards/rejected": -25.503124237060547, |
| "step": 5370 |
| }, |
| { |
| "epoch": 3.46218216929514, |
| "grad_norm": 0.017031868523837528, |
| "learning_rate": 1.3409529942047648e-07, |
| "logits/chosen": -1.152441382408142, |
| "logits/rejected": -1.169335961341858, |
| "logps/chosen": -466.92498779296875, |
| "logps/rejected": -623.4000244140625, |
| "loss": 0.0088, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -6.587695121765137, |
| "rewards/margins": 18.764062881469727, |
| "rewards/rejected": -25.3515625, |
| "step": 5380 |
| }, |
| { |
| "epoch": 3.4686192468619246, |
| "grad_norm": 0.001165475350785279, |
| "learning_rate": 1.3248551191242756e-07, |
| "logits/chosen": -1.215234398841858, |
| "logits/rejected": -1.2233397960662842, |
| "logps/chosen": -442.29998779296875, |
| "logps/rejected": -611.2000122070312, |
| "loss": 0.013, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -6.246972560882568, |
| "rewards/margins": 18.779687881469727, |
| "rewards/rejected": -25.021875381469727, |
| "step": 5390 |
| }, |
| { |
| "epoch": 3.475056324428709, |
| "grad_norm": 0.005232958057430995, |
| "learning_rate": 1.308757244043786e-07, |
| "logits/chosen": -1.1653320789337158, |
| "logits/rejected": -1.1648437976837158, |
| "logps/chosen": -410.17498779296875, |
| "logps/rejected": -596.4500122070312, |
| "loss": 0.0134, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -6.278515815734863, |
| "rewards/margins": 17.900781631469727, |
| "rewards/rejected": -24.192188262939453, |
| "step": 5400 |
| }, |
| { |
| "epoch": 3.481493401995494, |
| "grad_norm": 0.012513547122515394, |
| "learning_rate": 1.2926593689632968e-07, |
| "logits/chosen": -1.149658203125, |
| "logits/rejected": -1.196874976158142, |
| "logps/chosen": -399.11248779296875, |
| "logps/rejected": -591.7249755859375, |
| "loss": 0.0173, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -6.450390815734863, |
| "rewards/margins": 19.028125762939453, |
| "rewards/rejected": -25.479686737060547, |
| "step": 5410 |
| }, |
| { |
| "epoch": 3.4879304795622788, |
| "grad_norm": 0.020245274961717487, |
| "learning_rate": 1.2765614938828072e-07, |
| "logits/chosen": -1.2703125476837158, |
| "logits/rejected": -1.290917992591858, |
| "logps/chosen": -479.1000061035156, |
| "logps/rejected": -626.1500244140625, |
| "loss": 0.0087, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -7.063818454742432, |
| "rewards/margins": 20.067188262939453, |
| "rewards/rejected": -27.118749618530273, |
| "step": 5420 |
| }, |
| { |
| "epoch": 3.4943675571290633, |
| "grad_norm": 0.017114813119376918, |
| "learning_rate": 1.260463618802318e-07, |
| "logits/chosen": -1.210693359375, |
| "logits/rejected": -1.2346680164337158, |
| "logps/chosen": -458.2250061035156, |
| "logps/rejected": -611.75, |
| "loss": 0.0261, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -6.143652439117432, |
| "rewards/margins": 19.665624618530273, |
| "rewards/rejected": -25.8125, |
| "step": 5430 |
| }, |
| { |
| "epoch": 3.5008046346958483, |
| "grad_norm": 0.018696061591920828, |
| "learning_rate": 1.2443657437218287e-07, |
| "logits/chosen": -1.2316405773162842, |
| "logits/rejected": -1.231054663658142, |
| "logps/chosen": -460.2749938964844, |
| "logps/rejected": -670.75, |
| "loss": 0.0044, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -6.941992282867432, |
| "rewards/margins": 20.125, |
| "rewards/rejected": -27.071874618530273, |
| "step": 5440 |
| }, |
| { |
| "epoch": 3.507241712262633, |
| "grad_norm": 0.9238072536774803, |
| "learning_rate": 1.2282678686413394e-07, |
| "logits/chosen": -1.198486328125, |
| "logits/rejected": -1.195947289466858, |
| "logps/chosen": -401.70001220703125, |
| "logps/rejected": -588.2999877929688, |
| "loss": 0.0091, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -6.305371284484863, |
| "rewards/margins": 20.212499618530273, |
| "rewards/rejected": -26.515625, |
| "step": 5450 |
| }, |
| { |
| "epoch": 3.5136787898294175, |
| "grad_norm": 0.059459388376309825, |
| "learning_rate": 1.21216999356085e-07, |
| "logits/chosen": -1.1687500476837158, |
| "logits/rejected": -1.204199194908142, |
| "logps/chosen": -428.75, |
| "logps/rejected": -604.9500122070312, |
| "loss": 0.0087, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -6.602441310882568, |
| "rewards/margins": 19.134374618530273, |
| "rewards/rejected": -25.737499237060547, |
| "step": 5460 |
| }, |
| { |
| "epoch": 3.520115867396202, |
| "grad_norm": 0.27459831092472686, |
| "learning_rate": 1.1960721184803606e-07, |
| "logits/chosen": -1.142822265625, |
| "logits/rejected": -1.189208984375, |
| "logps/chosen": -410.29998779296875, |
| "logps/rejected": -570.4000244140625, |
| "loss": 0.013, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -7.4951171875, |
| "rewards/margins": 18.196874618530273, |
| "rewards/rejected": -25.700000762939453, |
| "step": 5470 |
| }, |
| { |
| "epoch": 3.5265529449629867, |
| "grad_norm": 0.009273735594422142, |
| "learning_rate": 1.1799742433998712e-07, |
| "logits/chosen": -1.131933569908142, |
| "logits/rejected": -1.1398437023162842, |
| "logps/chosen": -420.9375, |
| "logps/rejected": -592.5999755859375, |
| "loss": 0.0216, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -6.311841011047363, |
| "rewards/margins": 18.828125, |
| "rewards/rejected": -25.142187118530273, |
| "step": 5480 |
| }, |
| { |
| "epoch": 3.5329900225297717, |
| "grad_norm": 0.0528647089110024, |
| "learning_rate": 1.1638763683193818e-07, |
| "logits/chosen": -1.117700219154358, |
| "logits/rejected": -1.1470947265625, |
| "logps/chosen": -438.29998779296875, |
| "logps/rejected": -586.5999755859375, |
| "loss": 0.0153, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -6.416796684265137, |
| "rewards/margins": 19.087499618530273, |
| "rewards/rejected": -25.515625, |
| "step": 5490 |
| }, |
| { |
| "epoch": 3.5394271000965563, |
| "grad_norm": 0.055401282213416295, |
| "learning_rate": 1.1477784932388924e-07, |
| "logits/chosen": -1.223168969154358, |
| "logits/rejected": -1.2292969226837158, |
| "logps/chosen": -404.4125061035156, |
| "logps/rejected": -573.75, |
| "loss": 0.0132, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -7.193163871765137, |
| "rewards/margins": 18.621875762939453, |
| "rewards/rejected": -25.798437118530273, |
| "step": 5500 |
| }, |
| { |
| "epoch": 3.545864177663341, |
| "grad_norm": 0.002659691570256985, |
| "learning_rate": 1.131680618158403e-07, |
| "logits/chosen": -1.1658203601837158, |
| "logits/rejected": -1.186425805091858, |
| "logps/chosen": -443.2250061035156, |
| "logps/rejected": -587.3499755859375, |
| "loss": 0.0173, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -5.725976467132568, |
| "rewards/margins": 18.939062118530273, |
| "rewards/rejected": -24.674999237060547, |
| "step": 5510 |
| }, |
| { |
| "epoch": 3.5523012552301254, |
| "grad_norm": 0.014521885488236713, |
| "learning_rate": 1.1155827430779136e-07, |
| "logits/chosen": -1.2326171398162842, |
| "logits/rejected": -1.237036108970642, |
| "logps/chosen": -446.38751220703125, |
| "logps/rejected": -592.9500122070312, |
| "loss": 0.013, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -5.961035251617432, |
| "rewards/margins": 18.581249237060547, |
| "rewards/rejected": -24.543750762939453, |
| "step": 5520 |
| }, |
| { |
| "epoch": 3.55873833279691, |
| "grad_norm": 0.025977628087194685, |
| "learning_rate": 1.0994848679974244e-07, |
| "logits/chosen": -1.169042944908142, |
| "logits/rejected": -1.169335961341858, |
| "logps/chosen": -384.70001220703125, |
| "logps/rejected": -554.3499755859375, |
| "loss": 0.0131, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -5.560546875, |
| "rewards/margins": 18.770313262939453, |
| "rewards/rejected": -24.334375381469727, |
| "step": 5530 |
| }, |
| { |
| "epoch": 3.565175410363695, |
| "grad_norm": 0.0046389029281187245, |
| "learning_rate": 1.083386992916935e-07, |
| "logits/chosen": -1.123803734779358, |
| "logits/rejected": -1.149511694908142, |
| "logps/chosen": -401.0249938964844, |
| "logps/rejected": -595.0, |
| "loss": 0.0163, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -6.404040336608887, |
| "rewards/margins": 20.435937881469727, |
| "rewards/rejected": -26.846874237060547, |
| "step": 5540 |
| }, |
| { |
| "epoch": 3.5716124879304796, |
| "grad_norm": 0.017966434361620324, |
| "learning_rate": 1.0672891178364456e-07, |
| "logits/chosen": -1.214257836341858, |
| "logits/rejected": -1.2414062023162842, |
| "logps/chosen": -414.375, |
| "logps/rejected": -604.8499755859375, |
| "loss": 0.0173, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -6.5302734375, |
| "rewards/margins": 18.482812881469727, |
| "rewards/rejected": -25.018749237060547, |
| "step": 5550 |
| }, |
| { |
| "epoch": 3.578049565497264, |
| "grad_norm": 0.014762859371399277, |
| "learning_rate": 1.0511912427559562e-07, |
| "logits/chosen": -1.200341820716858, |
| "logits/rejected": -1.205175757408142, |
| "logps/chosen": -407.8500061035156, |
| "logps/rejected": -590.7000122070312, |
| "loss": 0.0141, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -5.133984565734863, |
| "rewards/margins": 18.453125, |
| "rewards/rejected": -23.579687118530273, |
| "step": 5560 |
| }, |
| { |
| "epoch": 3.584486643064049, |
| "grad_norm": 0.344430572394362, |
| "learning_rate": 1.0350933676754668e-07, |
| "logits/chosen": -1.171777367591858, |
| "logits/rejected": -1.159082055091858, |
| "logps/chosen": -397.9750061035156, |
| "logps/rejected": -590.5999755859375, |
| "loss": 0.0087, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -5.848437309265137, |
| "rewards/margins": 18.840625762939453, |
| "rewards/rejected": -24.703125, |
| "step": 5570 |
| }, |
| { |
| "epoch": 3.5909237206308338, |
| "grad_norm": 0.009410808435903622, |
| "learning_rate": 1.0189954925949774e-07, |
| "logits/chosen": -1.302636742591858, |
| "logits/rejected": -1.300878882408142, |
| "logps/chosen": -390.6499938964844, |
| "logps/rejected": -587.9500122070312, |
| "loss": 0.0043, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -6.140234470367432, |
| "rewards/margins": 18.835155487060547, |
| "rewards/rejected": -24.9765625, |
| "step": 5580 |
| }, |
| { |
| "epoch": 3.5973607981976183, |
| "grad_norm": 0.47452165527225904, |
| "learning_rate": 1.0028976175144881e-07, |
| "logits/chosen": -1.177978515625, |
| "logits/rejected": -1.1656982898712158, |
| "logps/chosen": -415.0, |
| "logps/rejected": -600.7999877929688, |
| "loss": 0.0133, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -6.214453220367432, |
| "rewards/margins": 19.087499618530273, |
| "rewards/rejected": -25.3125, |
| "step": 5590 |
| }, |
| { |
| "epoch": 3.603797875764403, |
| "grad_norm": 0.05161000381239699, |
| "learning_rate": 9.867997424339987e-08, |
| "logits/chosen": -1.251953125, |
| "logits/rejected": -1.2570312023162842, |
| "logps/chosen": -446.625, |
| "logps/rejected": -611.25, |
| "loss": 0.013, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -5.92431640625, |
| "rewards/margins": 18.439062118530273, |
| "rewards/rejected": -24.360937118530273, |
| "step": 5600 |
| }, |
| { |
| "epoch": 3.6102349533311875, |
| "grad_norm": 0.009289587395709877, |
| "learning_rate": 9.707018673535093e-08, |
| "logits/chosen": -1.2129395008087158, |
| "logits/rejected": -1.230371117591858, |
| "logps/chosen": -432.8500061035156, |
| "logps/rejected": -588.7999877929688, |
| "loss": 0.013, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -5.639208793640137, |
| "rewards/margins": 18.514062881469727, |
| "rewards/rejected": -24.134374618530273, |
| "step": 5610 |
| }, |
| { |
| "epoch": 3.6166720308979725, |
| "grad_norm": 0.011055816630815653, |
| "learning_rate": 9.5460399227302e-08, |
| "logits/chosen": -1.1920287609100342, |
| "logits/rejected": -1.178613305091858, |
| "logps/chosen": -444.6499938964844, |
| "logps/rejected": -599.0, |
| "loss": 0.0216, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -5.476122856140137, |
| "rewards/margins": 18.553125381469727, |
| "rewards/rejected": -24.029687881469727, |
| "step": 5620 |
| }, |
| { |
| "epoch": 3.623109108464757, |
| "grad_norm": 0.561158217771827, |
| "learning_rate": 9.385061171925305e-08, |
| "logits/chosen": -1.147802710533142, |
| "logits/rejected": -1.180761694908142, |
| "logps/chosen": -487.7250061035156, |
| "logps/rejected": -629.0999755859375, |
| "loss": 0.0132, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -6.1337890625, |
| "rewards/margins": 18.4140625, |
| "rewards/rejected": -24.549999237060547, |
| "step": 5630 |
| }, |
| { |
| "epoch": 3.6295461860315417, |
| "grad_norm": 0.09391178158930319, |
| "learning_rate": 9.224082421120411e-08, |
| "logits/chosen": -1.2050292491912842, |
| "logits/rejected": -1.23583984375, |
| "logps/chosen": -410.8999938964844, |
| "logps/rejected": -601.0999755859375, |
| "loss": 0.0303, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -5.978369235992432, |
| "rewards/margins": 19.447656631469727, |
| "rewards/rejected": -25.4296875, |
| "step": 5640 |
| }, |
| { |
| "epoch": 3.6359832635983262, |
| "grad_norm": 0.005353638341178634, |
| "learning_rate": 9.063103670315519e-08, |
| "logits/chosen": -1.254248023033142, |
| "logits/rejected": -1.2638671398162842, |
| "logps/chosen": -437.1875, |
| "logps/rejected": -608.2000122070312, |
| "loss": 0.0087, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -5.216015815734863, |
| "rewards/margins": 18.854686737060547, |
| "rewards/rejected": -24.067188262939453, |
| "step": 5650 |
| }, |
| { |
| "epoch": 3.642420341165111, |
| "grad_norm": 0.0796077309007433, |
| "learning_rate": 8.902124919510625e-08, |
| "logits/chosen": -1.177978515625, |
| "logits/rejected": -1.2056152820587158, |
| "logps/chosen": -475.1499938964844, |
| "logps/rejected": -607.5, |
| "loss": 0.0087, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -6.043749809265137, |
| "rewards/margins": 19.282812118530273, |
| "rewards/rejected": -25.309375762939453, |
| "step": 5660 |
| }, |
| { |
| "epoch": 3.648857418731896, |
| "grad_norm": 0.00354886488066356, |
| "learning_rate": 8.741146168705731e-08, |
| "logits/chosen": -1.2111327648162842, |
| "logits/rejected": -1.214453101158142, |
| "logps/chosen": -421.1499938964844, |
| "logps/rejected": -611.0999755859375, |
| "loss": 0.0043, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -5.758008003234863, |
| "rewards/margins": 19.325000762939453, |
| "rewards/rejected": -25.079687118530273, |
| "step": 5670 |
| }, |
| { |
| "epoch": 3.6552944962986804, |
| "grad_norm": 0.011692323247531729, |
| "learning_rate": 8.580167417900837e-08, |
| "logits/chosen": -1.193945288658142, |
| "logits/rejected": -1.203710913658142, |
| "logps/chosen": -440.8999938964844, |
| "logps/rejected": -606.0, |
| "loss": 0.013, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -6.059960842132568, |
| "rewards/margins": 18.110937118530273, |
| "rewards/rejected": -24.171875, |
| "step": 5680 |
| }, |
| { |
| "epoch": 3.661731573865465, |
| "grad_norm": 0.0032497001850268115, |
| "learning_rate": 8.419188667095943e-08, |
| "logits/chosen": -1.242285132408142, |
| "logits/rejected": -1.253173828125, |
| "logps/chosen": -448.1000061035156, |
| "logps/rejected": -609.4000244140625, |
| "loss": 0.0087, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -6.205370903015137, |
| "rewards/margins": 19.162500381469727, |
| "rewards/rejected": -25.389062881469727, |
| "step": 5690 |
| }, |
| { |
| "epoch": 3.66816865143225, |
| "grad_norm": 0.021910077756192244, |
| "learning_rate": 8.258209916291049e-08, |
| "logits/chosen": -1.19921875, |
| "logits/rejected": -1.2140624523162842, |
| "logps/chosen": -414.42498779296875, |
| "logps/rejected": -615.6500244140625, |
| "loss": 0.0044, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -6.513964653015137, |
| "rewards/margins": 19.879688262939453, |
| "rewards/rejected": -26.390625, |
| "step": 5700 |
| }, |
| { |
| "epoch": 3.6746057289990346, |
| "grad_norm": 0.028081385474183675, |
| "learning_rate": 8.097231165486156e-08, |
| "logits/chosen": -1.206298828125, |
| "logits/rejected": -1.247802734375, |
| "logps/chosen": -469.45001220703125, |
| "logps/rejected": -612.3499755859375, |
| "loss": 0.0044, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -5.853906154632568, |
| "rewards/margins": 18.9140625, |
| "rewards/rejected": -24.764062881469727, |
| "step": 5710 |
| }, |
| { |
| "epoch": 3.681042806565819, |
| "grad_norm": 0.009291367600511937, |
| "learning_rate": 7.936252414681262e-08, |
| "logits/chosen": -1.1588866710662842, |
| "logits/rejected": -1.1977050304412842, |
| "logps/chosen": -427.6000061035156, |
| "logps/rejected": -570.5499877929688, |
| "loss": 0.0174, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -5.973535060882568, |
| "rewards/margins": 18.046875, |
| "rewards/rejected": -24.015625, |
| "step": 5720 |
| }, |
| { |
| "epoch": 3.6874798841326037, |
| "grad_norm": 1.0300650774799196, |
| "learning_rate": 7.775273663876368e-08, |
| "logits/chosen": -1.214990258216858, |
| "logits/rejected": -1.264257788658142, |
| "logps/chosen": -436.8999938964844, |
| "logps/rejected": -600.75, |
| "loss": 0.0086, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -6.286913871765137, |
| "rewards/margins": 19.489063262939453, |
| "rewards/rejected": -25.778125762939453, |
| "step": 5730 |
| }, |
| { |
| "epoch": 3.6939169616993883, |
| "grad_norm": 0.1391319359241606, |
| "learning_rate": 7.614294913071474e-08, |
| "logits/chosen": -1.2268555164337158, |
| "logits/rejected": -1.248144507408142, |
| "logps/chosen": -445.8500061035156, |
| "logps/rejected": -629.2999877929688, |
| "loss": 0.0259, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -6.4736328125, |
| "rewards/margins": 19.696874618530273, |
| "rewards/rejected": -26.178125381469727, |
| "step": 5740 |
| }, |
| { |
| "epoch": 3.7003540392661733, |
| "grad_norm": 0.36740609289586745, |
| "learning_rate": 7.45331616226658e-08, |
| "logits/chosen": -1.179589867591858, |
| "logits/rejected": -1.18310546875, |
| "logps/chosen": -397.2250061035156, |
| "logps/rejected": -576.75, |
| "loss": 0.0217, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -6.327441215515137, |
| "rewards/margins": 18.165624618530273, |
| "rewards/rejected": -24.484375, |
| "step": 5750 |
| }, |
| { |
| "epoch": 3.706791116832958, |
| "grad_norm": 1.4062043903030401, |
| "learning_rate": 7.292337411461686e-08, |
| "logits/chosen": -1.1822540760040283, |
| "logits/rejected": -1.2383301258087158, |
| "logps/chosen": -418.6499938964844, |
| "logps/rejected": -598.6500244140625, |
| "loss": 0.0046, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -5.955420017242432, |
| "rewards/margins": 19.715625762939453, |
| "rewards/rejected": -25.673437118530273, |
| "step": 5760 |
| }, |
| { |
| "epoch": 3.7132281943997425, |
| "grad_norm": 0.010906458911690259, |
| "learning_rate": 7.131358660656794e-08, |
| "logits/chosen": -1.23486328125, |
| "logits/rejected": -1.24169921875, |
| "logps/chosen": -412.95001220703125, |
| "logps/rejected": -583.5499877929688, |
| "loss": 0.013, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -7.052734375, |
| "rewards/margins": 18.771875381469727, |
| "rewards/rejected": -25.815624237060547, |
| "step": 5770 |
| }, |
| { |
| "epoch": 3.719665271966527, |
| "grad_norm": 0.4194311473667511, |
| "learning_rate": 6.9703799098519e-08, |
| "logits/chosen": -1.142797827720642, |
| "logits/rejected": -1.152197241783142, |
| "logps/chosen": -451.17498779296875, |
| "logps/rejected": -613.25, |
| "loss": 0.013, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -6.297070503234863, |
| "rewards/margins": 18.793750762939453, |
| "rewards/rejected": -25.09375, |
| "step": 5780 |
| }, |
| { |
| "epoch": 3.7261023495333117, |
| "grad_norm": 0.14153168186128046, |
| "learning_rate": 6.809401159047006e-08, |
| "logits/chosen": -1.154870629310608, |
| "logits/rejected": -1.183935523033142, |
| "logps/chosen": -441.25, |
| "logps/rejected": -601.25, |
| "loss": 0.0087, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -6.050488471984863, |
| "rewards/margins": 19.293750762939453, |
| "rewards/rejected": -25.354686737060547, |
| "step": 5790 |
| }, |
| { |
| "epoch": 3.7325394271000967, |
| "grad_norm": 0.31575897374121853, |
| "learning_rate": 6.648422408242112e-08, |
| "logits/chosen": -1.235742211341858, |
| "logits/rejected": -1.2470214366912842, |
| "logps/chosen": -438.8500061035156, |
| "logps/rejected": -564.3499755859375, |
| "loss": 0.0047, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -5.980322360992432, |
| "rewards/margins": 18.651561737060547, |
| "rewards/rejected": -24.625, |
| "step": 5800 |
| }, |
| { |
| "epoch": 3.7389765046668813, |
| "grad_norm": 0.06287240602018677, |
| "learning_rate": 6.487443657437218e-08, |
| "logits/chosen": -1.195947289466858, |
| "logits/rejected": -1.205468773841858, |
| "logps/chosen": -417.57501220703125, |
| "logps/rejected": -602.0999755859375, |
| "loss": 0.0196, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -5.490429878234863, |
| "rewards/margins": 18.731250762939453, |
| "rewards/rejected": -24.228124618530273, |
| "step": 5810 |
| }, |
| { |
| "epoch": 3.745413582233666, |
| "grad_norm": 0.0005135924625372928, |
| "learning_rate": 6.326464906632324e-08, |
| "logits/chosen": -1.2028687000274658, |
| "logits/rejected": -1.223486304283142, |
| "logps/chosen": -466.2749938964844, |
| "logps/rejected": -638.0999755859375, |
| "loss": 0.0087, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -5.878710746765137, |
| "rewards/margins": 20.1484375, |
| "rewards/rejected": -26.024999618530273, |
| "step": 5820 |
| }, |
| { |
| "epoch": 3.751850659800451, |
| "grad_norm": 0.03898064290236112, |
| "learning_rate": 6.165486155827431e-08, |
| "logits/chosen": -1.201318383216858, |
| "logits/rejected": -1.215429663658142, |
| "logps/chosen": -429.57501220703125, |
| "logps/rejected": -586.4500122070312, |
| "loss": 0.018, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -5.779492378234863, |
| "rewards/margins": 19.001562118530273, |
| "rewards/rejected": -24.768749237060547, |
| "step": 5830 |
| }, |
| { |
| "epoch": 3.7582877373672354, |
| "grad_norm": 0.013577934214215267, |
| "learning_rate": 6.004507405022537e-08, |
| "logits/chosen": -1.228417992591858, |
| "logits/rejected": -1.2648437023162842, |
| "logps/chosen": -484.8999938964844, |
| "logps/rejected": -625.7999877929688, |
| "loss": 0.005, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -5.214037895202637, |
| "rewards/margins": 19.340625762939453, |
| "rewards/rejected": -24.549999237060547, |
| "step": 5840 |
| }, |
| { |
| "epoch": 3.76472481493402, |
| "grad_norm": 0.016377009732415706, |
| "learning_rate": 5.843528654217643e-08, |
| "logits/chosen": -1.1907227039337158, |
| "logits/rejected": -1.204492211341858, |
| "logps/chosen": -413.07501220703125, |
| "logps/rejected": -563.0999755859375, |
| "loss": 0.009, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -5.839453220367432, |
| "rewards/margins": 17.390625, |
| "rewards/rejected": -23.234375, |
| "step": 5850 |
| }, |
| { |
| "epoch": 3.7711618925008046, |
| "grad_norm": 0.015952358445255128, |
| "learning_rate": 5.682549903412749e-08, |
| "logits/chosen": -1.08544921875, |
| "logits/rejected": -1.106835961341858, |
| "logps/chosen": -406.57501220703125, |
| "logps/rejected": -596.4000244140625, |
| "loss": 0.0217, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -5.909936428070068, |
| "rewards/margins": 19.125, |
| "rewards/rejected": -25.037500381469727, |
| "step": 5860 |
| }, |
| { |
| "epoch": 3.777598970067589, |
| "grad_norm": 0.006201655476189211, |
| "learning_rate": 5.521571152607855e-08, |
| "logits/chosen": -1.1427733898162842, |
| "logits/rejected": -1.167578101158142, |
| "logps/chosen": -459.1000061035156, |
| "logps/rejected": -633.2999877929688, |
| "loss": 0.0217, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -5.592871189117432, |
| "rewards/margins": 20.014062881469727, |
| "rewards/rejected": -25.606250762939453, |
| "step": 5870 |
| }, |
| { |
| "epoch": 3.784036047634374, |
| "grad_norm": 0.009195903984204546, |
| "learning_rate": 5.360592401802962e-08, |
| "logits/chosen": -1.1323730945587158, |
| "logits/rejected": -1.120214819908142, |
| "logps/chosen": -447.07501220703125, |
| "logps/rejected": -620.5999755859375, |
| "loss": 0.0043, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -6.411718845367432, |
| "rewards/margins": 19.521875381469727, |
| "rewards/rejected": -25.934375762939453, |
| "step": 5880 |
| }, |
| { |
| "epoch": 3.7904731252011588, |
| "grad_norm": 6.727471051585618, |
| "learning_rate": 5.199613650998068e-08, |
| "logits/chosen": -1.192773461341858, |
| "logits/rejected": -1.21337890625, |
| "logps/chosen": -386.1499938964844, |
| "logps/rejected": -546.25, |
| "loss": 0.0133, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -5.852783203125, |
| "rewards/margins": 18.420312881469727, |
| "rewards/rejected": -24.274999618530273, |
| "step": 5890 |
| }, |
| { |
| "epoch": 3.7969102027679433, |
| "grad_norm": 0.02099655547602189, |
| "learning_rate": 5.038634900193174e-08, |
| "logits/chosen": -1.214257836341858, |
| "logits/rejected": -1.22705078125, |
| "logps/chosen": -445.125, |
| "logps/rejected": -612.5, |
| "loss": 0.013, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -5.768261909484863, |
| "rewards/margins": 17.332813262939453, |
| "rewards/rejected": -23.095312118530273, |
| "step": 5900 |
| }, |
| { |
| "epoch": 3.803347280334728, |
| "grad_norm": 0.004007814604864058, |
| "learning_rate": 4.8776561493882806e-08, |
| "logits/chosen": -1.2330811023712158, |
| "logits/rejected": -1.25048828125, |
| "logps/chosen": -476.54998779296875, |
| "logps/rejected": -619.9000244140625, |
| "loss": 0.013, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -6.557909965515137, |
| "rewards/margins": 18.684375762939453, |
| "rewards/rejected": -25.256250381469727, |
| "step": 5910 |
| }, |
| { |
| "epoch": 3.8097843579015125, |
| "grad_norm": 0.16369300391096503, |
| "learning_rate": 4.7166773985833866e-08, |
| "logits/chosen": -1.169335961341858, |
| "logits/rejected": -1.1863281726837158, |
| "logps/chosen": -452.32501220703125, |
| "logps/rejected": -593.3250122070312, |
| "loss": 0.0228, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -6.1298828125, |
| "rewards/margins": 18.404687881469727, |
| "rewards/rejected": -24.515625, |
| "step": 5920 |
| }, |
| { |
| "epoch": 3.8162214354682975, |
| "grad_norm": 0.00922647797366643, |
| "learning_rate": 4.5556986477784926e-08, |
| "logits/chosen": -1.1843993663787842, |
| "logits/rejected": -1.206201195716858, |
| "logps/chosen": -441.1499938964844, |
| "logps/rejected": -621.0999755859375, |
| "loss": 0.0173, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -5.243261814117432, |
| "rewards/margins": 19.825000762939453, |
| "rewards/rejected": -25.057811737060547, |
| "step": 5930 |
| }, |
| { |
| "epoch": 3.822658513035082, |
| "grad_norm": 0.013267277094511646, |
| "learning_rate": 4.394719896973599e-08, |
| "logits/chosen": -1.101464867591858, |
| "logits/rejected": -1.147607445716858, |
| "logps/chosen": -440.5249938964844, |
| "logps/rejected": -587.0999755859375, |
| "loss": 0.0303, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -5.225390434265137, |
| "rewards/margins": 18.484375, |
| "rewards/rejected": -23.71875, |
| "step": 5940 |
| }, |
| { |
| "epoch": 3.8290955906018667, |
| "grad_norm": 0.09662645701310577, |
| "learning_rate": 4.233741146168705e-08, |
| "logits/chosen": -1.1639404296875, |
| "logits/rejected": -1.1750977039337158, |
| "logps/chosen": -431.9750061035156, |
| "logps/rejected": -587.3499755859375, |
| "loss": 0.0259, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -5.033447265625, |
| "rewards/margins": 19.165624618530273, |
| "rewards/rejected": -24.212499618530273, |
| "step": 5950 |
| }, |
| { |
| "epoch": 3.8355326681686517, |
| "grad_norm": 0.008851252217354692, |
| "learning_rate": 4.072762395363811e-08, |
| "logits/chosen": -1.260156273841858, |
| "logits/rejected": -1.277929663658142, |
| "logps/chosen": -436.2250061035156, |
| "logps/rejected": -586.7999877929688, |
| "loss": 0.0086, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -6.388281345367432, |
| "rewards/margins": 18.201562881469727, |
| "rewards/rejected": -24.606250762939453, |
| "step": 5960 |
| }, |
| { |
| "epoch": 3.8419697457354363, |
| "grad_norm": 0.0248495029820189, |
| "learning_rate": 3.911783644558918e-08, |
| "logits/chosen": -1.160009741783142, |
| "logits/rejected": -1.194433569908142, |
| "logps/chosen": -444.6000061035156, |
| "logps/rejected": -621.7000122070312, |
| "loss": 0.0173, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -5.681836128234863, |
| "rewards/margins": 19.104686737060547, |
| "rewards/rejected": -24.806249618530273, |
| "step": 5970 |
| }, |
| { |
| "epoch": 3.848406823302221, |
| "grad_norm": 0.03249204827309899, |
| "learning_rate": 3.750804893754024e-08, |
| "logits/chosen": -1.224755883216858, |
| "logits/rejected": -1.24853515625, |
| "logps/chosen": -451.82501220703125, |
| "logps/rejected": -622.8499755859375, |
| "loss": 0.013, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -5.946679592132568, |
| "rewards/margins": 19.3359375, |
| "rewards/rejected": -25.296875, |
| "step": 5980 |
| }, |
| { |
| "epoch": 3.8548439008690054, |
| "grad_norm": 0.0045911375877971785, |
| "learning_rate": 3.58982614294913e-08, |
| "logits/chosen": -1.176025390625, |
| "logits/rejected": -1.178564429283142, |
| "logps/chosen": -412.375, |
| "logps/rejected": -592.3499755859375, |
| "loss": 0.0087, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -5.462890625, |
| "rewards/margins": 18.596874237060547, |
| "rewards/rejected": -24.068750381469727, |
| "step": 5990 |
| }, |
| { |
| "epoch": 3.86128097843579, |
| "grad_norm": 0.022481239633330877, |
| "learning_rate": 3.428847392144237e-08, |
| "logits/chosen": -1.226953148841858, |
| "logits/rejected": -1.242822289466858, |
| "logps/chosen": -441.20001220703125, |
| "logps/rejected": -612.3499755859375, |
| "loss": 0.0043, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -6.344152927398682, |
| "rewards/margins": 18.314062118530273, |
| "rewards/rejected": -24.653125762939453, |
| "step": 6000 |
| }, |
| { |
| "epoch": 3.867718056002575, |
| "grad_norm": 0.007146640596074133, |
| "learning_rate": 3.267868641339343e-08, |
| "logits/chosen": -1.117163062095642, |
| "logits/rejected": -1.1201903820037842, |
| "logps/chosen": -428.3125, |
| "logps/rejected": -588.6500244140625, |
| "loss": 0.0173, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -5.737353324890137, |
| "rewards/margins": 18.498437881469727, |
| "rewards/rejected": -24.25, |
| "step": 6010 |
| }, |
| { |
| "epoch": 3.8741551335693596, |
| "grad_norm": 0.007787262767055397, |
| "learning_rate": 3.1068898905344494e-08, |
| "logits/chosen": -1.100195288658142, |
| "logits/rejected": -1.133642554283142, |
| "logps/chosen": -393.17498779296875, |
| "logps/rejected": -556.2000122070312, |
| "loss": 0.0217, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -6.217382907867432, |
| "rewards/margins": 17.943750381469727, |
| "rewards/rejected": -24.15625, |
| "step": 6020 |
| }, |
| { |
| "epoch": 3.880592211136144, |
| "grad_norm": 0.02099978633163225, |
| "learning_rate": 2.9459111397295555e-08, |
| "logits/chosen": -1.2048828601837158, |
| "logits/rejected": -1.2155272960662842, |
| "logps/chosen": -444.3500061035156, |
| "logps/rejected": -610.6500244140625, |
| "loss": 0.0218, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -6.603125095367432, |
| "rewards/margins": 19.350000381469727, |
| "rewards/rejected": -25.959375381469727, |
| "step": 6030 |
| }, |
| { |
| "epoch": 3.8870292887029287, |
| "grad_norm": 0.09482176849813918, |
| "learning_rate": 2.7849323889246618e-08, |
| "logits/chosen": -1.2528808116912842, |
| "logits/rejected": -1.2414062023162842, |
| "logps/chosen": -433.2250061035156, |
| "logps/rejected": -581.5999755859375, |
| "loss": 0.013, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -5.998827934265137, |
| "rewards/margins": 18.329687118530273, |
| "rewards/rejected": -24.332813262939453, |
| "step": 6040 |
| }, |
| { |
| "epoch": 3.8934663662697133, |
| "grad_norm": 0.8030153239793042, |
| "learning_rate": 2.623953638119768e-08, |
| "logits/chosen": -1.220117211341858, |
| "logits/rejected": -1.2244141101837158, |
| "logps/chosen": -459.1499938964844, |
| "logps/rejected": -607.8499755859375, |
| "loss": 0.0087, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -5.8914794921875, |
| "rewards/margins": 18.568750381469727, |
| "rewards/rejected": -24.464061737060547, |
| "step": 6050 |
| }, |
| { |
| "epoch": 3.8999034438364983, |
| "grad_norm": 1.5912180308467658, |
| "learning_rate": 2.4629748873148742e-08, |
| "logits/chosen": -1.1505858898162842, |
| "logits/rejected": -1.1716797351837158, |
| "logps/chosen": -406.6187438964844, |
| "logps/rejected": -567.1500244140625, |
| "loss": 0.0303, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -5.146173000335693, |
| "rewards/margins": 19.529687881469727, |
| "rewards/rejected": -24.676563262939453, |
| "step": 6060 |
| }, |
| { |
| "epoch": 3.906340521403283, |
| "grad_norm": 0.0016637072817598324, |
| "learning_rate": 2.3019961365099805e-08, |
| "logits/chosen": -1.2268555164337158, |
| "logits/rejected": -1.233789086341858, |
| "logps/chosen": -451.6000061035156, |
| "logps/rejected": -651.7999877929688, |
| "loss": 0.0089, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -5.985547065734863, |
| "rewards/margins": 19.717187881469727, |
| "rewards/rejected": -25.6953125, |
| "step": 6070 |
| }, |
| { |
| "epoch": 3.9127775989700675, |
| "grad_norm": 0.007263086315974875, |
| "learning_rate": 2.141017385705087e-08, |
| "logits/chosen": -1.098291039466858, |
| "logits/rejected": -1.1204102039337158, |
| "logps/chosen": -413.7250061035156, |
| "logps/rejected": -584.9500122070312, |
| "loss": 0.0232, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -6.582226753234863, |
| "rewards/margins": 18.423437118530273, |
| "rewards/rejected": -25.003124237060547, |
| "step": 6080 |
| }, |
| { |
| "epoch": 3.9192146765368525, |
| "grad_norm": 0.005372894252220103, |
| "learning_rate": 1.980038634900193e-08, |
| "logits/chosen": -1.1232421398162842, |
| "logits/rejected": -1.110595703125, |
| "logps/chosen": -416.1000061035156, |
| "logps/rejected": -576.9500122070312, |
| "loss": 0.0216, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -5.376269340515137, |
| "rewards/margins": 19.579687118530273, |
| "rewards/rejected": -24.953125, |
| "step": 6090 |
| }, |
| { |
| "epoch": 3.925651754103637, |
| "grad_norm": 1.223081214457884, |
| "learning_rate": 1.8190598840952993e-08, |
| "logits/chosen": -1.138671875, |
| "logits/rejected": -1.160668969154358, |
| "logps/chosen": -426.2749938964844, |
| "logps/rejected": -576.2999877929688, |
| "loss": 0.0174, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -5.99658203125, |
| "rewards/margins": 19.203125, |
| "rewards/rejected": -25.189062118530273, |
| "step": 6100 |
| }, |
| { |
| "epoch": 3.9320888316704217, |
| "grad_norm": 0.24373284704903744, |
| "learning_rate": 1.6580811332904056e-08, |
| "logits/chosen": -1.2075684070587158, |
| "logits/rejected": -1.2453124523162842, |
| "logps/chosen": -456.2749938964844, |
| "logps/rejected": -626.9500122070312, |
| "loss": 0.0087, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -6.045752048492432, |
| "rewards/margins": 19.284374237060547, |
| "rewards/rejected": -25.317188262939453, |
| "step": 6110 |
| }, |
| { |
| "epoch": 3.9385259092372062, |
| "grad_norm": 0.030940312380477432, |
| "learning_rate": 1.497102382485512e-08, |
| "logits/chosen": -1.2628905773162842, |
| "logits/rejected": -1.289648413658142, |
| "logps/chosen": -446.42498779296875, |
| "logps/rejected": -595.75, |
| "loss": 0.013, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -5.348242282867432, |
| "rewards/margins": 19.128124237060547, |
| "rewards/rejected": -24.459375381469727, |
| "step": 6120 |
| }, |
| { |
| "epoch": 3.944962986803991, |
| "grad_norm": 0.030722284899537575, |
| "learning_rate": 1.3361236316806181e-08, |
| "logits/chosen": -1.1930663585662842, |
| "logits/rejected": -1.2233397960662842, |
| "logps/chosen": -457.6000061035156, |
| "logps/rejected": -600.6500244140625, |
| "loss": 0.0011, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -6.046191215515137, |
| "rewards/margins": 18.890625, |
| "rewards/rejected": -24.9375, |
| "step": 6130 |
| }, |
| { |
| "epoch": 3.951400064370776, |
| "grad_norm": 0.01006862708136734, |
| "learning_rate": 1.1751448808757243e-08, |
| "logits/chosen": -1.186792016029358, |
| "logits/rejected": -1.175756812095642, |
| "logps/chosen": -449.125, |
| "logps/rejected": -613.1500244140625, |
| "loss": 0.0173, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -6.257226467132568, |
| "rewards/margins": 18.479686737060547, |
| "rewards/rejected": -24.739063262939453, |
| "step": 6140 |
| }, |
| { |
| "epoch": 3.9578371419375604, |
| "grad_norm": 0.0269485574039851, |
| "learning_rate": 1.0141661300708307e-08, |
| "logits/chosen": -1.1842772960662842, |
| "logits/rejected": -1.1896483898162842, |
| "logps/chosen": -394.6000061035156, |
| "logps/rejected": -543.3499755859375, |
| "loss": 0.0087, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -6.286718845367432, |
| "rewards/margins": 17.590625762939453, |
| "rewards/rejected": -23.871875762939453, |
| "step": 6150 |
| }, |
| { |
| "epoch": 3.964274219504345, |
| "grad_norm": 0.015970088391158554, |
| "learning_rate": 8.531873792659369e-09, |
| "logits/chosen": -1.1664307117462158, |
| "logits/rejected": -1.1992676258087158, |
| "logps/chosen": -432.0, |
| "logps/rejected": -593.6500244140625, |
| "loss": 0.0087, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -5.511328220367432, |
| "rewards/margins": 19.0390625, |
| "rewards/rejected": -24.556249618530273, |
| "step": 6160 |
| }, |
| { |
| "epoch": 3.9707112970711296, |
| "grad_norm": 0.06187758075741397, |
| "learning_rate": 6.922086284610431e-09, |
| "logits/chosen": -1.233984351158142, |
| "logits/rejected": -1.266992211341858, |
| "logps/chosen": -442.20001220703125, |
| "logps/rejected": -558.5499877929688, |
| "loss": 0.0173, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -5.187597751617432, |
| "rewards/margins": 18.693750381469727, |
| "rewards/rejected": -23.881250381469727, |
| "step": 6170 |
| }, |
| { |
| "epoch": 3.977148374637914, |
| "grad_norm": 0.25424001056387396, |
| "learning_rate": 5.312298776561494e-09, |
| "logits/chosen": -1.1482422351837158, |
| "logits/rejected": -1.1685059070587158, |
| "logps/chosen": -441.0249938964844, |
| "logps/rejected": -610.4000244140625, |
| "loss": 0.013, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -6.648241996765137, |
| "rewards/margins": 18.834375381469727, |
| "rewards/rejected": -25.481250762939453, |
| "step": 6180 |
| }, |
| { |
| "epoch": 3.983585452204699, |
| "grad_norm": 0.006728702542557314, |
| "learning_rate": 3.7025112685125563e-09, |
| "logits/chosen": -1.1803710460662842, |
| "logits/rejected": -1.1916015148162842, |
| "logps/chosen": -434.8999938964844, |
| "logps/rejected": -569.0499877929688, |
| "loss": 0.0087, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -5.6787109375, |
| "rewards/margins": 19.334375381469727, |
| "rewards/rejected": -25.006250381469727, |
| "step": 6190 |
| }, |
| { |
| "epoch": 3.9900225297714838, |
| "grad_norm": 0.0492094714617133, |
| "learning_rate": 2.0927237604636186e-09, |
| "logits/chosen": -1.182037353515625, |
| "logits/rejected": -1.1907927989959717, |
| "logps/chosen": -433.1000061035156, |
| "logps/rejected": -609.4000244140625, |
| "loss": 0.0152, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -6.009033203125, |
| "rewards/margins": 18.553125381469727, |
| "rewards/rejected": -24.575000762939453, |
| "step": 6200 |
| }, |
| { |
| "epoch": 3.9964596073382683, |
| "grad_norm": 0.025755020406968258, |
| "learning_rate": 4.829362524146813e-10, |
| "logits/chosen": -1.1433594226837158, |
| "logits/rejected": -1.1748046875, |
| "logps/chosen": -440.6499938964844, |
| "logps/rejected": -625.7999877929688, |
| "loss": 0.0186, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -6.156201362609863, |
| "rewards/margins": 18.5078125, |
| "rewards/rejected": -24.65625, |
| "step": 6210 |
| } |
| ], |
| "logging_steps": 10, |
| "max_steps": 6212, |
| "num_input_tokens_seen": 0, |
| "num_train_epochs": 4, |
| "save_steps": 500, |
| "stateful_callbacks": { |
| "TrainerControl": { |
| "args": { |
| "should_epoch_stop": false, |
| "should_evaluate": false, |
| "should_log": false, |
| "should_save": true, |
| "should_training_stop": true |
| }, |
| "attributes": {} |
| } |
| }, |
| "total_flos": 0.0, |
| "train_batch_size": 1, |
| "trial_name": null, |
| "trial_params": null |
| } |
|
|
