| { |
| "best_global_step": null, |
| "best_metric": null, |
| "best_model_checkpoint": null, |
| "epoch": 3.9977470228516254, |
| "eval_steps": 500, |
| "global_step": 6212, |
| "is_hyper_param_search": false, |
| "is_local_process_zero": true, |
| "is_world_process_zero": true, |
| "log_history": [ |
| { |
| "epoch": 0.006437077566784679, |
| "grad_norm": 120.148749533682, |
| "learning_rate": 9.98551191242756e-07, |
| "logits/chosen": -0.355703741312027, |
| "logits/rejected": -0.254995733499527, |
| "logps/chosen": -393.3999938964844, |
| "logps/rejected": -391.75, |
| "loss": 0.685, |
| "rewards/accuracies": 0.3187499940395355, |
| "rewards/chosen": -0.06628093868494034, |
| "rewards/margins": 0.03150329738855362, |
| "rewards/rejected": -0.09776153415441513, |
| "step": 10 |
| }, |
| { |
| "epoch": 0.012874155133569359, |
| "grad_norm": 136.4607706440334, |
| "learning_rate": 9.96941403734707e-07, |
| "logits/chosen": -0.30156248807907104, |
| "logits/rejected": -0.20438233017921448, |
| "logps/chosen": -390.7749938964844, |
| "logps/rejected": -367.82501220703125, |
| "loss": 0.6442, |
| "rewards/accuracies": 0.5062500238418579, |
| "rewards/chosen": -0.373260498046875, |
| "rewards/margins": 0.16265258193016052, |
| "rewards/rejected": -0.5358031988143921, |
| "step": 20 |
| }, |
| { |
| "epoch": 0.01931123270035404, |
| "grad_norm": 118.52871882220536, |
| "learning_rate": 9.953316162266581e-07, |
| "logits/chosen": -0.30244141817092896, |
| "logits/rejected": -0.159698486328125, |
| "logps/chosen": -372.29998779296875, |
| "logps/rejected": -383.07501220703125, |
| "loss": 0.5924, |
| "rewards/accuracies": 0.6312500238418579, |
| "rewards/chosen": -0.501721203327179, |
| "rewards/margins": 0.529766857624054, |
| "rewards/rejected": -1.0309875011444092, |
| "step": 30 |
| }, |
| { |
| "epoch": 0.025748310267138717, |
| "grad_norm": 116.21829428894532, |
| "learning_rate": 9.93721828718609e-07, |
| "logits/chosen": -0.24136868119239807, |
| "logits/rejected": -0.17485961318016052, |
| "logps/chosen": -417.4750061035156, |
| "logps/rejected": -370.67498779296875, |
| "loss": 0.6302, |
| "rewards/accuracies": 0.6625000238418579, |
| "rewards/chosen": -0.2794189453125, |
| "rewards/margins": 0.44072264432907104, |
| "rewards/rejected": -0.7199767827987671, |
| "step": 40 |
| }, |
| { |
| "epoch": 0.0321853878339234, |
| "grad_norm": 96.9030889861934, |
| "learning_rate": 9.9211204121056e-07, |
| "logits/chosen": -0.4029174745082855, |
| "logits/rejected": -0.37464600801467896, |
| "logps/chosen": -387.54998779296875, |
| "logps/rejected": -365.95001220703125, |
| "loss": 0.5407, |
| "rewards/accuracies": 0.6812499761581421, |
| "rewards/chosen": 0.0676727294921875, |
| "rewards/margins": 0.6604553461074829, |
| "rewards/rejected": -0.5927703976631165, |
| "step": 50 |
| }, |
| { |
| "epoch": 0.03862246540070808, |
| "grad_norm": 105.8888517935353, |
| "learning_rate": 9.905022537025112e-07, |
| "logits/chosen": -0.313241571187973, |
| "logits/rejected": -0.26282960176467896, |
| "logps/chosen": -391.3999938964844, |
| "logps/rejected": -382.7250061035156, |
| "loss": 0.6136, |
| "rewards/accuracies": 0.65625, |
| "rewards/chosen": -0.2638183534145355, |
| "rewards/margins": 0.5556396245956421, |
| "rewards/rejected": -0.8190063238143921, |
| "step": 60 |
| }, |
| { |
| "epoch": 0.04505954296749276, |
| "grad_norm": 99.58255393276856, |
| "learning_rate": 9.888924661944624e-07, |
| "logits/chosen": -0.44835203886032104, |
| "logits/rejected": -0.41258543729782104, |
| "logps/chosen": -382.17498779296875, |
| "logps/rejected": -386.0249938964844, |
| "loss": 0.4908, |
| "rewards/accuracies": 0.71875, |
| "rewards/chosen": -0.30500489473342896, |
| "rewards/margins": 0.9129272699356079, |
| "rewards/rejected": -1.218530297279358, |
| "step": 70 |
| }, |
| { |
| "epoch": 0.051496620534277435, |
| "grad_norm": 106.44025678025184, |
| "learning_rate": 9.872826786864134e-07, |
| "logits/chosen": -0.4493164122104645, |
| "logits/rejected": -0.4206787049770355, |
| "logps/chosen": -433.0874938964844, |
| "logps/rejected": -442.45001220703125, |
| "loss": 0.6078, |
| "rewards/accuracies": 0.699999988079071, |
| "rewards/chosen": -0.9035400152206421, |
| "rewards/margins": 0.800549328327179, |
| "rewards/rejected": -1.703710913658142, |
| "step": 80 |
| }, |
| { |
| "epoch": 0.057933698101062116, |
| "grad_norm": 92.89298805386811, |
| "learning_rate": 9.856728911783643e-07, |
| "logits/chosen": -0.4051757752895355, |
| "logits/rejected": -0.3540100157260895, |
| "logps/chosen": -428.7250061035156, |
| "logps/rejected": -396.54998779296875, |
| "loss": 0.5945, |
| "rewards/accuracies": 0.699999988079071, |
| "rewards/chosen": -0.931018054485321, |
| "rewards/margins": 0.9133056402206421, |
| "rewards/rejected": -1.844335913658142, |
| "step": 90 |
| }, |
| { |
| "epoch": 0.0643707756678468, |
| "grad_norm": 87.56823461172453, |
| "learning_rate": 9.840631036703153e-07, |
| "logits/chosen": -0.302725225687027, |
| "logits/rejected": -0.3061630129814148, |
| "logps/chosen": -370.9750061035156, |
| "logps/rejected": -346.82501220703125, |
| "loss": 0.6311, |
| "rewards/accuracies": 0.625, |
| "rewards/chosen": -0.04775695875287056, |
| "rewards/margins": 0.561596691608429, |
| "rewards/rejected": -0.609820544719696, |
| "step": 100 |
| }, |
| { |
| "epoch": 0.07080785323463147, |
| "grad_norm": 102.28505747962548, |
| "learning_rate": 9.824533161622665e-07, |
| "logits/chosen": -0.42945557832717896, |
| "logits/rejected": -0.3567871153354645, |
| "logps/chosen": -381.375, |
| "logps/rejected": -348.8374938964844, |
| "loss": 0.5631, |
| "rewards/accuracies": 0.65625, |
| "rewards/chosen": 0.6863037347793579, |
| "rewards/margins": 0.6331787109375, |
| "rewards/rejected": 0.05278930813074112, |
| "step": 110 |
| }, |
| { |
| "epoch": 0.07724493080141616, |
| "grad_norm": 93.19653155872714, |
| "learning_rate": 9.808435286542177e-07, |
| "logits/chosen": -0.291656494140625, |
| "logits/rejected": -0.24359130859375, |
| "logps/chosen": -397.17498779296875, |
| "logps/rejected": -375.875, |
| "loss": 0.6141, |
| "rewards/accuracies": 0.675000011920929, |
| "rewards/chosen": 0.5700576901435852, |
| "rewards/margins": 0.64056396484375, |
| "rewards/rejected": -0.07020263373851776, |
| "step": 120 |
| }, |
| { |
| "epoch": 0.08368200836820083, |
| "grad_norm": 113.81728799309346, |
| "learning_rate": 9.792337411461686e-07, |
| "logits/chosen": -0.22374725341796875, |
| "logits/rejected": -0.15942077338695526, |
| "logps/chosen": -377.5249938964844, |
| "logps/rejected": -374.8500061035156, |
| "loss": 0.6194, |
| "rewards/accuracies": 0.612500011920929, |
| "rewards/chosen": 0.31868743896484375, |
| "rewards/margins": 0.6328979730606079, |
| "rewards/rejected": -0.3145996034145355, |
| "step": 130 |
| }, |
| { |
| "epoch": 0.09011908593498552, |
| "grad_norm": 120.17148752358271, |
| "learning_rate": 9.776239536381196e-07, |
| "logits/chosen": -0.14885863661766052, |
| "logits/rejected": -0.18120726943016052, |
| "logps/chosen": -420.25, |
| "logps/rejected": -373.75, |
| "loss": 0.6022, |
| "rewards/accuracies": 0.637499988079071, |
| "rewards/chosen": 0.45744019746780396, |
| "rewards/margins": 0.652172863483429, |
| "rewards/rejected": -0.194671630859375, |
| "step": 140 |
| }, |
| { |
| "epoch": 0.0965561635017702, |
| "grad_norm": 92.41902832635517, |
| "learning_rate": 9.760141661300708e-07, |
| "logits/chosen": -0.1698760986328125, |
| "logits/rejected": -0.10188598930835724, |
| "logps/chosen": -378.0249938964844, |
| "logps/rejected": -344.20001220703125, |
| "loss": 0.6055, |
| "rewards/accuracies": 0.612500011920929, |
| "rewards/chosen": 1.228124976158142, |
| "rewards/margins": 0.593334972858429, |
| "rewards/rejected": 0.63494873046875, |
| "step": 150 |
| }, |
| { |
| "epoch": 0.10299324106855487, |
| "grad_norm": 97.91240000919029, |
| "learning_rate": 9.744043786220218e-07, |
| "logits/chosen": -0.32441407442092896, |
| "logits/rejected": -0.2691238522529602, |
| "logps/chosen": -376.2749938964844, |
| "logps/rejected": -358.82501220703125, |
| "loss": 0.6683, |
| "rewards/accuracies": 0.5687500238418579, |
| "rewards/chosen": 0.8737030029296875, |
| "rewards/margins": 0.4753784239292145, |
| "rewards/rejected": 0.39832764863967896, |
| "step": 160 |
| }, |
| { |
| "epoch": 0.10943031863533956, |
| "grad_norm": 78.29197136247424, |
| "learning_rate": 9.72794591113973e-07, |
| "logits/chosen": -0.42585450410842896, |
| "logits/rejected": -0.3639587461948395, |
| "logps/chosen": -395.125, |
| "logps/rejected": -392.6499938964844, |
| "loss": 0.5872, |
| "rewards/accuracies": 0.6499999761581421, |
| "rewards/chosen": 0.34979248046875, |
| "rewards/margins": 0.704418957233429, |
| "rewards/rejected": -0.35491639375686646, |
| "step": 170 |
| }, |
| { |
| "epoch": 0.11586739620212423, |
| "grad_norm": 108.97542441210452, |
| "learning_rate": 9.71184803605924e-07, |
| "logits/chosen": -0.5166260004043579, |
| "logits/rejected": -0.480621337890625, |
| "logps/chosen": -407.79998779296875, |
| "logps/rejected": -407.92498779296875, |
| "loss": 0.598, |
| "rewards/accuracies": 0.637499988079071, |
| "rewards/chosen": -0.3631347715854645, |
| "rewards/margins": 0.6824706792831421, |
| "rewards/rejected": -1.0447876453399658, |
| "step": 180 |
| }, |
| { |
| "epoch": 0.12230447376890892, |
| "grad_norm": 147.32131536244177, |
| "learning_rate": 9.69575016097875e-07, |
| "logits/chosen": -0.596240222454071, |
| "logits/rejected": -0.5375427007675171, |
| "logps/chosen": -405.4750061035156, |
| "logps/rejected": -410.7749938964844, |
| "loss": 0.6098, |
| "rewards/accuracies": 0.675000011920929, |
| "rewards/chosen": -1.025366187095642, |
| "rewards/margins": 0.804125964641571, |
| "rewards/rejected": -1.829003930091858, |
| "step": 190 |
| }, |
| { |
| "epoch": 0.1287415513356936, |
| "grad_norm": 125.45923451084157, |
| "learning_rate": 9.67965228589826e-07, |
| "logits/chosen": -0.534393310546875, |
| "logits/rejected": -0.564160168170929, |
| "logps/chosen": -438.3999938964844, |
| "logps/rejected": -387.8500061035156, |
| "loss": 0.5597, |
| "rewards/accuracies": 0.65625, |
| "rewards/chosen": -0.656054675579071, |
| "rewards/margins": 0.9250732660293579, |
| "rewards/rejected": -1.580810546875, |
| "step": 200 |
| }, |
| { |
| "epoch": 0.13517862890247828, |
| "grad_norm": 113.1081364866241, |
| "learning_rate": 9.663554410817772e-07, |
| "logits/chosen": -0.364013671875, |
| "logits/rejected": -0.33275145292282104, |
| "logps/chosen": -373.8125, |
| "logps/rejected": -379.6000061035156, |
| "loss": 0.6145, |
| "rewards/accuracies": 0.6499999761581421, |
| "rewards/chosen": -0.13249512016773224, |
| "rewards/margins": 0.6836303472518921, |
| "rewards/rejected": -0.81549072265625, |
| "step": 210 |
| }, |
| { |
| "epoch": 0.14161570646926294, |
| "grad_norm": 89.87804619747781, |
| "learning_rate": 9.647456535737282e-07, |
| "logits/chosen": -0.34938353300094604, |
| "logits/rejected": -0.27142030000686646, |
| "logps/chosen": -396.3500061035156, |
| "logps/rejected": -397.07501220703125, |
| "loss": 0.5988, |
| "rewards/accuracies": 0.637499988079071, |
| "rewards/chosen": 0.4713592529296875, |
| "rewards/margins": 0.753369152545929, |
| "rewards/rejected": -0.2815612852573395, |
| "step": 220 |
| }, |
| { |
| "epoch": 0.14805278403604763, |
| "grad_norm": 85.7888258251216, |
| "learning_rate": 9.631358660656794e-07, |
| "logits/chosen": -0.3939758241176605, |
| "logits/rejected": -0.29407960176467896, |
| "logps/chosen": -368.375, |
| "logps/rejected": -365.3374938964844, |
| "loss": 0.6138, |
| "rewards/accuracies": 0.6312500238418579, |
| "rewards/chosen": 0.47608643770217896, |
| "rewards/margins": 0.6263427734375, |
| "rewards/rejected": -0.15087279677391052, |
| "step": 230 |
| }, |
| { |
| "epoch": 0.15448986160283232, |
| "grad_norm": 115.72031881125797, |
| "learning_rate": 9.615260785576303e-07, |
| "logits/chosen": -0.3743301331996918, |
| "logits/rejected": -0.3862457275390625, |
| "logps/chosen": -401.25, |
| "logps/rejected": -371.5375061035156, |
| "loss": 0.6564, |
| "rewards/accuracies": 0.668749988079071, |
| "rewards/chosen": -0.10862579196691513, |
| "rewards/margins": 0.71014404296875, |
| "rewards/rejected": -0.8185669183731079, |
| "step": 240 |
| }, |
| { |
| "epoch": 0.160926939169617, |
| "grad_norm": 110.59209582564404, |
| "learning_rate": 9.599162910495813e-07, |
| "logits/chosen": -0.43714600801467896, |
| "logits/rejected": -0.355712890625, |
| "logps/chosen": -436.4750061035156, |
| "logps/rejected": -401.6000061035156, |
| "loss": 0.5783, |
| "rewards/accuracies": 0.71875, |
| "rewards/chosen": -0.17133179306983948, |
| "rewards/margins": 0.7469482421875, |
| "rewards/rejected": -0.9180663824081421, |
| "step": 250 |
| }, |
| { |
| "epoch": 0.16736401673640167, |
| "grad_norm": 113.88687975240751, |
| "learning_rate": 9.583065035415325e-07, |
| "logits/chosen": -0.309173583984375, |
| "logits/rejected": -0.21812744438648224, |
| "logps/chosen": -385.32501220703125, |
| "logps/rejected": -362.07501220703125, |
| "loss": 0.5577, |
| "rewards/accuracies": 0.6499999761581421, |
| "rewards/chosen": -0.28889161348342896, |
| "rewards/margins": 0.845550537109375, |
| "rewards/rejected": -1.134130835533142, |
| "step": 260 |
| }, |
| { |
| "epoch": 0.17380109430318635, |
| "grad_norm": 129.1658487670197, |
| "learning_rate": 9.566967160334835e-07, |
| "logits/chosen": -0.47778016328811646, |
| "logits/rejected": -0.32499390840530396, |
| "logps/chosen": -355.2749938964844, |
| "logps/rejected": -364.875, |
| "loss": 0.5486, |
| "rewards/accuracies": 0.625, |
| "rewards/chosen": 0.720959484577179, |
| "rewards/margins": 0.8039306402206421, |
| "rewards/rejected": -0.08214721828699112, |
| "step": 270 |
| }, |
| { |
| "epoch": 0.18023817186997104, |
| "grad_norm": 90.70276370808507, |
| "learning_rate": 9.550869285254346e-07, |
| "logits/chosen": -0.3791870176792145, |
| "logits/rejected": -0.36989134550094604, |
| "logps/chosen": -387.2749938964844, |
| "logps/rejected": -386.75, |
| "loss": 0.6349, |
| "rewards/accuracies": 0.6000000238418579, |
| "rewards/chosen": 0.5383666753768921, |
| "rewards/margins": 0.5897369384765625, |
| "rewards/rejected": -0.05165405198931694, |
| "step": 280 |
| }, |
| { |
| "epoch": 0.1866752494367557, |
| "grad_norm": 81.87512178252723, |
| "learning_rate": 9.534771410173856e-07, |
| "logits/chosen": -0.56195068359375, |
| "logits/rejected": -0.5644897222518921, |
| "logps/chosen": -394.875, |
| "logps/rejected": -392.125, |
| "loss": 0.6867, |
| "rewards/accuracies": 0.6000000238418579, |
| "rewards/chosen": -0.09564208984375, |
| "rewards/margins": 0.6316894292831421, |
| "rewards/rejected": -0.727764904499054, |
| "step": 290 |
| }, |
| { |
| "epoch": 0.1931123270035404, |
| "grad_norm": 88.15168582376563, |
| "learning_rate": 9.518673535093368e-07, |
| "logits/chosen": -0.4905151426792145, |
| "logits/rejected": -0.439178466796875, |
| "logps/chosen": -408.54998779296875, |
| "logps/rejected": -378.92498779296875, |
| "loss": 0.6145, |
| "rewards/accuracies": 0.6625000238418579, |
| "rewards/chosen": -0.671856701374054, |
| "rewards/margins": 0.8810058832168579, |
| "rewards/rejected": -1.552545189857483, |
| "step": 300 |
| }, |
| { |
| "epoch": 0.19954940457032508, |
| "grad_norm": 88.1085775229601, |
| "learning_rate": 9.502575660012879e-07, |
| "logits/chosen": -0.566027820110321, |
| "logits/rejected": -0.456756591796875, |
| "logps/chosen": -393.625, |
| "logps/rejected": -406.1000061035156, |
| "loss": 0.6173, |
| "rewards/accuracies": 0.625, |
| "rewards/chosen": -0.507843017578125, |
| "rewards/margins": 0.672717273235321, |
| "rewards/rejected": -1.1804687976837158, |
| "step": 310 |
| }, |
| { |
| "epoch": 0.20598648213710974, |
| "grad_norm": 99.78525688570807, |
| "learning_rate": 9.486477784932388e-07, |
| "logits/chosen": -0.629040539264679, |
| "logits/rejected": -0.511334240436554, |
| "logps/chosen": -405.1499938964844, |
| "logps/rejected": -413.45001220703125, |
| "loss": 0.4823, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -0.08429565280675888, |
| "rewards/margins": 1.1553833484649658, |
| "rewards/rejected": -1.239740014076233, |
| "step": 320 |
| }, |
| { |
| "epoch": 0.21242355970389443, |
| "grad_norm": 99.88587933762932, |
| "learning_rate": 9.470379909851899e-07, |
| "logits/chosen": -0.5509277582168579, |
| "logits/rejected": -0.489593505859375, |
| "logps/chosen": -395.5625, |
| "logps/rejected": -385.76251220703125, |
| "loss": 0.6205, |
| "rewards/accuracies": 0.6625000238418579, |
| "rewards/chosen": -0.47585755586624146, |
| "rewards/margins": 0.841845691204071, |
| "rewards/rejected": -1.317297339439392, |
| "step": 330 |
| }, |
| { |
| "epoch": 0.21886063727067911, |
| "grad_norm": 88.83656417221579, |
| "learning_rate": 9.454282034771411e-07, |
| "logits/chosen": -0.6428283452987671, |
| "logits/rejected": -0.588879406452179, |
| "logps/chosen": -389.3500061035156, |
| "logps/rejected": -391.79998779296875, |
| "loss": 0.5652, |
| "rewards/accuracies": 0.6812499761581421, |
| "rewards/chosen": -0.403717041015625, |
| "rewards/margins": 0.948803722858429, |
| "rewards/rejected": -1.352783203125, |
| "step": 340 |
| }, |
| { |
| "epoch": 0.2252977148374638, |
| "grad_norm": 73.64497342881143, |
| "learning_rate": 9.43818415969092e-07, |
| "logits/chosen": -0.3382400572299957, |
| "logits/rejected": -0.28192442655563354, |
| "logps/chosen": -409.54998779296875, |
| "logps/rejected": -392.29998779296875, |
| "loss": 0.625, |
| "rewards/accuracies": 0.6312500238418579, |
| "rewards/chosen": -0.04579772800207138, |
| "rewards/margins": 0.7647460699081421, |
| "rewards/rejected": -0.8106445074081421, |
| "step": 350 |
| }, |
| { |
| "epoch": 0.23173479240424846, |
| "grad_norm": 148.36214529047115, |
| "learning_rate": 9.422086284610431e-07, |
| "logits/chosen": -0.552001953125, |
| "logits/rejected": -0.4778381288051605, |
| "logps/chosen": -370.20001220703125, |
| "logps/rejected": -380.3999938964844, |
| "loss": 0.6407, |
| "rewards/accuracies": 0.606249988079071, |
| "rewards/chosen": 0.3538192808628082, |
| "rewards/margins": 0.731701672077179, |
| "rewards/rejected": -0.37797242403030396, |
| "step": 360 |
| }, |
| { |
| "epoch": 0.23817186997103315, |
| "grad_norm": 97.71575688934269, |
| "learning_rate": 9.405988409529941e-07, |
| "logits/chosen": -0.50762939453125, |
| "logits/rejected": -0.46068114042282104, |
| "logps/chosen": -413.625, |
| "logps/rejected": -370.82501220703125, |
| "loss": 0.593, |
| "rewards/accuracies": 0.65625, |
| "rewards/chosen": 0.3553710877895355, |
| "rewards/margins": 0.780346691608429, |
| "rewards/rejected": -0.42524415254592896, |
| "step": 370 |
| }, |
| { |
| "epoch": 0.24460894753781784, |
| "grad_norm": 100.48668697477665, |
| "learning_rate": 9.389890534449453e-07, |
| "logits/chosen": -0.606915295124054, |
| "logits/rejected": -0.584973156452179, |
| "logps/chosen": -405.54998779296875, |
| "logps/rejected": -396.4750061035156, |
| "loss": 0.533, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": 0.153045654296875, |
| "rewards/margins": 0.961181640625, |
| "rewards/rejected": -0.80743408203125, |
| "step": 380 |
| }, |
| { |
| "epoch": 0.2510460251046025, |
| "grad_norm": 109.3924143916782, |
| "learning_rate": 9.373792659368963e-07, |
| "logits/chosen": -0.518872082233429, |
| "logits/rejected": -0.5242980718612671, |
| "logps/chosen": -392.75, |
| "logps/rejected": -383.5249938964844, |
| "loss": 0.5842, |
| "rewards/accuracies": 0.675000011920929, |
| "rewards/chosen": -0.884783923625946, |
| "rewards/margins": 1.0972778797149658, |
| "rewards/rejected": -1.9823486804962158, |
| "step": 390 |
| }, |
| { |
| "epoch": 0.2574831026713872, |
| "grad_norm": 129.0262252885059, |
| "learning_rate": 9.357694784288473e-07, |
| "logits/chosen": -0.5479461550712585, |
| "logits/rejected": -0.42854613065719604, |
| "logps/chosen": -394.875, |
| "logps/rejected": -397.5, |
| "loss": 0.6954, |
| "rewards/accuracies": 0.6625000238418579, |
| "rewards/chosen": -0.8142364621162415, |
| "rewards/margins": 1.1148192882537842, |
| "rewards/rejected": -1.9297668933868408, |
| "step": 400 |
| }, |
| { |
| "epoch": 0.2639201802381719, |
| "grad_norm": 113.95512234335068, |
| "learning_rate": 9.341596909207984e-07, |
| "logits/chosen": -0.4556030333042145, |
| "logits/rejected": -0.48054808378219604, |
| "logps/chosen": -397.9750061035156, |
| "logps/rejected": -381.45001220703125, |
| "loss": 0.5552, |
| "rewards/accuracies": 0.6812499761581421, |
| "rewards/chosen": 0.02728271484375, |
| "rewards/margins": 0.9527587890625, |
| "rewards/rejected": -0.9253570437431335, |
| "step": 410 |
| }, |
| { |
| "epoch": 0.27035725780495656, |
| "grad_norm": 79.07068780985843, |
| "learning_rate": 9.325499034127496e-07, |
| "logits/chosen": -0.5902328491210938, |
| "logits/rejected": -0.46696776151657104, |
| "logps/chosen": -417.3500061035156, |
| "logps/rejected": -416.42498779296875, |
| "loss": 0.5326, |
| "rewards/accuracies": 0.7124999761581421, |
| "rewards/chosen": 0.26826173067092896, |
| "rewards/margins": 0.956695556640625, |
| "rewards/rejected": -0.6884826421737671, |
| "step": 420 |
| }, |
| { |
| "epoch": 0.27679433537174125, |
| "grad_norm": 104.45515554912211, |
| "learning_rate": 9.309401159047005e-07, |
| "logits/chosen": -0.6111496090888977, |
| "logits/rejected": -0.5257537961006165, |
| "logps/chosen": -376.3999938964844, |
| "logps/rejected": -381.0249938964844, |
| "loss": 0.6838, |
| "rewards/accuracies": 0.65625, |
| "rewards/chosen": -0.22331543266773224, |
| "rewards/margins": 0.8266357183456421, |
| "rewards/rejected": -1.048437476158142, |
| "step": 430 |
| }, |
| { |
| "epoch": 0.2832314129385259, |
| "grad_norm": 130.9915025437971, |
| "learning_rate": 9.293303283966516e-07, |
| "logits/chosen": -0.5809875726699829, |
| "logits/rejected": -0.5356170535087585, |
| "logps/chosen": -417.29998779296875, |
| "logps/rejected": -412.8500061035156, |
| "loss": 0.6227, |
| "rewards/accuracies": 0.6312500238418579, |
| "rewards/chosen": 0.02593383751809597, |
| "rewards/margins": 0.669720470905304, |
| "rewards/rejected": -0.6444946527481079, |
| "step": 440 |
| }, |
| { |
| "epoch": 0.28966849050531057, |
| "grad_norm": 120.75500230202465, |
| "learning_rate": 9.277205408886027e-07, |
| "logits/chosen": -0.48475342988967896, |
| "logits/rejected": -0.34586793184280396, |
| "logps/chosen": -385.7749938964844, |
| "logps/rejected": -398.82501220703125, |
| "loss": 0.49, |
| "rewards/accuracies": 0.762499988079071, |
| "rewards/chosen": 0.11443176120519638, |
| "rewards/margins": 1.0911865234375, |
| "rewards/rejected": -0.976458728313446, |
| "step": 450 |
| }, |
| { |
| "epoch": 0.29610556807209526, |
| "grad_norm": 100.75697791096788, |
| "learning_rate": 9.261107533805537e-07, |
| "logits/chosen": -0.4248901307582855, |
| "logits/rejected": -0.42832642793655396, |
| "logps/chosen": -407.67498779296875, |
| "logps/rejected": -397.32501220703125, |
| "loss": 0.6917, |
| "rewards/accuracies": 0.59375, |
| "rewards/chosen": -0.14867553114891052, |
| "rewards/margins": 0.6075439453125, |
| "rewards/rejected": -0.7563232183456421, |
| "step": 460 |
| }, |
| { |
| "epoch": 0.30254264563887995, |
| "grad_norm": 87.40797987193615, |
| "learning_rate": 9.245009658725048e-07, |
| "logits/chosen": -0.4753662049770355, |
| "logits/rejected": -0.39787596464157104, |
| "logps/chosen": -421.61248779296875, |
| "logps/rejected": -382.1000061035156, |
| "loss": 0.5759, |
| "rewards/accuracies": 0.699999988079071, |
| "rewards/chosen": 0.2565246522426605, |
| "rewards/margins": 0.8692871332168579, |
| "rewards/rejected": -0.613067626953125, |
| "step": 470 |
| }, |
| { |
| "epoch": 0.30897972320566464, |
| "grad_norm": 112.18096019861129, |
| "learning_rate": 9.228911783644559e-07, |
| "logits/chosen": -0.39225465059280396, |
| "logits/rejected": -0.397390753030777, |
| "logps/chosen": -387.75, |
| "logps/rejected": -393.2250061035156, |
| "loss": 0.5672, |
| "rewards/accuracies": 0.668749988079071, |
| "rewards/chosen": 0.3129638731479645, |
| "rewards/margins": 0.839489758014679, |
| "rewards/rejected": -0.526226818561554, |
| "step": 480 |
| }, |
| { |
| "epoch": 0.3154168007724493, |
| "grad_norm": 115.73521559457446, |
| "learning_rate": 9.212813908564069e-07, |
| "logits/chosen": -0.313497930765152, |
| "logits/rejected": -0.29547119140625, |
| "logps/chosen": -391.04998779296875, |
| "logps/rejected": -381.4750061035156, |
| "loss": 0.6692, |
| "rewards/accuracies": 0.606249988079071, |
| "rewards/chosen": -0.16169433295726776, |
| "rewards/margins": 0.6041320562362671, |
| "rewards/rejected": -0.7651427984237671, |
| "step": 490 |
| }, |
| { |
| "epoch": 0.321853878339234, |
| "grad_norm": 83.27425077008014, |
| "learning_rate": 9.19671603348358e-07, |
| "logits/chosen": -0.55694580078125, |
| "logits/rejected": -0.45402830839157104, |
| "logps/chosen": -399.0625, |
| "logps/rejected": -395.3500061035156, |
| "loss": 0.7226, |
| "rewards/accuracies": 0.550000011920929, |
| "rewards/chosen": -0.279684454202652, |
| "rewards/margins": 0.715710461139679, |
| "rewards/rejected": -0.9955230951309204, |
| "step": 500 |
| }, |
| { |
| "epoch": 0.32829095590601864, |
| "grad_norm": 126.11980958122653, |
| "learning_rate": 9.18061815840309e-07, |
| "logits/chosen": -0.4515624940395355, |
| "logits/rejected": -0.3348388671875, |
| "logps/chosen": -386.6000061035156, |
| "logps/rejected": -394.3999938964844, |
| "loss": 0.6056, |
| "rewards/accuracies": 0.6937500238418579, |
| "rewards/chosen": 0.01708373986184597, |
| "rewards/margins": 0.7932494878768921, |
| "rewards/rejected": -0.775280773639679, |
| "step": 510 |
| }, |
| { |
| "epoch": 0.33472803347280333, |
| "grad_norm": 78.44116756435817, |
| "learning_rate": 9.164520283322601e-07, |
| "logits/chosen": -0.4293670654296875, |
| "logits/rejected": -0.425506591796875, |
| "logps/chosen": -390.07501220703125, |
| "logps/rejected": -382.375, |
| "loss": 0.5929, |
| "rewards/accuracies": 0.6625000238418579, |
| "rewards/chosen": 0.020965576171875, |
| "rewards/margins": 0.8412841558456421, |
| "rewards/rejected": -0.8202239871025085, |
| "step": 520 |
| }, |
| { |
| "epoch": 0.341165111039588, |
| "grad_norm": 141.31003997813028, |
| "learning_rate": 9.148422408242112e-07, |
| "logits/chosen": -0.477294921875, |
| "logits/rejected": -0.38500672578811646, |
| "logps/chosen": -390.2250061035156, |
| "logps/rejected": -406.29998779296875, |
| "loss": 0.5424, |
| "rewards/accuracies": 0.65625, |
| "rewards/chosen": -0.162872314453125, |
| "rewards/margins": 0.97357177734375, |
| "rewards/rejected": -1.136993408203125, |
| "step": 530 |
| }, |
| { |
| "epoch": 0.3476021886063727, |
| "grad_norm": 113.57080133316278, |
| "learning_rate": 9.132324533161622e-07, |
| "logits/chosen": -0.5093628168106079, |
| "logits/rejected": -0.4897399842739105, |
| "logps/chosen": -373.2250061035156, |
| "logps/rejected": -362.6499938964844, |
| "loss": 0.6143, |
| "rewards/accuracies": 0.6499999761581421, |
| "rewards/chosen": -0.4917846620082855, |
| "rewards/margins": 0.7836669683456421, |
| "rewards/rejected": -1.2767822742462158, |
| "step": 540 |
| }, |
| { |
| "epoch": 0.3540392661731574, |
| "grad_norm": 117.73430272365195, |
| "learning_rate": 9.116226658081133e-07, |
| "logits/chosen": -0.47328490018844604, |
| "logits/rejected": -0.46728515625, |
| "logps/chosen": -399.4750061035156, |
| "logps/rejected": -374.75, |
| "loss": 0.5563, |
| "rewards/accuracies": 0.675000011920929, |
| "rewards/chosen": -0.50042724609375, |
| "rewards/margins": 0.7510131597518921, |
| "rewards/rejected": -1.251708984375, |
| "step": 550 |
| }, |
| { |
| "epoch": 0.3604763437399421, |
| "grad_norm": 98.10679036552621, |
| "learning_rate": 9.100128783000644e-07, |
| "logits/chosen": -0.5118408203125, |
| "logits/rejected": -0.5103088617324829, |
| "logps/chosen": -380.7749938964844, |
| "logps/rejected": -405.2250061035156, |
| "loss": 0.5441, |
| "rewards/accuracies": 0.7250000238418579, |
| "rewards/chosen": -0.18172912299633026, |
| "rewards/margins": 0.9223388433456421, |
| "rewards/rejected": -1.1035888195037842, |
| "step": 560 |
| }, |
| { |
| "epoch": 0.3669134213067268, |
| "grad_norm": 153.64255025464655, |
| "learning_rate": 9.084030907920153e-07, |
| "logits/chosen": -0.5662018060684204, |
| "logits/rejected": -0.44904786348342896, |
| "logps/chosen": -394.625, |
| "logps/rejected": -409.8500061035156, |
| "loss": 0.7296, |
| "rewards/accuracies": 0.59375, |
| "rewards/chosen": -0.20541992783546448, |
| "rewards/margins": 0.5594848394393921, |
| "rewards/rejected": -0.764599621295929, |
| "step": 570 |
| }, |
| { |
| "epoch": 0.3733504988735114, |
| "grad_norm": 101.23911656488093, |
| "learning_rate": 9.067933032839665e-07, |
| "logits/chosen": -0.6080871820449829, |
| "logits/rejected": -0.4863525331020355, |
| "logps/chosen": -367.67498779296875, |
| "logps/rejected": -397.67498779296875, |
| "loss": 0.5976, |
| "rewards/accuracies": 0.6812499761581421, |
| "rewards/chosen": -0.013964843936264515, |
| "rewards/margins": 0.9591064453125, |
| "rewards/rejected": -0.9730590581893921, |
| "step": 580 |
| }, |
| { |
| "epoch": 0.3797875764402961, |
| "grad_norm": 91.53741263281876, |
| "learning_rate": 9.051835157759176e-07, |
| "logits/chosen": -0.6253296136856079, |
| "logits/rejected": -0.5479980707168579, |
| "logps/chosen": -425.5, |
| "logps/rejected": -417.42498779296875, |
| "loss": 0.6109, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": -0.3316894471645355, |
| "rewards/margins": 0.900891125202179, |
| "rewards/rejected": -1.232812523841858, |
| "step": 590 |
| }, |
| { |
| "epoch": 0.3862246540070808, |
| "grad_norm": 116.3183413353052, |
| "learning_rate": 9.035737282678686e-07, |
| "logits/chosen": -0.42241746187210083, |
| "logits/rejected": -0.4776352047920227, |
| "logps/chosen": -360.375, |
| "logps/rejected": -344.6000061035156, |
| "loss": 0.6352, |
| "rewards/accuracies": 0.65625, |
| "rewards/chosen": -0.26518553495407104, |
| "rewards/margins": 0.810314953327179, |
| "rewards/rejected": -1.0747559070587158, |
| "step": 600 |
| }, |
| { |
| "epoch": 0.39266173157386547, |
| "grad_norm": 82.65123414122117, |
| "learning_rate": 9.019639407598196e-07, |
| "logits/chosen": -0.46138304471969604, |
| "logits/rejected": -0.43217772245407104, |
| "logps/chosen": -396.92498779296875, |
| "logps/rejected": -400.29998779296875, |
| "loss": 0.5881, |
| "rewards/accuracies": 0.7124999761581421, |
| "rewards/chosen": 0.21893310546875, |
| "rewards/margins": 0.983294665813446, |
| "rewards/rejected": -0.76446533203125, |
| "step": 610 |
| }, |
| { |
| "epoch": 0.39909880914065016, |
| "grad_norm": 129.45561570069154, |
| "learning_rate": 9.003541532517708e-07, |
| "logits/chosen": -0.41582030057907104, |
| "logits/rejected": -0.4110961854457855, |
| "logps/chosen": -391.1499938964844, |
| "logps/rejected": -393.82501220703125, |
| "loss": 0.5721, |
| "rewards/accuracies": 0.65625, |
| "rewards/chosen": -0.1370849609375, |
| "rewards/margins": 0.884106457233429, |
| "rewards/rejected": -1.020452857017517, |
| "step": 620 |
| }, |
| { |
| "epoch": 0.40553588670743485, |
| "grad_norm": 141.85369123816798, |
| "learning_rate": 8.987443657437218e-07, |
| "logits/chosen": -0.4580749571323395, |
| "logits/rejected": -0.3816772401332855, |
| "logps/chosen": -412.5, |
| "logps/rejected": -405.8374938964844, |
| "loss": 0.626, |
| "rewards/accuracies": 0.6312500238418579, |
| "rewards/chosen": -0.09260253608226776, |
| "rewards/margins": 0.9054931402206421, |
| "rewards/rejected": -0.998791515827179, |
| "step": 630 |
| }, |
| { |
| "epoch": 0.4119729642742195, |
| "grad_norm": 130.38405351194206, |
| "learning_rate": 8.971345782356729e-07, |
| "logits/chosen": -0.41401976346969604, |
| "logits/rejected": -0.3881790041923523, |
| "logps/chosen": -403.625, |
| "logps/rejected": -415.8500061035156, |
| "loss": 0.6293, |
| "rewards/accuracies": 0.65625, |
| "rewards/chosen": 0.07443847507238388, |
| "rewards/margins": 0.769213855266571, |
| "rewards/rejected": -0.6953796148300171, |
| "step": 640 |
| }, |
| { |
| "epoch": 0.41841004184100417, |
| "grad_norm": 100.14647718943294, |
| "learning_rate": 8.955247907276239e-07, |
| "logits/chosen": -0.30842286348342896, |
| "logits/rejected": -0.25592344999313354, |
| "logps/chosen": -380.1499938964844, |
| "logps/rejected": -370.1499938964844, |
| "loss": 0.5838, |
| "rewards/accuracies": 0.65625, |
| "rewards/chosen": 0.139495849609375, |
| "rewards/margins": 0.7997680902481079, |
| "rewards/rejected": -0.659741222858429, |
| "step": 650 |
| }, |
| { |
| "epoch": 0.42484711940778885, |
| "grad_norm": 95.21246423581125, |
| "learning_rate": 8.93915003219575e-07, |
| "logits/chosen": -0.3635665774345398, |
| "logits/rejected": -0.3023620545864105, |
| "logps/chosen": -410.5249938964844, |
| "logps/rejected": -384.20001220703125, |
| "loss": 0.4839, |
| "rewards/accuracies": 0.7250000238418579, |
| "rewards/chosen": -0.09310607612133026, |
| "rewards/margins": 1.2048461437225342, |
| "rewards/rejected": -1.2967650890350342, |
| "step": 660 |
| }, |
| { |
| "epoch": 0.43128419697457354, |
| "grad_norm": 94.10347466811871, |
| "learning_rate": 8.923052157115261e-07, |
| "logits/chosen": -0.4094909727573395, |
| "logits/rejected": -0.4021057188510895, |
| "logps/chosen": -414.54998779296875, |
| "logps/rejected": -401.32501220703125, |
| "loss": 0.5611, |
| "rewards/accuracies": 0.706250011920929, |
| "rewards/chosen": -0.547119140625, |
| "rewards/margins": 1.0107300281524658, |
| "rewards/rejected": -1.5586426258087158, |
| "step": 670 |
| }, |
| { |
| "epoch": 0.43772127454135823, |
| "grad_norm": 175.0582870965575, |
| "learning_rate": 8.90695428203477e-07, |
| "logits/chosen": -0.3139282166957855, |
| "logits/rejected": -0.2869323790073395, |
| "logps/chosen": -427.2749938964844, |
| "logps/rejected": -393.32501220703125, |
| "loss": 0.6173, |
| "rewards/accuracies": 0.706250011920929, |
| "rewards/chosen": -0.773022472858429, |
| "rewards/margins": 1.0076782703399658, |
| "rewards/rejected": -1.779541015625, |
| "step": 680 |
| }, |
| { |
| "epoch": 0.4441583521081429, |
| "grad_norm": 50.71727713444255, |
| "learning_rate": 8.890856406954281e-07, |
| "logits/chosen": -0.47215956449508667, |
| "logits/rejected": -0.3836425840854645, |
| "logps/chosen": -404.8999938964844, |
| "logps/rejected": -391.1000061035156, |
| "loss": 0.6279, |
| "rewards/accuracies": 0.6937500238418579, |
| "rewards/chosen": -0.34046632051467896, |
| "rewards/margins": 0.947644054889679, |
| "rewards/rejected": -1.288793921470642, |
| "step": 690 |
| }, |
| { |
| "epoch": 0.4505954296749276, |
| "grad_norm": 105.3352248318369, |
| "learning_rate": 8.874758531873793e-07, |
| "logits/chosen": -0.4835266172885895, |
| "logits/rejected": -0.434173583984375, |
| "logps/chosen": -388.54998779296875, |
| "logps/rejected": -413.3999938964844, |
| "loss": 0.5098, |
| "rewards/accuracies": 0.7124999761581421, |
| "rewards/chosen": -0.7118164300918579, |
| "rewards/margins": 1.2132079601287842, |
| "rewards/rejected": -1.926367163658142, |
| "step": 700 |
| }, |
| { |
| "epoch": 0.45703250724171224, |
| "grad_norm": 88.37902482946126, |
| "learning_rate": 8.858660656793303e-07, |
| "logits/chosen": -0.496194452047348, |
| "logits/rejected": -0.47770994901657104, |
| "logps/chosen": -432.4750061035156, |
| "logps/rejected": -435.57501220703125, |
| "loss": 0.52, |
| "rewards/accuracies": 0.731249988079071, |
| "rewards/chosen": -0.845947265625, |
| "rewards/margins": 1.1395752429962158, |
| "rewards/rejected": -1.98681640625, |
| "step": 710 |
| }, |
| { |
| "epoch": 0.4634695848084969, |
| "grad_norm": 77.67888124551067, |
| "learning_rate": 8.842562781712813e-07, |
| "logits/chosen": -0.460174560546875, |
| "logits/rejected": -0.4686080813407898, |
| "logps/chosen": -378.375, |
| "logps/rejected": -356.2250061035156, |
| "loss": 0.5882, |
| "rewards/accuracies": 0.6937500238418579, |
| "rewards/chosen": -0.4576416015625, |
| "rewards/margins": 0.9515380859375, |
| "rewards/rejected": -1.40814208984375, |
| "step": 720 |
| }, |
| { |
| "epoch": 0.4699066623752816, |
| "grad_norm": 90.44837256848525, |
| "learning_rate": 8.826464906632324e-07, |
| "logits/chosen": -0.536785900592804, |
| "logits/rejected": -0.4457763731479645, |
| "logps/chosen": -407.32501220703125, |
| "logps/rejected": -387.125, |
| "loss": 0.5575, |
| "rewards/accuracies": 0.6499999761581421, |
| "rewards/chosen": -0.03801880031824112, |
| "rewards/margins": 0.9344238042831421, |
| "rewards/rejected": -0.9721740484237671, |
| "step": 730 |
| }, |
| { |
| "epoch": 0.4763437399420663, |
| "grad_norm": 122.21877336180194, |
| "learning_rate": 8.810367031551835e-07, |
| "logits/chosen": -0.49167174100875854, |
| "logits/rejected": -0.45794677734375, |
| "logps/chosen": -437.375, |
| "logps/rejected": -380.2749938964844, |
| "loss": 0.5935, |
| "rewards/accuracies": 0.6625000238418579, |
| "rewards/chosen": -0.37174683809280396, |
| "rewards/margins": 0.9723144769668579, |
| "rewards/rejected": -1.3447265625, |
| "step": 740 |
| }, |
| { |
| "epoch": 0.482780817508851, |
| "grad_norm": 54.57698292330573, |
| "learning_rate": 8.794269156471346e-07, |
| "logits/chosen": -0.389068603515625, |
| "logits/rejected": -0.3659606873989105, |
| "logps/chosen": -387.01251220703125, |
| "logps/rejected": -370.7250061035156, |
| "loss": 0.5743, |
| "rewards/accuracies": 0.6812499761581421, |
| "rewards/chosen": 0.4018188416957855, |
| "rewards/margins": 1.0439453125, |
| "rewards/rejected": -0.6425536870956421, |
| "step": 750 |
| }, |
| { |
| "epoch": 0.4892178950756357, |
| "grad_norm": 108.4704052573543, |
| "learning_rate": 8.778171281390856e-07, |
| "logits/chosen": -0.32984620332717896, |
| "logits/rejected": -0.32059937715530396, |
| "logps/chosen": -405.2749938964844, |
| "logps/rejected": -385.75, |
| "loss": 0.6011, |
| "rewards/accuracies": 0.668749988079071, |
| "rewards/chosen": 0.867169201374054, |
| "rewards/margins": 0.833203136920929, |
| "rewards/rejected": 0.03310547024011612, |
| "step": 760 |
| }, |
| { |
| "epoch": 0.49565497264242037, |
| "grad_norm": 113.89195458322894, |
| "learning_rate": 8.762073406310366e-07, |
| "logits/chosen": -0.4198974668979645, |
| "logits/rejected": -0.2990661561489105, |
| "logps/chosen": -367.3500061035156, |
| "logps/rejected": -376.4750061035156, |
| "loss": 0.6727, |
| "rewards/accuracies": 0.606249988079071, |
| "rewards/chosen": 0.2831481993198395, |
| "rewards/margins": 0.803393542766571, |
| "rewards/rejected": -0.5206359624862671, |
| "step": 770 |
| }, |
| { |
| "epoch": 0.502092050209205, |
| "grad_norm": 106.63234570247351, |
| "learning_rate": 8.745975531229878e-07, |
| "logits/chosen": -0.511187732219696, |
| "logits/rejected": -0.39805299043655396, |
| "logps/chosen": -393.4750061035156, |
| "logps/rejected": -409.8500061035156, |
| "loss": 0.6488, |
| "rewards/accuracies": 0.7250000238418579, |
| "rewards/chosen": -0.47346192598342896, |
| "rewards/margins": 1.167700171470642, |
| "rewards/rejected": -1.642236351966858, |
| "step": 780 |
| }, |
| { |
| "epoch": 0.5085291277759897, |
| "grad_norm": 118.68916702899207, |
| "learning_rate": 8.729877656149389e-07, |
| "logits/chosen": -0.517321765422821, |
| "logits/rejected": -0.4344238340854645, |
| "logps/chosen": -386.0, |
| "logps/rejected": -381.79998779296875, |
| "loss": 0.5568, |
| "rewards/accuracies": 0.7250000238418579, |
| "rewards/chosen": -0.2811523377895355, |
| "rewards/margins": 1.21490478515625, |
| "rewards/rejected": -1.4958617687225342, |
| "step": 790 |
| }, |
| { |
| "epoch": 0.5149662053427744, |
| "grad_norm": 111.05063246601398, |
| "learning_rate": 8.713779781068898e-07, |
| "logits/chosen": -0.6059936285018921, |
| "logits/rejected": -0.43763428926467896, |
| "logps/chosen": -368.1000061035156, |
| "logps/rejected": -392.0, |
| "loss": 0.5174, |
| "rewards/accuracies": 0.731249988079071, |
| "rewards/chosen": -0.22099609673023224, |
| "rewards/margins": 1.20947265625, |
| "rewards/rejected": -1.4312012195587158, |
| "step": 800 |
| }, |
| { |
| "epoch": 0.5214032829095591, |
| "grad_norm": 134.76359626873952, |
| "learning_rate": 8.697681905988409e-07, |
| "logits/chosen": -0.41099852323532104, |
| "logits/rejected": -0.34346312284469604, |
| "logps/chosen": -442.3999938964844, |
| "logps/rejected": -411.125, |
| "loss": 0.6765, |
| "rewards/accuracies": 0.625, |
| "rewards/chosen": -0.4715026915073395, |
| "rewards/margins": 0.7641357183456421, |
| "rewards/rejected": -1.235327124595642, |
| "step": 810 |
| }, |
| { |
| "epoch": 0.5278403604763438, |
| "grad_norm": 87.4331158621849, |
| "learning_rate": 8.681584030907921e-07, |
| "logits/chosen": -0.5297912359237671, |
| "logits/rejected": -0.46336668729782104, |
| "logps/chosen": -436.5, |
| "logps/rejected": -380.875, |
| "loss": 0.6107, |
| "rewards/accuracies": 0.6625000238418579, |
| "rewards/chosen": -0.07208862155675888, |
| "rewards/margins": 0.814453125, |
| "rewards/rejected": -0.887280285358429, |
| "step": 820 |
| }, |
| { |
| "epoch": 0.5342774380431284, |
| "grad_norm": 97.84610962781957, |
| "learning_rate": 8.66548615582743e-07, |
| "logits/chosen": -0.2605529725551605, |
| "logits/rejected": -0.24983826279640198, |
| "logps/chosen": -381.17498779296875, |
| "logps/rejected": -379.57501220703125, |
| "loss": 0.6787, |
| "rewards/accuracies": 0.6312500238418579, |
| "rewards/chosen": -0.168365478515625, |
| "rewards/margins": 0.7323974370956421, |
| "rewards/rejected": -0.901623547077179, |
| "step": 830 |
| }, |
| { |
| "epoch": 0.5407145156099131, |
| "grad_norm": 125.91504124019485, |
| "learning_rate": 8.649388280746941e-07, |
| "logits/chosen": -0.2859359681606293, |
| "logits/rejected": -0.2567504942417145, |
| "logps/chosen": -426.9750061035156, |
| "logps/rejected": -394.67498779296875, |
| "loss": 0.5673, |
| "rewards/accuracies": 0.6937500238418579, |
| "rewards/chosen": 0.7361084222793579, |
| "rewards/margins": 0.9647582769393921, |
| "rewards/rejected": -0.22756347060203552, |
| "step": 840 |
| }, |
| { |
| "epoch": 0.5471515931766978, |
| "grad_norm": 75.95692268992741, |
| "learning_rate": 8.633290405666451e-07, |
| "logits/chosen": -0.3863723874092102, |
| "logits/rejected": -0.3452392518520355, |
| "logps/chosen": -380.29998779296875, |
| "logps/rejected": -372.5, |
| "loss": 0.6117, |
| "rewards/accuracies": 0.643750011920929, |
| "rewards/chosen": 0.423837274312973, |
| "rewards/margins": 0.813854992389679, |
| "rewards/rejected": -0.39080810546875, |
| "step": 850 |
| }, |
| { |
| "epoch": 0.5535886707434825, |
| "grad_norm": 91.75699602139787, |
| "learning_rate": 8.617192530585963e-07, |
| "logits/chosen": -0.3606933653354645, |
| "logits/rejected": -0.2834411561489105, |
| "logps/chosen": -438.625, |
| "logps/rejected": -439.375, |
| "loss": 0.6135, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": 0.11137084662914276, |
| "rewards/margins": 1.0670897960662842, |
| "rewards/rejected": -0.956835925579071, |
| "step": 860 |
| }, |
| { |
| "epoch": 0.5600257483102672, |
| "grad_norm": 103.29713595686344, |
| "learning_rate": 8.601094655505473e-07, |
| "logits/chosen": -0.43974608182907104, |
| "logits/rejected": -0.35341185331344604, |
| "logps/chosen": -408.4750061035156, |
| "logps/rejected": -420.125, |
| "loss": 0.6552, |
| "rewards/accuracies": 0.625, |
| "rewards/chosen": -0.21293334662914276, |
| "rewards/margins": 0.87274169921875, |
| "rewards/rejected": -1.0856201648712158, |
| "step": 870 |
| }, |
| { |
| "epoch": 0.5664628258770518, |
| "grad_norm": 111.65403956546032, |
| "learning_rate": 8.584996780424983e-07, |
| "logits/chosen": -0.4928222596645355, |
| "logits/rejected": -0.4177490174770355, |
| "logps/chosen": -390.67498779296875, |
| "logps/rejected": -373.375, |
| "loss": 0.5652, |
| "rewards/accuracies": 0.6625000238418579, |
| "rewards/chosen": -0.01723022386431694, |
| "rewards/margins": 0.909130871295929, |
| "rewards/rejected": -0.9266723394393921, |
| "step": 880 |
| }, |
| { |
| "epoch": 0.5728999034438365, |
| "grad_norm": 87.66305388602096, |
| "learning_rate": 8.568898905344494e-07, |
| "logits/chosen": -0.4807495176792145, |
| "logits/rejected": -0.4421447813510895, |
| "logps/chosen": -370.2250061035156, |
| "logps/rejected": -359.9750061035156, |
| "loss": 0.5363, |
| "rewards/accuracies": 0.6499999761581421, |
| "rewards/chosen": 0.06788329780101776, |
| "rewards/margins": 1.0357787609100342, |
| "rewards/rejected": -0.9671386480331421, |
| "step": 890 |
| }, |
| { |
| "epoch": 0.5793369810106211, |
| "grad_norm": 102.03952260297537, |
| "learning_rate": 8.552801030264006e-07, |
| "logits/chosen": -0.38992422819137573, |
| "logits/rejected": -0.3139907717704773, |
| "logps/chosen": -403.6875, |
| "logps/rejected": -385.67498779296875, |
| "loss": 0.6003, |
| "rewards/accuracies": 0.637499988079071, |
| "rewards/chosen": -0.638165295124054, |
| "rewards/margins": 0.8839966058731079, |
| "rewards/rejected": -1.521630883216858, |
| "step": 900 |
| }, |
| { |
| "epoch": 0.5857740585774058, |
| "grad_norm": 80.7078532242885, |
| "learning_rate": 8.536703155183515e-07, |
| "logits/chosen": -0.41375732421875, |
| "logits/rejected": -0.3028625547885895, |
| "logps/chosen": -437.8500061035156, |
| "logps/rejected": -443.67498779296875, |
| "loss": 0.6257, |
| "rewards/accuracies": 0.668749988079071, |
| "rewards/chosen": -0.715228259563446, |
| "rewards/margins": 1.041479468345642, |
| "rewards/rejected": -1.7564208507537842, |
| "step": 910 |
| }, |
| { |
| "epoch": 0.5922111361441905, |
| "grad_norm": 96.49118838896231, |
| "learning_rate": 8.520605280103026e-07, |
| "logits/chosen": -0.46844482421875, |
| "logits/rejected": -0.394622802734375, |
| "logps/chosen": -372.82501220703125, |
| "logps/rejected": -397.29998779296875, |
| "loss": 0.6404, |
| "rewards/accuracies": 0.59375, |
| "rewards/chosen": -0.819348156452179, |
| "rewards/margins": 0.7739013433456421, |
| "rewards/rejected": -1.5933349132537842, |
| "step": 920 |
| }, |
| { |
| "epoch": 0.5986482137109752, |
| "grad_norm": 129.53803628893053, |
| "learning_rate": 8.504507405022537e-07, |
| "logits/chosen": -0.3647918701171875, |
| "logits/rejected": -0.33575439453125, |
| "logps/chosen": -398.875, |
| "logps/rejected": -406.5, |
| "loss": 0.6191, |
| "rewards/accuracies": 0.675000011920929, |
| "rewards/chosen": -0.511859118938446, |
| "rewards/margins": 0.87646484375, |
| "rewards/rejected": -1.388574242591858, |
| "step": 930 |
| }, |
| { |
| "epoch": 0.6050852912777599, |
| "grad_norm": 66.14097164824625, |
| "learning_rate": 8.488409529942047e-07, |
| "logits/chosen": -0.5127807855606079, |
| "logits/rejected": -0.4483596682548523, |
| "logps/chosen": -383.13751220703125, |
| "logps/rejected": -380.57501220703125, |
| "loss": 0.5433, |
| "rewards/accuracies": 0.706250011920929, |
| "rewards/chosen": -0.20880126953125, |
| "rewards/margins": 1.076025366783142, |
| "rewards/rejected": -1.2833983898162842, |
| "step": 940 |
| }, |
| { |
| "epoch": 0.6115223688445446, |
| "grad_norm": 81.37765408363661, |
| "learning_rate": 8.472311654861558e-07, |
| "logits/chosen": -0.553210437297821, |
| "logits/rejected": -0.4498535096645355, |
| "logps/chosen": -416.01251220703125, |
| "logps/rejected": -411.3999938964844, |
| "loss": 0.5936, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": -0.11196289211511612, |
| "rewards/margins": 1.044183373451233, |
| "rewards/rejected": -1.1564514636993408, |
| "step": 950 |
| }, |
| { |
| "epoch": 0.6179594464113293, |
| "grad_norm": 97.50277556563883, |
| "learning_rate": 8.456213779781069e-07, |
| "logits/chosen": -0.5756286382675171, |
| "logits/rejected": -0.43685150146484375, |
| "logps/chosen": -352.9624938964844, |
| "logps/rejected": -374.8999938964844, |
| "loss": 0.6536, |
| "rewards/accuracies": 0.668749988079071, |
| "rewards/chosen": -0.16705933213233948, |
| "rewards/margins": 0.99993896484375, |
| "rewards/rejected": -1.167504906654358, |
| "step": 960 |
| }, |
| { |
| "epoch": 0.624396523978114, |
| "grad_norm": 123.65054219366036, |
| "learning_rate": 8.440115904700579e-07, |
| "logits/chosen": -0.441427618265152, |
| "logits/rejected": -0.43394166231155396, |
| "logps/chosen": -421.0249938964844, |
| "logps/rejected": -407.875, |
| "loss": 0.6009, |
| "rewards/accuracies": 0.643750011920929, |
| "rewards/chosen": 0.04934082180261612, |
| "rewards/margins": 1.0316894054412842, |
| "rewards/rejected": -0.9807189702987671, |
| "step": 970 |
| }, |
| { |
| "epoch": 0.6308336015448986, |
| "grad_norm": 111.92329982226788, |
| "learning_rate": 8.42401802962009e-07, |
| "logits/chosen": -0.414886474609375, |
| "logits/rejected": -0.33685302734375, |
| "logps/chosen": -420.3999938964844, |
| "logps/rejected": -406.45001220703125, |
| "loss": 0.495, |
| "rewards/accuracies": 0.7250000238418579, |
| "rewards/chosen": -0.03255004808306694, |
| "rewards/margins": 1.1223876476287842, |
| "rewards/rejected": -1.154748558998108, |
| "step": 980 |
| }, |
| { |
| "epoch": 0.6372706791116833, |
| "grad_norm": 138.39278292998316, |
| "learning_rate": 8.4079201545396e-07, |
| "logits/chosen": -0.537518322467804, |
| "logits/rejected": -0.4464355409145355, |
| "logps/chosen": -392.3500061035156, |
| "logps/rejected": -378.45001220703125, |
| "loss": 0.6678, |
| "rewards/accuracies": 0.65625, |
| "rewards/chosen": -0.09423828125, |
| "rewards/margins": 0.8354126214981079, |
| "rewards/rejected": -0.929370105266571, |
| "step": 990 |
| }, |
| { |
| "epoch": 0.643707756678468, |
| "grad_norm": 93.25190853268431, |
| "learning_rate": 8.391822279459111e-07, |
| "logits/chosen": -0.43569716811180115, |
| "logits/rejected": -0.413809210062027, |
| "logps/chosen": -411.92498779296875, |
| "logps/rejected": -398.32501220703125, |
| "loss": 0.5936, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": -0.24852904677391052, |
| "rewards/margins": 0.984545886516571, |
| "rewards/rejected": -1.232568383216858, |
| "step": 1000 |
| }, |
| { |
| "epoch": 0.6501448342452526, |
| "grad_norm": 111.30115525182288, |
| "learning_rate": 8.375724404378622e-07, |
| "logits/chosen": -0.431182861328125, |
| "logits/rejected": -0.44279783964157104, |
| "logps/chosen": -366.3500061035156, |
| "logps/rejected": -351.6000061035156, |
| "loss": 0.632, |
| "rewards/accuracies": 0.6625000238418579, |
| "rewards/chosen": -0.13744811713695526, |
| "rewards/margins": 0.904370129108429, |
| "rewards/rejected": -1.0412108898162842, |
| "step": 1010 |
| }, |
| { |
| "epoch": 0.6565819118120373, |
| "grad_norm": 79.98866194834122, |
| "learning_rate": 8.359626529298132e-07, |
| "logits/chosen": -0.4437606930732727, |
| "logits/rejected": -0.34916990995407104, |
| "logps/chosen": -390.875, |
| "logps/rejected": -379.57501220703125, |
| "loss": 0.5612, |
| "rewards/accuracies": 0.6812499761581421, |
| "rewards/chosen": -0.12116698920726776, |
| "rewards/margins": 1.0020020008087158, |
| "rewards/rejected": -1.123510718345642, |
| "step": 1020 |
| }, |
| { |
| "epoch": 0.663018989378822, |
| "grad_norm": 112.77159877359298, |
| "learning_rate": 8.343528654217643e-07, |
| "logits/chosen": -0.41150206327438354, |
| "logits/rejected": -0.35490721464157104, |
| "logps/chosen": -406.1499938964844, |
| "logps/rejected": -412.1499938964844, |
| "loss": 0.5863, |
| "rewards/accuracies": 0.6812499761581421, |
| "rewards/chosen": -0.333587646484375, |
| "rewards/margins": 1.02294921875, |
| "rewards/rejected": -1.356530785560608, |
| "step": 1030 |
| }, |
| { |
| "epoch": 0.6694560669456067, |
| "grad_norm": 94.66475003385241, |
| "learning_rate": 8.327430779137154e-07, |
| "logits/chosen": -0.5471832156181335, |
| "logits/rejected": -0.49498289823532104, |
| "logps/chosen": -405.1000061035156, |
| "logps/rejected": -382.75, |
| "loss": 0.6681, |
| "rewards/accuracies": 0.6187499761581421, |
| "rewards/chosen": -0.3744140565395355, |
| "rewards/margins": 0.890625, |
| "rewards/rejected": -1.265710473060608, |
| "step": 1040 |
| }, |
| { |
| "epoch": 0.6758931445123914, |
| "grad_norm": 113.3209291863091, |
| "learning_rate": 8.311332904056663e-07, |
| "logits/chosen": -0.4117431640625, |
| "logits/rejected": -0.3825225830078125, |
| "logps/chosen": -390.6000061035156, |
| "logps/rejected": -367.07501220703125, |
| "loss": 0.6439, |
| "rewards/accuracies": 0.668749988079071, |
| "rewards/chosen": 0.29993897676467896, |
| "rewards/margins": 0.8597656488418579, |
| "rewards/rejected": -0.559405505657196, |
| "step": 1050 |
| }, |
| { |
| "epoch": 0.682330222079176, |
| "grad_norm": 107.76705576693296, |
| "learning_rate": 8.295235028976175e-07, |
| "logits/chosen": -0.4195312559604645, |
| "logits/rejected": -0.320770263671875, |
| "logps/chosen": -364.67498779296875, |
| "logps/rejected": -381.95001220703125, |
| "loss": 0.641, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": 0.639392077922821, |
| "rewards/margins": 0.7391113042831421, |
| "rewards/rejected": -0.09996338188648224, |
| "step": 1060 |
| }, |
| { |
| "epoch": 0.6887672996459607, |
| "grad_norm": 77.47875882433111, |
| "learning_rate": 8.279137153895686e-07, |
| "logits/chosen": -0.3383941650390625, |
| "logits/rejected": -0.2616821229457855, |
| "logps/chosen": -406.0, |
| "logps/rejected": -407.9624938964844, |
| "loss": 0.5825, |
| "rewards/accuracies": 0.6499999761581421, |
| "rewards/chosen": 0.4314331114292145, |
| "rewards/margins": 1.0717284679412842, |
| "rewards/rejected": -0.641308605670929, |
| "step": 1070 |
| }, |
| { |
| "epoch": 0.6952043772127454, |
| "grad_norm": 76.5335562611094, |
| "learning_rate": 8.263039278815196e-07, |
| "logits/chosen": -0.556866466999054, |
| "logits/rejected": -0.565563976764679, |
| "logps/chosen": -384.8999938964844, |
| "logps/rejected": -364.32501220703125, |
| "loss": 0.5466, |
| "rewards/accuracies": 0.7250000238418579, |
| "rewards/chosen": -0.08623047173023224, |
| "rewards/margins": 1.076501488685608, |
| "rewards/rejected": -1.1626465320587158, |
| "step": 1080 |
| }, |
| { |
| "epoch": 0.7016414547795301, |
| "grad_norm": 115.11461300840443, |
| "learning_rate": 8.246941403734706e-07, |
| "logits/chosen": -0.3494495451450348, |
| "logits/rejected": -0.2852233946323395, |
| "logps/chosen": -389.57501220703125, |
| "logps/rejected": -434.2250061035156, |
| "loss": 0.6759, |
| "rewards/accuracies": 0.643750011920929, |
| "rewards/chosen": -0.4667602479457855, |
| "rewards/margins": 1.0223877429962158, |
| "rewards/rejected": -1.4884827136993408, |
| "step": 1090 |
| }, |
| { |
| "epoch": 0.7080785323463148, |
| "grad_norm": 71.06290101898851, |
| "learning_rate": 8.230843528654218e-07, |
| "logits/chosen": -0.32899171113967896, |
| "logits/rejected": -0.2647033631801605, |
| "logps/chosen": -367.1499938964844, |
| "logps/rejected": -361.1000061035156, |
| "loss": 0.5871, |
| "rewards/accuracies": 0.7124999761581421, |
| "rewards/chosen": -0.29622191190719604, |
| "rewards/margins": 1.125097632408142, |
| "rewards/rejected": -1.4209716320037842, |
| "step": 1100 |
| }, |
| { |
| "epoch": 0.7145156099130995, |
| "grad_norm": 166.9870480842849, |
| "learning_rate": 8.214745653573728e-07, |
| "logits/chosen": -0.23881225287914276, |
| "logits/rejected": -0.19378051161766052, |
| "logps/chosen": -444.25, |
| "logps/rejected": -428.42498779296875, |
| "loss": 0.5704, |
| "rewards/accuracies": 0.7124999761581421, |
| "rewards/chosen": -0.15664367377758026, |
| "rewards/margins": 1.1732177734375, |
| "rewards/rejected": -1.3302733898162842, |
| "step": 1110 |
| }, |
| { |
| "epoch": 0.7209526874798842, |
| "grad_norm": 124.491828151764, |
| "learning_rate": 8.198647778493239e-07, |
| "logits/chosen": -0.33013916015625, |
| "logits/rejected": -0.33750611543655396, |
| "logps/chosen": -405.875, |
| "logps/rejected": -386.6499938964844, |
| "loss": 0.564, |
| "rewards/accuracies": 0.737500011920929, |
| "rewards/chosen": 0.40095216035842896, |
| "rewards/margins": 1.118994116783142, |
| "rewards/rejected": -0.7183593511581421, |
| "step": 1120 |
| }, |
| { |
| "epoch": 0.7273897650466689, |
| "grad_norm": 110.50543265252429, |
| "learning_rate": 8.182549903412748e-07, |
| "logits/chosen": -0.589404284954071, |
| "logits/rejected": -0.531262218952179, |
| "logps/chosen": -369.25, |
| "logps/rejected": -372.875, |
| "loss": 0.6616, |
| "rewards/accuracies": 0.6499999761581421, |
| "rewards/chosen": 0.40489500761032104, |
| "rewards/margins": 0.701403796672821, |
| "rewards/rejected": -0.2970825135707855, |
| "step": 1130 |
| }, |
| { |
| "epoch": 0.7338268426134535, |
| "grad_norm": 90.54134404500911, |
| "learning_rate": 8.16645202833226e-07, |
| "logits/chosen": -0.606860339641571, |
| "logits/rejected": -0.4567016661167145, |
| "logps/chosen": -386.2749938964844, |
| "logps/rejected": -407.8500061035156, |
| "loss": 0.6294, |
| "rewards/accuracies": 0.668749988079071, |
| "rewards/chosen": 0.02239990234375, |
| "rewards/margins": 0.90380859375, |
| "rewards/rejected": -0.8811706304550171, |
| "step": 1140 |
| }, |
| { |
| "epoch": 0.7402639201802381, |
| "grad_norm": 88.36571147021662, |
| "learning_rate": 8.150354153251771e-07, |
| "logits/chosen": -0.674603283405304, |
| "logits/rejected": -0.5813964605331421, |
| "logps/chosen": -394.17498779296875, |
| "logps/rejected": -400.57501220703125, |
| "loss": 0.5355, |
| "rewards/accuracies": 0.6937500238418579, |
| "rewards/chosen": 0.03326416015625, |
| "rewards/margins": 1.2031738758087158, |
| "rewards/rejected": -1.169226050376892, |
| "step": 1150 |
| }, |
| { |
| "epoch": 0.7467009977470228, |
| "grad_norm": 100.58385711551767, |
| "learning_rate": 8.13425627817128e-07, |
| "logits/chosen": -0.5221923589706421, |
| "logits/rejected": -0.591015636920929, |
| "logps/chosen": -403.54998779296875, |
| "logps/rejected": -372.4750061035156, |
| "loss": 0.5793, |
| "rewards/accuracies": 0.668749988079071, |
| "rewards/chosen": 0.03043212927877903, |
| "rewards/margins": 1.0827758312225342, |
| "rewards/rejected": -1.051733374595642, |
| "step": 1160 |
| }, |
| { |
| "epoch": 0.7531380753138075, |
| "grad_norm": 112.17749936907586, |
| "learning_rate": 8.118158403090791e-07, |
| "logits/chosen": -0.3908630311489105, |
| "logits/rejected": -0.3272033631801605, |
| "logps/chosen": -403.5625, |
| "logps/rejected": -372.82501220703125, |
| "loss": 0.5899, |
| "rewards/accuracies": 0.65625, |
| "rewards/chosen": 0.306427001953125, |
| "rewards/margins": 0.9740966558456421, |
| "rewards/rejected": -0.6680663824081421, |
| "step": 1170 |
| }, |
| { |
| "epoch": 0.7595751528805922, |
| "grad_norm": 127.6681016766061, |
| "learning_rate": 8.102060528010303e-07, |
| "logits/chosen": -0.5384277105331421, |
| "logits/rejected": -0.44244384765625, |
| "logps/chosen": -417.7749938964844, |
| "logps/rejected": -427.9750061035156, |
| "loss": 0.5494, |
| "rewards/accuracies": 0.6812499761581421, |
| "rewards/chosen": -0.11136779934167862, |
| "rewards/margins": 1.2354004383087158, |
| "rewards/rejected": -1.345434546470642, |
| "step": 1180 |
| }, |
| { |
| "epoch": 0.7660122304473769, |
| "grad_norm": 129.67816740062835, |
| "learning_rate": 8.085962652929813e-07, |
| "logits/chosen": -0.5533996820449829, |
| "logits/rejected": -0.47967529296875, |
| "logps/chosen": -411.57501220703125, |
| "logps/rejected": -389.75, |
| "loss": 0.5818, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -0.500091552734375, |
| "rewards/margins": 1.339941382408142, |
| "rewards/rejected": -1.841210961341858, |
| "step": 1190 |
| }, |
| { |
| "epoch": 0.7724493080141616, |
| "grad_norm": 117.49110874092457, |
| "learning_rate": 8.069864777849323e-07, |
| "logits/chosen": -0.551342785358429, |
| "logits/rejected": -0.5241241455078125, |
| "logps/chosen": -392.8500061035156, |
| "logps/rejected": -422.04998779296875, |
| "loss": 0.7403, |
| "rewards/accuracies": 0.6499999761581421, |
| "rewards/chosen": -0.41949462890625, |
| "rewards/margins": 0.7759643793106079, |
| "rewards/rejected": -1.195459008216858, |
| "step": 1200 |
| }, |
| { |
| "epoch": 0.7788863855809462, |
| "grad_norm": 109.08732097693981, |
| "learning_rate": 8.053766902768834e-07, |
| "logits/chosen": -0.46064454317092896, |
| "logits/rejected": -0.485626220703125, |
| "logps/chosen": -373.01251220703125, |
| "logps/rejected": -372.8500061035156, |
| "loss": 0.568, |
| "rewards/accuracies": 0.637499988079071, |
| "rewards/chosen": 0.09330444037914276, |
| "rewards/margins": 0.924389660358429, |
| "rewards/rejected": -0.831207275390625, |
| "step": 1210 |
| }, |
| { |
| "epoch": 0.7853234631477309, |
| "grad_norm": 88.08741679244518, |
| "learning_rate": 8.037669027688345e-07, |
| "logits/chosen": -0.526684582233429, |
| "logits/rejected": -0.443817138671875, |
| "logps/chosen": -390.4125061035156, |
| "logps/rejected": -400.20001220703125, |
| "loss": 0.585, |
| "rewards/accuracies": 0.643750011920929, |
| "rewards/chosen": 0.05859375, |
| "rewards/margins": 1.100195288658142, |
| "rewards/rejected": -1.043003797531128, |
| "step": 1220 |
| }, |
| { |
| "epoch": 0.7917605407145156, |
| "grad_norm": 85.4114147916735, |
| "learning_rate": 8.021571152607856e-07, |
| "logits/chosen": -0.5202575922012329, |
| "logits/rejected": -0.497567743062973, |
| "logps/chosen": -388.7250061035156, |
| "logps/rejected": -388.82501220703125, |
| "loss": 0.6121, |
| "rewards/accuracies": 0.6499999761581421, |
| "rewards/chosen": -0.305633544921875, |
| "rewards/margins": 1.0255553722381592, |
| "rewards/rejected": -1.33056640625, |
| "step": 1230 |
| }, |
| { |
| "epoch": 0.7981976182813003, |
| "grad_norm": 90.00657522496898, |
| "learning_rate": 8.005473277527366e-07, |
| "logits/chosen": -0.48822021484375, |
| "logits/rejected": -0.3925414979457855, |
| "logps/chosen": -408.04998779296875, |
| "logps/rejected": -410.32501220703125, |
| "loss": 0.6074, |
| "rewards/accuracies": 0.6499999761581421, |
| "rewards/chosen": -0.34266358613967896, |
| "rewards/margins": 1.035668969154358, |
| "rewards/rejected": -1.377264380455017, |
| "step": 1240 |
| }, |
| { |
| "epoch": 0.804634695848085, |
| "grad_norm": 134.29669498014678, |
| "learning_rate": 7.989375402446876e-07, |
| "logits/chosen": -0.5795685052871704, |
| "logits/rejected": -0.4964965879917145, |
| "logps/chosen": -370.5249938964844, |
| "logps/rejected": -359.1499938964844, |
| "loss": 0.592, |
| "rewards/accuracies": 0.6812499761581421, |
| "rewards/chosen": -0.27702027559280396, |
| "rewards/margins": 1.0216064453125, |
| "rewards/rejected": -1.297296166419983, |
| "step": 1250 |
| }, |
| { |
| "epoch": 0.8110717734148697, |
| "grad_norm": 108.18684260484262, |
| "learning_rate": 7.973277527366388e-07, |
| "logits/chosen": -0.513226330280304, |
| "logits/rejected": -0.379913330078125, |
| "logps/chosen": -397.07501220703125, |
| "logps/rejected": -388.54998779296875, |
| "loss": 0.5746, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": -0.31059569120407104, |
| "rewards/margins": 1.1084473133087158, |
| "rewards/rejected": -1.4196045398712158, |
| "step": 1260 |
| }, |
| { |
| "epoch": 0.8175088509816544, |
| "grad_norm": 82.29538007059577, |
| "learning_rate": 7.957179652285899e-07, |
| "logits/chosen": -0.579394519329071, |
| "logits/rejected": -0.4920410215854645, |
| "logps/chosen": -375.67498779296875, |
| "logps/rejected": -382.1499938964844, |
| "loss": 0.5641, |
| "rewards/accuracies": 0.699999988079071, |
| "rewards/chosen": -0.262350469827652, |
| "rewards/margins": 0.981903076171875, |
| "rewards/rejected": -1.243798851966858, |
| "step": 1270 |
| }, |
| { |
| "epoch": 0.823945928548439, |
| "grad_norm": 70.67470972637592, |
| "learning_rate": 7.941081777205408e-07, |
| "logits/chosen": -0.411407470703125, |
| "logits/rejected": -0.3772827088832855, |
| "logps/chosen": -411.0874938964844, |
| "logps/rejected": -406.32501220703125, |
| "loss": 0.5696, |
| "rewards/accuracies": 0.675000011920929, |
| "rewards/chosen": -0.6337951421737671, |
| "rewards/margins": 1.3263428211212158, |
| "rewards/rejected": -1.959710717201233, |
| "step": 1280 |
| }, |
| { |
| "epoch": 0.8303830061152236, |
| "grad_norm": 66.33995522828697, |
| "learning_rate": 7.924983902124919e-07, |
| "logits/chosen": -0.4928649961948395, |
| "logits/rejected": -0.3884643614292145, |
| "logps/chosen": -400.17498779296875, |
| "logps/rejected": -394.5249938964844, |
| "loss": 0.4922, |
| "rewards/accuracies": 0.7437499761581421, |
| "rewards/chosen": -0.31853026151657104, |
| "rewards/margins": 1.355432152748108, |
| "rewards/rejected": -1.675195336341858, |
| "step": 1290 |
| }, |
| { |
| "epoch": 0.8368200836820083, |
| "grad_norm": 137.14048441459954, |
| "learning_rate": 7.908886027044431e-07, |
| "logits/chosen": -0.493743896484375, |
| "logits/rejected": -0.510913074016571, |
| "logps/chosen": -406.95001220703125, |
| "logps/rejected": -407.5249938964844, |
| "loss": 0.6906, |
| "rewards/accuracies": 0.668749988079071, |
| "rewards/chosen": -0.07345275580883026, |
| "rewards/margins": 0.911694347858429, |
| "rewards/rejected": -0.9843505620956421, |
| "step": 1300 |
| }, |
| { |
| "epoch": 0.843257161248793, |
| "grad_norm": 100.496851606368, |
| "learning_rate": 7.89278815196394e-07, |
| "logits/chosen": -0.5165466070175171, |
| "logits/rejected": -0.4143218994140625, |
| "logps/chosen": -380.13751220703125, |
| "logps/rejected": -397.29998779296875, |
| "loss": 0.5728, |
| "rewards/accuracies": 0.675000011920929, |
| "rewards/chosen": -0.074462890625, |
| "rewards/margins": 1.19537353515625, |
| "rewards/rejected": -1.2693359851837158, |
| "step": 1310 |
| }, |
| { |
| "epoch": 0.8496942388155777, |
| "grad_norm": 116.04790050118325, |
| "learning_rate": 7.876690276883451e-07, |
| "logits/chosen": -0.698803722858429, |
| "logits/rejected": -0.688671886920929, |
| "logps/chosen": -421.04998779296875, |
| "logps/rejected": -413.125, |
| "loss": 0.7025, |
| "rewards/accuracies": 0.6312500238418579, |
| "rewards/chosen": -0.4968017637729645, |
| "rewards/margins": 0.7909301519393921, |
| "rewards/rejected": -1.2884399890899658, |
| "step": 1320 |
| }, |
| { |
| "epoch": 0.8561313163823624, |
| "grad_norm": 100.22228137099864, |
| "learning_rate": 7.860592401802961e-07, |
| "logits/chosen": -0.582684338092804, |
| "logits/rejected": -0.553387463092804, |
| "logps/chosen": -432.4750061035156, |
| "logps/rejected": -412.70001220703125, |
| "loss": 0.6611, |
| "rewards/accuracies": 0.643750011920929, |
| "rewards/chosen": -0.3724426329135895, |
| "rewards/margins": 0.9818054437637329, |
| "rewards/rejected": -1.3540771007537842, |
| "step": 1330 |
| }, |
| { |
| "epoch": 0.8625683939491471, |
| "grad_norm": 125.79634444278314, |
| "learning_rate": 7.844494526722473e-07, |
| "logits/chosen": -0.588531494140625, |
| "logits/rejected": -0.48869627714157104, |
| "logps/chosen": -408.95001220703125, |
| "logps/rejected": -401.375, |
| "loss": 0.6163, |
| "rewards/accuracies": 0.6812499761581421, |
| "rewards/chosen": 0.09281005710363388, |
| "rewards/margins": 0.8982909917831421, |
| "rewards/rejected": -0.8051513433456421, |
| "step": 1340 |
| }, |
| { |
| "epoch": 0.8690054715159318, |
| "grad_norm": 96.97619232024756, |
| "learning_rate": 7.828396651641983e-07, |
| "logits/chosen": -0.48669129610061646, |
| "logits/rejected": -0.503399670124054, |
| "logps/chosen": -400.8999938964844, |
| "logps/rejected": -364.65625, |
| "loss": 0.541, |
| "rewards/accuracies": 0.7250000238418579, |
| "rewards/chosen": 0.749493420124054, |
| "rewards/margins": 1.071374535560608, |
| "rewards/rejected": -0.3219665586948395, |
| "step": 1350 |
| }, |
| { |
| "epoch": 0.8754425490827165, |
| "grad_norm": 50.91802213424051, |
| "learning_rate": 7.812298776561493e-07, |
| "logits/chosen": -0.41199952363967896, |
| "logits/rejected": -0.3683837950229645, |
| "logps/chosen": -421.1625061035156, |
| "logps/rejected": -407.70001220703125, |
| "loss": 0.5879, |
| "rewards/accuracies": 0.7250000238418579, |
| "rewards/chosen": 0.3733276426792145, |
| "rewards/margins": 0.9334716796875, |
| "rewards/rejected": -0.5606689453125, |
| "step": 1360 |
| }, |
| { |
| "epoch": 0.8818796266495011, |
| "grad_norm": 84.63648593866137, |
| "learning_rate": 7.796200901481004e-07, |
| "logits/chosen": -0.5473464727401733, |
| "logits/rejected": -0.453460693359375, |
| "logps/chosen": -361.1000061035156, |
| "logps/rejected": -371.1499938964844, |
| "loss": 0.6046, |
| "rewards/accuracies": 0.6937500238418579, |
| "rewards/chosen": -0.09481201320886612, |
| "rewards/margins": 0.9881957769393921, |
| "rewards/rejected": -1.0829894542694092, |
| "step": 1370 |
| }, |
| { |
| "epoch": 0.8883167042162858, |
| "grad_norm": 101.58494761812099, |
| "learning_rate": 7.780103026400516e-07, |
| "logits/chosen": -0.629565417766571, |
| "logits/rejected": -0.557690441608429, |
| "logps/chosen": -395.07501220703125, |
| "logps/rejected": -398.375, |
| "loss": 0.7146, |
| "rewards/accuracies": 0.6187499761581421, |
| "rewards/chosen": -0.5787719488143921, |
| "rewards/margins": 0.736572265625, |
| "rewards/rejected": -1.315161108970642, |
| "step": 1380 |
| }, |
| { |
| "epoch": 0.8947537817830705, |
| "grad_norm": 87.62319214477031, |
| "learning_rate": 7.764005151320025e-07, |
| "logits/chosen": -0.5127319097518921, |
| "logits/rejected": -0.43252867460250854, |
| "logps/chosen": -368.42498779296875, |
| "logps/rejected": -369.7250061035156, |
| "loss": 0.5064, |
| "rewards/accuracies": 0.6812499761581421, |
| "rewards/chosen": -0.35234373807907104, |
| "rewards/margins": 1.072973608970642, |
| "rewards/rejected": -1.4260375499725342, |
| "step": 1390 |
| }, |
| { |
| "epoch": 0.9011908593498552, |
| "grad_norm": 83.85747627730278, |
| "learning_rate": 7.747907276239536e-07, |
| "logits/chosen": -0.561938464641571, |
| "logits/rejected": -0.5323241949081421, |
| "logps/chosen": -406.7749938964844, |
| "logps/rejected": -393.75, |
| "loss": 0.6053, |
| "rewards/accuracies": 0.675000011920929, |
| "rewards/chosen": 0.02264404296875, |
| "rewards/margins": 0.788708508014679, |
| "rewards/rejected": -0.766406238079071, |
| "step": 1400 |
| }, |
| { |
| "epoch": 0.9076279369166399, |
| "grad_norm": 111.55283894267552, |
| "learning_rate": 7.731809401159047e-07, |
| "logits/chosen": -0.38446044921875, |
| "logits/rejected": -0.36006468534469604, |
| "logps/chosen": -403.9125061035156, |
| "logps/rejected": -411.3999938964844, |
| "loss": 0.704, |
| "rewards/accuracies": 0.612500011920929, |
| "rewards/chosen": -0.322662353515625, |
| "rewards/margins": 0.836181640625, |
| "rewards/rejected": -1.159490942955017, |
| "step": 1410 |
| }, |
| { |
| "epoch": 0.9140650144834245, |
| "grad_norm": 81.33181594569466, |
| "learning_rate": 7.715711526078557e-07, |
| "logits/chosen": -0.4296203553676605, |
| "logits/rejected": -0.4065185487270355, |
| "logps/chosen": -376.75, |
| "logps/rejected": -370.1000061035156, |
| "loss": 0.5695, |
| "rewards/accuracies": 0.643750011920929, |
| "rewards/chosen": 0.09414062649011612, |
| "rewards/margins": 1.0592772960662842, |
| "rewards/rejected": -0.965771496295929, |
| "step": 1420 |
| }, |
| { |
| "epoch": 0.9205020920502092, |
| "grad_norm": 127.5199541941311, |
| "learning_rate": 7.699613650998068e-07, |
| "logits/chosen": -0.518847644329071, |
| "logits/rejected": -0.4928344786167145, |
| "logps/chosen": -398.2250061035156, |
| "logps/rejected": -400.6499938964844, |
| "loss": 0.6342, |
| "rewards/accuracies": 0.65625, |
| "rewards/chosen": 0.19356688857078552, |
| "rewards/margins": 0.814056396484375, |
| "rewards/rejected": -0.621685802936554, |
| "step": 1430 |
| }, |
| { |
| "epoch": 0.9269391696169939, |
| "grad_norm": 105.59882986085388, |
| "learning_rate": 7.683515775917579e-07, |
| "logits/chosen": -0.591418445110321, |
| "logits/rejected": -0.5198699831962585, |
| "logps/chosen": -386.79998779296875, |
| "logps/rejected": -398.45001220703125, |
| "loss": 0.6236, |
| "rewards/accuracies": 0.6187499761581421, |
| "rewards/chosen": -0.12529906630516052, |
| "rewards/margins": 0.8775573968887329, |
| "rewards/rejected": -1.0029296875, |
| "step": 1440 |
| }, |
| { |
| "epoch": 0.9333762471837785, |
| "grad_norm": 106.76321232568708, |
| "learning_rate": 7.667417900837089e-07, |
| "logits/chosen": -0.515380859375, |
| "logits/rejected": -0.508331298828125, |
| "logps/chosen": -400.29998779296875, |
| "logps/rejected": -381.04998779296875, |
| "loss": 0.6011, |
| "rewards/accuracies": 0.699999988079071, |
| "rewards/chosen": -0.46058350801467896, |
| "rewards/margins": 1.276391625404358, |
| "rewards/rejected": -1.736718773841858, |
| "step": 1450 |
| }, |
| { |
| "epoch": 0.9398133247505632, |
| "grad_norm": 86.114741385391, |
| "learning_rate": 7.6513200257566e-07, |
| "logits/chosen": -0.5946594476699829, |
| "logits/rejected": -0.632550060749054, |
| "logps/chosen": -411.7250061035156, |
| "logps/rejected": -386.75, |
| "loss": 0.6807, |
| "rewards/accuracies": 0.668749988079071, |
| "rewards/chosen": -0.6214355230331421, |
| "rewards/margins": 1.0218627452850342, |
| "rewards/rejected": -1.6437499523162842, |
| "step": 1460 |
| }, |
| { |
| "epoch": 0.9462504023173479, |
| "grad_norm": 120.05871392937466, |
| "learning_rate": 7.63522215067611e-07, |
| "logits/chosen": -0.4129577577114105, |
| "logits/rejected": -0.4022201597690582, |
| "logps/chosen": -414.25, |
| "logps/rejected": -421.57501220703125, |
| "loss": 0.5926, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": -0.5174010992050171, |
| "rewards/margins": 0.9172607660293579, |
| "rewards/rejected": -1.4355347156524658, |
| "step": 1470 |
| }, |
| { |
| "epoch": 0.9526874798841326, |
| "grad_norm": 113.01818753168125, |
| "learning_rate": 7.619124275595621e-07, |
| "logits/chosen": -0.5003722906112671, |
| "logits/rejected": -0.5239623785018921, |
| "logps/chosen": -432.6499938964844, |
| "logps/rejected": -437.6000061035156, |
| "loss": 0.5754, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": -0.4602111876010895, |
| "rewards/margins": 0.9864257574081421, |
| "rewards/rejected": -1.4470946788787842, |
| "step": 1480 |
| }, |
| { |
| "epoch": 0.9591245574509173, |
| "grad_norm": 122.1368477312503, |
| "learning_rate": 7.603026400515131e-07, |
| "logits/chosen": -0.6516479253768921, |
| "logits/rejected": -0.61376953125, |
| "logps/chosen": -432.625, |
| "logps/rejected": -431.8500061035156, |
| "loss": 0.5486, |
| "rewards/accuracies": 0.706250011920929, |
| "rewards/chosen": -0.658294677734375, |
| "rewards/margins": 1.1542479991912842, |
| "rewards/rejected": -1.810888648033142, |
| "step": 1490 |
| }, |
| { |
| "epoch": 0.965561635017702, |
| "grad_norm": 86.20161715780311, |
| "learning_rate": 7.586928525434642e-07, |
| "logits/chosen": -0.38746339082717896, |
| "logits/rejected": -0.42163389921188354, |
| "logps/chosen": -413.32501220703125, |
| "logps/rejected": -402.8500061035156, |
| "loss": 0.5953, |
| "rewards/accuracies": 0.6187499761581421, |
| "rewards/chosen": -0.622509777545929, |
| "rewards/margins": 1.0743408203125, |
| "rewards/rejected": -1.697534203529358, |
| "step": 1500 |
| }, |
| { |
| "epoch": 0.9719987125844867, |
| "grad_norm": 99.30873180754713, |
| "learning_rate": 7.570830650354153e-07, |
| "logits/chosen": -0.606640636920929, |
| "logits/rejected": -0.558026134967804, |
| "logps/chosen": -387.8999938964844, |
| "logps/rejected": -382.3500061035156, |
| "loss": 0.5288, |
| "rewards/accuracies": 0.731249988079071, |
| "rewards/chosen": 0.001861572265625, |
| "rewards/margins": 1.194787621498108, |
| "rewards/rejected": -1.1917235851287842, |
| "step": 1510 |
| }, |
| { |
| "epoch": 0.9784357901512714, |
| "grad_norm": 106.16738318159852, |
| "learning_rate": 7.554732775273664e-07, |
| "logits/chosen": -0.563079833984375, |
| "logits/rejected": -0.569580078125, |
| "logps/chosen": -377.875, |
| "logps/rejected": -406.20001220703125, |
| "loss": 0.6417, |
| "rewards/accuracies": 0.6937500238418579, |
| "rewards/chosen": 0.25871580839157104, |
| "rewards/margins": 0.8615356683731079, |
| "rewards/rejected": -0.6027892827987671, |
| "step": 1520 |
| }, |
| { |
| "epoch": 0.984872867718056, |
| "grad_norm": 95.76390458870594, |
| "learning_rate": 7.538634900193173e-07, |
| "logits/chosen": -0.6464599370956421, |
| "logits/rejected": -0.660693347454071, |
| "logps/chosen": -382.92498779296875, |
| "logps/rejected": -366.92498779296875, |
| "loss": 0.5379, |
| "rewards/accuracies": 0.699999988079071, |
| "rewards/chosen": 0.21351318061351776, |
| "rewards/margins": 1.159826636314392, |
| "rewards/rejected": -0.946704089641571, |
| "step": 1530 |
| }, |
| { |
| "epoch": 0.9913099452848407, |
| "grad_norm": 76.0710448944881, |
| "learning_rate": 7.522537025112685e-07, |
| "logits/chosen": -0.5799926519393921, |
| "logits/rejected": -0.603985607624054, |
| "logps/chosen": -397.8500061035156, |
| "logps/rejected": -408.20001220703125, |
| "loss": 0.472, |
| "rewards/accuracies": 0.762499988079071, |
| "rewards/chosen": -0.10820923000574112, |
| "rewards/margins": 1.3258178234100342, |
| "rewards/rejected": -1.434454321861267, |
| "step": 1540 |
| }, |
| { |
| "epoch": 0.9977470228516253, |
| "grad_norm": 93.32557561962946, |
| "learning_rate": 7.506439150032196e-07, |
| "logits/chosen": -0.4845016598701477, |
| "logits/rejected": -0.4236694276332855, |
| "logps/chosen": -425.95001220703125, |
| "logps/rejected": -426.3999938964844, |
| "loss": 0.5096, |
| "rewards/accuracies": 0.7437499761581421, |
| "rewards/chosen": -0.77288818359375, |
| "rewards/margins": 1.4271972179412842, |
| "rewards/rejected": -2.2017579078674316, |
| "step": 1550 |
| }, |
| { |
| "epoch": 1.0038622465400708, |
| "grad_norm": 9.50912731413506, |
| "learning_rate": 7.490341274951706e-07, |
| "logits/chosen": -0.5405594706535339, |
| "logits/rejected": -0.5138292908668518, |
| "logps/chosen": -389.7894592285156, |
| "logps/rejected": -399.7105407714844, |
| "loss": 0.2632, |
| "rewards/accuracies": 0.8552631735801697, |
| "rewards/chosen": 0.8779168128967285, |
| "rewards/margins": 3.9738898277282715, |
| "rewards/rejected": -3.09220814704895, |
| "step": 1560 |
| }, |
| { |
| "epoch": 1.0102993241068554, |
| "grad_norm": 2.5775668594464056, |
| "learning_rate": 7.474243399871216e-07, |
| "logits/chosen": -0.461325079202652, |
| "logits/rejected": -0.45458984375, |
| "logps/chosen": -402.7875061035156, |
| "logps/rejected": -451.29998779296875, |
| "loss": 0.0428, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": 1.9087402820587158, |
| "rewards/margins": 6.967968940734863, |
| "rewards/rejected": -5.064453125, |
| "step": 1570 |
| }, |
| { |
| "epoch": 1.0167364016736402, |
| "grad_norm": 4.95068747477217, |
| "learning_rate": 7.458145524790728e-07, |
| "logits/chosen": -0.824658215045929, |
| "logits/rejected": -0.7963622808456421, |
| "logps/chosen": -372.0, |
| "logps/rejected": -424.07501220703125, |
| "loss": 0.0222, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": 1.25982666015625, |
| "rewards/margins": 6.735547065734863, |
| "rewards/rejected": -5.477929592132568, |
| "step": 1580 |
| }, |
| { |
| "epoch": 1.0231734792404248, |
| "grad_norm": 4.905295674057439, |
| "learning_rate": 7.442047649710238e-07, |
| "logits/chosen": -0.8260864019393921, |
| "logits/rejected": -0.7978149652481079, |
| "logps/chosen": -351.17498779296875, |
| "logps/rejected": -405.95001220703125, |
| "loss": 0.0479, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": 0.3309326171875, |
| "rewards/margins": 7.040234565734863, |
| "rewards/rejected": -6.708593845367432, |
| "step": 1590 |
| }, |
| { |
| "epoch": 1.0296105568072096, |
| "grad_norm": 12.17548781312339, |
| "learning_rate": 7.425949774629749e-07, |
| "logits/chosen": -0.8370605707168579, |
| "logits/rejected": -0.80029296875, |
| "logps/chosen": -419.3999938964844, |
| "logps/rejected": -455.375, |
| "loss": 0.0311, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": 0.5588623285293579, |
| "rewards/margins": 8.084375381469727, |
| "rewards/rejected": -7.52734375, |
| "step": 1600 |
| }, |
| { |
| "epoch": 1.0360476343739942, |
| "grad_norm": 8.010004501370812, |
| "learning_rate": 7.409851899549258e-07, |
| "logits/chosen": -0.905322253704071, |
| "logits/rejected": -0.7925049066543579, |
| "logps/chosen": -382.04998779296875, |
| "logps/rejected": -436.42498779296875, |
| "loss": 0.0456, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -0.29853516817092896, |
| "rewards/margins": 7.053124904632568, |
| "rewards/rejected": -7.353125095367432, |
| "step": 1610 |
| }, |
| { |
| "epoch": 1.042484711940779, |
| "grad_norm": 8.732308996703201, |
| "learning_rate": 7.39375402446877e-07, |
| "logits/chosen": -1.030371069908142, |
| "logits/rejected": -1.013916015625, |
| "logps/chosen": -378.20001220703125, |
| "logps/rejected": -450.3500061035156, |
| "loss": 0.058, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -0.46614378690719604, |
| "rewards/margins": 7.793749809265137, |
| "rewards/rejected": -8.259765625, |
| "step": 1620 |
| }, |
| { |
| "epoch": 1.0489217895075635, |
| "grad_norm": 19.427693209919855, |
| "learning_rate": 7.377656149388281e-07, |
| "logits/chosen": -1.0540039539337158, |
| "logits/rejected": -1.018334984779358, |
| "logps/chosen": -421.75, |
| "logps/rejected": -484.2250061035156, |
| "loss": 0.0313, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -1.264062523841858, |
| "rewards/margins": 7.595312595367432, |
| "rewards/rejected": -8.857812881469727, |
| "step": 1630 |
| }, |
| { |
| "epoch": 1.0553588670743483, |
| "grad_norm": 15.795151791665331, |
| "learning_rate": 7.36155827430779e-07, |
| "logits/chosen": -1.109399437904358, |
| "logits/rejected": -1.030297875404358, |
| "logps/chosen": -401.98748779296875, |
| "logps/rejected": -455.2749938964844, |
| "loss": 0.0463, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -0.14042969048023224, |
| "rewards/margins": 7.012499809265137, |
| "rewards/rejected": -7.153515815734863, |
| "step": 1640 |
| }, |
| { |
| "epoch": 1.061795944641133, |
| "grad_norm": 6.0961203516699705, |
| "learning_rate": 7.345460399227301e-07, |
| "logits/chosen": -0.962207019329071, |
| "logits/rejected": -0.8519836664199829, |
| "logps/chosen": -408.0249938964844, |
| "logps/rejected": -468.54998779296875, |
| "loss": 0.0202, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": 1.3169066905975342, |
| "rewards/margins": 8.044530868530273, |
| "rewards/rejected": -6.726953029632568, |
| "step": 1650 |
| }, |
| { |
| "epoch": 1.0682330222079175, |
| "grad_norm": 45.25198219601515, |
| "learning_rate": 7.329362524146813e-07, |
| "logits/chosen": -1.0541870594024658, |
| "logits/rejected": -0.8856567144393921, |
| "logps/chosen": -365.29998779296875, |
| "logps/rejected": -451.875, |
| "loss": 0.0427, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": 0.071807861328125, |
| "rewards/margins": 7.198046684265137, |
| "rewards/rejected": -7.125781059265137, |
| "step": 1660 |
| }, |
| { |
| "epoch": 1.0746700997747023, |
| "grad_norm": 1.944497198300728, |
| "learning_rate": 7.313264649066323e-07, |
| "logits/chosen": -1.158349633216858, |
| "logits/rejected": -1.0755188465118408, |
| "logps/chosen": -401.3999938964844, |
| "logps/rejected": -458.7250061035156, |
| "loss": 0.02, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": 0.23220214247703552, |
| "rewards/margins": 8.360937118530273, |
| "rewards/rejected": -8.125781059265137, |
| "step": 1670 |
| }, |
| { |
| "epoch": 1.0811071773414869, |
| "grad_norm": 28.97664404589722, |
| "learning_rate": 7.297166773985833e-07, |
| "logits/chosen": -1.06744384765625, |
| "logits/rejected": -0.9425598382949829, |
| "logps/chosen": -391.29998779296875, |
| "logps/rejected": -484.29998779296875, |
| "loss": 0.0268, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -0.704028308391571, |
| "rewards/margins": 8.478906631469727, |
| "rewards/rejected": -9.181249618530273, |
| "step": 1680 |
| }, |
| { |
| "epoch": 1.0875442549082717, |
| "grad_norm": 1.517312981756122, |
| "learning_rate": 7.281068898905344e-07, |
| "logits/chosen": -1.2833251953125, |
| "logits/rejected": -1.215576171875, |
| "logps/chosen": -421.125, |
| "logps/rejected": -513.0499877929688, |
| "loss": 0.0239, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -1.653222680091858, |
| "rewards/margins": 8.629687309265137, |
| "rewards/rejected": -10.280468940734863, |
| "step": 1690 |
| }, |
| { |
| "epoch": 1.0939813324750562, |
| "grad_norm": 58.53522510792332, |
| "learning_rate": 7.264971023824855e-07, |
| "logits/chosen": -1.225439429283142, |
| "logits/rejected": -1.212548851966858, |
| "logps/chosen": -389.76251220703125, |
| "logps/rejected": -454.42498779296875, |
| "loss": 0.0443, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -2.330340623855591, |
| "rewards/margins": 8.215624809265137, |
| "rewards/rejected": -10.545312881469727, |
| "step": 1700 |
| }, |
| { |
| "epoch": 1.100418410041841, |
| "grad_norm": 10.209902191563296, |
| "learning_rate": 7.248873148744366e-07, |
| "logits/chosen": -1.172216773033142, |
| "logits/rejected": -1.174768090248108, |
| "logps/chosen": -442.8999938964844, |
| "logps/rejected": -483.5, |
| "loss": 0.0329, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -3.423828125, |
| "rewards/margins": 8.693750381469727, |
| "rewards/rejected": -12.121874809265137, |
| "step": 1710 |
| }, |
| { |
| "epoch": 1.1068554876086256, |
| "grad_norm": 6.412558464603696, |
| "learning_rate": 7.232775273663876e-07, |
| "logits/chosen": -1.18695068359375, |
| "logits/rejected": -1.145483374595642, |
| "logps/chosen": -446.07501220703125, |
| "logps/rejected": -516.9500122070312, |
| "loss": 0.0462, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -1.7499511241912842, |
| "rewards/margins": 9.45703125, |
| "rewards/rejected": -11.206250190734863, |
| "step": 1720 |
| }, |
| { |
| "epoch": 1.1132925651754104, |
| "grad_norm": 6.194697986377926, |
| "learning_rate": 7.216677398583386e-07, |
| "logits/chosen": -1.2653319835662842, |
| "logits/rejected": -1.1570556163787842, |
| "logps/chosen": -385.2875061035156, |
| "logps/rejected": -478.7250061035156, |
| "loss": 0.0395, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -1.419775366783142, |
| "rewards/margins": 9.310937881469727, |
| "rewards/rejected": -10.722265243530273, |
| "step": 1730 |
| }, |
| { |
| "epoch": 1.119729642742195, |
| "grad_norm": 18.76884531324279, |
| "learning_rate": 7.200579523502898e-07, |
| "logits/chosen": -0.91015625, |
| "logits/rejected": -0.889569103717804, |
| "logps/chosen": -409.9375, |
| "logps/rejected": -524.0750122070312, |
| "loss": 0.0629, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -0.8971313238143921, |
| "rewards/margins": 9.75390625, |
| "rewards/rejected": -10.6484375, |
| "step": 1740 |
| }, |
| { |
| "epoch": 1.1261667203089798, |
| "grad_norm": 14.602574707428438, |
| "learning_rate": 7.184481648422408e-07, |
| "logits/chosen": -1.213189721107483, |
| "logits/rejected": -1.109521508216858, |
| "logps/chosen": -387.75, |
| "logps/rejected": -466.17498779296875, |
| "loss": 0.0592, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -0.6850067377090454, |
| "rewards/margins": 8.534375190734863, |
| "rewards/rejected": -9.213281631469727, |
| "step": 1750 |
| }, |
| { |
| "epoch": 1.1326037978757644, |
| "grad_norm": 3.094115436762287, |
| "learning_rate": 7.168383773341918e-07, |
| "logits/chosen": -1.167382836341858, |
| "logits/rejected": -1.086084008216858, |
| "logps/chosen": -416.79998779296875, |
| "logps/rejected": -465.6000061035156, |
| "loss": 0.0345, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -0.8143554925918579, |
| "rewards/margins": 7.749218940734863, |
| "rewards/rejected": -8.564062118530273, |
| "step": 1760 |
| }, |
| { |
| "epoch": 1.1390408754425492, |
| "grad_norm": 10.132894836108427, |
| "learning_rate": 7.152285898261429e-07, |
| "logits/chosen": -1.207275390625, |
| "logits/rejected": -1.1055176258087158, |
| "logps/chosen": -394.67498779296875, |
| "logps/rejected": -474.95001220703125, |
| "loss": 0.0675, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -0.510723888874054, |
| "rewards/margins": 8.532031059265137, |
| "rewards/rejected": -9.044530868530273, |
| "step": 1770 |
| }, |
| { |
| "epoch": 1.1454779530093337, |
| "grad_norm": 10.402058711985868, |
| "learning_rate": 7.136188023180941e-07, |
| "logits/chosen": -1.149804711341858, |
| "logits/rejected": -1.090429663658142, |
| "logps/chosen": -412.04998779296875, |
| "logps/rejected": -478.375, |
| "loss": 0.0272, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -0.9574218988418579, |
| "rewards/margins": 8.774218559265137, |
| "rewards/rejected": -9.733593940734863, |
| "step": 1780 |
| }, |
| { |
| "epoch": 1.1519150305761183, |
| "grad_norm": 9.618457091702922, |
| "learning_rate": 7.12009014810045e-07, |
| "logits/chosen": -1.2156250476837158, |
| "logits/rejected": -1.124487280845642, |
| "logps/chosen": -433.54998779296875, |
| "logps/rejected": -492.70001220703125, |
| "loss": 0.0305, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -2.349658250808716, |
| "rewards/margins": 9.159375190734863, |
| "rewards/rejected": -11.506250381469727, |
| "step": 1790 |
| }, |
| { |
| "epoch": 1.1583521081429031, |
| "grad_norm": 51.505885719901904, |
| "learning_rate": 7.103992273019961e-07, |
| "logits/chosen": -1.0395629405975342, |
| "logits/rejected": -1.0539062023162842, |
| "logps/chosen": -441.7749938964844, |
| "logps/rejected": -504.1000061035156, |
| "loss": 0.0509, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -1.985009789466858, |
| "rewards/margins": 9.35546875, |
| "rewards/rejected": -11.33984375, |
| "step": 1800 |
| }, |
| { |
| "epoch": 1.164789185709688, |
| "grad_norm": 17.55209647214659, |
| "learning_rate": 7.087894397939471e-07, |
| "logits/chosen": -1.1386840343475342, |
| "logits/rejected": -1.068383812904358, |
| "logps/chosen": -419.9375, |
| "logps/rejected": -479.0, |
| "loss": 0.0465, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -1.9088013172149658, |
| "rewards/margins": 8.859375, |
| "rewards/rejected": -10.772656440734863, |
| "step": 1810 |
| }, |
| { |
| "epoch": 1.1712262632764725, |
| "grad_norm": 6.571822176847439, |
| "learning_rate": 7.071796522858983e-07, |
| "logits/chosen": -1.186547875404358, |
| "logits/rejected": -1.1846923828125, |
| "logps/chosen": -429.7250061035156, |
| "logps/rejected": -510.3999938964844, |
| "loss": 0.0325, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -2.0152587890625, |
| "rewards/margins": 9.216796875, |
| "rewards/rejected": -11.228906631469727, |
| "step": 1820 |
| }, |
| { |
| "epoch": 1.177663340843257, |
| "grad_norm": 0.3676066268885366, |
| "learning_rate": 7.055698647778493e-07, |
| "logits/chosen": -1.108911156654358, |
| "logits/rejected": -1.08203125, |
| "logps/chosen": -376.25, |
| "logps/rejected": -438.95001220703125, |
| "loss": 0.0673, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -1.5198547840118408, |
| "rewards/margins": 9.435155868530273, |
| "rewards/rejected": -10.961718559265137, |
| "step": 1830 |
| }, |
| { |
| "epoch": 1.1841004184100419, |
| "grad_norm": 2.125598068147854, |
| "learning_rate": 7.039600772698003e-07, |
| "logits/chosen": -1.168554663658142, |
| "logits/rejected": -1.048956274986267, |
| "logps/chosen": -396.26251220703125, |
| "logps/rejected": -497.45001220703125, |
| "loss": 0.0404, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -1.5553467273712158, |
| "rewards/margins": 9.450780868530273, |
| "rewards/rejected": -11.004687309265137, |
| "step": 1840 |
| }, |
| { |
| "epoch": 1.1905374959768265, |
| "grad_norm": 0.9847677719024663, |
| "learning_rate": 7.023502897617514e-07, |
| "logits/chosen": -1.0049316883087158, |
| "logits/rejected": -1.020361304283142, |
| "logps/chosen": -432.57501220703125, |
| "logps/rejected": -481.1000061035156, |
| "loss": 0.035, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -1.0696899890899658, |
| "rewards/margins": 8.899218559265137, |
| "rewards/rejected": -9.970312118530273, |
| "step": 1850 |
| }, |
| { |
| "epoch": 1.1969745735436113, |
| "grad_norm": 10.926894639122786, |
| "learning_rate": 7.007405022537025e-07, |
| "logits/chosen": -1.159753441810608, |
| "logits/rejected": -1.1105468273162842, |
| "logps/chosen": -382.26251220703125, |
| "logps/rejected": -479.1499938964844, |
| "loss": 0.0432, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -1.564386010169983, |
| "rewards/margins": 9.151171684265137, |
| "rewards/rejected": -10.711718559265137, |
| "step": 1860 |
| }, |
| { |
| "epoch": 1.2034116511103958, |
| "grad_norm": 22.471944298023597, |
| "learning_rate": 6.991307147456535e-07, |
| "logits/chosen": -1.157171607017517, |
| "logits/rejected": -1.1033935546875, |
| "logps/chosen": -407.3999938964844, |
| "logps/rejected": -464.625, |
| "loss": 0.0363, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -1.2672119140625, |
| "rewards/margins": 8.650390625, |
| "rewards/rejected": -9.915624618530273, |
| "step": 1870 |
| }, |
| { |
| "epoch": 1.2098487286771806, |
| "grad_norm": 32.325103664777046, |
| "learning_rate": 6.975209272376046e-07, |
| "logits/chosen": -1.300506591796875, |
| "logits/rejected": -1.2924315929412842, |
| "logps/chosen": -378.26251220703125, |
| "logps/rejected": -446.67498779296875, |
| "loss": 0.034, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -0.806591808795929, |
| "rewards/margins": 8.588671684265137, |
| "rewards/rejected": -9.397656440734863, |
| "step": 1880 |
| }, |
| { |
| "epoch": 1.2162858062439652, |
| "grad_norm": 1.3775173281719182, |
| "learning_rate": 6.959111397295557e-07, |
| "logits/chosen": -1.290136694908142, |
| "logits/rejected": -1.186279296875, |
| "logps/chosen": -358.61248779296875, |
| "logps/rejected": -473.2250061035156, |
| "loss": 0.017, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -1.0308837890625, |
| "rewards/margins": 8.96484375, |
| "rewards/rejected": -9.995312690734863, |
| "step": 1890 |
| }, |
| { |
| "epoch": 1.22272288381075, |
| "grad_norm": 2.507719213393242, |
| "learning_rate": 6.943013522215067e-07, |
| "logits/chosen": -1.1824219226837158, |
| "logits/rejected": -1.1527831554412842, |
| "logps/chosen": -413.125, |
| "logps/rejected": -463.6000061035156, |
| "loss": 0.0423, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -0.471221923828125, |
| "rewards/margins": 8.40234375, |
| "rewards/rejected": -8.869531631469727, |
| "step": 1900 |
| }, |
| { |
| "epoch": 1.2291599613775346, |
| "grad_norm": 3.332872366437569, |
| "learning_rate": 6.926915647134578e-07, |
| "logits/chosen": -1.093164086341858, |
| "logits/rejected": -1.084130883216858, |
| "logps/chosen": -416.95001220703125, |
| "logps/rejected": -494.6000061035156, |
| "loss": 0.0504, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": 0.06691894680261612, |
| "rewards/margins": 9.274609565734863, |
| "rewards/rejected": -9.211328506469727, |
| "step": 1910 |
| }, |
| { |
| "epoch": 1.2355970389443192, |
| "grad_norm": 10.003925914268681, |
| "learning_rate": 6.910817772054089e-07, |
| "logits/chosen": -1.184240698814392, |
| "logits/rejected": -1.1048583984375, |
| "logps/chosen": -381.6499938964844, |
| "logps/rejected": -468.20001220703125, |
| "loss": 0.0246, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -0.55438232421875, |
| "rewards/margins": 8.818750381469727, |
| "rewards/rejected": -9.37109375, |
| "step": 1920 |
| }, |
| { |
| "epoch": 1.242034116511104, |
| "grad_norm": 4.014129022864646, |
| "learning_rate": 6.894719896973599e-07, |
| "logits/chosen": -1.094751000404358, |
| "logits/rejected": -1.173437476158142, |
| "logps/chosen": -411.29998779296875, |
| "logps/rejected": -489.54998779296875, |
| "loss": 0.0307, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -0.4810852110385895, |
| "rewards/margins": 9.17578125, |
| "rewards/rejected": -9.653124809265137, |
| "step": 1930 |
| }, |
| { |
| "epoch": 1.2484711940778888, |
| "grad_norm": 4.534028289345323, |
| "learning_rate": 6.87862202189311e-07, |
| "logits/chosen": -1.2288086414337158, |
| "logits/rejected": -1.117089867591858, |
| "logps/chosen": -383.875, |
| "logps/rejected": -484.8999938964844, |
| "loss": 0.0235, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -0.5280822515487671, |
| "rewards/margins": 8.389843940734863, |
| "rewards/rejected": -8.9140625, |
| "step": 1940 |
| }, |
| { |
| "epoch": 1.2549082716446733, |
| "grad_norm": 3.2091697666172885, |
| "learning_rate": 6.86252414681262e-07, |
| "logits/chosen": -1.0226958990097046, |
| "logits/rejected": -1.001196265220642, |
| "logps/chosen": -406.5249938964844, |
| "logps/rejected": -485.6000061035156, |
| "loss": 0.0613, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -0.984326183795929, |
| "rewards/margins": 8.4453125, |
| "rewards/rejected": -9.425000190734863, |
| "step": 1950 |
| }, |
| { |
| "epoch": 1.261345349211458, |
| "grad_norm": 7.466707070222194, |
| "learning_rate": 6.846426271732131e-07, |
| "logits/chosen": -1.146093726158142, |
| "logits/rejected": -1.05426025390625, |
| "logps/chosen": -388.75, |
| "logps/rejected": -493.45001220703125, |
| "loss": 0.0215, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -1.274377465248108, |
| "rewards/margins": 9.146093368530273, |
| "rewards/rejected": -10.423437118530273, |
| "step": 1960 |
| }, |
| { |
| "epoch": 1.2677824267782427, |
| "grad_norm": 5.797398529102235, |
| "learning_rate": 6.830328396651641e-07, |
| "logits/chosen": -1.2474486827850342, |
| "logits/rejected": -1.279638648033142, |
| "logps/chosen": -372.625, |
| "logps/rejected": -458.3500061035156, |
| "loss": 0.0129, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -1.103295922279358, |
| "rewards/margins": 9.251562118530273, |
| "rewards/rejected": -10.357812881469727, |
| "step": 1970 |
| }, |
| { |
| "epoch": 1.2742195043450273, |
| "grad_norm": 10.928389947545064, |
| "learning_rate": 6.814230521571152e-07, |
| "logits/chosen": -1.1201660633087158, |
| "logits/rejected": -1.0519287586212158, |
| "logps/chosen": -430.6000061035156, |
| "logps/rejected": -529.6500244140625, |
| "loss": 0.027, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -2.0275511741638184, |
| "rewards/margins": 9.775781631469727, |
| "rewards/rejected": -11.805468559265137, |
| "step": 1980 |
| }, |
| { |
| "epoch": 1.280656581911812, |
| "grad_norm": 4.255669736046018, |
| "learning_rate": 6.798132646490663e-07, |
| "logits/chosen": -1.1365234851837158, |
| "logits/rejected": -1.049340844154358, |
| "logps/chosen": -400.5249938964844, |
| "logps/rejected": -514.8499755859375, |
| "loss": 0.0298, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -2.0904541015625, |
| "rewards/margins": 10.901562690734863, |
| "rewards/rejected": -12.986719131469727, |
| "step": 1990 |
| }, |
| { |
| "epoch": 1.2870936594785967, |
| "grad_norm": 3.103866395229069, |
| "learning_rate": 6.782034771410174e-07, |
| "logits/chosen": -1.3879883289337158, |
| "logits/rejected": -1.3746337890625, |
| "logps/chosen": -408.0, |
| "logps/rejected": -502.1000061035156, |
| "loss": 0.0142, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -3.5293211936950684, |
| "rewards/margins": 9.793749809265137, |
| "rewards/rejected": -13.319531440734863, |
| "step": 2000 |
| }, |
| { |
| "epoch": 1.2935307370453815, |
| "grad_norm": 0.8230170263848153, |
| "learning_rate": 6.765936896329683e-07, |
| "logits/chosen": -1.2859375476837158, |
| "logits/rejected": -1.1643555164337158, |
| "logps/chosen": -387.42498779296875, |
| "logps/rejected": -492.29998779296875, |
| "loss": 0.0448, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -3.2607421875, |
| "rewards/margins": 9.910937309265137, |
| "rewards/rejected": -13.181249618530273, |
| "step": 2010 |
| }, |
| { |
| "epoch": 1.299967814612166, |
| "grad_norm": 5.879176568986368, |
| "learning_rate": 6.749839021249195e-07, |
| "logits/chosen": -1.177392601966858, |
| "logits/rejected": -1.2256958484649658, |
| "logps/chosen": -459.20001220703125, |
| "logps/rejected": -532.5, |
| "loss": 0.0224, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -2.456787109375, |
| "rewards/margins": 10.708593368530273, |
| "rewards/rejected": -13.160937309265137, |
| "step": 2020 |
| }, |
| { |
| "epoch": 1.3064048921789508, |
| "grad_norm": 5.093843499121855, |
| "learning_rate": 6.733741146168706e-07, |
| "logits/chosen": -1.142187476158142, |
| "logits/rejected": -1.0554687976837158, |
| "logps/chosen": -387.4624938964844, |
| "logps/rejected": -494.82501220703125, |
| "loss": 0.028, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -1.9534790515899658, |
| "rewards/margins": 10.528124809265137, |
| "rewards/rejected": -12.490625381469727, |
| "step": 2030 |
| }, |
| { |
| "epoch": 1.3128419697457354, |
| "grad_norm": 2.8755689696089064, |
| "learning_rate": 6.717643271088216e-07, |
| "logits/chosen": -1.171655297279358, |
| "logits/rejected": -1.1729247570037842, |
| "logps/chosen": -423.4750061035156, |
| "logps/rejected": -488.5, |
| "loss": 0.0359, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -2.997210741043091, |
| "rewards/margins": 9.702343940734863, |
| "rewards/rejected": -12.702343940734863, |
| "step": 2040 |
| }, |
| { |
| "epoch": 1.31927904731252, |
| "grad_norm": 1.221889850434017, |
| "learning_rate": 6.701545396007726e-07, |
| "logits/chosen": -1.307226538658142, |
| "logits/rejected": -1.285620093345642, |
| "logps/chosen": -385.79998779296875, |
| "logps/rejected": -469.7749938964844, |
| "loss": 0.024, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -2.591259717941284, |
| "rewards/margins": 10.02734375, |
| "rewards/rejected": -12.621874809265137, |
| "step": 2050 |
| }, |
| { |
| "epoch": 1.3257161248793048, |
| "grad_norm": 0.8793129216435185, |
| "learning_rate": 6.685447520927238e-07, |
| "logits/chosen": -1.21142578125, |
| "logits/rejected": -1.1825683116912842, |
| "logps/chosen": -401.79998779296875, |
| "logps/rejected": -494.1499938964844, |
| "loss": 0.0195, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -2.934521436691284, |
| "rewards/margins": 10.438281059265137, |
| "rewards/rejected": -13.370312690734863, |
| "step": 2060 |
| }, |
| { |
| "epoch": 1.3321532024460896, |
| "grad_norm": 8.274962453414453, |
| "learning_rate": 6.669349645846748e-07, |
| "logits/chosen": -1.2031738758087158, |
| "logits/rejected": -1.2007324695587158, |
| "logps/chosen": -439.70001220703125, |
| "logps/rejected": -500.79998779296875, |
| "loss": 0.0168, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -2.692675828933716, |
| "rewards/margins": 10.457812309265137, |
| "rewards/rejected": -13.154687881469727, |
| "step": 2070 |
| }, |
| { |
| "epoch": 1.3385902800128742, |
| "grad_norm": 10.593918303266882, |
| "learning_rate": 6.653251770766258e-07, |
| "logits/chosen": -1.194555640220642, |
| "logits/rejected": -1.203466773033142, |
| "logps/chosen": -431.67498779296875, |
| "logps/rejected": -506.1000061035156, |
| "loss": 0.0328, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -3.3885254859924316, |
| "rewards/margins": 9.889062881469727, |
| "rewards/rejected": -13.275781631469727, |
| "step": 2080 |
| }, |
| { |
| "epoch": 1.3450273575796587, |
| "grad_norm": 1.6942636133623765, |
| "learning_rate": 6.637153895685768e-07, |
| "logits/chosen": -1.262109398841858, |
| "logits/rejected": -1.240087866783142, |
| "logps/chosen": -437.17498779296875, |
| "logps/rejected": -549.9749755859375, |
| "loss": 0.0386, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -4.543066501617432, |
| "rewards/margins": 10.63671875, |
| "rewards/rejected": -15.1875, |
| "step": 2090 |
| }, |
| { |
| "epoch": 1.3514644351464435, |
| "grad_norm": 2.8523703263937756, |
| "learning_rate": 6.62105602060528e-07, |
| "logits/chosen": -1.2749512195587158, |
| "logits/rejected": -1.1814696788787842, |
| "logps/chosen": -435.875, |
| "logps/rejected": -522.375, |
| "loss": 0.0246, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -4.801953315734863, |
| "rewards/margins": 10.099218368530273, |
| "rewards/rejected": -14.903124809265137, |
| "step": 2100 |
| }, |
| { |
| "epoch": 1.3579015127132281, |
| "grad_norm": 1.7207516817060962, |
| "learning_rate": 6.604958145524791e-07, |
| "logits/chosen": -1.212646484375, |
| "logits/rejected": -1.260156273841858, |
| "logps/chosen": -449.1499938964844, |
| "logps/rejected": -507.25, |
| "loss": 0.0462, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -3.799609422683716, |
| "rewards/margins": 9.243359565734863, |
| "rewards/rejected": -13.044530868530273, |
| "step": 2110 |
| }, |
| { |
| "epoch": 1.364338590280013, |
| "grad_norm": 3.0061085616418426, |
| "learning_rate": 6.5888602704443e-07, |
| "logits/chosen": -1.1973755359649658, |
| "logits/rejected": -1.115209937095642, |
| "logps/chosen": -402.6499938964844, |
| "logps/rejected": -483.8999938964844, |
| "loss": 0.0421, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -1.74658203125, |
| "rewards/margins": 9.484375, |
| "rewards/rejected": -11.233593940734863, |
| "step": 2120 |
| }, |
| { |
| "epoch": 1.3707756678467975, |
| "grad_norm": 64.93152042342604, |
| "learning_rate": 6.572762395363811e-07, |
| "logits/chosen": -1.217919945716858, |
| "logits/rejected": -1.1572754383087158, |
| "logps/chosen": -421.07501220703125, |
| "logps/rejected": -513.8250122070312, |
| "loss": 0.05, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -1.381683349609375, |
| "rewards/margins": 9.639843940734863, |
| "rewards/rejected": -11.022656440734863, |
| "step": 2130 |
| }, |
| { |
| "epoch": 1.3772127454135823, |
| "grad_norm": 3.5821901931613813, |
| "learning_rate": 6.556664520283323e-07, |
| "logits/chosen": -1.005407691001892, |
| "logits/rejected": -1.0377929210662842, |
| "logps/chosen": -422.7250061035156, |
| "logps/rejected": -481.8999938964844, |
| "loss": 0.0427, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -1.526123046875, |
| "rewards/margins": 9.578125, |
| "rewards/rejected": -11.103124618530273, |
| "step": 2140 |
| }, |
| { |
| "epoch": 1.3836498229803669, |
| "grad_norm": 0.8762611536208239, |
| "learning_rate": 6.540566645202833e-07, |
| "logits/chosen": -1.129541039466858, |
| "logits/rejected": -1.1396713256835938, |
| "logps/chosen": -412.04998779296875, |
| "logps/rejected": -483.8999938964844, |
| "loss": 0.0181, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -2.1922364234924316, |
| "rewards/margins": 9.396093368530273, |
| "rewards/rejected": -11.584375381469727, |
| "step": 2150 |
| }, |
| { |
| "epoch": 1.3900869005471517, |
| "grad_norm": 3.386415010320299, |
| "learning_rate": 6.524468770122343e-07, |
| "logits/chosen": -1.171350121498108, |
| "logits/rejected": -1.138671875, |
| "logps/chosen": -396.4750061035156, |
| "logps/rejected": -510.3500061035156, |
| "loss": 0.0284, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -2.336010694503784, |
| "rewards/margins": 10.917187690734863, |
| "rewards/rejected": -13.254687309265137, |
| "step": 2160 |
| }, |
| { |
| "epoch": 1.3965239781139362, |
| "grad_norm": 2.3665712353635295, |
| "learning_rate": 6.508370895041854e-07, |
| "logits/chosen": -1.1177246570587158, |
| "logits/rejected": -1.11614990234375, |
| "logps/chosen": -418.98748779296875, |
| "logps/rejected": -483.45001220703125, |
| "loss": 0.0435, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -2.291699171066284, |
| "rewards/margins": 9.687108993530273, |
| "rewards/rejected": -11.983593940734863, |
| "step": 2170 |
| }, |
| { |
| "epoch": 1.4029610556807208, |
| "grad_norm": 36.91063615603027, |
| "learning_rate": 6.492273019961365e-07, |
| "logits/chosen": -1.1660645008087158, |
| "logits/rejected": -1.140771508216858, |
| "logps/chosen": -402.875, |
| "logps/rejected": -478.54998779296875, |
| "loss": 0.0456, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -2.0533447265625, |
| "rewards/margins": 9.824999809265137, |
| "rewards/rejected": -11.871874809265137, |
| "step": 2180 |
| }, |
| { |
| "epoch": 1.4093981332475056, |
| "grad_norm": 5.039995786070021, |
| "learning_rate": 6.476175144880875e-07, |
| "logits/chosen": -1.183496117591858, |
| "logits/rejected": -1.175573706626892, |
| "logps/chosen": -433.45001220703125, |
| "logps/rejected": -497.70001220703125, |
| "loss": 0.0263, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -2.1736817359924316, |
| "rewards/margins": 10.38671875, |
| "rewards/rejected": -12.557812690734863, |
| "step": 2190 |
| }, |
| { |
| "epoch": 1.4158352108142904, |
| "grad_norm": 72.88654802005208, |
| "learning_rate": 6.460077269800386e-07, |
| "logits/chosen": -1.1180541515350342, |
| "logits/rejected": -1.0501708984375, |
| "logps/chosen": -396.45001220703125, |
| "logps/rejected": -520.5999755859375, |
| "loss": 0.0921, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -2.112744092941284, |
| "rewards/margins": 10.481249809265137, |
| "rewards/rejected": -12.600000381469727, |
| "step": 2200 |
| }, |
| { |
| "epoch": 1.422272288381075, |
| "grad_norm": 51.573026979366915, |
| "learning_rate": 6.443979394719896e-07, |
| "logits/chosen": -1.1914551258087158, |
| "logits/rejected": -1.08184814453125, |
| "logps/chosen": -440.70001220703125, |
| "logps/rejected": -513.5, |
| "loss": 0.0262, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -2.119824171066284, |
| "rewards/margins": 9.930468559265137, |
| "rewards/rejected": -12.053906440734863, |
| "step": 2210 |
| }, |
| { |
| "epoch": 1.4287093659478596, |
| "grad_norm": 9.30589119660866, |
| "learning_rate": 6.427881519639408e-07, |
| "logits/chosen": -1.039739966392517, |
| "logits/rejected": -1.1082274913787842, |
| "logps/chosen": -432.375, |
| "logps/rejected": -514.9500122070312, |
| "loss": 0.0313, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -2.3951172828674316, |
| "rewards/margins": 10.203125, |
| "rewards/rejected": -12.59765625, |
| "step": 2220 |
| }, |
| { |
| "epoch": 1.4351464435146444, |
| "grad_norm": 0.7939990982699106, |
| "learning_rate": 6.411783644558918e-07, |
| "logits/chosen": -1.29931640625, |
| "logits/rejected": -1.2560303211212158, |
| "logps/chosen": -426.5249938964844, |
| "logps/rejected": -490.1000061035156, |
| "loss": 0.0193, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -2.1973876953125, |
| "rewards/margins": 9.762499809265137, |
| "rewards/rejected": -11.960156440734863, |
| "step": 2230 |
| }, |
| { |
| "epoch": 1.441583521081429, |
| "grad_norm": 8.793565470628264, |
| "learning_rate": 6.395685769478428e-07, |
| "logits/chosen": -1.236547827720642, |
| "logits/rejected": -1.1888916492462158, |
| "logps/chosen": -385.1000061035156, |
| "logps/rejected": -509.2250061035156, |
| "loss": 0.0553, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -1.98095703125, |
| "rewards/margins": 9.751562118530273, |
| "rewards/rejected": -11.740625381469727, |
| "step": 2240 |
| }, |
| { |
| "epoch": 1.4480205986482138, |
| "grad_norm": 17.46647417026905, |
| "learning_rate": 6.379587894397939e-07, |
| "logits/chosen": -1.1052520275115967, |
| "logits/rejected": -1.061315894126892, |
| "logps/chosen": -431.54998779296875, |
| "logps/rejected": -496.3999938964844, |
| "loss": 0.029, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -2.140625, |
| "rewards/margins": 10.21484375, |
| "rewards/rejected": -12.354687690734863, |
| "step": 2250 |
| }, |
| { |
| "epoch": 1.4544576762149983, |
| "grad_norm": 10.559629096936263, |
| "learning_rate": 6.363490019317451e-07, |
| "logits/chosen": -1.1336669921875, |
| "logits/rejected": -1.0379517078399658, |
| "logps/chosen": -417.54998779296875, |
| "logps/rejected": -510.2749938964844, |
| "loss": 0.0498, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -2.4303832054138184, |
| "rewards/margins": 9.861719131469727, |
| "rewards/rejected": -12.288281440734863, |
| "step": 2260 |
| }, |
| { |
| "epoch": 1.4608947537817831, |
| "grad_norm": 0.22125735571588645, |
| "learning_rate": 6.34739214423696e-07, |
| "logits/chosen": -1.0536987781524658, |
| "logits/rejected": -0.9582275152206421, |
| "logps/chosen": -435.875, |
| "logps/rejected": -532.0499877929688, |
| "loss": 0.0376, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -2.0049805641174316, |
| "rewards/margins": 10.857030868530273, |
| "rewards/rejected": -12.866406440734863, |
| "step": 2270 |
| }, |
| { |
| "epoch": 1.4673318313485677, |
| "grad_norm": 1.4944651668112314, |
| "learning_rate": 6.331294269156471e-07, |
| "logits/chosen": -1.1802246570587158, |
| "logits/rejected": -1.088720679283142, |
| "logps/chosen": -426.01251220703125, |
| "logps/rejected": -527.0, |
| "loss": 0.0319, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -2.1170716285705566, |
| "rewards/margins": 10.889062881469727, |
| "rewards/rejected": -13.003125190734863, |
| "step": 2280 |
| }, |
| { |
| "epoch": 1.4737689089153525, |
| "grad_norm": 11.46093255002139, |
| "learning_rate": 6.315196394075981e-07, |
| "logits/chosen": -1.1174499988555908, |
| "logits/rejected": -1.0461242198944092, |
| "logps/chosen": -410.5, |
| "logps/rejected": -511.6000061035156, |
| "loss": 0.0372, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -2.679003953933716, |
| "rewards/margins": 9.881640434265137, |
| "rewards/rejected": -12.564844131469727, |
| "step": 2290 |
| }, |
| { |
| "epoch": 1.480205986482137, |
| "grad_norm": 51.42125186732637, |
| "learning_rate": 6.299098518995493e-07, |
| "logits/chosen": -1.050195336341858, |
| "logits/rejected": -1.0292479991912842, |
| "logps/chosen": -428.67498779296875, |
| "logps/rejected": -520.5250244140625, |
| "loss": 0.0885, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -2.9413084983825684, |
| "rewards/margins": 10.017969131469727, |
| "rewards/rejected": -12.961718559265137, |
| "step": 2300 |
| }, |
| { |
| "epoch": 1.4866430640489217, |
| "grad_norm": 3.7485614615505276, |
| "learning_rate": 6.283000643915003e-07, |
| "logits/chosen": -1.2371094226837158, |
| "logits/rejected": -1.2183837890625, |
| "logps/chosen": -388.9750061035156, |
| "logps/rejected": -493.32501220703125, |
| "loss": 0.0681, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -3.2401976585388184, |
| "rewards/margins": 10.273046493530273, |
| "rewards/rejected": -13.505468368530273, |
| "step": 2310 |
| }, |
| { |
| "epoch": 1.4930801416157065, |
| "grad_norm": 1.7017620437153345, |
| "learning_rate": 6.266902768834513e-07, |
| "logits/chosen": -1.0468018054962158, |
| "logits/rejected": -1.0492737293243408, |
| "logps/chosen": -432.6000061035156, |
| "logps/rejected": -519.7999877929688, |
| "loss": 0.0401, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -2.44488525390625, |
| "rewards/margins": 9.664843559265137, |
| "rewards/rejected": -12.108593940734863, |
| "step": 2320 |
| }, |
| { |
| "epoch": 1.4995172191824913, |
| "grad_norm": 86.14101039384126, |
| "learning_rate": 6.250804893754024e-07, |
| "logits/chosen": -1.1101806163787842, |
| "logits/rejected": -1.0790283679962158, |
| "logps/chosen": -407.4750061035156, |
| "logps/rejected": -465.6000061035156, |
| "loss": 0.0615, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -1.9099853038787842, |
| "rewards/margins": 9.540624618530273, |
| "rewards/rejected": -11.442968368530273, |
| "step": 2330 |
| }, |
| { |
| "epoch": 1.5059542967492758, |
| "grad_norm": 0.13614339960276306, |
| "learning_rate": 6.234707018673535e-07, |
| "logits/chosen": -1.0773437023162842, |
| "logits/rejected": -1.0900757312774658, |
| "logps/chosen": -410.125, |
| "logps/rejected": -490.1499938964844, |
| "loss": 0.0155, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -2.8124022483825684, |
| "rewards/margins": 10.106249809265137, |
| "rewards/rejected": -12.918749809265137, |
| "step": 2340 |
| }, |
| { |
| "epoch": 1.5123913743160604, |
| "grad_norm": 0.48357819528400076, |
| "learning_rate": 6.218609143593045e-07, |
| "logits/chosen": -1.0822265148162842, |
| "logits/rejected": -1.067724585533142, |
| "logps/chosen": -413.92498779296875, |
| "logps/rejected": -505.25, |
| "loss": 0.0272, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -2.4931578636169434, |
| "rewards/margins": 9.866406440734863, |
| "rewards/rejected": -12.359375, |
| "step": 2350 |
| }, |
| { |
| "epoch": 1.5188284518828452, |
| "grad_norm": 0.33636474504491165, |
| "learning_rate": 6.202511268512556e-07, |
| "logits/chosen": -1.079614281654358, |
| "logits/rejected": -1.0338866710662842, |
| "logps/chosen": -416.45001220703125, |
| "logps/rejected": -499.5, |
| "loss": 0.0392, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -2.1781249046325684, |
| "rewards/margins": 9.346094131469727, |
| "rewards/rejected": -11.530468940734863, |
| "step": 2360 |
| }, |
| { |
| "epoch": 1.52526552944963, |
| "grad_norm": 16.095215751344575, |
| "learning_rate": 6.186413393432067e-07, |
| "logits/chosen": -1.261962890625, |
| "logits/rejected": -1.1941649913787842, |
| "logps/chosen": -441.92498779296875, |
| "logps/rejected": -498.70001220703125, |
| "loss": 0.0341, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -2.4649901390075684, |
| "rewards/margins": 9.321874618530273, |
| "rewards/rejected": -11.787500381469727, |
| "step": 2370 |
| }, |
| { |
| "epoch": 1.5317026070164146, |
| "grad_norm": 9.772434603393211, |
| "learning_rate": 6.170315518351577e-07, |
| "logits/chosen": -1.0902588367462158, |
| "logits/rejected": -1.061254858970642, |
| "logps/chosen": -409.875, |
| "logps/rejected": -496.70001220703125, |
| "loss": 0.0878, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -2.263427734375, |
| "rewards/margins": 8.946874618530273, |
| "rewards/rejected": -11.206250190734863, |
| "step": 2380 |
| }, |
| { |
| "epoch": 1.5381396845831992, |
| "grad_norm": 0.9308667811456769, |
| "learning_rate": 6.154217643271088e-07, |
| "logits/chosen": -0.999707043170929, |
| "logits/rejected": -1.0982666015625, |
| "logps/chosen": -429.1499938964844, |
| "logps/rejected": -519.4000244140625, |
| "loss": 0.0208, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -2.0730957984924316, |
| "rewards/margins": 9.200780868530273, |
| "rewards/rejected": -11.283594131469727, |
| "step": 2390 |
| }, |
| { |
| "epoch": 1.544576762149984, |
| "grad_norm": 0.9182332000756648, |
| "learning_rate": 6.138119768190599e-07, |
| "logits/chosen": -1.144189476966858, |
| "logits/rejected": -1.1719238758087158, |
| "logps/chosen": -381.63751220703125, |
| "logps/rejected": -450.1000061035156, |
| "loss": 0.0341, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -1.660675048828125, |
| "rewards/margins": 8.57421875, |
| "rewards/rejected": -10.234375, |
| "step": 2400 |
| }, |
| { |
| "epoch": 1.5510138397167685, |
| "grad_norm": 1.5948090467797094, |
| "learning_rate": 6.122021893110108e-07, |
| "logits/chosen": -0.9137908816337585, |
| "logits/rejected": -0.9247192144393921, |
| "logps/chosen": -402.7749938964844, |
| "logps/rejected": -499.6499938964844, |
| "loss": 0.0415, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -2.481677293777466, |
| "rewards/margins": 10.053125381469727, |
| "rewards/rejected": -12.5390625, |
| "step": 2410 |
| }, |
| { |
| "epoch": 1.5574509172835533, |
| "grad_norm": 4.060398294292906, |
| "learning_rate": 6.10592401802962e-07, |
| "logits/chosen": -1.104638695716858, |
| "logits/rejected": -1.0888671875, |
| "logps/chosen": -418.32501220703125, |
| "logps/rejected": -524.9500122070312, |
| "loss": 0.0628, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -3.686840772628784, |
| "rewards/margins": 9.7265625, |
| "rewards/rejected": -13.417187690734863, |
| "step": 2420 |
| }, |
| { |
| "epoch": 1.563887994850338, |
| "grad_norm": 5.311380971214449, |
| "learning_rate": 6.08982614294913e-07, |
| "logits/chosen": -1.1854248046875, |
| "logits/rejected": -1.120947241783142, |
| "logps/chosen": -420.7250061035156, |
| "logps/rejected": -513.7000122070312, |
| "loss": 0.0106, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -4.440527439117432, |
| "rewards/margins": 9.75, |
| "rewards/rejected": -14.190625190734863, |
| "step": 2430 |
| }, |
| { |
| "epoch": 1.5703250724171225, |
| "grad_norm": 17.733404283489456, |
| "learning_rate": 6.073728267868641e-07, |
| "logits/chosen": -1.014990210533142, |
| "logits/rejected": -1.0634276866912842, |
| "logps/chosen": -461.3500061035156, |
| "logps/rejected": -524.875, |
| "loss": 0.0547, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -4.6923828125, |
| "rewards/margins": 9.534375190734863, |
| "rewards/rejected": -14.231249809265137, |
| "step": 2440 |
| }, |
| { |
| "epoch": 1.5767621499839073, |
| "grad_norm": 20.86849854267267, |
| "learning_rate": 6.057630392788152e-07, |
| "logits/chosen": -0.96435546875, |
| "logits/rejected": -0.9388672113418579, |
| "logps/chosen": -460.82501220703125, |
| "logps/rejected": -535.4500122070312, |
| "loss": 0.0447, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -5.363085746765137, |
| "rewards/margins": 9.359375, |
| "rewards/rejected": -14.7265625, |
| "step": 2450 |
| }, |
| { |
| "epoch": 1.583199227550692, |
| "grad_norm": 1.338263306923686, |
| "learning_rate": 6.041532517707662e-07, |
| "logits/chosen": -1.238916039466858, |
| "logits/rejected": -1.2165038585662842, |
| "logps/chosen": -457.3999938964844, |
| "logps/rejected": -521.2999877929688, |
| "loss": 0.0253, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -4.875586032867432, |
| "rewards/margins": 9.38671875, |
| "rewards/rejected": -14.262499809265137, |
| "step": 2460 |
| }, |
| { |
| "epoch": 1.5896363051174767, |
| "grad_norm": 9.504956640733559, |
| "learning_rate": 6.025434642627173e-07, |
| "logits/chosen": -1.211645483970642, |
| "logits/rejected": -1.083642601966858, |
| "logps/chosen": -448.2749938964844, |
| "logps/rejected": -554.5499877929688, |
| "loss": 0.0256, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -3.560546875, |
| "rewards/margins": 10.532031059265137, |
| "rewards/rejected": -14.095312118530273, |
| "step": 2470 |
| }, |
| { |
| "epoch": 1.5960733826842612, |
| "grad_norm": 0.09616702281724097, |
| "learning_rate": 6.009336767546684e-07, |
| "logits/chosen": -1.2487304210662842, |
| "logits/rejected": -1.2554931640625, |
| "logps/chosen": -453.25, |
| "logps/rejected": -547.0, |
| "loss": 0.0243, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -3.2339844703674316, |
| "rewards/margins": 10.850000381469727, |
| "rewards/rejected": -14.079687118530273, |
| "step": 2480 |
| }, |
| { |
| "epoch": 1.602510460251046, |
| "grad_norm": 0.21078523404962335, |
| "learning_rate": 5.993238892466194e-07, |
| "logits/chosen": -1.168969750404358, |
| "logits/rejected": -1.160742163658142, |
| "logps/chosen": -415.6000061035156, |
| "logps/rejected": -514.375, |
| "loss": 0.0303, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -2.8856444358825684, |
| "rewards/margins": 10.192187309265137, |
| "rewards/rejected": -13.088281631469727, |
| "step": 2490 |
| }, |
| { |
| "epoch": 1.6089475378178308, |
| "grad_norm": 144.16364379486112, |
| "learning_rate": 5.977141017385705e-07, |
| "logits/chosen": -1.3957030773162842, |
| "logits/rejected": -1.3458983898162842, |
| "logps/chosen": -420.45001220703125, |
| "logps/rejected": -524.0, |
| "loss": 0.0809, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -2.9536375999450684, |
| "rewards/margins": 10.486719131469727, |
| "rewards/rejected": -13.443750381469727, |
| "step": 2500 |
| }, |
| { |
| "epoch": 1.6153846153846154, |
| "grad_norm": 4.473077383117404, |
| "learning_rate": 5.961043142305216e-07, |
| "logits/chosen": -1.168542504310608, |
| "logits/rejected": -1.0180175304412842, |
| "logps/chosen": -429.8500061035156, |
| "logps/rejected": -490.8500061035156, |
| "loss": 0.0567, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": -3.0799803733825684, |
| "rewards/margins": 10.00390625, |
| "rewards/rejected": -13.075780868530273, |
| "step": 2510 |
| }, |
| { |
| "epoch": 1.6218216929514, |
| "grad_norm": 9.22345622696017, |
| "learning_rate": 5.944945267224725e-07, |
| "logits/chosen": -1.2401611804962158, |
| "logits/rejected": -1.2089354991912842, |
| "logps/chosen": -440.7749938964844, |
| "logps/rejected": -529.5499877929688, |
| "loss": 0.0189, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -3.308886766433716, |
| "rewards/margins": 10.943750381469727, |
| "rewards/rejected": -14.245312690734863, |
| "step": 2520 |
| }, |
| { |
| "epoch": 1.6282587705181848, |
| "grad_norm": 29.57432630776374, |
| "learning_rate": 5.928847392144237e-07, |
| "logits/chosen": -1.148168921470642, |
| "logits/rejected": -1.056127905845642, |
| "logps/chosen": -429.59375, |
| "logps/rejected": -529.5750122070312, |
| "loss": 0.0311, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -2.960583448410034, |
| "rewards/margins": 11.469531059265137, |
| "rewards/rejected": -14.431249618530273, |
| "step": 2530 |
| }, |
| { |
| "epoch": 1.6346958480849694, |
| "grad_norm": 0.5562405862110368, |
| "learning_rate": 5.912749517063748e-07, |
| "logits/chosen": -1.2148926258087158, |
| "logits/rejected": -1.18359375, |
| "logps/chosen": -431.7749938964844, |
| "logps/rejected": -494.3999938964844, |
| "loss": 0.0154, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -2.9759278297424316, |
| "rewards/margins": 10.875781059265137, |
| "rewards/rejected": -13.85546875, |
| "step": 2540 |
| }, |
| { |
| "epoch": 1.6411329256517542, |
| "grad_norm": 15.841928719800434, |
| "learning_rate": 5.896651641983258e-07, |
| "logits/chosen": -1.23388671875, |
| "logits/rejected": -1.24755859375, |
| "logps/chosen": -411.125, |
| "logps/rejected": -476.17498779296875, |
| "loss": 0.0112, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -2.725146532058716, |
| "rewards/margins": 10.572656631469727, |
| "rewards/rejected": -13.301562309265137, |
| "step": 2550 |
| }, |
| { |
| "epoch": 1.6475700032185387, |
| "grad_norm": 1.6307612196797676, |
| "learning_rate": 5.880553766902768e-07, |
| "logits/chosen": -1.1931641101837158, |
| "logits/rejected": -1.09033203125, |
| "logps/chosen": -433.5, |
| "logps/rejected": -527.2999877929688, |
| "loss": 0.0265, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -2.443164110183716, |
| "rewards/margins": 10.084375381469727, |
| "rewards/rejected": -12.530468940734863, |
| "step": 2560 |
| }, |
| { |
| "epoch": 1.6540070807853233, |
| "grad_norm": 0.364667536975264, |
| "learning_rate": 5.86445589182228e-07, |
| "logits/chosen": -1.055761694908142, |
| "logits/rejected": -0.978259265422821, |
| "logps/chosen": -436.8500061035156, |
| "logps/rejected": -506.1499938964844, |
| "loss": 0.0377, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -3.1280274391174316, |
| "rewards/margins": 10.255468368530273, |
| "rewards/rejected": -13.382031440734863, |
| "step": 2570 |
| }, |
| { |
| "epoch": 1.6604441583521081, |
| "grad_norm": 0.3025825721261884, |
| "learning_rate": 5.84835801674179e-07, |
| "logits/chosen": -1.1936523914337158, |
| "logits/rejected": -1.1957519054412842, |
| "logps/chosen": -406.7250061035156, |
| "logps/rejected": -491.5, |
| "loss": 0.0341, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -2.4527344703674316, |
| "rewards/margins": 10.291406631469727, |
| "rewards/rejected": -12.748437881469727, |
| "step": 2580 |
| }, |
| { |
| "epoch": 1.666881235918893, |
| "grad_norm": 19.681767482833102, |
| "learning_rate": 5.832260141661301e-07, |
| "logits/chosen": -1.162451148033142, |
| "logits/rejected": -1.109948754310608, |
| "logps/chosen": -448.0249938964844, |
| "logps/rejected": -533.5, |
| "loss": 0.0319, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -1.914453148841858, |
| "rewards/margins": 11.007031440734863, |
| "rewards/rejected": -12.921875, |
| "step": 2590 |
| }, |
| { |
| "epoch": 1.6733183134856775, |
| "grad_norm": 9.266326635548827, |
| "learning_rate": 5.81616226658081e-07, |
| "logits/chosen": -1.170861840248108, |
| "logits/rejected": -1.265234351158142, |
| "logps/chosen": -435.5249938964844, |
| "logps/rejected": -496.25, |
| "loss": 0.0233, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -3.002148389816284, |
| "rewards/margins": 10.118749618530273, |
| "rewards/rejected": -13.122655868530273, |
| "step": 2600 |
| }, |
| { |
| "epoch": 1.679755391052462, |
| "grad_norm": 31.49903463002844, |
| "learning_rate": 5.800064391500322e-07, |
| "logits/chosen": -1.1318848133087158, |
| "logits/rejected": -1.146704077720642, |
| "logps/chosen": -446.7250061035156, |
| "logps/rejected": -529.8499755859375, |
| "loss": 0.0281, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -3.250903367996216, |
| "rewards/margins": 9.783594131469727, |
| "rewards/rejected": -13.034375190734863, |
| "step": 2610 |
| }, |
| { |
| "epoch": 1.6861924686192469, |
| "grad_norm": 3.051142014151063, |
| "learning_rate": 5.783966516419833e-07, |
| "logits/chosen": -1.339379906654358, |
| "logits/rejected": -1.3441894054412842, |
| "logps/chosen": -412.8999938964844, |
| "logps/rejected": -495.8500061035156, |
| "loss": 0.0326, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -2.869140625, |
| "rewards/margins": 9.931249618530273, |
| "rewards/rejected": -12.800000190734863, |
| "step": 2620 |
| }, |
| { |
| "epoch": 1.6926295461860317, |
| "grad_norm": 11.85133822050956, |
| "learning_rate": 5.767868641339343e-07, |
| "logits/chosen": -1.2352173328399658, |
| "logits/rejected": -1.2333252429962158, |
| "logps/chosen": -383.20001220703125, |
| "logps/rejected": -490.5, |
| "loss": 0.0427, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -2.8724608421325684, |
| "rewards/margins": 10.013671875, |
| "rewards/rejected": -12.888280868530273, |
| "step": 2630 |
| }, |
| { |
| "epoch": 1.6990666237528163, |
| "grad_norm": 12.413854103296234, |
| "learning_rate": 5.751770766258853e-07, |
| "logits/chosen": -1.1151611804962158, |
| "logits/rejected": -1.1124267578125, |
| "logps/chosen": -456.92498779296875, |
| "logps/rejected": -524.7000122070312, |
| "loss": 0.0636, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -2.9258790016174316, |
| "rewards/margins": 10.004687309265137, |
| "rewards/rejected": -12.926562309265137, |
| "step": 2640 |
| }, |
| { |
| "epoch": 1.7055037013196008, |
| "grad_norm": 0.6649208434305837, |
| "learning_rate": 5.735672891178365e-07, |
| "logits/chosen": -1.223883032798767, |
| "logits/rejected": -1.2350585460662842, |
| "logps/chosen": -454.7749938964844, |
| "logps/rejected": -504.625, |
| "loss": 0.0407, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -3.3885254859924316, |
| "rewards/margins": 10.134374618530273, |
| "rewards/rejected": -13.522656440734863, |
| "step": 2650 |
| }, |
| { |
| "epoch": 1.7119407788863856, |
| "grad_norm": 10.023564861820988, |
| "learning_rate": 5.719575016097875e-07, |
| "logits/chosen": -1.145532250404358, |
| "logits/rejected": -1.044458031654358, |
| "logps/chosen": -376.42498779296875, |
| "logps/rejected": -523.5, |
| "loss": 0.0365, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -3.2010741233825684, |
| "rewards/margins": 10.46875, |
| "rewards/rejected": -13.666406631469727, |
| "step": 2660 |
| }, |
| { |
| "epoch": 1.7183778564531704, |
| "grad_norm": 2.6239722773404455, |
| "learning_rate": 5.703477141017385e-07, |
| "logits/chosen": -1.1627929210662842, |
| "logits/rejected": -1.143164038658142, |
| "logps/chosen": -413.5, |
| "logps/rejected": -508.1000061035156, |
| "loss": 0.0199, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -2.8783202171325684, |
| "rewards/margins": 10.829687118530273, |
| "rewards/rejected": -13.701562881469727, |
| "step": 2670 |
| }, |
| { |
| "epoch": 1.724814934019955, |
| "grad_norm": 1.6985887936660748, |
| "learning_rate": 5.687379265936896e-07, |
| "logits/chosen": -1.31072998046875, |
| "logits/rejected": -1.2086303234100342, |
| "logps/chosen": -415.95001220703125, |
| "logps/rejected": -505.82501220703125, |
| "loss": 0.0204, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -3.9234375953674316, |
| "rewards/margins": 10.600781440734863, |
| "rewards/rejected": -14.532812118530273, |
| "step": 2680 |
| }, |
| { |
| "epoch": 1.7312520115867396, |
| "grad_norm": 5.877398352222268, |
| "learning_rate": 5.671281390856407e-07, |
| "logits/chosen": -1.31103515625, |
| "logits/rejected": -1.377294898033142, |
| "logps/chosen": -441.0249938964844, |
| "logps/rejected": -518.0999755859375, |
| "loss": 0.0388, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -3.5423827171325684, |
| "rewards/margins": 10.294530868530273, |
| "rewards/rejected": -13.833593368530273, |
| "step": 2690 |
| }, |
| { |
| "epoch": 1.7376890891535242, |
| "grad_norm": 2.0088822694870303, |
| "learning_rate": 5.655183515775918e-07, |
| "logits/chosen": -1.268896460533142, |
| "logits/rejected": -1.2238280773162842, |
| "logps/chosen": -469.25, |
| "logps/rejected": -546.2999877929688, |
| "loss": 0.019, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -3.117431640625, |
| "rewards/margins": 10.917187690734863, |
| "rewards/rejected": -14.029687881469727, |
| "step": 2700 |
| }, |
| { |
| "epoch": 1.744126166720309, |
| "grad_norm": 16.8533267821526, |
| "learning_rate": 5.639085640695428e-07, |
| "logits/chosen": -0.975415050983429, |
| "logits/rejected": -1.056909203529358, |
| "logps/chosen": -441.8999938964844, |
| "logps/rejected": -514.4500122070312, |
| "loss": 0.0765, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -3.0760741233825684, |
| "rewards/margins": 9.939844131469727, |
| "rewards/rejected": -13.021875381469727, |
| "step": 2710 |
| }, |
| { |
| "epoch": 1.7505632442870938, |
| "grad_norm": 5.240569520603171, |
| "learning_rate": 5.622987765614938e-07, |
| "logits/chosen": -1.237890601158142, |
| "logits/rejected": -1.2314941883087158, |
| "logps/chosen": -447.70001220703125, |
| "logps/rejected": -537.2999877929688, |
| "loss": 0.0531, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -3.1049561500549316, |
| "rewards/margins": 10.680468559265137, |
| "rewards/rejected": -13.784375190734863, |
| "step": 2720 |
| }, |
| { |
| "epoch": 1.7570003218538783, |
| "grad_norm": 93.09000389797974, |
| "learning_rate": 5.60688989053445e-07, |
| "logits/chosen": -1.2188202142715454, |
| "logits/rejected": -1.24371337890625, |
| "logps/chosen": -438.1499938964844, |
| "logps/rejected": -509.8500061035156, |
| "loss": 0.0831, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -3.014416456222534, |
| "rewards/margins": 10.219922065734863, |
| "rewards/rejected": -13.2421875, |
| "step": 2730 |
| }, |
| { |
| "epoch": 1.763437399420663, |
| "grad_norm": 29.50554946684019, |
| "learning_rate": 5.590792015453961e-07, |
| "logits/chosen": -1.0483887195587158, |
| "logits/rejected": -1.1110107898712158, |
| "logps/chosen": -466.57501220703125, |
| "logps/rejected": -534.2999877929688, |
| "loss": 0.0388, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -2.966796875, |
| "rewards/margins": 10.001562118530273, |
| "rewards/rejected": -12.971875190734863, |
| "step": 2740 |
| }, |
| { |
| "epoch": 1.7698744769874477, |
| "grad_norm": 64.6632678215838, |
| "learning_rate": 5.57469414037347e-07, |
| "logits/chosen": -1.145898461341858, |
| "logits/rejected": -1.103369116783142, |
| "logps/chosen": -462.1499938964844, |
| "logps/rejected": -504.70001220703125, |
| "loss": 0.0951, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": -2.512646436691284, |
| "rewards/margins": 9.794530868530273, |
| "rewards/rejected": -12.314062118530273, |
| "step": 2750 |
| }, |
| { |
| "epoch": 1.7763115545542325, |
| "grad_norm": 24.573201788811367, |
| "learning_rate": 5.558596265292981e-07, |
| "logits/chosen": -1.016626000404358, |
| "logits/rejected": -1.0913879871368408, |
| "logps/chosen": -415.7749938964844, |
| "logps/rejected": -472.42498779296875, |
| "loss": 0.029, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -2.5428709983825684, |
| "rewards/margins": 9.334375381469727, |
| "rewards/rejected": -11.889062881469727, |
| "step": 2760 |
| }, |
| { |
| "epoch": 1.782748632121017, |
| "grad_norm": 43.91628486074585, |
| "learning_rate": 5.542498390212492e-07, |
| "logits/chosen": -1.0629394054412842, |
| "logits/rejected": -1.07763671875, |
| "logps/chosen": -405.95001220703125, |
| "logps/rejected": -476.0, |
| "loss": 0.0884, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -2.224804639816284, |
| "rewards/margins": 8.9296875, |
| "rewards/rejected": -11.15625, |
| "step": 2770 |
| }, |
| { |
| "epoch": 1.7891857096878017, |
| "grad_norm": 9.913401725690312, |
| "learning_rate": 5.526400515132002e-07, |
| "logits/chosen": -1.155664086341858, |
| "logits/rejected": -1.091650366783142, |
| "logps/chosen": -439.20001220703125, |
| "logps/rejected": -552.6500244140625, |
| "loss": 0.0172, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -2.5236573219299316, |
| "rewards/margins": 10.727343559265137, |
| "rewards/rejected": -13.243749618530273, |
| "step": 2780 |
| }, |
| { |
| "epoch": 1.7956227872545865, |
| "grad_norm": 7.294597773629929, |
| "learning_rate": 5.510302640051513e-07, |
| "logits/chosen": -1.221435546875, |
| "logits/rejected": -1.2138550281524658, |
| "logps/chosen": -426.54998779296875, |
| "logps/rejected": -503.54998779296875, |
| "loss": 0.0266, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -2.975512742996216, |
| "rewards/margins": 10.671875, |
| "rewards/rejected": -13.646093368530273, |
| "step": 2790 |
| }, |
| { |
| "epoch": 1.8020598648213713, |
| "grad_norm": 8.088395142009128, |
| "learning_rate": 5.494204764971023e-07, |
| "logits/chosen": -1.28759765625, |
| "logits/rejected": -1.1837890148162842, |
| "logps/chosen": -442.4750061035156, |
| "logps/rejected": -535.6500244140625, |
| "loss": 0.0138, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -3.1391358375549316, |
| "rewards/margins": 10.840624809265137, |
| "rewards/rejected": -13.982812881469727, |
| "step": 2800 |
| }, |
| { |
| "epoch": 1.8084969423881558, |
| "grad_norm": 60.413121566780774, |
| "learning_rate": 5.478106889890535e-07, |
| "logits/chosen": -1.274511694908142, |
| "logits/rejected": -1.209326148033142, |
| "logps/chosen": -400.3999938964844, |
| "logps/rejected": -526.0499877929688, |
| "loss": 0.0493, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -2.994873046875, |
| "rewards/margins": 10.5390625, |
| "rewards/rejected": -13.534375190734863, |
| "step": 2810 |
| }, |
| { |
| "epoch": 1.8149340199549404, |
| "grad_norm": 0.8057869454662108, |
| "learning_rate": 5.462009014810045e-07, |
| "logits/chosen": -1.1865966320037842, |
| "logits/rejected": -1.200463891029358, |
| "logps/chosen": -484.79998779296875, |
| "logps/rejected": -544.5999755859375, |
| "loss": 0.0229, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -3.5977416038513184, |
| "rewards/margins": 11.043749809265137, |
| "rewards/rejected": -14.645312309265137, |
| "step": 2820 |
| }, |
| { |
| "epoch": 1.821371097521725, |
| "grad_norm": 5.680742689207317, |
| "learning_rate": 5.445911139729555e-07, |
| "logits/chosen": -1.3112671375274658, |
| "logits/rejected": -1.2607543468475342, |
| "logps/chosen": -429.54998779296875, |
| "logps/rejected": -533.4749755859375, |
| "loss": 0.0226, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -3.511523485183716, |
| "rewards/margins": 10.745312690734863, |
| "rewards/rejected": -14.251562118530273, |
| "step": 2830 |
| }, |
| { |
| "epoch": 1.8278081750885098, |
| "grad_norm": 33.65858455149658, |
| "learning_rate": 5.429813264649066e-07, |
| "logits/chosen": -1.2681396007537842, |
| "logits/rejected": -1.2060546875, |
| "logps/chosen": -436.20001220703125, |
| "logps/rejected": -532.8499755859375, |
| "loss": 0.0227, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -3.87255859375, |
| "rewards/margins": 10.264062881469727, |
| "rewards/rejected": -14.135937690734863, |
| "step": 2840 |
| }, |
| { |
| "epoch": 1.8342452526552946, |
| "grad_norm": 37.95912695465393, |
| "learning_rate": 5.413715389568578e-07, |
| "logits/chosen": -1.2689940929412842, |
| "logits/rejected": -1.2407745122909546, |
| "logps/chosen": -405.79998779296875, |
| "logps/rejected": -500.42498779296875, |
| "loss": 0.0678, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -3.3427734375, |
| "rewards/margins": 10.258593559265137, |
| "rewards/rejected": -13.604687690734863, |
| "step": 2850 |
| }, |
| { |
| "epoch": 1.8406823302220792, |
| "grad_norm": 0.7695568415638987, |
| "learning_rate": 5.397617514488087e-07, |
| "logits/chosen": -1.241064429283142, |
| "logits/rejected": -1.177160620689392, |
| "logps/chosen": -422.20001220703125, |
| "logps/rejected": -527.25, |
| "loss": 0.0195, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -3.3658204078674316, |
| "rewards/margins": 9.84375, |
| "rewards/rejected": -13.209375381469727, |
| "step": 2860 |
| }, |
| { |
| "epoch": 1.8471194077888637, |
| "grad_norm": 44.51667510286836, |
| "learning_rate": 5.381519639407598e-07, |
| "logits/chosen": -1.228906273841858, |
| "logits/rejected": -1.2727782726287842, |
| "logps/chosen": -411.8500061035156, |
| "logps/rejected": -548.1500244140625, |
| "loss": 0.0228, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -3.72265625, |
| "rewards/margins": 11.209375381469727, |
| "rewards/rejected": -14.935937881469727, |
| "step": 2870 |
| }, |
| { |
| "epoch": 1.8535564853556485, |
| "grad_norm": 19.27693244417542, |
| "learning_rate": 5.365421764327109e-07, |
| "logits/chosen": -1.263916015625, |
| "logits/rejected": -1.2194092273712158, |
| "logps/chosen": -433.1499938964844, |
| "logps/rejected": -524.4000244140625, |
| "loss": 0.0441, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -3.2273926734924316, |
| "rewards/margins": 10.41796875, |
| "rewards/rejected": -13.649999618530273, |
| "step": 2880 |
| }, |
| { |
| "epoch": 1.8599935629224333, |
| "grad_norm": 1.7913933764246404, |
| "learning_rate": 5.34932388924662e-07, |
| "logits/chosen": -1.196044921875, |
| "logits/rejected": -1.24072265625, |
| "logps/chosen": -446.32501220703125, |
| "logps/rejected": -523.7625122070312, |
| "loss": 0.0436, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -3.5137572288513184, |
| "rewards/margins": 9.908594131469727, |
| "rewards/rejected": -13.43359375, |
| "step": 2890 |
| }, |
| { |
| "epoch": 1.866430640489218, |
| "grad_norm": 17.728406330203313, |
| "learning_rate": 5.33322601416613e-07, |
| "logits/chosen": -1.225341796875, |
| "logits/rejected": -1.1571776866912842, |
| "logps/chosen": -434.375, |
| "logps/rejected": -531.1500244140625, |
| "loss": 0.0393, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -3.3700194358825684, |
| "rewards/margins": 9.891406059265137, |
| "rewards/rejected": -13.256250381469727, |
| "step": 2900 |
| }, |
| { |
| "epoch": 1.8728677180560025, |
| "grad_norm": 0.5585488010994464, |
| "learning_rate": 5.31712813908564e-07, |
| "logits/chosen": -1.30419921875, |
| "logits/rejected": -1.1737792491912842, |
| "logps/chosen": -408.4750061035156, |
| "logps/rejected": -544.0, |
| "loss": 0.0093, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -3.195849657058716, |
| "rewards/margins": 10.068750381469727, |
| "rewards/rejected": -13.264062881469727, |
| "step": 2910 |
| }, |
| { |
| "epoch": 1.8793047956227873, |
| "grad_norm": 8.289949602627718, |
| "learning_rate": 5.301030264005151e-07, |
| "logits/chosen": -1.1561279296875, |
| "logits/rejected": -1.15283203125, |
| "logps/chosen": -437.0, |
| "logps/rejected": -503.9750061035156, |
| "loss": 0.0224, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -3.2488770484924316, |
| "rewards/margins": 10.243749618530273, |
| "rewards/rejected": -13.497655868530273, |
| "step": 2920 |
| }, |
| { |
| "epoch": 1.885741873189572, |
| "grad_norm": 12.8256364369073, |
| "learning_rate": 5.284932388924662e-07, |
| "logits/chosen": -1.1316649913787842, |
| "logits/rejected": -1.084375023841858, |
| "logps/chosen": -409.3500061035156, |
| "logps/rejected": -496.8500061035156, |
| "loss": 0.0614, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -2.7437500953674316, |
| "rewards/margins": 10.496874809265137, |
| "rewards/rejected": -13.240625381469727, |
| "step": 2930 |
| }, |
| { |
| "epoch": 1.8921789507563567, |
| "grad_norm": 17.60736216733103, |
| "learning_rate": 5.268834513844172e-07, |
| "logits/chosen": -1.112817406654358, |
| "logits/rejected": -1.1421387195587158, |
| "logps/chosen": -415.07501220703125, |
| "logps/rejected": -523.0999755859375, |
| "loss": 0.0665, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -2.8383545875549316, |
| "rewards/margins": 10.260156631469727, |
| "rewards/rejected": -13.095312118530273, |
| "step": 2940 |
| }, |
| { |
| "epoch": 1.8986160283231412, |
| "grad_norm": 5.864322661168903, |
| "learning_rate": 5.252736638763683e-07, |
| "logits/chosen": -1.0784180164337158, |
| "logits/rejected": -1.1527099609375, |
| "logps/chosen": -448.7250061035156, |
| "logps/rejected": -503.42498779296875, |
| "loss": 0.0329, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -2.317370653152466, |
| "rewards/margins": 10.392969131469727, |
| "rewards/rejected": -12.712499618530273, |
| "step": 2950 |
| }, |
| { |
| "epoch": 1.9050531058899258, |
| "grad_norm": 2.2553254096477526, |
| "learning_rate": 5.236638763683194e-07, |
| "logits/chosen": -1.139184594154358, |
| "logits/rejected": -1.069677710533142, |
| "logps/chosen": -419.82501220703125, |
| "logps/rejected": -553.4000244140625, |
| "loss": 0.024, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -3.17431640625, |
| "rewards/margins": 10.610937118530273, |
| "rewards/rejected": -13.784375190734863, |
| "step": 2960 |
| }, |
| { |
| "epoch": 1.9114901834567106, |
| "grad_norm": 0.2734892226087745, |
| "learning_rate": 5.220540888602704e-07, |
| "logits/chosen": -1.1643555164337158, |
| "logits/rejected": -1.127343773841858, |
| "logps/chosen": -461.42498779296875, |
| "logps/rejected": -548.75, |
| "loss": 0.0337, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -2.8843016624450684, |
| "rewards/margins": 10.59765625, |
| "rewards/rejected": -13.469531059265137, |
| "step": 2970 |
| }, |
| { |
| "epoch": 1.9179272610234954, |
| "grad_norm": 0.3547205065818246, |
| "learning_rate": 5.204443013522215e-07, |
| "logits/chosen": -1.235681176185608, |
| "logits/rejected": -1.1963622570037842, |
| "logps/chosen": -427.5249938964844, |
| "logps/rejected": -508.6000061035156, |
| "loss": 0.0353, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -3.3380095958709717, |
| "rewards/margins": 9.974218368530273, |
| "rewards/rejected": -13.3125, |
| "step": 2980 |
| }, |
| { |
| "epoch": 1.92436433859028, |
| "grad_norm": 1.5271220311955436, |
| "learning_rate": 5.188345138441726e-07, |
| "logits/chosen": -1.204736351966858, |
| "logits/rejected": -1.214941382408142, |
| "logps/chosen": -442.8999938964844, |
| "logps/rejected": -505.25, |
| "loss": 0.0294, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -3.086230516433716, |
| "rewards/margins": 9.942968368530273, |
| "rewards/rejected": -13.029687881469727, |
| "step": 2990 |
| }, |
| { |
| "epoch": 1.9308014161570646, |
| "grad_norm": 25.90175001475661, |
| "learning_rate": 5.172247263361235e-07, |
| "logits/chosen": -1.0989258289337158, |
| "logits/rejected": -1.1459472179412842, |
| "logps/chosen": -468.57501220703125, |
| "logps/rejected": -547.7999877929688, |
| "loss": 0.0221, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -2.743847608566284, |
| "rewards/margins": 10.381250381469727, |
| "rewards/rejected": -13.124218940734863, |
| "step": 3000 |
| }, |
| { |
| "epoch": 1.9372384937238494, |
| "grad_norm": 19.517585448598766, |
| "learning_rate": 5.156149388280747e-07, |
| "logits/chosen": -1.1027343273162842, |
| "logits/rejected": -1.085351586341858, |
| "logps/chosen": -437.7749938964844, |
| "logps/rejected": -489.45001220703125, |
| "loss": 0.0294, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -3.750927686691284, |
| "rewards/margins": 10.114843368530273, |
| "rewards/rejected": -13.860937118530273, |
| "step": 3010 |
| }, |
| { |
| "epoch": 1.9436755712906342, |
| "grad_norm": 0.9713312965614613, |
| "learning_rate": 5.140051513200258e-07, |
| "logits/chosen": -1.3384277820587158, |
| "logits/rejected": -1.333593726158142, |
| "logps/chosen": -425.875, |
| "logps/rejected": -504.5249938964844, |
| "loss": 0.0193, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -3.0125975608825684, |
| "rewards/margins": 10.182812690734863, |
| "rewards/rejected": -13.189062118530273, |
| "step": 3020 |
| }, |
| { |
| "epoch": 1.9501126488574188, |
| "grad_norm": 2.3402658749481597, |
| "learning_rate": 5.123953638119768e-07, |
| "logits/chosen": -1.146459937095642, |
| "logits/rejected": -1.138208031654358, |
| "logps/chosen": -456.75, |
| "logps/rejected": -536.25, |
| "loss": 0.0299, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -3.2066407203674316, |
| "rewards/margins": 10.0703125, |
| "rewards/rejected": -13.2734375, |
| "step": 3030 |
| }, |
| { |
| "epoch": 1.9565497264242033, |
| "grad_norm": 6.230436374880822, |
| "learning_rate": 5.107855763039278e-07, |
| "logits/chosen": -1.300146460533142, |
| "logits/rejected": -1.32470703125, |
| "logps/chosen": -432.42498779296875, |
| "logps/rejected": -520.25, |
| "loss": 0.0378, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -3.313647508621216, |
| "rewards/margins": 10.26953125, |
| "rewards/rejected": -13.578125, |
| "step": 3040 |
| }, |
| { |
| "epoch": 1.9629868039909881, |
| "grad_norm": 19.401894404149196, |
| "learning_rate": 5.09175788795879e-07, |
| "logits/chosen": -1.282006859779358, |
| "logits/rejected": -1.2487304210662842, |
| "logps/chosen": -418.70001220703125, |
| "logps/rejected": -506.25, |
| "loss": 0.0218, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -2.7337646484375, |
| "rewards/margins": 10.192187309265137, |
| "rewards/rejected": -12.925000190734863, |
| "step": 3050 |
| }, |
| { |
| "epoch": 1.969423881557773, |
| "grad_norm": 1.7424478766604075, |
| "learning_rate": 5.0756600128783e-07, |
| "logits/chosen": -1.264672875404358, |
| "logits/rejected": -1.3064696788787842, |
| "logps/chosen": -436.07501220703125, |
| "logps/rejected": -522.4500122070312, |
| "loss": 0.0272, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -3.3886475563049316, |
| "rewards/margins": 11.032031059265137, |
| "rewards/rejected": -14.428125381469727, |
| "step": 3060 |
| }, |
| { |
| "epoch": 1.9758609591245575, |
| "grad_norm": 2.521297330850847, |
| "learning_rate": 5.059562137797811e-07, |
| "logits/chosen": -1.414209008216858, |
| "logits/rejected": -1.430566430091858, |
| "logps/chosen": -435.625, |
| "logps/rejected": -480.79998779296875, |
| "loss": 0.0243, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -3.6727051734924316, |
| "rewards/margins": 10.212499618530273, |
| "rewards/rejected": -13.895312309265137, |
| "step": 3070 |
| }, |
| { |
| "epoch": 1.982298036691342, |
| "grad_norm": 2.72166667410134, |
| "learning_rate": 5.04346426271732e-07, |
| "logits/chosen": -1.379296898841858, |
| "logits/rejected": -1.362695336341858, |
| "logps/chosen": -469.20001220703125, |
| "logps/rejected": -536.2000122070312, |
| "loss": 0.0405, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -3.7247071266174316, |
| "rewards/margins": 10.990625381469727, |
| "rewards/rejected": -14.723437309265137, |
| "step": 3080 |
| }, |
| { |
| "epoch": 1.9887351142581267, |
| "grad_norm": 0.8886574474010416, |
| "learning_rate": 5.027366387636832e-07, |
| "logits/chosen": -1.266259789466858, |
| "logits/rejected": -1.205468773841858, |
| "logps/chosen": -400.5, |
| "logps/rejected": -498.1000061035156, |
| "loss": 0.0436, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -3.1731934547424316, |
| "rewards/margins": 10.158984184265137, |
| "rewards/rejected": -13.330469131469727, |
| "step": 3090 |
| }, |
| { |
| "epoch": 1.9951721918249115, |
| "grad_norm": 41.25117206925145, |
| "learning_rate": 5.011268512556343e-07, |
| "logits/chosen": -1.372460961341858, |
| "logits/rejected": -1.3676269054412842, |
| "logps/chosen": -432.4750061035156, |
| "logps/rejected": -510.2749938964844, |
| "loss": 0.0286, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -2.969555616378784, |
| "rewards/margins": 10.041406631469727, |
| "rewards/rejected": -13.005468368530273, |
| "step": 3100 |
| }, |
| { |
| "epoch": 2.001287415513357, |
| "grad_norm": 0.27539900977102494, |
| "learning_rate": 4.995170637475852e-07, |
| "logits/chosen": -1.356959342956543, |
| "logits/rejected": -1.2509572505950928, |
| "logps/chosen": -395.3947448730469, |
| "logps/rejected": -513.9473876953125, |
| "loss": 0.0239, |
| "rewards/accuracies": 0.9934210777282715, |
| "rewards/chosen": -3.39103627204895, |
| "rewards/margins": 10.711348533630371, |
| "rewards/rejected": -14.103618621826172, |
| "step": 3110 |
| }, |
| { |
| "epoch": 2.0077244930801417, |
| "grad_norm": 0.22702723200509595, |
| "learning_rate": 4.979072762395364e-07, |
| "logits/chosen": -1.2954590320587158, |
| "logits/rejected": -1.284912109375, |
| "logps/chosen": -437.5625, |
| "logps/rejected": -536.5999755859375, |
| "loss": 0.0136, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -2.2456297874450684, |
| "rewards/margins": 12.434374809265137, |
| "rewards/rejected": -14.678125381469727, |
| "step": 3120 |
| }, |
| { |
| "epoch": 2.0141615706469262, |
| "grad_norm": 0.5386680666844332, |
| "learning_rate": 4.962974887314874e-07, |
| "logits/chosen": -1.340429663658142, |
| "logits/rejected": -1.352783203125, |
| "logps/chosen": -406.57501220703125, |
| "logps/rejected": -504.79998779296875, |
| "loss": 0.024, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -2.758251905441284, |
| "rewards/margins": 12.631250381469727, |
| "rewards/rejected": -15.3828125, |
| "step": 3130 |
| }, |
| { |
| "epoch": 2.020598648213711, |
| "grad_norm": 0.11976598731527363, |
| "learning_rate": 4.946877012234385e-07, |
| "logits/chosen": -1.393090844154358, |
| "logits/rejected": -1.280542016029358, |
| "logps/chosen": -386.95001220703125, |
| "logps/rejected": -517.2750244140625, |
| "loss": 0.0164, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -2.921679735183716, |
| "rewards/margins": 12.131250381469727, |
| "rewards/rejected": -15.059374809265137, |
| "step": 3140 |
| }, |
| { |
| "epoch": 2.027035725780496, |
| "grad_norm": 0.5061178833401264, |
| "learning_rate": 4.930779137153895e-07, |
| "logits/chosen": -1.325439453125, |
| "logits/rejected": -1.2689208984375, |
| "logps/chosen": -405.45001220703125, |
| "logps/rejected": -520.5499877929688, |
| "loss": 0.0142, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -2.393078565597534, |
| "rewards/margins": 13.182812690734863, |
| "rewards/rejected": -15.5703125, |
| "step": 3150 |
| }, |
| { |
| "epoch": 2.0334728033472804, |
| "grad_norm": 0.1420213399458517, |
| "learning_rate": 4.914681262073406e-07, |
| "logits/chosen": -1.451928734779358, |
| "logits/rejected": -1.342871069908142, |
| "logps/chosen": -416.125, |
| "logps/rejected": -536.0, |
| "loss": 0.0143, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -3.326171875, |
| "rewards/margins": 12.823437690734863, |
| "rewards/rejected": -16.157812118530273, |
| "step": 3160 |
| }, |
| { |
| "epoch": 2.039909880914065, |
| "grad_norm": 0.2870207266885512, |
| "learning_rate": 4.898583386992917e-07, |
| "logits/chosen": -1.3640625476837158, |
| "logits/rejected": -1.3269531726837158, |
| "logps/chosen": -388.75, |
| "logps/rejected": -527.5499877929688, |
| "loss": 0.0262, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -2.991943359375, |
| "rewards/margins": 13.064062118530273, |
| "rewards/rejected": -16.067188262939453, |
| "step": 3170 |
| }, |
| { |
| "epoch": 2.0463469584808496, |
| "grad_norm": 1.988033208700941, |
| "learning_rate": 4.882485511912428e-07, |
| "logits/chosen": -1.2886474132537842, |
| "logits/rejected": -1.2509765625, |
| "logps/chosen": -459.875, |
| "logps/rejected": -595.5, |
| "loss": 0.0135, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -4.087500095367432, |
| "rewards/margins": 14.385937690734863, |
| "rewards/rejected": -18.487499237060547, |
| "step": 3180 |
| }, |
| { |
| "epoch": 2.052784036047634, |
| "grad_norm": 0.07799476058924272, |
| "learning_rate": 4.866387636831938e-07, |
| "logits/chosen": -1.529931664466858, |
| "logits/rejected": -1.4558594226837158, |
| "logps/chosen": -403.70001220703125, |
| "logps/rejected": -542.6500244140625, |
| "loss": 0.0223, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -3.908984422683716, |
| "rewards/margins": 13.6796875, |
| "rewards/rejected": -17.590625762939453, |
| "step": 3190 |
| }, |
| { |
| "epoch": 2.059221113614419, |
| "grad_norm": 0.20873721083774974, |
| "learning_rate": 4.850289761751449e-07, |
| "logits/chosen": -1.4627196788787842, |
| "logits/rejected": -1.3955078125, |
| "logps/chosen": -422.3999938964844, |
| "logps/rejected": -528.75, |
| "loss": 0.021, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -4.801171779632568, |
| "rewards/margins": 11.931249618530273, |
| "rewards/rejected": -16.729686737060547, |
| "step": 3200 |
| }, |
| { |
| "epoch": 2.0656581911812038, |
| "grad_norm": 0.060018781366832474, |
| "learning_rate": 4.834191886670959e-07, |
| "logits/chosen": -1.366235375404358, |
| "logits/rejected": -1.3874022960662842, |
| "logps/chosen": -471.3500061035156, |
| "logps/rejected": -567.9000244140625, |
| "loss": 0.0046, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -4.335375785827637, |
| "rewards/margins": 13.503125190734863, |
| "rewards/rejected": -17.84375, |
| "step": 3210 |
| }, |
| { |
| "epoch": 2.0720952687479883, |
| "grad_norm": 0.17169944145131982, |
| "learning_rate": 4.818094011590471e-07, |
| "logits/chosen": -1.2931396961212158, |
| "logits/rejected": -1.2729003429412842, |
| "logps/chosen": -409.17498779296875, |
| "logps/rejected": -550.0, |
| "loss": 0.0219, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -4.220117092132568, |
| "rewards/margins": 13.587499618530273, |
| "rewards/rejected": -17.803125381469727, |
| "step": 3220 |
| }, |
| { |
| "epoch": 2.078532346314773, |
| "grad_norm": 1.0753436316182554, |
| "learning_rate": 4.80199613650998e-07, |
| "logits/chosen": -1.357519507408142, |
| "logits/rejected": -1.4267578125, |
| "logps/chosen": -478.92498779296875, |
| "logps/rejected": -579.7750244140625, |
| "loss": 0.0138, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -4.186474800109863, |
| "rewards/margins": 14.008593559265137, |
| "rewards/rejected": -18.1953125, |
| "step": 3230 |
| }, |
| { |
| "epoch": 2.084969423881558, |
| "grad_norm": 0.6837663530863362, |
| "learning_rate": 4.785898261429491e-07, |
| "logits/chosen": -1.5143554210662842, |
| "logits/rejected": -1.5537109375, |
| "logps/chosen": -429.25, |
| "logps/rejected": -556.2999877929688, |
| "loss": 0.009, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -5.096581935882568, |
| "rewards/margins": 14.401562690734863, |
| "rewards/rejected": -19.4921875, |
| "step": 3240 |
| }, |
| { |
| "epoch": 2.0914065014483425, |
| "grad_norm": 0.02844425579096925, |
| "learning_rate": 4.769800386349002e-07, |
| "logits/chosen": -1.4525146484375, |
| "logits/rejected": -1.429443359375, |
| "logps/chosen": -459.95001220703125, |
| "logps/rejected": -593.4500122070312, |
| "loss": 0.0045, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -5.351403713226318, |
| "rewards/margins": 14.448437690734863, |
| "rewards/rejected": -19.807811737060547, |
| "step": 3250 |
| }, |
| { |
| "epoch": 2.097843579015127, |
| "grad_norm": 0.3300029059084895, |
| "learning_rate": 4.7537025112685124e-07, |
| "logits/chosen": -1.3985106945037842, |
| "logits/rejected": -1.303955078125, |
| "logps/chosen": -421.3125, |
| "logps/rejected": -553.4000244140625, |
| "loss": 0.0243, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -5.023241996765137, |
| "rewards/margins": 14.096094131469727, |
| "rewards/rejected": -19.121875762939453, |
| "step": 3260 |
| }, |
| { |
| "epoch": 2.1042806565819117, |
| "grad_norm": 0.6014401403345411, |
| "learning_rate": 4.7376046361880226e-07, |
| "logits/chosen": -1.3398926258087158, |
| "logits/rejected": -1.359106421470642, |
| "logps/chosen": -459.375, |
| "logps/rejected": -577.7999877929688, |
| "loss": 0.0262, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -4.295947074890137, |
| "rewards/margins": 14.603124618530273, |
| "rewards/rejected": -18.899999618530273, |
| "step": 3270 |
| }, |
| { |
| "epoch": 2.1107177341486967, |
| "grad_norm": 0.10699440633427301, |
| "learning_rate": 4.721506761107534e-07, |
| "logits/chosen": -1.5061523914337158, |
| "logits/rejected": -1.502050757408142, |
| "logps/chosen": -465.36248779296875, |
| "logps/rejected": -586.4000244140625, |
| "loss": 0.0004, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -4.754296779632568, |
| "rewards/margins": 14.556249618530273, |
| "rewards/rejected": -19.314062118530273, |
| "step": 3280 |
| }, |
| { |
| "epoch": 2.1171548117154813, |
| "grad_norm": 0.19169893872270757, |
| "learning_rate": 4.705408886027044e-07, |
| "logits/chosen": -1.4147460460662842, |
| "logits/rejected": -1.367285132408142, |
| "logps/chosen": -404.5249938964844, |
| "logps/rejected": -550.4500122070312, |
| "loss": 0.0318, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -5.122460842132568, |
| "rewards/margins": 13.771093368530273, |
| "rewards/rejected": -18.904687881469727, |
| "step": 3290 |
| }, |
| { |
| "epoch": 2.123591889282266, |
| "grad_norm": 0.013682100436810481, |
| "learning_rate": 4.689311010946555e-07, |
| "logits/chosen": -1.5072510242462158, |
| "logits/rejected": -1.472900390625, |
| "logps/chosen": -410.8999938964844, |
| "logps/rejected": -573.75, |
| "loss": 0.0183, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -4.404589653015137, |
| "rewards/margins": 15.071874618530273, |
| "rewards/rejected": -19.4765625, |
| "step": 3300 |
| }, |
| { |
| "epoch": 2.1300289668490504, |
| "grad_norm": 0.8987373864874652, |
| "learning_rate": 4.6732131358660656e-07, |
| "logits/chosen": -1.425048828125, |
| "logits/rejected": -1.3694305419921875, |
| "logps/chosen": -422.6625061035156, |
| "logps/rejected": -558.9500122070312, |
| "loss": 0.0219, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -4.257226467132568, |
| "rewards/margins": 14.396875381469727, |
| "rewards/rejected": -18.660938262939453, |
| "step": 3310 |
| }, |
| { |
| "epoch": 2.136466044415835, |
| "grad_norm": 0.11133976729955616, |
| "learning_rate": 4.6571152607855763e-07, |
| "logits/chosen": -1.274999976158142, |
| "logits/rejected": -1.236730933189392, |
| "logps/chosen": -453.875, |
| "logps/rejected": -573.0, |
| "loss": 0.0233, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -4.51824951171875, |
| "rewards/margins": 14.482812881469727, |
| "rewards/rejected": -19.018749237060547, |
| "step": 3320 |
| }, |
| { |
| "epoch": 2.14290312198262, |
| "grad_norm": 0.2967003025922388, |
| "learning_rate": 4.6410173857050865e-07, |
| "logits/chosen": -1.469140648841858, |
| "logits/rejected": -1.446630835533142, |
| "logps/chosen": -456.5, |
| "logps/rejected": -600.0, |
| "loss": 0.0146, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -6.058423042297363, |
| "rewards/margins": 14.389843940734863, |
| "rewards/rejected": -20.453125, |
| "step": 3330 |
| }, |
| { |
| "epoch": 2.1493401995494046, |
| "grad_norm": 0.05110981471707289, |
| "learning_rate": 4.624919510624598e-07, |
| "logits/chosen": -1.611718773841858, |
| "logits/rejected": -1.576757788658142, |
| "logps/chosen": -438.0, |
| "logps/rejected": -569.5, |
| "loss": 0.0133, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -6.008447170257568, |
| "rewards/margins": 14.025781631469727, |
| "rewards/rejected": -20.032812118530273, |
| "step": 3340 |
| }, |
| { |
| "epoch": 2.155777277116189, |
| "grad_norm": 0.036902779629885446, |
| "learning_rate": 4.608821635544108e-07, |
| "logits/chosen": -1.3733398914337158, |
| "logits/rejected": -1.411157250404358, |
| "logps/chosen": -469.25, |
| "logps/rejected": -565.0999755859375, |
| "loss": 0.0105, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -4.664453029632568, |
| "rewards/margins": 14.110937118530273, |
| "rewards/rejected": -18.768749237060547, |
| "step": 3350 |
| }, |
| { |
| "epoch": 2.1622143546829737, |
| "grad_norm": 7.206483217930865, |
| "learning_rate": 4.5927237604636187e-07, |
| "logits/chosen": -1.4674193859100342, |
| "logits/rejected": -1.5466797351837158, |
| "logps/chosen": -471.32501220703125, |
| "logps/rejected": -585.9000244140625, |
| "loss": 0.015, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -4.689843654632568, |
| "rewards/margins": 14.893750190734863, |
| "rewards/rejected": -19.5703125, |
| "step": 3360 |
| }, |
| { |
| "epoch": 2.1686514322497588, |
| "grad_norm": 0.030603692001684026, |
| "learning_rate": 4.576625885383129e-07, |
| "logits/chosen": -1.348242163658142, |
| "logits/rejected": -1.382665991783142, |
| "logps/chosen": -479.29998779296875, |
| "logps/rejected": -601.3499755859375, |
| "loss": 0.0045, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -4.857617378234863, |
| "rewards/margins": 13.970312118530273, |
| "rewards/rejected": -18.817188262939453, |
| "step": 3370 |
| }, |
| { |
| "epoch": 2.1750885098165433, |
| "grad_norm": 0.39450551026115094, |
| "learning_rate": 4.56052801030264e-07, |
| "logits/chosen": -1.4818847179412842, |
| "logits/rejected": -1.4171874523162842, |
| "logps/chosen": -451.67498779296875, |
| "logps/rejected": -576.8499755859375, |
| "loss": 0.0092, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -4.820263862609863, |
| "rewards/margins": 14.928125381469727, |
| "rewards/rejected": -19.743749618530273, |
| "step": 3380 |
| }, |
| { |
| "epoch": 2.181525587383328, |
| "grad_norm": 0.02651151421531908, |
| "learning_rate": 4.5444301352221504e-07, |
| "logits/chosen": -1.459375023841858, |
| "logits/rejected": -1.380834937095642, |
| "logps/chosen": -486.875, |
| "logps/rejected": -623.2999877929688, |
| "loss": 0.0136, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -5.310156345367432, |
| "rewards/margins": 14.821093559265137, |
| "rewards/rejected": -20.131250381469727, |
| "step": 3390 |
| }, |
| { |
| "epoch": 2.1879626649501125, |
| "grad_norm": 0.7239760804949088, |
| "learning_rate": 4.528332260141661e-07, |
| "logits/chosen": -1.419042944908142, |
| "logits/rejected": -1.378027319908142, |
| "logps/chosen": -444.7749938964844, |
| "logps/rejected": -567.25, |
| "loss": 0.0102, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -5.276953220367432, |
| "rewards/margins": 14.184374809265137, |
| "rewards/rejected": -19.470312118530273, |
| "step": 3400 |
| }, |
| { |
| "epoch": 2.1943997425168975, |
| "grad_norm": 0.3051785555739404, |
| "learning_rate": 4.512234385061172e-07, |
| "logits/chosen": -1.3330566883087158, |
| "logits/rejected": -1.3825194835662842, |
| "logps/chosen": -495.20001220703125, |
| "logps/rejected": -576.7000122070312, |
| "loss": 0.0186, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -5.559960842132568, |
| "rewards/margins": 13.477343559265137, |
| "rewards/rejected": -19.0390625, |
| "step": 3410 |
| }, |
| { |
| "epoch": 2.200836820083682, |
| "grad_norm": 0.4322230346946436, |
| "learning_rate": 4.4961365099806826e-07, |
| "logits/chosen": -1.5175292491912842, |
| "logits/rejected": -1.524560570716858, |
| "logps/chosen": -440.82501220703125, |
| "logps/rejected": -538.5, |
| "loss": 0.0008, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -4.824462890625, |
| "rewards/margins": 15.010937690734863, |
| "rewards/rejected": -19.825000762939453, |
| "step": 3420 |
| }, |
| { |
| "epoch": 2.2072738976504667, |
| "grad_norm": 0.10696448497825761, |
| "learning_rate": 4.480038634900193e-07, |
| "logits/chosen": -1.335412621498108, |
| "logits/rejected": -1.334326148033142, |
| "logps/chosen": -423.07501220703125, |
| "logps/rejected": -552.4000244140625, |
| "loss": 0.0234, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -4.78515625, |
| "rewards/margins": 13.764843940734863, |
| "rewards/rejected": -18.559375762939453, |
| "step": 3430 |
| }, |
| { |
| "epoch": 2.2137109752172512, |
| "grad_norm": 0.21681989288448078, |
| "learning_rate": 4.4639407598197035e-07, |
| "logits/chosen": -1.4794921875, |
| "logits/rejected": -1.447509765625, |
| "logps/chosen": -454.75, |
| "logps/rejected": -577.2999877929688, |
| "loss": 0.0018, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -3.880908250808716, |
| "rewards/margins": 13.978124618530273, |
| "rewards/rejected": -17.854686737060547, |
| "step": 3440 |
| }, |
| { |
| "epoch": 2.220148052784036, |
| "grad_norm": 0.9328545258206828, |
| "learning_rate": 4.447842884739214e-07, |
| "logits/chosen": -1.3971436023712158, |
| "logits/rejected": -1.3560791015625, |
| "logps/chosen": -416.2250061035156, |
| "logps/rejected": -537.7999877929688, |
| "loss": 0.0243, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -4.1676025390625, |
| "rewards/margins": 13.984375, |
| "rewards/rejected": -18.140625, |
| "step": 3450 |
| }, |
| { |
| "epoch": 2.226585130350821, |
| "grad_norm": 0.0617367345109128, |
| "learning_rate": 4.431745009658725e-07, |
| "logits/chosen": -1.474951148033142, |
| "logits/rejected": -1.4214355945587158, |
| "logps/chosen": -435.5, |
| "logps/rejected": -569.7999877929688, |
| "loss": 0.0045, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -4.1029052734375, |
| "rewards/margins": 15.614062309265137, |
| "rewards/rejected": -19.728124618530273, |
| "step": 3460 |
| }, |
| { |
| "epoch": 2.2330222079176054, |
| "grad_norm": 0.5705440382514301, |
| "learning_rate": 4.415647134578235e-07, |
| "logits/chosen": -1.481347680091858, |
| "logits/rejected": -1.438745141029358, |
| "logps/chosen": -443.07501220703125, |
| "logps/rejected": -575.9500122070312, |
| "loss": 0.0094, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -4.044140815734863, |
| "rewards/margins": 14.544530868530273, |
| "rewards/rejected": -18.587499618530273, |
| "step": 3470 |
| }, |
| { |
| "epoch": 2.23945928548439, |
| "grad_norm": 0.10399795101666638, |
| "learning_rate": 4.3995492594977465e-07, |
| "logits/chosen": -1.46966552734375, |
| "logits/rejected": -1.4376952648162842, |
| "logps/chosen": -419.95001220703125, |
| "logps/rejected": -553.4500122070312, |
| "loss": 0.0137, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -5.474023342132568, |
| "rewards/margins": 14.16796875, |
| "rewards/rejected": -19.639062881469727, |
| "step": 3480 |
| }, |
| { |
| "epoch": 2.2458963630511746, |
| "grad_norm": 0.0054108280993078565, |
| "learning_rate": 4.3834513844172567e-07, |
| "logits/chosen": -1.3809814453125, |
| "logits/rejected": -1.3763916492462158, |
| "logps/chosen": -464.2250061035156, |
| "logps/rejected": -585.7999877929688, |
| "loss": 0.0219, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -5.171679496765137, |
| "rewards/margins": 14.655468940734863, |
| "rewards/rejected": -19.834375381469727, |
| "step": 3490 |
| }, |
| { |
| "epoch": 2.2523334406179596, |
| "grad_norm": 1.3424048828171435, |
| "learning_rate": 4.3673535093367674e-07, |
| "logits/chosen": -1.5671875476837158, |
| "logits/rejected": -1.4344971179962158, |
| "logps/chosen": -401.8500061035156, |
| "logps/rejected": -547.0999755859375, |
| "loss": 0.0046, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -5.40234375, |
| "rewards/margins": 14.580469131469727, |
| "rewards/rejected": -19.975000381469727, |
| "step": 3500 |
| }, |
| { |
| "epoch": 2.258770518184744, |
| "grad_norm": 0.22660963382492813, |
| "learning_rate": 4.3512556342562776e-07, |
| "logits/chosen": -1.388818383216858, |
| "logits/rejected": -1.3882324695587158, |
| "logps/chosen": -457.29998779296875, |
| "logps/rejected": -589.75, |
| "loss": 0.014, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -5.254492282867432, |
| "rewards/margins": 14.98828125, |
| "rewards/rejected": -20.240625381469727, |
| "step": 3510 |
| }, |
| { |
| "epoch": 2.2652075957515287, |
| "grad_norm": 0.010223293776609103, |
| "learning_rate": 4.335157759175789e-07, |
| "logits/chosen": -1.366784691810608, |
| "logits/rejected": -1.3222167491912842, |
| "logps/chosen": -428.20001220703125, |
| "logps/rejected": -580.5499877929688, |
| "loss": 0.0147, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -5.898046970367432, |
| "rewards/margins": 14.146875381469727, |
| "rewards/rejected": -20.043750762939453, |
| "step": 3520 |
| }, |
| { |
| "epoch": 2.2716446733183133, |
| "grad_norm": 0.1341115353978205, |
| "learning_rate": 4.319059884095299e-07, |
| "logits/chosen": -1.426171898841858, |
| "logits/rejected": -1.4014160633087158, |
| "logps/chosen": -436.57501220703125, |
| "logps/rejected": -562.5499877929688, |
| "loss": 0.0258, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -5.5546875, |
| "rewards/margins": 13.90625, |
| "rewards/rejected": -19.456249237060547, |
| "step": 3530 |
| }, |
| { |
| "epoch": 2.2780817508850983, |
| "grad_norm": 0.5206039308552399, |
| "learning_rate": 4.30296200901481e-07, |
| "logits/chosen": -1.269140601158142, |
| "logits/rejected": -1.228662133216858, |
| "logps/chosen": -458.54998779296875, |
| "logps/rejected": -594.4000244140625, |
| "loss": 0.0132, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -4.688867092132568, |
| "rewards/margins": 14.796875, |
| "rewards/rejected": -19.4921875, |
| "step": 3540 |
| }, |
| { |
| "epoch": 2.284518828451883, |
| "grad_norm": 0.08394243831322193, |
| "learning_rate": 4.2868641339343205e-07, |
| "logits/chosen": -1.3561522960662842, |
| "logits/rejected": -1.393457055091858, |
| "logps/chosen": -450.95001220703125, |
| "logps/rejected": -563.25, |
| "loss": 0.0141, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -4.6201171875, |
| "rewards/margins": 14.989843368530273, |
| "rewards/rejected": -19.607812881469727, |
| "step": 3550 |
| }, |
| { |
| "epoch": 2.2909559060186675, |
| "grad_norm": 8.998990771782054, |
| "learning_rate": 4.2707662588538313e-07, |
| "logits/chosen": -1.405358910560608, |
| "logits/rejected": -1.375634789466858, |
| "logps/chosen": -451.3500061035156, |
| "logps/rejected": -581.4500122070312, |
| "loss": 0.0191, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -5.491601467132568, |
| "rewards/margins": 13.482030868530273, |
| "rewards/rejected": -18.962499618530273, |
| "step": 3560 |
| }, |
| { |
| "epoch": 2.297392983585452, |
| "grad_norm": 0.005113373828258078, |
| "learning_rate": 4.2546683837733415e-07, |
| "logits/chosen": -1.2739746570587158, |
| "logits/rejected": -1.2206299304962158, |
| "logps/chosen": -465.67498779296875, |
| "logps/rejected": -565.2999877929688, |
| "loss": 0.0184, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -5.182031154632568, |
| "rewards/margins": 13.671875, |
| "rewards/rejected": -18.846874237060547, |
| "step": 3570 |
| }, |
| { |
| "epoch": 2.3038300611522367, |
| "grad_norm": 0.015383738968627606, |
| "learning_rate": 4.238570508692853e-07, |
| "logits/chosen": -1.414007544517517, |
| "logits/rejected": -1.310766577720642, |
| "logps/chosen": -438.1000061035156, |
| "logps/rejected": -581.3499755859375, |
| "loss": 0.0142, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -4.516161918640137, |
| "rewards/margins": 14.2421875, |
| "rewards/rejected": -18.760936737060547, |
| "step": 3580 |
| }, |
| { |
| "epoch": 2.3102671387190217, |
| "grad_norm": 0.8851160683604858, |
| "learning_rate": 4.222472633612363e-07, |
| "logits/chosen": -1.290747046470642, |
| "logits/rejected": -1.285546898841858, |
| "logps/chosen": -444.20001220703125, |
| "logps/rejected": -576.9500122070312, |
| "loss": 0.0089, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -3.9532227516174316, |
| "rewards/margins": 14.285937309265137, |
| "rewards/rejected": -18.235937118530273, |
| "step": 3590 |
| }, |
| { |
| "epoch": 2.3167042162858063, |
| "grad_norm": 5.46888938749729, |
| "learning_rate": 4.2063747585318737e-07, |
| "logits/chosen": -1.5020263195037842, |
| "logits/rejected": -1.449243187904358, |
| "logps/chosen": -405.04998779296875, |
| "logps/rejected": -551.5999755859375, |
| "loss": 0.006, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -4.069287300109863, |
| "rewards/margins": 15.004687309265137, |
| "rewards/rejected": -19.075000762939453, |
| "step": 3600 |
| }, |
| { |
| "epoch": 2.323141293852591, |
| "grad_norm": 0.012621281699271462, |
| "learning_rate": 4.190276883451384e-07, |
| "logits/chosen": -1.4910156726837158, |
| "logits/rejected": -1.3923828601837158, |
| "logps/chosen": -460.20001220703125, |
| "logps/rejected": -594.75, |
| "loss": 0.0046, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -4.56787109375, |
| "rewards/margins": 15.345312118530273, |
| "rewards/rejected": -19.918750762939453, |
| "step": 3610 |
| }, |
| { |
| "epoch": 2.329578371419376, |
| "grad_norm": 0.09103788380750466, |
| "learning_rate": 4.174179008370895e-07, |
| "logits/chosen": -1.559228539466858, |
| "logits/rejected": -1.4150390625, |
| "logps/chosen": -441.67498779296875, |
| "logps/rejected": -589.2999877929688, |
| "loss": 0.0133, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -4.918164253234863, |
| "rewards/margins": 14.6640625, |
| "rewards/rejected": -19.571874618530273, |
| "step": 3620 |
| }, |
| { |
| "epoch": 2.3360154489861604, |
| "grad_norm": 0.08436509622869823, |
| "learning_rate": 4.1580811332904054e-07, |
| "logits/chosen": -1.425500512123108, |
| "logits/rejected": -1.492285132408142, |
| "logps/chosen": -427.7250061035156, |
| "logps/rejected": -524.8499755859375, |
| "loss": 0.0109, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -5.557812690734863, |
| "rewards/margins": 13.44140625, |
| "rewards/rejected": -18.996875762939453, |
| "step": 3630 |
| }, |
| { |
| "epoch": 2.342452526552945, |
| "grad_norm": 0.05612137480682725, |
| "learning_rate": 4.141983258209916e-07, |
| "logits/chosen": -1.3402099609375, |
| "logits/rejected": -1.349877953529358, |
| "logps/chosen": -430.20001220703125, |
| "logps/rejected": -541.9000244140625, |
| "loss": 0.0315, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -4.4638671875, |
| "rewards/margins": 14.018750190734863, |
| "rewards/rejected": -18.493749618530273, |
| "step": 3640 |
| }, |
| { |
| "epoch": 2.3488896041197296, |
| "grad_norm": 0.11473868118981836, |
| "learning_rate": 4.125885383129427e-07, |
| "logits/chosen": -1.3649413585662842, |
| "logits/rejected": -1.3944091796875, |
| "logps/chosen": -429.54998779296875, |
| "logps/rejected": -552.625, |
| "loss": 0.0182, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -4.501757621765137, |
| "rewards/margins": 14.725000381469727, |
| "rewards/rejected": -19.235937118530273, |
| "step": 3650 |
| }, |
| { |
| "epoch": 2.355326681686514, |
| "grad_norm": 0.017842754368858602, |
| "learning_rate": 4.1097875080489376e-07, |
| "logits/chosen": -1.515722632408142, |
| "logits/rejected": -1.5048828125, |
| "logps/chosen": -429.4750061035156, |
| "logps/rejected": -568.2999877929688, |
| "loss": 0.0088, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -5.205859184265137, |
| "rewards/margins": 14.600000381469727, |
| "rewards/rejected": -19.814062118530273, |
| "step": 3660 |
| }, |
| { |
| "epoch": 2.361763759253299, |
| "grad_norm": 0.06156868848231738, |
| "learning_rate": 4.093689632968448e-07, |
| "logits/chosen": -1.4617431163787842, |
| "logits/rejected": -1.3827393054962158, |
| "logps/chosen": -452.04998779296875, |
| "logps/rejected": -592.2000122070312, |
| "loss": 0.0068, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -5.0029296875, |
| "rewards/margins": 14.176562309265137, |
| "rewards/rejected": -19.173437118530273, |
| "step": 3670 |
| }, |
| { |
| "epoch": 2.3682008368200838, |
| "grad_norm": 0.2134348709808394, |
| "learning_rate": 4.0775917578879585e-07, |
| "logits/chosen": -1.25048828125, |
| "logits/rejected": -1.260229468345642, |
| "logps/chosen": -404.6499938964844, |
| "logps/rejected": -542.4749755859375, |
| "loss": 0.0388, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -4.767675876617432, |
| "rewards/margins": 14.387499809265137, |
| "rewards/rejected": -19.143749237060547, |
| "step": 3680 |
| }, |
| { |
| "epoch": 2.3746379143868683, |
| "grad_norm": 0.024387450116424302, |
| "learning_rate": 4.061493882807469e-07, |
| "logits/chosen": -1.446386694908142, |
| "logits/rejected": -1.4363281726837158, |
| "logps/chosen": -447.82501220703125, |
| "logps/rejected": -565.5999755859375, |
| "loss": 0.0051, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -4.927538871765137, |
| "rewards/margins": 14.706250190734863, |
| "rewards/rejected": -19.637500762939453, |
| "step": 3690 |
| }, |
| { |
| "epoch": 2.381074991953653, |
| "grad_norm": 0.829475183316574, |
| "learning_rate": 4.04539600772698e-07, |
| "logits/chosen": -1.2946288585662842, |
| "logits/rejected": -1.253698706626892, |
| "logps/chosen": -454.45001220703125, |
| "logps/rejected": -569.9000244140625, |
| "loss": 0.0139, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -4.502636909484863, |
| "rewards/margins": 14.334375381469727, |
| "rewards/rejected": -18.84375, |
| "step": 3700 |
| }, |
| { |
| "epoch": 2.3875120695204375, |
| "grad_norm": 0.13775127608429233, |
| "learning_rate": 4.02929813264649e-07, |
| "logits/chosen": -1.381323218345642, |
| "logits/rejected": -1.313330054283142, |
| "logps/chosen": -441.6499938964844, |
| "logps/rejected": -579.4500122070312, |
| "loss": 0.0221, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -5.333447456359863, |
| "rewards/margins": 13.985937118530273, |
| "rewards/rejected": -19.317188262939453, |
| "step": 3710 |
| }, |
| { |
| "epoch": 2.3939491470872225, |
| "grad_norm": 0.15299805925174, |
| "learning_rate": 4.0132002575660014e-07, |
| "logits/chosen": -1.2962157726287842, |
| "logits/rejected": -1.2700684070587158, |
| "logps/chosen": -455.5874938964844, |
| "logps/rejected": -606.4249877929688, |
| "loss": 0.0022, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -4.905468940734863, |
| "rewards/margins": 15.782812118530273, |
| "rewards/rejected": -20.676563262939453, |
| "step": 3720 |
| }, |
| { |
| "epoch": 2.400386224654007, |
| "grad_norm": 0.030978548871580143, |
| "learning_rate": 3.9971023824855116e-07, |
| "logits/chosen": -1.489404320716858, |
| "logits/rejected": -1.3497192859649658, |
| "logps/chosen": -438.375, |
| "logps/rejected": -563.75, |
| "loss": 0.0088, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -5.1630859375, |
| "rewards/margins": 15.171875, |
| "rewards/rejected": -20.346874237060547, |
| "step": 3730 |
| }, |
| { |
| "epoch": 2.4068233022207917, |
| "grad_norm": 0.9386875809684314, |
| "learning_rate": 3.9810045074050224e-07, |
| "logits/chosen": -1.415917992591858, |
| "logits/rejected": -1.47314453125, |
| "logps/chosen": -514.9500122070312, |
| "logps/rejected": -586.7999877929688, |
| "loss": 0.006, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -5.189062595367432, |
| "rewards/margins": 15.08984375, |
| "rewards/rejected": -20.267187118530273, |
| "step": 3740 |
| }, |
| { |
| "epoch": 2.4132603797875767, |
| "grad_norm": 0.3241365396354039, |
| "learning_rate": 3.9649066323245326e-07, |
| "logits/chosen": -1.2907226085662842, |
| "logits/rejected": -1.242285132408142, |
| "logps/chosen": -421.29998779296875, |
| "logps/rejected": -540.5750122070312, |
| "loss": 0.014, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -4.833788871765137, |
| "rewards/margins": 14.451562881469727, |
| "rewards/rejected": -19.279687881469727, |
| "step": 3750 |
| }, |
| { |
| "epoch": 2.4196974573543613, |
| "grad_norm": 0.03959823546564496, |
| "learning_rate": 3.948808757244044e-07, |
| "logits/chosen": -1.2924072742462158, |
| "logits/rejected": -1.270178198814392, |
| "logps/chosen": -467.54998779296875, |
| "logps/rejected": -593.0499877929688, |
| "loss": 0.0191, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -5.218847751617432, |
| "rewards/margins": 15.098437309265137, |
| "rewards/rejected": -20.306249618530273, |
| "step": 3760 |
| }, |
| { |
| "epoch": 2.426134534921146, |
| "grad_norm": 1.582910224004781, |
| "learning_rate": 3.932710882163554e-07, |
| "logits/chosen": -1.408837914466858, |
| "logits/rejected": -1.3612792491912842, |
| "logps/chosen": -480.75, |
| "logps/rejected": -588.0, |
| "loss": 0.005, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -5.593554496765137, |
| "rewards/margins": 14.28125, |
| "rewards/rejected": -19.871875762939453, |
| "step": 3770 |
| }, |
| { |
| "epoch": 2.4325716124879304, |
| "grad_norm": 0.24893890647759087, |
| "learning_rate": 3.916613007083065e-07, |
| "logits/chosen": -1.3599121570587158, |
| "logits/rejected": -1.336328148841858, |
| "logps/chosen": -418.25, |
| "logps/rejected": -554.9500122070312, |
| "loss": 0.0183, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -5.775781154632568, |
| "rewards/margins": 14.360937118530273, |
| "rewards/rejected": -20.1328125, |
| "step": 3780 |
| }, |
| { |
| "epoch": 2.439008690054715, |
| "grad_norm": 0.07087116627113998, |
| "learning_rate": 3.9005151320025755e-07, |
| "logits/chosen": -1.3931884765625, |
| "logits/rejected": -1.2934081554412842, |
| "logps/chosen": -428.25, |
| "logps/rejected": -572.9000244140625, |
| "loss": 0.0091, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -5.252148628234863, |
| "rewards/margins": 14.998437881469727, |
| "rewards/rejected": -20.243749618530273, |
| "step": 3790 |
| }, |
| { |
| "epoch": 2.4454457676215, |
| "grad_norm": 0.3172570642850163, |
| "learning_rate": 3.884417256922086e-07, |
| "logits/chosen": -1.341406226158142, |
| "logits/rejected": -1.246435523033142, |
| "logps/chosen": -432.07501220703125, |
| "logps/rejected": -576.2999877929688, |
| "loss": 0.0015, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -4.865942478179932, |
| "rewards/margins": 14.267187118530273, |
| "rewards/rejected": -19.135936737060547, |
| "step": 3800 |
| }, |
| { |
| "epoch": 2.4518828451882846, |
| "grad_norm": 0.49587830961694207, |
| "learning_rate": 3.8683193818415965e-07, |
| "logits/chosen": -1.348486304283142, |
| "logits/rejected": -1.3211791515350342, |
| "logps/chosen": -446.42498779296875, |
| "logps/rejected": -613.9500122070312, |
| "loss": 0.0132, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -5.450243949890137, |
| "rewards/margins": 14.667187690734863, |
| "rewards/rejected": -20.114063262939453, |
| "step": 3810 |
| }, |
| { |
| "epoch": 2.458319922755069, |
| "grad_norm": 0.3107367308417446, |
| "learning_rate": 3.8522215067611077e-07, |
| "logits/chosen": -1.394775390625, |
| "logits/rejected": -1.355859398841858, |
| "logps/chosen": -460.8500061035156, |
| "logps/rejected": -605.2999877929688, |
| "loss": 0.0136, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -6.498242378234863, |
| "rewards/margins": 14.278124809265137, |
| "rewards/rejected": -20.779687881469727, |
| "step": 3820 |
| }, |
| { |
| "epoch": 2.4647570003218537, |
| "grad_norm": 0.3902869119151466, |
| "learning_rate": 3.836123631680618e-07, |
| "logits/chosen": -1.3835937976837158, |
| "logits/rejected": -1.3350098133087158, |
| "logps/chosen": -406.8374938964844, |
| "logps/rejected": -568.0499877929688, |
| "loss": 0.0178, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -4.628857612609863, |
| "rewards/margins": 15.328125, |
| "rewards/rejected": -19.953125, |
| "step": 3830 |
| }, |
| { |
| "epoch": 2.4711940778886383, |
| "grad_norm": 0.1618820448239353, |
| "learning_rate": 3.8200257566001287e-07, |
| "logits/chosen": -1.397070288658142, |
| "logits/rejected": -1.353247046470642, |
| "logps/chosen": -435.79998779296875, |
| "logps/rejected": -551.9500122070312, |
| "loss": 0.0046, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -4.294921875, |
| "rewards/margins": 15.198437690734863, |
| "rewards/rejected": -19.490625381469727, |
| "step": 3840 |
| }, |
| { |
| "epoch": 2.4776311554554233, |
| "grad_norm": 2.8558153197608798, |
| "learning_rate": 3.803927881519639e-07, |
| "logits/chosen": -1.366601586341858, |
| "logits/rejected": -1.249121069908142, |
| "logps/chosen": -461.7250061035156, |
| "logps/rejected": -604.3499755859375, |
| "loss": 0.0154, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -4.662499904632568, |
| "rewards/margins": 14.569531440734863, |
| "rewards/rejected": -19.239063262939453, |
| "step": 3850 |
| }, |
| { |
| "epoch": 2.484068233022208, |
| "grad_norm": 0.044976528435669044, |
| "learning_rate": 3.78783000643915e-07, |
| "logits/chosen": -1.35009765625, |
| "logits/rejected": -1.372216820716858, |
| "logps/chosen": -392.25, |
| "logps/rejected": -532.1500244140625, |
| "loss": 0.0263, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -3.923535108566284, |
| "rewards/margins": 14.418749809265137, |
| "rewards/rejected": -18.346874237060547, |
| "step": 3860 |
| }, |
| { |
| "epoch": 2.4905053105889925, |
| "grad_norm": 0.15667488871433924, |
| "learning_rate": 3.7717321313586603e-07, |
| "logits/chosen": -1.1943359375, |
| "logits/rejected": -1.2462279796600342, |
| "logps/chosen": -487.73748779296875, |
| "logps/rejected": -584.1500244140625, |
| "loss": 0.0077, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -5.42529296875, |
| "rewards/margins": 14.572656631469727, |
| "rewards/rejected": -20.003124237060547, |
| "step": 3870 |
| }, |
| { |
| "epoch": 2.4969423881557775, |
| "grad_norm": 2.1052668132008105, |
| "learning_rate": 3.755634256278171e-07, |
| "logits/chosen": -1.1891601085662842, |
| "logits/rejected": -1.1065857410430908, |
| "logps/chosen": -449.3999938964844, |
| "logps/rejected": -576.2999877929688, |
| "loss": 0.0134, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -4.512597560882568, |
| "rewards/margins": 14.981249809265137, |
| "rewards/rejected": -19.498437881469727, |
| "step": 3880 |
| }, |
| { |
| "epoch": 2.503379465722562, |
| "grad_norm": 0.0165471960968611, |
| "learning_rate": 3.739536381197682e-07, |
| "logits/chosen": -1.490625023841858, |
| "logits/rejected": -1.427392601966858, |
| "logps/chosen": -423.5249938964844, |
| "logps/rejected": -547.0499877929688, |
| "loss": 0.0057, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -4.746874809265137, |
| "rewards/margins": 14.676562309265137, |
| "rewards/rejected": -19.4140625, |
| "step": 3890 |
| }, |
| { |
| "epoch": 2.5098165432893467, |
| "grad_norm": 0.129753036072525, |
| "learning_rate": 3.7234385061171925e-07, |
| "logits/chosen": -1.241845726966858, |
| "logits/rejected": -1.1749756336212158, |
| "logps/chosen": -431.82501220703125, |
| "logps/rejected": -553.5499877929688, |
| "loss": 0.0148, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -4.623144626617432, |
| "rewards/margins": 15.4296875, |
| "rewards/rejected": -20.051563262939453, |
| "step": 3900 |
| }, |
| { |
| "epoch": 2.5162536208561312, |
| "grad_norm": 0.050428040791496605, |
| "learning_rate": 3.707340631036703e-07, |
| "logits/chosen": -1.32916259765625, |
| "logits/rejected": -1.31201171875, |
| "logps/chosen": -426.625, |
| "logps/rejected": -531.4000244140625, |
| "loss": 0.0094, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -4.797461032867432, |
| "rewards/margins": 14.034375190734863, |
| "rewards/rejected": -18.814062118530273, |
| "step": 3910 |
| }, |
| { |
| "epoch": 2.522690698422916, |
| "grad_norm": 0.31166637951000387, |
| "learning_rate": 3.6912427559562135e-07, |
| "logits/chosen": -1.361718773841858, |
| "logits/rejected": -1.3713867664337158, |
| "logps/chosen": -455.17498779296875, |
| "logps/rejected": -571.7000122070312, |
| "loss": 0.0047, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -5.124707221984863, |
| "rewards/margins": 13.5546875, |
| "rewards/rejected": -18.6875, |
| "step": 3920 |
| }, |
| { |
| "epoch": 2.529127775989701, |
| "grad_norm": 0.0121789469953649, |
| "learning_rate": 3.675144880875724e-07, |
| "logits/chosen": -1.3695189952850342, |
| "logits/rejected": -1.3829834461212158, |
| "logps/chosen": -466.5249938964844, |
| "logps/rejected": -574.7999877929688, |
| "loss": 0.009, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -4.450585842132568, |
| "rewards/margins": 15.112500190734863, |
| "rewards/rejected": -19.543750762939453, |
| "step": 3930 |
| }, |
| { |
| "epoch": 2.5355648535564854, |
| "grad_norm": 0.012160818687281362, |
| "learning_rate": 3.659047005795235e-07, |
| "logits/chosen": -1.3375060558319092, |
| "logits/rejected": -1.347924828529358, |
| "logps/chosen": -463.75, |
| "logps/rejected": -590.0, |
| "loss": 0.0089, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -4.125586032867432, |
| "rewards/margins": 15.189062118530273, |
| "rewards/rejected": -19.3046875, |
| "step": 3940 |
| }, |
| { |
| "epoch": 2.54200193112327, |
| "grad_norm": 0.45713523530019656, |
| "learning_rate": 3.642949130714745e-07, |
| "logits/chosen": -1.29791259765625, |
| "logits/rejected": -1.298828125, |
| "logps/chosen": -397.7749938964844, |
| "logps/rejected": -532.2249755859375, |
| "loss": 0.0132, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -4.905029296875, |
| "rewards/margins": 14.159375190734863, |
| "rewards/rejected": -19.0703125, |
| "step": 3950 |
| }, |
| { |
| "epoch": 2.5484390086900546, |
| "grad_norm": 1.7615668379004694, |
| "learning_rate": 3.6268512556342564e-07, |
| "logits/chosen": -1.1576111316680908, |
| "logits/rejected": -1.118920922279358, |
| "logps/chosen": -432.45001220703125, |
| "logps/rejected": -583.9500122070312, |
| "loss": 0.018, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -4.877148628234863, |
| "rewards/margins": 14.981249809265137, |
| "rewards/rejected": -19.857812881469727, |
| "step": 3960 |
| }, |
| { |
| "epoch": 2.554876086256839, |
| "grad_norm": 0.3323977393705623, |
| "learning_rate": 3.6107533805537666e-07, |
| "logits/chosen": -1.42822265625, |
| "logits/rejected": -1.427001953125, |
| "logps/chosen": -473.125, |
| "logps/rejected": -588.75, |
| "loss": 0.0057, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -5.215624809265137, |
| "rewards/margins": 14.40625, |
| "rewards/rejected": -19.620311737060547, |
| "step": 3970 |
| }, |
| { |
| "epoch": 2.561313163823624, |
| "grad_norm": 1.3083611989549269, |
| "learning_rate": 3.5946555054732774e-07, |
| "logits/chosen": -1.272802710533142, |
| "logits/rejected": -1.229638695716858, |
| "logps/chosen": -440.2250061035156, |
| "logps/rejected": -558.5999755859375, |
| "loss": 0.011, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -4.670947074890137, |
| "rewards/margins": 14.982812881469727, |
| "rewards/rejected": -19.657812118530273, |
| "step": 3980 |
| }, |
| { |
| "epoch": 2.5677502413904087, |
| "grad_norm": 0.05097514420930869, |
| "learning_rate": 3.5785576303927876e-07, |
| "logits/chosen": -1.182641625404358, |
| "logits/rejected": -1.045190453529358, |
| "logps/chosen": -443.0, |
| "logps/rejected": -605.0499877929688, |
| "loss": 0.0088, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -4.727636814117432, |
| "rewards/margins": 14.396875381469727, |
| "rewards/rejected": -19.120311737060547, |
| "step": 3990 |
| }, |
| { |
| "epoch": 2.5741873189571933, |
| "grad_norm": 1.3984619068469475, |
| "learning_rate": 3.562459755312299e-07, |
| "logits/chosen": -1.3110840320587158, |
| "logits/rejected": -1.2914550304412842, |
| "logps/chosen": -429.17498779296875, |
| "logps/rejected": -573.9500122070312, |
| "loss": 0.0002, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -4.596386909484863, |
| "rewards/margins": 15.034375190734863, |
| "rewards/rejected": -19.621875762939453, |
| "step": 4000 |
| }, |
| { |
| "epoch": 2.5806243965239783, |
| "grad_norm": 0.29561421683564104, |
| "learning_rate": 3.546361880231809e-07, |
| "logits/chosen": -1.2984130382537842, |
| "logits/rejected": -1.3658447265625, |
| "logps/chosen": -486.92498779296875, |
| "logps/rejected": -587.2999877929688, |
| "loss": 0.0094, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -4.917004585266113, |
| "rewards/margins": 13.660937309265137, |
| "rewards/rejected": -18.565624237060547, |
| "step": 4010 |
| }, |
| { |
| "epoch": 2.587061474090763, |
| "grad_norm": 0.09885666252294395, |
| "learning_rate": 3.53026400515132e-07, |
| "logits/chosen": -1.289819359779358, |
| "logits/rejected": -1.254052758216858, |
| "logps/chosen": -424.79998779296875, |
| "logps/rejected": -540.1500244140625, |
| "loss": 0.0088, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -4.140038967132568, |
| "rewards/margins": 14.110156059265137, |
| "rewards/rejected": -18.2421875, |
| "step": 4020 |
| }, |
| { |
| "epoch": 2.5934985516575475, |
| "grad_norm": 0.030440607615156004, |
| "learning_rate": 3.5141661300708305e-07, |
| "logits/chosen": -1.291113257408142, |
| "logits/rejected": -1.300927758216858, |
| "logps/chosen": -391.38751220703125, |
| "logps/rejected": -535.9249877929688, |
| "loss": 0.0294, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -4.266406059265137, |
| "rewards/margins": 13.657812118530273, |
| "rewards/rejected": -17.924999237060547, |
| "step": 4030 |
| }, |
| { |
| "epoch": 2.599935629224332, |
| "grad_norm": 0.24943713233428713, |
| "learning_rate": 3.498068254990341e-07, |
| "logits/chosen": -1.39892578125, |
| "logits/rejected": -1.3342773914337158, |
| "logps/chosen": -439.70001220703125, |
| "logps/rejected": -563.0499877929688, |
| "loss": 0.0007, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -4.022900581359863, |
| "rewards/margins": 14.159375190734863, |
| "rewards/rejected": -18.184375762939453, |
| "step": 4040 |
| }, |
| { |
| "epoch": 2.6063727067911167, |
| "grad_norm": 0.061725888483107956, |
| "learning_rate": 3.4819703799098514e-07, |
| "logits/chosen": -1.3244140148162842, |
| "logits/rejected": -1.3451659679412842, |
| "logps/chosen": -456.625, |
| "logps/rejected": -591.5, |
| "loss": 0.0117, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -5.069726467132568, |
| "rewards/margins": 13.715624809265137, |
| "rewards/rejected": -18.793750762939453, |
| "step": 4050 |
| }, |
| { |
| "epoch": 2.6128097843579017, |
| "grad_norm": 0.5718095183597939, |
| "learning_rate": 3.4658725048293627e-07, |
| "logits/chosen": -1.252783179283142, |
| "logits/rejected": -1.3213379383087158, |
| "logps/chosen": -468.29998779296875, |
| "logps/rejected": -577.4500122070312, |
| "loss": 0.0133, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -4.600293159484863, |
| "rewards/margins": 14.832812309265137, |
| "rewards/rejected": -19.428125381469727, |
| "step": 4060 |
| }, |
| { |
| "epoch": 2.6192468619246863, |
| "grad_norm": 0.3079163291219139, |
| "learning_rate": 3.449774629748873e-07, |
| "logits/chosen": -1.2415282726287842, |
| "logits/rejected": -1.212371826171875, |
| "logps/chosen": -399.42498779296875, |
| "logps/rejected": -548.4500122070312, |
| "loss": 0.0265, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -4.042443752288818, |
| "rewards/margins": 14.8515625, |
| "rewards/rejected": -18.887500762939453, |
| "step": 4070 |
| }, |
| { |
| "epoch": 2.625683939491471, |
| "grad_norm": 3.5841809337112283, |
| "learning_rate": 3.4336767546683836e-07, |
| "logits/chosen": -1.2685546875, |
| "logits/rejected": -1.18829345703125, |
| "logps/chosen": -458.29998779296875, |
| "logps/rejected": -608.8499755859375, |
| "loss": 0.0133, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -4.5087890625, |
| "rewards/margins": 15.016406059265137, |
| "rewards/rejected": -19.53125, |
| "step": 4080 |
| }, |
| { |
| "epoch": 2.6321210170582554, |
| "grad_norm": 0.0027121742646580927, |
| "learning_rate": 3.417578879587894e-07, |
| "logits/chosen": -1.316503882408142, |
| "logits/rejected": -1.361291527748108, |
| "logps/chosen": -496.57501220703125, |
| "logps/rejected": -602.5, |
| "loss": 0.0002, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -4.669360160827637, |
| "rewards/margins": 15.223437309265137, |
| "rewards/rejected": -19.8984375, |
| "step": 4090 |
| }, |
| { |
| "epoch": 2.63855809462504, |
| "grad_norm": 0.2180399250102561, |
| "learning_rate": 3.401481004507405e-07, |
| "logits/chosen": -1.2664673328399658, |
| "logits/rejected": -1.2131226062774658, |
| "logps/chosen": -429.07501220703125, |
| "logps/rejected": -573.2000122070312, |
| "loss": 0.0111, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -4.9736328125, |
| "rewards/margins": 14.734375, |
| "rewards/rejected": -19.704687118530273, |
| "step": 4100 |
| }, |
| { |
| "epoch": 2.644995172191825, |
| "grad_norm": 0.13297865408922263, |
| "learning_rate": 3.3853831294269153e-07, |
| "logits/chosen": -1.3541991710662842, |
| "logits/rejected": -1.32293701171875, |
| "logps/chosen": -452.875, |
| "logps/rejected": -573.5499877929688, |
| "loss": 0.0089, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -5.601660251617432, |
| "rewards/margins": 14.8515625, |
| "rewards/rejected": -20.4375, |
| "step": 4110 |
| }, |
| { |
| "epoch": 2.6514322497586096, |
| "grad_norm": 1.011735046778201, |
| "learning_rate": 3.369285254346426e-07, |
| "logits/chosen": -1.26678466796875, |
| "logits/rejected": -1.1880309581756592, |
| "logps/chosen": -428.42498779296875, |
| "logps/rejected": -568.6500244140625, |
| "loss": 0.0144, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -5.233202934265137, |
| "rewards/margins": 14.040624618530273, |
| "rewards/rejected": -19.270313262939453, |
| "step": 4120 |
| }, |
| { |
| "epoch": 2.657869327325394, |
| "grad_norm": 0.005354655550657211, |
| "learning_rate": 3.353187379265937e-07, |
| "logits/chosen": -1.412255883216858, |
| "logits/rejected": -1.4703369140625, |
| "logps/chosen": -433.32501220703125, |
| "logps/rejected": -578.5499877929688, |
| "loss": 0.0109, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -6.357812404632568, |
| "rewards/margins": 15.625, |
| "rewards/rejected": -21.981250762939453, |
| "step": 4130 |
| }, |
| { |
| "epoch": 2.664306404892179, |
| "grad_norm": 0.019422961933523768, |
| "learning_rate": 3.3370895041854475e-07, |
| "logits/chosen": -1.3243896961212158, |
| "logits/rejected": -1.340576171875, |
| "logps/chosen": -488.17498779296875, |
| "logps/rejected": -617.7999877929688, |
| "loss": 0.0224, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -5.976758003234863, |
| "rewards/margins": 15.4453125, |
| "rewards/rejected": -21.418750762939453, |
| "step": 4140 |
| }, |
| { |
| "epoch": 2.6707434824589638, |
| "grad_norm": 0.08342933651698449, |
| "learning_rate": 3.3209916291049577e-07, |
| "logits/chosen": -1.3651123046875, |
| "logits/rejected": -1.2025878429412842, |
| "logps/chosen": -419.2250061035156, |
| "logps/rejected": -547.7750244140625, |
| "loss": 0.0313, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -6.464453220367432, |
| "rewards/margins": 14.140625, |
| "rewards/rejected": -20.607812881469727, |
| "step": 4150 |
| }, |
| { |
| "epoch": 2.6771805600257483, |
| "grad_norm": 0.019996989322876788, |
| "learning_rate": 3.3048937540244685e-07, |
| "logits/chosen": -1.3374512195587158, |
| "logits/rejected": -1.401300072669983, |
| "logps/chosen": -504.0, |
| "logps/rejected": -627.9500122070312, |
| "loss": 0.0045, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -6.466796875, |
| "rewards/margins": 15.621874809265137, |
| "rewards/rejected": -22.09375, |
| "step": 4160 |
| }, |
| { |
| "epoch": 2.683617637592533, |
| "grad_norm": 0.06308040397789559, |
| "learning_rate": 3.288795878943979e-07, |
| "logits/chosen": -1.39990234375, |
| "logits/rejected": -1.395751953125, |
| "logps/chosen": -466.82501220703125, |
| "logps/rejected": -630.25, |
| "loss": 0.0047, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -6.091406345367432, |
| "rewards/margins": 15.3828125, |
| "rewards/rejected": -21.484375, |
| "step": 4170 |
| }, |
| { |
| "epoch": 2.6900547151593175, |
| "grad_norm": 0.029933522576406545, |
| "learning_rate": 3.27269800386349e-07, |
| "logits/chosen": -1.191162109375, |
| "logits/rejected": -1.2812378406524658, |
| "logps/chosen": -438.8500061035156, |
| "logps/rejected": -537.8499755859375, |
| "loss": 0.0217, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -5.103320121765137, |
| "rewards/margins": 14.745312690734863, |
| "rewards/rejected": -19.868749618530273, |
| "step": 4180 |
| }, |
| { |
| "epoch": 2.6964917927261025, |
| "grad_norm": 0.05590209351303259, |
| "learning_rate": 3.256600128783e-07, |
| "logits/chosen": -1.3470947742462158, |
| "logits/rejected": -1.3581664562225342, |
| "logps/chosen": -456.04998779296875, |
| "logps/rejected": -588.0, |
| "loss": 0.0167, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -5.517285346984863, |
| "rewards/margins": 14.671875, |
| "rewards/rejected": -20.1875, |
| "step": 4190 |
| }, |
| { |
| "epoch": 2.702928870292887, |
| "grad_norm": 0.06055438613372513, |
| "learning_rate": 3.2405022537025114e-07, |
| "logits/chosen": -1.4084594249725342, |
| "logits/rejected": -1.3955810070037842, |
| "logps/chosen": -451.92498779296875, |
| "logps/rejected": -603.2999877929688, |
| "loss": 0.005, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -5.625244140625, |
| "rewards/margins": 15.265625, |
| "rewards/rejected": -20.890625, |
| "step": 4200 |
| }, |
| { |
| "epoch": 2.7093659478596717, |
| "grad_norm": 0.20635665056149888, |
| "learning_rate": 3.2244043786220216e-07, |
| "logits/chosen": -1.1822509765625, |
| "logits/rejected": -1.2685058116912842, |
| "logps/chosen": -504.4750061035156, |
| "logps/rejected": -604.9500122070312, |
| "loss": 0.0062, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -5.189843654632568, |
| "rewards/margins": 13.864843368530273, |
| "rewards/rejected": -19.042186737060547, |
| "step": 4210 |
| }, |
| { |
| "epoch": 2.7158030254264562, |
| "grad_norm": 0.1802602821863471, |
| "learning_rate": 3.2083065035415323e-07, |
| "logits/chosen": -1.2764160633087158, |
| "logits/rejected": -1.2879638671875, |
| "logps/chosen": -439.0, |
| "logps/rejected": -542.375, |
| "loss": 0.0133, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -3.7996826171875, |
| "rewards/margins": 15.301562309265137, |
| "rewards/rejected": -19.095312118530273, |
| "step": 4220 |
| }, |
| { |
| "epoch": 2.722240102993241, |
| "grad_norm": 0.3710075476245095, |
| "learning_rate": 3.1922086284610425e-07, |
| "logits/chosen": -1.3041260242462158, |
| "logits/rejected": -1.244390845298767, |
| "logps/chosen": -435.8500061035156, |
| "logps/rejected": -574.6500244140625, |
| "loss": 0.0176, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -4.208447456359863, |
| "rewards/margins": 14.787500381469727, |
| "rewards/rejected": -18.995311737060547, |
| "step": 4230 |
| }, |
| { |
| "epoch": 2.728677180560026, |
| "grad_norm": 0.02462359575135724, |
| "learning_rate": 3.176110753380554e-07, |
| "logits/chosen": -1.2622802257537842, |
| "logits/rejected": -1.265625, |
| "logps/chosen": -438.82501220703125, |
| "logps/rejected": -584.7000122070312, |
| "loss": 0.0132, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -5.3291015625, |
| "rewards/margins": 14.46875, |
| "rewards/rejected": -19.795312881469727, |
| "step": 4240 |
| }, |
| { |
| "epoch": 2.7351142581268104, |
| "grad_norm": 5.662666104787528, |
| "learning_rate": 3.160012878300064e-07, |
| "logits/chosen": -1.2684326171875, |
| "logits/rejected": -1.2008788585662842, |
| "logps/chosen": -439.5, |
| "logps/rejected": -556.4000244140625, |
| "loss": 0.0009, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -4.8134765625, |
| "rewards/margins": 14.362500190734863, |
| "rewards/rejected": -19.171875, |
| "step": 4250 |
| }, |
| { |
| "epoch": 2.741551335693595, |
| "grad_norm": 1.1277858727349865, |
| "learning_rate": 3.143915003219575e-07, |
| "logits/chosen": -1.4165527820587158, |
| "logits/rejected": -1.386132836341858, |
| "logps/chosen": -443.8500061035156, |
| "logps/rejected": -562.5999755859375, |
| "loss": 0.0135, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -4.987890720367432, |
| "rewards/margins": 15.0078125, |
| "rewards/rejected": -20.003124237060547, |
| "step": 4260 |
| }, |
| { |
| "epoch": 2.74798841326038, |
| "grad_norm": 0.44038614901284245, |
| "learning_rate": 3.1278171281390855e-07, |
| "logits/chosen": -1.246728539466858, |
| "logits/rejected": -1.220605492591858, |
| "logps/chosen": -448.3999938964844, |
| "logps/rejected": -604.5999755859375, |
| "loss": 0.0287, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -5.421484470367432, |
| "rewards/margins": 15.09375, |
| "rewards/rejected": -20.524999618530273, |
| "step": 4270 |
| }, |
| { |
| "epoch": 2.7544254908271646, |
| "grad_norm": 0.705087466283429, |
| "learning_rate": 3.111719253058596e-07, |
| "logits/chosen": -1.40234375, |
| "logits/rejected": -1.387451171875, |
| "logps/chosen": -467.75, |
| "logps/rejected": -564.1500244140625, |
| "loss": 0.0088, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -5.703125, |
| "rewards/margins": 14.893750190734863, |
| "rewards/rejected": -20.603124618530273, |
| "step": 4280 |
| }, |
| { |
| "epoch": 2.760862568393949, |
| "grad_norm": 0.0236042677920065, |
| "learning_rate": 3.0956213779781064e-07, |
| "logits/chosen": -1.1304199695587158, |
| "logits/rejected": -1.1470947265625, |
| "logps/chosen": -438.6000061035156, |
| "logps/rejected": -589.3499755859375, |
| "loss": 0.0233, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -5.782519340515137, |
| "rewards/margins": 15.456250190734863, |
| "rewards/rejected": -21.231250762939453, |
| "step": 4290 |
| }, |
| { |
| "epoch": 2.7672996459607337, |
| "grad_norm": 0.07157308722250051, |
| "learning_rate": 3.0795235028976177e-07, |
| "logits/chosen": -1.402978539466858, |
| "logits/rejected": -1.389013648033142, |
| "logps/chosen": -481.6499938964844, |
| "logps/rejected": -644.3499755859375, |
| "loss": 0.0089, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -5.412109375, |
| "rewards/margins": 15.732812881469727, |
| "rewards/rejected": -21.143749237060547, |
| "step": 4300 |
| }, |
| { |
| "epoch": 2.7737367235275183, |
| "grad_norm": 0.3446378662940137, |
| "learning_rate": 3.063425627817128e-07, |
| "logits/chosen": -1.2295410633087158, |
| "logits/rejected": -1.161035180091858, |
| "logps/chosen": -450.875, |
| "logps/rejected": -587.4000244140625, |
| "loss": 0.0219, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -5.231054782867432, |
| "rewards/margins": 14.50390625, |
| "rewards/rejected": -19.739063262939453, |
| "step": 4310 |
| }, |
| { |
| "epoch": 2.7801738010943033, |
| "grad_norm": 0.746745080249211, |
| "learning_rate": 3.0473277527366386e-07, |
| "logits/chosen": -1.383813500404358, |
| "logits/rejected": -1.3329589366912842, |
| "logps/chosen": -429.8999938964844, |
| "logps/rejected": -574.5, |
| "loss": 0.0116, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -5.476366996765137, |
| "rewards/margins": 15.09765625, |
| "rewards/rejected": -20.565624237060547, |
| "step": 4320 |
| }, |
| { |
| "epoch": 2.786610878661088, |
| "grad_norm": 0.16843258668629368, |
| "learning_rate": 3.031229877656149e-07, |
| "logits/chosen": -1.270361304283142, |
| "logits/rejected": -1.3173339366912842, |
| "logps/chosen": -462.95001220703125, |
| "logps/rejected": -580.25, |
| "loss": 0.0091, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -6.013379096984863, |
| "rewards/margins": 14.896875381469727, |
| "rewards/rejected": -20.910938262939453, |
| "step": 4330 |
| }, |
| { |
| "epoch": 2.7930479562278725, |
| "grad_norm": 0.12653159072498812, |
| "learning_rate": 3.01513200257566e-07, |
| "logits/chosen": -1.2772705554962158, |
| "logits/rejected": -1.27880859375, |
| "logps/chosen": -432.42498779296875, |
| "logps/rejected": -612.4500122070312, |
| "loss": 0.0132, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -5.567187309265137, |
| "rewards/margins": 16.659374237060547, |
| "rewards/rejected": -22.225000381469727, |
| "step": 4340 |
| }, |
| { |
| "epoch": 2.799485033794657, |
| "grad_norm": 0.37246028512753815, |
| "learning_rate": 2.9990341274951703e-07, |
| "logits/chosen": -1.361853003501892, |
| "logits/rejected": -1.354736328125, |
| "logps/chosen": -440.42498779296875, |
| "logps/rejected": -566.625, |
| "loss": 0.0218, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -5.515210151672363, |
| "rewards/margins": 14.96875, |
| "rewards/rejected": -20.4765625, |
| "step": 4350 |
| }, |
| { |
| "epoch": 2.8059221113614417, |
| "grad_norm": 0.09673175563129986, |
| "learning_rate": 2.982936252414681e-07, |
| "logits/chosen": -1.337988257408142, |
| "logits/rejected": -1.355627417564392, |
| "logps/chosen": -438.25, |
| "logps/rejected": -598.7999877929688, |
| "loss": 0.0262, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -6.223046779632568, |
| "rewards/margins": 14.789843559265137, |
| "rewards/rejected": -21.010936737060547, |
| "step": 4360 |
| }, |
| { |
| "epoch": 2.8123591889282267, |
| "grad_norm": 3.947378292821696, |
| "learning_rate": 2.966838377334192e-07, |
| "logits/chosen": -1.321386694908142, |
| "logits/rejected": -1.417724609375, |
| "logps/chosen": -460.5, |
| "logps/rejected": -601.1500244140625, |
| "loss": 0.0224, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -5.823095798492432, |
| "rewards/margins": 15.607812881469727, |
| "rewards/rejected": -21.435937881469727, |
| "step": 4370 |
| }, |
| { |
| "epoch": 2.8187962664950112, |
| "grad_norm": 0.3405088810035094, |
| "learning_rate": 2.9507405022537025e-07, |
| "logits/chosen": -1.5154297351837158, |
| "logits/rejected": -1.470312476158142, |
| "logps/chosen": -490.07501220703125, |
| "logps/rejected": -605.0499877929688, |
| "loss": 0.0234, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -6.825585842132568, |
| "rewards/margins": 14.551562309265137, |
| "rewards/rejected": -21.388280868530273, |
| "step": 4380 |
| }, |
| { |
| "epoch": 2.825233344061796, |
| "grad_norm": 0.03158191215788186, |
| "learning_rate": 2.9346426271732127e-07, |
| "logits/chosen": -1.3859374523162842, |
| "logits/rejected": -1.3014647960662842, |
| "logps/chosen": -439.67498779296875, |
| "logps/rejected": -598.9500122070312, |
| "loss": 0.0132, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -7.13330078125, |
| "rewards/margins": 14.612500190734863, |
| "rewards/rejected": -21.743749618530273, |
| "step": 4390 |
| }, |
| { |
| "epoch": 2.831670421628581, |
| "grad_norm": 0.013435590273287331, |
| "learning_rate": 2.9185447520927234e-07, |
| "logits/chosen": -1.489160180091858, |
| "logits/rejected": -1.3664062023162842, |
| "logps/chosen": -448.8500061035156, |
| "logps/rejected": -628.4500122070312, |
| "loss": 0.0103, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -6.942968845367432, |
| "rewards/margins": 15.854687690734863, |
| "rewards/rejected": -22.801563262939453, |
| "step": 4400 |
| }, |
| { |
| "epoch": 2.8381074991953654, |
| "grad_norm": 7.778492191817943, |
| "learning_rate": 2.902446877012234e-07, |
| "logits/chosen": -1.2322266101837158, |
| "logits/rejected": -1.182885766029358, |
| "logps/chosen": -437.0, |
| "logps/rejected": -589.4749755859375, |
| "loss": 0.0136, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -5.081006050109863, |
| "rewards/margins": 15.537500381469727, |
| "rewards/rejected": -20.631250381469727, |
| "step": 4410 |
| }, |
| { |
| "epoch": 2.84454457676215, |
| "grad_norm": 11.508484818886531, |
| "learning_rate": 2.886349001931745e-07, |
| "logits/chosen": -1.2247803211212158, |
| "logits/rejected": -1.2336914539337158, |
| "logps/chosen": -494.57501220703125, |
| "logps/rejected": -620.4500122070312, |
| "loss": 0.019, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -5.497656345367432, |
| "rewards/margins": 15.034375190734863, |
| "rewards/rejected": -20.526561737060547, |
| "step": 4420 |
| }, |
| { |
| "epoch": 2.8509816543289346, |
| "grad_norm": 0.13620381372826948, |
| "learning_rate": 2.870251126851255e-07, |
| "logits/chosen": -1.3820312023162842, |
| "logits/rejected": -1.406957983970642, |
| "logps/chosen": -447.2250061035156, |
| "logps/rejected": -551.1500244140625, |
| "loss": 0.0447, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -4.809277534484863, |
| "rewards/margins": 15.220312118530273, |
| "rewards/rejected": -20.026561737060547, |
| "step": 4430 |
| }, |
| { |
| "epoch": 2.857418731895719, |
| "grad_norm": 1.6369283331049351, |
| "learning_rate": 2.8541532517707664e-07, |
| "logits/chosen": -1.418066382408142, |
| "logits/rejected": -1.3649413585662842, |
| "logps/chosen": -484.04998779296875, |
| "logps/rejected": -626.5, |
| "loss": 0.0091, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -5.284326076507568, |
| "rewards/margins": 15.056249618530273, |
| "rewards/rejected": -20.348438262939453, |
| "step": 4440 |
| }, |
| { |
| "epoch": 2.863855809462504, |
| "grad_norm": 0.4983365678119172, |
| "learning_rate": 2.8380553766902766e-07, |
| "logits/chosen": -1.380517601966858, |
| "logits/rejected": -1.409570336341858, |
| "logps/chosen": -474.20001220703125, |
| "logps/rejected": -602.0, |
| "loss": 0.0132, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -5.658203125, |
| "rewards/margins": 14.7578125, |
| "rewards/rejected": -20.412500381469727, |
| "step": 4450 |
| }, |
| { |
| "epoch": 2.8702928870292888, |
| "grad_norm": 0.012272657893228, |
| "learning_rate": 2.8219575016097873e-07, |
| "logits/chosen": -1.2690308094024658, |
| "logits/rejected": -1.293493628501892, |
| "logps/chosen": -463.70001220703125, |
| "logps/rejected": -576.3499755859375, |
| "loss": 0.0219, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -6.08447265625, |
| "rewards/margins": 13.734375, |
| "rewards/rejected": -19.831249237060547, |
| "step": 4460 |
| }, |
| { |
| "epoch": 2.8767299645960733, |
| "grad_norm": 0.4994090395645203, |
| "learning_rate": 2.8058596265292975e-07, |
| "logits/chosen": -1.3449218273162842, |
| "logits/rejected": -1.3623046875, |
| "logps/chosen": -472.07501220703125, |
| "logps/rejected": -585.7000122070312, |
| "loss": 0.0089, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -5.009667873382568, |
| "rewards/margins": 15.090624809265137, |
| "rewards/rejected": -20.104686737060547, |
| "step": 4470 |
| }, |
| { |
| "epoch": 2.883167042162858, |
| "grad_norm": 1.3278107046647045, |
| "learning_rate": 2.789761751448809e-07, |
| "logits/chosen": -1.4123046398162842, |
| "logits/rejected": -1.3676269054412842, |
| "logps/chosen": -425.82501220703125, |
| "logps/rejected": -577.0999755859375, |
| "loss": 0.0093, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -5.455273628234863, |
| "rewards/margins": 14.846875190734863, |
| "rewards/rejected": -20.314062118530273, |
| "step": 4480 |
| }, |
| { |
| "epoch": 2.8896041197296425, |
| "grad_norm": 0.12612088100463062, |
| "learning_rate": 2.773663876368319e-07, |
| "logits/chosen": -1.344335913658142, |
| "logits/rejected": -1.2717773914337158, |
| "logps/chosen": -453.5, |
| "logps/rejected": -588.5999755859375, |
| "loss": 0.0142, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -5.467871189117432, |
| "rewards/margins": 13.962499618530273, |
| "rewards/rejected": -19.442188262939453, |
| "step": 4490 |
| }, |
| { |
| "epoch": 2.8960411972964275, |
| "grad_norm": 0.8481392245674592, |
| "learning_rate": 2.7575660012878297e-07, |
| "logits/chosen": -1.2632324695587158, |
| "logits/rejected": -1.1241455078125, |
| "logps/chosen": -428.32501220703125, |
| "logps/rejected": -603.4500122070312, |
| "loss": 0.0049, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -4.707812309265137, |
| "rewards/margins": 14.428125381469727, |
| "rewards/rejected": -19.128124237060547, |
| "step": 4500 |
| }, |
| { |
| "epoch": 2.902478274863212, |
| "grad_norm": 1.0879764692019862, |
| "learning_rate": 2.7414681262073405e-07, |
| "logits/chosen": -1.439355492591858, |
| "logits/rejected": -1.404638648033142, |
| "logps/chosen": -452.3999938964844, |
| "logps/rejected": -564.5499877929688, |
| "loss": 0.009, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -5.261328220367432, |
| "rewards/margins": 14.337499618530273, |
| "rewards/rejected": -19.598438262939453, |
| "step": 4510 |
| }, |
| { |
| "epoch": 2.9089153524299967, |
| "grad_norm": 0.021872127522654484, |
| "learning_rate": 2.725370251126851e-07, |
| "logits/chosen": -1.362573266029358, |
| "logits/rejected": -1.354272484779358, |
| "logps/chosen": -479.29998779296875, |
| "logps/rejected": -610.2000122070312, |
| "loss": 0.0131, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -5.497460842132568, |
| "rewards/margins": 14.434374809265137, |
| "rewards/rejected": -19.934375762939453, |
| "step": 4520 |
| }, |
| { |
| "epoch": 2.9153524299967817, |
| "grad_norm": 1.0112163577958093, |
| "learning_rate": 2.7092723760463614e-07, |
| "logits/chosen": -1.4724609851837158, |
| "logits/rejected": -1.430151343345642, |
| "logps/chosen": -453.375, |
| "logps/rejected": -576.2000122070312, |
| "loss": 0.0134, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -5.37548828125, |
| "rewards/margins": 14.346875190734863, |
| "rewards/rejected": -19.717187881469727, |
| "step": 4530 |
| }, |
| { |
| "epoch": 2.9217895075635663, |
| "grad_norm": 0.14575457521705656, |
| "learning_rate": 2.6931745009658727e-07, |
| "logits/chosen": -1.3837401866912842, |
| "logits/rejected": -1.2932617664337158, |
| "logps/chosen": -442.4750061035156, |
| "logps/rejected": -626.25, |
| "loss": 0.0046, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -5.666601657867432, |
| "rewards/margins": 16.089061737060547, |
| "rewards/rejected": -21.760936737060547, |
| "step": 4540 |
| }, |
| { |
| "epoch": 2.928226585130351, |
| "grad_norm": 3.5094570543472843, |
| "learning_rate": 2.677076625885383e-07, |
| "logits/chosen": -1.3143310546875, |
| "logits/rejected": -1.331701636314392, |
| "logps/chosen": -436.95001220703125, |
| "logps/rejected": -571.2750244140625, |
| "loss": 0.0265, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -5.491796970367432, |
| "rewards/margins": 14.731249809265137, |
| "rewards/rejected": -20.229686737060547, |
| "step": 4550 |
| }, |
| { |
| "epoch": 2.9346636626971354, |
| "grad_norm": 0.9458534463592883, |
| "learning_rate": 2.6609787508048936e-07, |
| "logits/chosen": -1.504492163658142, |
| "logits/rejected": -1.54974365234375, |
| "logps/chosen": -457.7749938964844, |
| "logps/rejected": -572.2999877929688, |
| "loss": 0.0047, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -5.666760444641113, |
| "rewards/margins": 14.213281631469727, |
| "rewards/rejected": -19.884374618530273, |
| "step": 4560 |
| }, |
| { |
| "epoch": 2.94110074026392, |
| "grad_norm": 0.06625517986954294, |
| "learning_rate": 2.644880875724404e-07, |
| "logits/chosen": -1.408447265625, |
| "logits/rejected": -1.36572265625, |
| "logps/chosen": -461.07501220703125, |
| "logps/rejected": -605.0999755859375, |
| "loss": 0.0044, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -5.924218654632568, |
| "rewards/margins": 14.995312690734863, |
| "rewards/rejected": -20.915624618530273, |
| "step": 4570 |
| }, |
| { |
| "epoch": 2.947537817830705, |
| "grad_norm": 0.09220583655181887, |
| "learning_rate": 2.628783000643915e-07, |
| "logits/chosen": -1.278833031654358, |
| "logits/rejected": -1.3231933116912842, |
| "logps/chosen": -470.9750061035156, |
| "logps/rejected": -585.5, |
| "loss": 0.0218, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -5.672461032867432, |
| "rewards/margins": 14.517187118530273, |
| "rewards/rejected": -20.182811737060547, |
| "step": 4580 |
| }, |
| { |
| "epoch": 2.9539748953974896, |
| "grad_norm": 0.041605345092253, |
| "learning_rate": 2.6126851255634253e-07, |
| "logits/chosen": -1.3131592273712158, |
| "logits/rejected": -1.2116577625274658, |
| "logps/chosen": -420.125, |
| "logps/rejected": -592.4500122070312, |
| "loss": 0.0306, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -5.489062309265137, |
| "rewards/margins": 14.5078125, |
| "rewards/rejected": -19.996875762939453, |
| "step": 4590 |
| }, |
| { |
| "epoch": 2.960411972964274, |
| "grad_norm": 3.6711898336965794, |
| "learning_rate": 2.596587250482936e-07, |
| "logits/chosen": -1.2788574695587158, |
| "logits/rejected": -1.2344238758087158, |
| "logps/chosen": -449.9750061035156, |
| "logps/rejected": -574.6500244140625, |
| "loss": 0.0182, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -5.914575099945068, |
| "rewards/margins": 14.723437309265137, |
| "rewards/rejected": -20.639062881469727, |
| "step": 4600 |
| }, |
| { |
| "epoch": 2.9668490505310587, |
| "grad_norm": 1.0804216876420778, |
| "learning_rate": 2.580489375402447e-07, |
| "logits/chosen": -1.292724609375, |
| "logits/rejected": -1.2802307605743408, |
| "logps/chosen": -440.07501220703125, |
| "logps/rejected": -600.6500244140625, |
| "loss": 0.0133, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -5.880859375, |
| "rewards/margins": 15.024999618530273, |
| "rewards/rejected": -20.912500381469727, |
| "step": 4610 |
| }, |
| { |
| "epoch": 2.9732861280978433, |
| "grad_norm": 0.03330797945860044, |
| "learning_rate": 2.5643915003219575e-07, |
| "logits/chosen": -1.3818359375, |
| "logits/rejected": -1.422827124595642, |
| "logps/chosen": -465.6000061035156, |
| "logps/rejected": -555.2000122070312, |
| "loss": 0.0094, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -5.842187404632568, |
| "rewards/margins": 14.151562690734863, |
| "rewards/rejected": -19.9921875, |
| "step": 4620 |
| }, |
| { |
| "epoch": 2.9797232056646283, |
| "grad_norm": 0.1108094091516729, |
| "learning_rate": 2.5482936252414677e-07, |
| "logits/chosen": -1.2595703601837158, |
| "logits/rejected": -1.2952392101287842, |
| "logps/chosen": -506.25, |
| "logps/rejected": -613.0499877929688, |
| "loss": 0.0267, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -6.214062690734863, |
| "rewards/margins": 13.689062118530273, |
| "rewards/rejected": -19.90625, |
| "step": 4630 |
| }, |
| { |
| "epoch": 2.986160283231413, |
| "grad_norm": 0.18436801634071254, |
| "learning_rate": 2.5321957501609784e-07, |
| "logits/chosen": -1.3479492664337158, |
| "logits/rejected": -1.3339722156524658, |
| "logps/chosen": -509.875, |
| "logps/rejected": -626.7000122070312, |
| "loss": 0.0087, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -5.481249809265137, |
| "rewards/margins": 15.123437881469727, |
| "rewards/rejected": -20.595312118530273, |
| "step": 4640 |
| }, |
| { |
| "epoch": 2.9925973607981975, |
| "grad_norm": 0.48813441481765374, |
| "learning_rate": 2.516097875080489e-07, |
| "logits/chosen": -1.351660132408142, |
| "logits/rejected": -1.3151366710662842, |
| "logps/chosen": -487.20001220703125, |
| "logps/rejected": -625.4500122070312, |
| "loss": 0.0092, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -5.345800876617432, |
| "rewards/margins": 14.993749618530273, |
| "rewards/rejected": -20.332813262939453, |
| "step": 4650 |
| }, |
| { |
| "epoch": 2.9990344383649825, |
| "grad_norm": 0.31605485582269216, |
| "learning_rate": 2.5e-07, |
| "logits/chosen": -1.213165283203125, |
| "logits/rejected": -1.209228515625, |
| "logps/chosen": -454.17498779296875, |
| "logps/rejected": -600.2000122070312, |
| "loss": 0.0177, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -5.169140815734863, |
| "rewards/margins": 14.807812690734863, |
| "rewards/rejected": -19.971874237060547, |
| "step": 4660 |
| }, |
| { |
| "epoch": 3.005149662053428, |
| "grad_norm": 0.01528203368042275, |
| "learning_rate": 2.4839021249195106e-07, |
| "logits/chosen": -1.4519942998886108, |
| "logits/rejected": -1.3072702884674072, |
| "logps/chosen": -461.2105407714844, |
| "logps/rejected": -599.26318359375, |
| "loss": 0.0044, |
| "rewards/accuracies": 0.9934210777282715, |
| "rewards/chosen": -5.082647800445557, |
| "rewards/margins": 15.504934310913086, |
| "rewards/rejected": -20.588815689086914, |
| "step": 4670 |
| }, |
| { |
| "epoch": 3.0115867396202125, |
| "grad_norm": 0.01256528152297684, |
| "learning_rate": 2.4678042498390214e-07, |
| "logits/chosen": -1.3317382335662842, |
| "logits/rejected": -1.317724585533142, |
| "logps/chosen": -477.0, |
| "logps/rejected": -607.0, |
| "loss": 0.0001, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -4.975390434265137, |
| "rewards/margins": 15.810937881469727, |
| "rewards/rejected": -20.787500381469727, |
| "step": 4680 |
| }, |
| { |
| "epoch": 3.018023817186997, |
| "grad_norm": 0.02369916588108901, |
| "learning_rate": 2.4517063747585316e-07, |
| "logits/chosen": -1.265869140625, |
| "logits/rejected": -1.266992211341858, |
| "logps/chosen": -433.0, |
| "logps/rejected": -585.6500244140625, |
| "loss": 0.0173, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -4.644873142242432, |
| "rewards/margins": 15.395312309265137, |
| "rewards/rejected": -20.053125381469727, |
| "step": 4690 |
| }, |
| { |
| "epoch": 3.0244608947537817, |
| "grad_norm": 0.14270311658136978, |
| "learning_rate": 2.4356084996780423e-07, |
| "logits/chosen": -1.3472168445587158, |
| "logits/rejected": -1.3324706554412842, |
| "logps/chosen": -445.0249938964844, |
| "logps/rejected": -591.1500244140625, |
| "loss": 0.0173, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -4.390332221984863, |
| "rewards/margins": 15.764062881469727, |
| "rewards/rejected": -20.157812118530273, |
| "step": 4700 |
| }, |
| { |
| "epoch": 3.0308979723205667, |
| "grad_norm": 0.08858786606659473, |
| "learning_rate": 2.419510624597553e-07, |
| "logits/chosen": -1.201318383216858, |
| "logits/rejected": -1.170166015625, |
| "logps/chosen": -468.2749938964844, |
| "logps/rejected": -600.5499877929688, |
| "loss": 0.0087, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -5.220507621765137, |
| "rewards/margins": 14.752344131469727, |
| "rewards/rejected": -19.967187881469727, |
| "step": 4710 |
| }, |
| { |
| "epoch": 3.0373350498873513, |
| "grad_norm": 0.04611246505031775, |
| "learning_rate": 2.403412749517064e-07, |
| "logits/chosen": -1.4478027820587158, |
| "logits/rejected": -1.3209960460662842, |
| "logps/chosen": -456.5, |
| "logps/rejected": -586.3499755859375, |
| "loss": 0.0043, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -4.649609565734863, |
| "rewards/margins": 15.618749618530273, |
| "rewards/rejected": -20.285938262939453, |
| "step": 4720 |
| }, |
| { |
| "epoch": 3.043772127454136, |
| "grad_norm": 0.0179974008949943, |
| "learning_rate": 2.3873148744365745e-07, |
| "logits/chosen": -1.2719604969024658, |
| "logits/rejected": -1.3468506336212158, |
| "logps/chosen": -431.125, |
| "logps/rejected": -559.9500122070312, |
| "loss": 0.013, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -4.644189357757568, |
| "rewards/margins": 15.265625, |
| "rewards/rejected": -19.90625, |
| "step": 4730 |
| }, |
| { |
| "epoch": 3.0502092050209204, |
| "grad_norm": 0.028026501462890086, |
| "learning_rate": 2.371216999356085e-07, |
| "logits/chosen": -1.4129760265350342, |
| "logits/rejected": -1.3403809070587158, |
| "logps/chosen": -441.5, |
| "logps/rejected": -590.2999877929688, |
| "loss": 0.026, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -5.531640529632568, |
| "rewards/margins": 15.153124809265137, |
| "rewards/rejected": -20.676563262939453, |
| "step": 4740 |
| }, |
| { |
| "epoch": 3.056646282587705, |
| "grad_norm": 0.037780814798613345, |
| "learning_rate": 2.3551191242755954e-07, |
| "logits/chosen": -1.3036377429962158, |
| "logits/rejected": -1.3723876476287842, |
| "logps/chosen": -463.17498779296875, |
| "logps/rejected": -585.3499755859375, |
| "loss": 0.0044, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -4.96044921875, |
| "rewards/margins": 16.181249618530273, |
| "rewards/rejected": -21.135936737060547, |
| "step": 4750 |
| }, |
| { |
| "epoch": 3.06308336015449, |
| "grad_norm": 0.16236504668110097, |
| "learning_rate": 2.3390212491951062e-07, |
| "logits/chosen": -1.2888062000274658, |
| "logits/rejected": -1.3119995594024658, |
| "logps/chosen": -461.8500061035156, |
| "logps/rejected": -564.3499755859375, |
| "loss": 0.0216, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -4.878808498382568, |
| "rewards/margins": 14.649999618530273, |
| "rewards/rejected": -19.526561737060547, |
| "step": 4760 |
| }, |
| { |
| "epoch": 3.0695204377212746, |
| "grad_norm": 0.04648560942633776, |
| "learning_rate": 2.3229233741146166e-07, |
| "logits/chosen": -1.461889624595642, |
| "logits/rejected": -1.418554663658142, |
| "logps/chosen": -484.54998779296875, |
| "logps/rejected": -627.2000122070312, |
| "loss": 0.0006, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -5.915429592132568, |
| "rewards/margins": 15.75, |
| "rewards/rejected": -21.662500381469727, |
| "step": 4770 |
| }, |
| { |
| "epoch": 3.075957515288059, |
| "grad_norm": 0.015543564803289752, |
| "learning_rate": 2.3068254990341274e-07, |
| "logits/chosen": -1.3328125476837158, |
| "logits/rejected": -1.416015625, |
| "logps/chosen": -482.6499938964844, |
| "logps/rejected": -592.4000244140625, |
| "loss": 0.0044, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -5.499218940734863, |
| "rewards/margins": 14.957812309265137, |
| "rewards/rejected": -20.462499618530273, |
| "step": 4780 |
| }, |
| { |
| "epoch": 3.0823945928548437, |
| "grad_norm": 0.035501794461136674, |
| "learning_rate": 2.290727623953638e-07, |
| "logits/chosen": -1.3611328601837158, |
| "logits/rejected": -1.3807861804962158, |
| "logps/chosen": -478.70001220703125, |
| "logps/rejected": -603.4000244140625, |
| "loss": 0.013, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -5.576123237609863, |
| "rewards/margins": 16.162500381469727, |
| "rewards/rejected": -21.740625381469727, |
| "step": 4790 |
| }, |
| { |
| "epoch": 3.0888316704216288, |
| "grad_norm": 0.008840538506660322, |
| "learning_rate": 2.2746297488731486e-07, |
| "logits/chosen": -1.4937012195587158, |
| "logits/rejected": -1.428466796875, |
| "logps/chosen": -433.2250061035156, |
| "logps/rejected": -599.7999877929688, |
| "loss": 0.0044, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -5.576171875, |
| "rewards/margins": 15.5625, |
| "rewards/rejected": -21.131250381469727, |
| "step": 4800 |
| }, |
| { |
| "epoch": 3.0952687479884133, |
| "grad_norm": 0.027509396831896113, |
| "learning_rate": 2.2585318737926593e-07, |
| "logits/chosen": -1.369287133216858, |
| "logits/rejected": -1.391381859779358, |
| "logps/chosen": -449.375, |
| "logps/rejected": -557.9000244140625, |
| "loss": 0.0001, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -5.578906059265137, |
| "rewards/margins": 14.975000381469727, |
| "rewards/rejected": -20.556249618530273, |
| "step": 4810 |
| }, |
| { |
| "epoch": 3.101705825555198, |
| "grad_norm": 0.005417397762369448, |
| "learning_rate": 2.2424339987121698e-07, |
| "logits/chosen": -1.4420166015625, |
| "logits/rejected": -1.472998023033142, |
| "logps/chosen": -445.20001220703125, |
| "logps/rejected": -566.4500122070312, |
| "loss": 0.0001, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -5.31689453125, |
| "rewards/margins": 15.339062690734863, |
| "rewards/rejected": -20.665624618530273, |
| "step": 4820 |
| }, |
| { |
| "epoch": 3.1081429031219825, |
| "grad_norm": 0.008175503472805599, |
| "learning_rate": 2.2263361236316805e-07, |
| "logits/chosen": -1.332605004310608, |
| "logits/rejected": -1.258056640625, |
| "logps/chosen": -423.07501220703125, |
| "logps/rejected": -591.625, |
| "loss": 0.0044, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -5.3994140625, |
| "rewards/margins": 15.467187881469727, |
| "rewards/rejected": -20.871875762939453, |
| "step": 4830 |
| }, |
| { |
| "epoch": 3.1145799806887675, |
| "grad_norm": 0.020342841260593283, |
| "learning_rate": 2.210238248551191e-07, |
| "logits/chosen": -1.3341796398162842, |
| "logits/rejected": -1.253759741783142, |
| "logps/chosen": -454.07501220703125, |
| "logps/rejected": -584.4500122070312, |
| "loss": 0.013, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -5.464453220367432, |
| "rewards/margins": 15.890625, |
| "rewards/rejected": -21.348438262939453, |
| "step": 4840 |
| }, |
| { |
| "epoch": 3.121017058255552, |
| "grad_norm": 0.44937853593296095, |
| "learning_rate": 2.1941403734707017e-07, |
| "logits/chosen": -1.354339599609375, |
| "logits/rejected": -1.322021484375, |
| "logps/chosen": -427.7250061035156, |
| "logps/rejected": -601.1500244140625, |
| "loss": 0.0131, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -5.684668064117432, |
| "rewards/margins": 15.954687118530273, |
| "rewards/rejected": -21.635936737060547, |
| "step": 4850 |
| }, |
| { |
| "epoch": 3.1274541358223367, |
| "grad_norm": 0.015487845509029223, |
| "learning_rate": 2.1780424983902125e-07, |
| "logits/chosen": -1.283288598060608, |
| "logits/rejected": -1.2652099132537842, |
| "logps/chosen": -443.0, |
| "logps/rejected": -575.0250244140625, |
| "loss": 0.013, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -4.24462890625, |
| "rewards/margins": 15.1640625, |
| "rewards/rejected": -19.410938262939453, |
| "step": 4860 |
| }, |
| { |
| "epoch": 3.1338912133891212, |
| "grad_norm": 0.0063810008211449545, |
| "learning_rate": 2.161944623309723e-07, |
| "logits/chosen": -1.438085913658142, |
| "logits/rejected": -1.3974609375, |
| "logps/chosen": -411.04998779296875, |
| "logps/rejected": -565.2000122070312, |
| "loss": 0.0043, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -4.849951267242432, |
| "rewards/margins": 16.435937881469727, |
| "rewards/rejected": -21.3046875, |
| "step": 4870 |
| }, |
| { |
| "epoch": 3.140328290955906, |
| "grad_norm": 0.013303285180195717, |
| "learning_rate": 2.1458467482292337e-07, |
| "logits/chosen": -1.212042212486267, |
| "logits/rejected": -1.311767578125, |
| "logps/chosen": -498.7124938964844, |
| "logps/rejected": -615.1500244140625, |
| "loss": 0.0045, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -5.472216606140137, |
| "rewards/margins": 16.0625, |
| "rewards/rejected": -21.515625, |
| "step": 4880 |
| }, |
| { |
| "epoch": 3.146765368522691, |
| "grad_norm": 0.017089215511075364, |
| "learning_rate": 2.129748873148744e-07, |
| "logits/chosen": -1.324072241783142, |
| "logits/rejected": -1.324072241783142, |
| "logps/chosen": -455.6000061035156, |
| "logps/rejected": -584.0999755859375, |
| "loss": 0.0048, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -4.962304592132568, |
| "rewards/margins": 16.393749237060547, |
| "rewards/rejected": -21.356250762939453, |
| "step": 4890 |
| }, |
| { |
| "epoch": 3.1532024460894754, |
| "grad_norm": 0.012166861835420342, |
| "learning_rate": 2.1136509980682549e-07, |
| "logits/chosen": -1.31005859375, |
| "logits/rejected": -1.301611304283142, |
| "logps/chosen": -466.04998779296875, |
| "logps/rejected": -608.5999755859375, |
| "loss": 0.0043, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -5.631933689117432, |
| "rewards/margins": 15.435937881469727, |
| "rewards/rejected": -21.056249618530273, |
| "step": 4900 |
| }, |
| { |
| "epoch": 3.15963952365626, |
| "grad_norm": 0.0524800845723654, |
| "learning_rate": 2.0975531229877656e-07, |
| "logits/chosen": -1.3974487781524658, |
| "logits/rejected": -1.3639007806777954, |
| "logps/chosen": -464.29998779296875, |
| "logps/rejected": -610.0999755859375, |
| "loss": 0.0173, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -5.533984184265137, |
| "rewards/margins": 15.307812690734863, |
| "rewards/rejected": -20.832813262939453, |
| "step": 4910 |
| }, |
| { |
| "epoch": 3.1660766012230446, |
| "grad_norm": 0.015232174149713472, |
| "learning_rate": 2.081455247907276e-07, |
| "logits/chosen": -1.358789086341858, |
| "logits/rejected": -1.361425757408142, |
| "logps/chosen": -452.82501220703125, |
| "logps/rejected": -589.8499755859375, |
| "loss": 0.0087, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -5.072851657867432, |
| "rewards/margins": 15.634374618530273, |
| "rewards/rejected": -20.714061737060547, |
| "step": 4920 |
| }, |
| { |
| "epoch": 3.1725136787898296, |
| "grad_norm": 0.013631590366399621, |
| "learning_rate": 2.0653573728267868e-07, |
| "logits/chosen": -1.2882080078125, |
| "logits/rejected": -1.2051513195037842, |
| "logps/chosen": -416.79998779296875, |
| "logps/rejected": -583.0499877929688, |
| "loss": 0.0216, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -5.652539253234863, |
| "rewards/margins": 15.868749618530273, |
| "rewards/rejected": -21.526561737060547, |
| "step": 4930 |
| }, |
| { |
| "epoch": 3.178950756356614, |
| "grad_norm": 0.08496115304673249, |
| "learning_rate": 2.0492594977462973e-07, |
| "logits/chosen": -1.4072265625, |
| "logits/rejected": -1.3898437023162842, |
| "logps/chosen": -461.7749938964844, |
| "logps/rejected": -578.4000244140625, |
| "loss": 0.0087, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -5.668554782867432, |
| "rewards/margins": 14.5546875, |
| "rewards/rejected": -20.228124618530273, |
| "step": 4940 |
| }, |
| { |
| "epoch": 3.1853878339233987, |
| "grad_norm": 0.012240679996153669, |
| "learning_rate": 2.033161622665808e-07, |
| "logits/chosen": -1.3662598133087158, |
| "logits/rejected": -1.2894287109375, |
| "logps/chosen": -465.75, |
| "logps/rejected": -615.5499877929688, |
| "loss": 0.0045, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -5.158837795257568, |
| "rewards/margins": 16.399999618530273, |
| "rewards/rejected": -21.551563262939453, |
| "step": 4950 |
| }, |
| { |
| "epoch": 3.1918249114901833, |
| "grad_norm": 0.14014065961321068, |
| "learning_rate": 2.0170637475853185e-07, |
| "logits/chosen": -1.364648461341858, |
| "logits/rejected": -1.418341040611267, |
| "logps/chosen": -483.17498779296875, |
| "logps/rejected": -577.3499755859375, |
| "loss": 0.0087, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -5.236572265625, |
| "rewards/margins": 14.850000381469727, |
| "rewards/rejected": -20.087499618530273, |
| "step": 4960 |
| }, |
| { |
| "epoch": 3.1982619890569683, |
| "grad_norm": 0.050676021092912533, |
| "learning_rate": 2.0009658725048292e-07, |
| "logits/chosen": -1.2871582508087158, |
| "logits/rejected": -1.299414038658142, |
| "logps/chosen": -477.9750061035156, |
| "logps/rejected": -594.5, |
| "loss": 0.013, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -5.313867092132568, |
| "rewards/margins": 15.201562881469727, |
| "rewards/rejected": -20.520313262939453, |
| "step": 4970 |
| }, |
| { |
| "epoch": 3.204699066623753, |
| "grad_norm": 0.013817183605439995, |
| "learning_rate": 1.98486799742434e-07, |
| "logits/chosen": -1.536718726158142, |
| "logits/rejected": -1.51123046875, |
| "logps/chosen": -473.875, |
| "logps/rejected": -598.6500244140625, |
| "loss": 0.0087, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -5.576928615570068, |
| "rewards/margins": 15.256250381469727, |
| "rewards/rejected": -20.832813262939453, |
| "step": 4980 |
| }, |
| { |
| "epoch": 3.2111361441905375, |
| "grad_norm": 0.03400668043498249, |
| "learning_rate": 1.9687701223438504e-07, |
| "logits/chosen": -1.3931152820587158, |
| "logits/rejected": -1.377465844154358, |
| "logps/chosen": -458.79998779296875, |
| "logps/rejected": -588.0, |
| "loss": 0.013, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -5.489648342132568, |
| "rewards/margins": 16.287500381469727, |
| "rewards/rejected": -21.784374237060547, |
| "step": 4990 |
| }, |
| { |
| "epoch": 3.217573221757322, |
| "grad_norm": 0.01809042060378709, |
| "learning_rate": 1.9526722472633612e-07, |
| "logits/chosen": -1.2072632312774658, |
| "logits/rejected": -1.2759521007537842, |
| "logps/chosen": -438.79998779296875, |
| "logps/rejected": -576.0499877929688, |
| "loss": 0.0176, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -5.733984470367432, |
| "rewards/margins": 15.526562690734863, |
| "rewards/rejected": -21.268749237060547, |
| "step": 5000 |
| }, |
| { |
| "epoch": 3.2240102993241067, |
| "grad_norm": 0.03881644341284769, |
| "learning_rate": 1.9365743721828716e-07, |
| "logits/chosen": -1.415551781654358, |
| "logits/rejected": -1.366943359375, |
| "logps/chosen": -422.3999938964844, |
| "logps/rejected": -567.7999877929688, |
| "loss": 0.0216, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -6.197461128234863, |
| "rewards/margins": 15.357812881469727, |
| "rewards/rejected": -21.564062118530273, |
| "step": 5010 |
| }, |
| { |
| "epoch": 3.2304473768908917, |
| "grad_norm": 0.006722883336909701, |
| "learning_rate": 1.9204764971023824e-07, |
| "logits/chosen": -1.355737328529358, |
| "logits/rejected": -1.379052758216858, |
| "logps/chosen": -439.6000061035156, |
| "logps/rejected": -585.4000244140625, |
| "loss": 0.005, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -5.842871189117432, |
| "rewards/margins": 16.524999618530273, |
| "rewards/rejected": -22.362499237060547, |
| "step": 5020 |
| }, |
| { |
| "epoch": 3.2368844544576763, |
| "grad_norm": 0.05554583685844547, |
| "learning_rate": 1.904378622021893e-07, |
| "logits/chosen": -1.373681664466858, |
| "logits/rejected": -1.3494384288787842, |
| "logps/chosen": -459.8999938964844, |
| "logps/rejected": -601.5999755859375, |
| "loss": 0.0173, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -5.371484279632568, |
| "rewards/margins": 16.53125, |
| "rewards/rejected": -21.9140625, |
| "step": 5030 |
| }, |
| { |
| "epoch": 3.243321532024461, |
| "grad_norm": 0.26912842088688743, |
| "learning_rate": 1.8882807469414036e-07, |
| "logits/chosen": -1.445703148841858, |
| "logits/rejected": -1.390283226966858, |
| "logps/chosen": -426.8999938964844, |
| "logps/rejected": -562.25, |
| "loss": 0.0044, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -5.7451171875, |
| "rewards/margins": 15.753125190734863, |
| "rewards/rejected": -21.489063262939453, |
| "step": 5040 |
| }, |
| { |
| "epoch": 3.2497586095912454, |
| "grad_norm": 0.017693757670447882, |
| "learning_rate": 1.8721828718609143e-07, |
| "logits/chosen": -1.2490966320037842, |
| "logits/rejected": -1.2368652820587158, |
| "logps/chosen": -447.75, |
| "logps/rejected": -602.0, |
| "loss": 0.0216, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -5.513476371765137, |
| "rewards/margins": 15.207812309265137, |
| "rewards/rejected": -20.712499618530273, |
| "step": 5050 |
| }, |
| { |
| "epoch": 3.2561956871580304, |
| "grad_norm": 0.03682378147121155, |
| "learning_rate": 1.8560849967804248e-07, |
| "logits/chosen": -1.212988257408142, |
| "logits/rejected": -1.095739722251892, |
| "logps/chosen": -458.3500061035156, |
| "logps/rejected": -596.2000122070312, |
| "loss": 0.0277, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -6.671093940734863, |
| "rewards/margins": 14.649999618530273, |
| "rewards/rejected": -21.314062118530273, |
| "step": 5060 |
| }, |
| { |
| "epoch": 3.262632764724815, |
| "grad_norm": 0.009161269502864606, |
| "learning_rate": 1.8399871216999355e-07, |
| "logits/chosen": -1.393798828125, |
| "logits/rejected": -1.3373534679412842, |
| "logps/chosen": -453.1499938964844, |
| "logps/rejected": -614.2999877929688, |
| "loss": 0.0173, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -5.572265625, |
| "rewards/margins": 15.957812309265137, |
| "rewards/rejected": -21.524999618530273, |
| "step": 5070 |
| }, |
| { |
| "epoch": 3.2690698422915996, |
| "grad_norm": 0.0271239307373583, |
| "learning_rate": 1.823889246619446e-07, |
| "logits/chosen": -1.3394043445587158, |
| "logits/rejected": -1.3982422351837158, |
| "logps/chosen": -495.0249938964844, |
| "logps/rejected": -625.25, |
| "loss": 0.0043, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -6.257226467132568, |
| "rewards/margins": 16.135936737060547, |
| "rewards/rejected": -22.401561737060547, |
| "step": 5080 |
| }, |
| { |
| "epoch": 3.275506919858384, |
| "grad_norm": 0.057690101209044296, |
| "learning_rate": 1.8077913715389567e-07, |
| "logits/chosen": -1.2102539539337158, |
| "logits/rejected": -1.1611816883087158, |
| "logps/chosen": -440.6499938964844, |
| "logps/rejected": -608.5, |
| "loss": 0.0303, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -5.851953029632568, |
| "rewards/margins": 15.940625190734863, |
| "rewards/rejected": -21.792186737060547, |
| "step": 5090 |
| }, |
| { |
| "epoch": 3.281943997425169, |
| "grad_norm": 0.050727211447405836, |
| "learning_rate": 1.7916934964584674e-07, |
| "logits/chosen": -1.3723633289337158, |
| "logits/rejected": -1.359643578529358, |
| "logps/chosen": -432.7250061035156, |
| "logps/rejected": -613.7999877929688, |
| "loss": 0.0133, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -6.140038967132568, |
| "rewards/margins": 16.126562118530273, |
| "rewards/rejected": -22.274999618530273, |
| "step": 5100 |
| }, |
| { |
| "epoch": 3.2883810749919538, |
| "grad_norm": 0.056286448302638535, |
| "learning_rate": 1.775595621377978e-07, |
| "logits/chosen": -1.3057861328125, |
| "logits/rejected": -1.3517334461212158, |
| "logps/chosen": -460.0249938964844, |
| "logps/rejected": -584.7999877929688, |
| "loss": 0.0087, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -5.604498386383057, |
| "rewards/margins": 16.073436737060547, |
| "rewards/rejected": -21.681249618530273, |
| "step": 5110 |
| }, |
| { |
| "epoch": 3.2948181525587383, |
| "grad_norm": 0.010729072544668962, |
| "learning_rate": 1.7594977462974886e-07, |
| "logits/chosen": -1.285314917564392, |
| "logits/rejected": -1.218505859375, |
| "logps/chosen": -412.2250061035156, |
| "logps/rejected": -575.0499877929688, |
| "loss": 0.013, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -5.792431831359863, |
| "rewards/margins": 16.496875762939453, |
| "rewards/rejected": -22.299999237060547, |
| "step": 5120 |
| }, |
| { |
| "epoch": 3.301255230125523, |
| "grad_norm": 0.026680415200292355, |
| "learning_rate": 1.743399871216999e-07, |
| "logits/chosen": -1.564111351966858, |
| "logits/rejected": -1.472924828529358, |
| "logps/chosen": -477.17498779296875, |
| "logps/rejected": -609.5999755859375, |
| "loss": 0.0087, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -5.904296875, |
| "rewards/margins": 15.345312118530273, |
| "rewards/rejected": -21.264062881469727, |
| "step": 5130 |
| }, |
| { |
| "epoch": 3.3076923076923075, |
| "grad_norm": 0.010730868739063386, |
| "learning_rate": 1.7273019961365098e-07, |
| "logits/chosen": -1.5020020008087158, |
| "logits/rejected": -1.467138648033142, |
| "logps/chosen": -449.3500061035156, |
| "logps/rejected": -604.5499877929688, |
| "loss": 0.0043, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -5.293993949890137, |
| "rewards/margins": 16.642187118530273, |
| "rewards/rejected": -21.934375762939453, |
| "step": 5140 |
| }, |
| { |
| "epoch": 3.3141293852590925, |
| "grad_norm": 0.696373086142649, |
| "learning_rate": 1.7112041210560206e-07, |
| "logits/chosen": -1.325341820716858, |
| "logits/rejected": -1.296875, |
| "logps/chosen": -483.07501220703125, |
| "logps/rejected": -600.9000244140625, |
| "loss": 0.0131, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -5.895898342132568, |
| "rewards/margins": 16.443750381469727, |
| "rewards/rejected": -22.350000381469727, |
| "step": 5150 |
| }, |
| { |
| "epoch": 3.320566462825877, |
| "grad_norm": 0.015300880181736134, |
| "learning_rate": 1.695106245975531e-07, |
| "logits/chosen": -1.357812523841858, |
| "logits/rejected": -1.368798851966858, |
| "logps/chosen": -467.42498779296875, |
| "logps/rejected": -635.75, |
| "loss": 0.0043, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -5.926953315734863, |
| "rewards/margins": 15.146875381469727, |
| "rewards/rejected": -21.075000762939453, |
| "step": 5160 |
| }, |
| { |
| "epoch": 3.3270035403926617, |
| "grad_norm": 0.012682743786594078, |
| "learning_rate": 1.6790083708950418e-07, |
| "logits/chosen": -1.374902367591858, |
| "logits/rejected": -1.4656250476837158, |
| "logps/chosen": -461.25, |
| "logps/rejected": -608.0999755859375, |
| "loss": 0.013, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -5.766211032867432, |
| "rewards/margins": 16.5703125, |
| "rewards/rejected": -22.337499618530273, |
| "step": 5170 |
| }, |
| { |
| "epoch": 3.3334406179594462, |
| "grad_norm": 0.5267383779640614, |
| "learning_rate": 1.6629104958145523e-07, |
| "logits/chosen": -1.31268310546875, |
| "logits/rejected": -1.361962914466858, |
| "logps/chosen": -436.8999938964844, |
| "logps/rejected": -585.75, |
| "loss": 0.0144, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -6.013281345367432, |
| "rewards/margins": 16.279687881469727, |
| "rewards/rejected": -22.285938262939453, |
| "step": 5180 |
| }, |
| { |
| "epoch": 3.3398776955262313, |
| "grad_norm": 0.9351690134596885, |
| "learning_rate": 1.646812620734063e-07, |
| "logits/chosen": -1.395117163658142, |
| "logits/rejected": -1.318603515625, |
| "logps/chosen": -459.3500061035156, |
| "logps/rejected": -626.3499755859375, |
| "loss": 0.013, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -6.458788871765137, |
| "rewards/margins": 15.534375190734863, |
| "rewards/rejected": -21.998437881469727, |
| "step": 5190 |
| }, |
| { |
| "epoch": 3.346314773093016, |
| "grad_norm": 0.014883169326817438, |
| "learning_rate": 1.6307147456535735e-07, |
| "logits/chosen": -1.359643578529358, |
| "logits/rejected": -1.3416748046875, |
| "logps/chosen": -460.7250061035156, |
| "logps/rejected": -587.75, |
| "loss": 0.0044, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -6.137499809265137, |
| "rewards/margins": 16.204687118530273, |
| "rewards/rejected": -22.34375, |
| "step": 5200 |
| }, |
| { |
| "epoch": 3.3527518506598004, |
| "grad_norm": 0.01957165159711994, |
| "learning_rate": 1.6146168705730842e-07, |
| "logits/chosen": -1.322167992591858, |
| "logits/rejected": -1.290869116783142, |
| "logps/chosen": -460.82501220703125, |
| "logps/rejected": -628.9500122070312, |
| "loss": 0.0136, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -5.961035251617432, |
| "rewards/margins": 15.953125, |
| "rewards/rejected": -21.910938262939453, |
| "step": 5210 |
| }, |
| { |
| "epoch": 3.359188928226585, |
| "grad_norm": 0.012774135087520195, |
| "learning_rate": 1.598518995492595e-07, |
| "logits/chosen": -1.174560546875, |
| "logits/rejected": -1.1700439453125, |
| "logps/chosen": -449.7749938964844, |
| "logps/rejected": -588.7999877929688, |
| "loss": 0.026, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -6.241796970367432, |
| "rewards/margins": 15.850000381469727, |
| "rewards/rejected": -22.089061737060547, |
| "step": 5220 |
| }, |
| { |
| "epoch": 3.36562600579337, |
| "grad_norm": 0.014735395021663272, |
| "learning_rate": 1.5824211204121054e-07, |
| "logits/chosen": -1.305688500404358, |
| "logits/rejected": -1.288183569908142, |
| "logps/chosen": -472.3500061035156, |
| "logps/rejected": -614.75, |
| "loss": 0.0174, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -6.526562690734863, |
| "rewards/margins": 15.370312690734863, |
| "rewards/rejected": -21.885936737060547, |
| "step": 5230 |
| }, |
| { |
| "epoch": 3.3720630833601546, |
| "grad_norm": 0.006502822275108849, |
| "learning_rate": 1.566323245331616e-07, |
| "logits/chosen": -1.327978491783142, |
| "logits/rejected": -1.243432641029358, |
| "logps/chosen": -465.1499938964844, |
| "logps/rejected": -619.0, |
| "loss": 0.0173, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -6.119921684265137, |
| "rewards/margins": 17.014062881469727, |
| "rewards/rejected": -23.128124237060547, |
| "step": 5240 |
| }, |
| { |
| "epoch": 3.378500160926939, |
| "grad_norm": 0.8934758932006331, |
| "learning_rate": 1.5502253702511266e-07, |
| "logits/chosen": -1.241064429283142, |
| "logits/rejected": -1.192724585533142, |
| "logps/chosen": -438.375, |
| "logps/rejected": -603.9500122070312, |
| "loss": 0.0174, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -6.636328220367432, |
| "rewards/margins": 16.337499618530273, |
| "rewards/rejected": -22.964061737060547, |
| "step": 5250 |
| }, |
| { |
| "epoch": 3.3849372384937237, |
| "grad_norm": 0.048621505703303786, |
| "learning_rate": 1.5341274951706373e-07, |
| "logits/chosen": -1.193261742591858, |
| "logits/rejected": -1.1887695789337158, |
| "logps/chosen": -481.1000061035156, |
| "logps/rejected": -619.7999877929688, |
| "loss": 0.0217, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -6.281786918640137, |
| "rewards/margins": 16.637500762939453, |
| "rewards/rejected": -22.9296875, |
| "step": 5260 |
| }, |
| { |
| "epoch": 3.3913743160605083, |
| "grad_norm": 0.015198438176935388, |
| "learning_rate": 1.518029620090148e-07, |
| "logits/chosen": -1.45703125, |
| "logits/rejected": -1.4525878429412842, |
| "logps/chosen": -442.9750061035156, |
| "logps/rejected": -575.5499877929688, |
| "loss": 0.0087, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -7.261914253234863, |
| "rewards/margins": 16.228124618530273, |
| "rewards/rejected": -23.487499237060547, |
| "step": 5270 |
| }, |
| { |
| "epoch": 3.3978113936272933, |
| "grad_norm": 0.008875394717014798, |
| "learning_rate": 1.5019317450096585e-07, |
| "logits/chosen": -1.3145020008087158, |
| "logits/rejected": -1.286279320716858, |
| "logps/chosen": -425.7250061035156, |
| "logps/rejected": -603.5499877929688, |
| "loss": 0.013, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -6.4228515625, |
| "rewards/margins": 16.689062118530273, |
| "rewards/rejected": -23.103124618530273, |
| "step": 5280 |
| }, |
| { |
| "epoch": 3.404248471194078, |
| "grad_norm": 0.011357571829177772, |
| "learning_rate": 1.4858338699291693e-07, |
| "logits/chosen": -1.2973144054412842, |
| "logits/rejected": -1.2838134765625, |
| "logps/chosen": -422.8999938964844, |
| "logps/rejected": -564.9000244140625, |
| "loss": 0.013, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -6.235156059265137, |
| "rewards/margins": 15.962499618530273, |
| "rewards/rejected": -22.182811737060547, |
| "step": 5290 |
| }, |
| { |
| "epoch": 3.4106855487608625, |
| "grad_norm": 0.004420966085981854, |
| "learning_rate": 1.4697359948486797e-07, |
| "logits/chosen": -1.445556640625, |
| "logits/rejected": -1.355444312095642, |
| "logps/chosen": -482.45001220703125, |
| "logps/rejected": -588.1500244140625, |
| "loss": 0.0067, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -6.265820503234863, |
| "rewards/margins": 16.3125, |
| "rewards/rejected": -22.575000762939453, |
| "step": 5300 |
| }, |
| { |
| "epoch": 3.417122626327647, |
| "grad_norm": 0.0019139752673292122, |
| "learning_rate": 1.4536381197681905e-07, |
| "logits/chosen": -1.2787353992462158, |
| "logits/rejected": -1.2913818359375, |
| "logps/chosen": -484.75, |
| "logps/rejected": -638.75, |
| "loss": 0.001, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -6.626172065734863, |
| "rewards/margins": 15.928125381469727, |
| "rewards/rejected": -22.557811737060547, |
| "step": 5310 |
| }, |
| { |
| "epoch": 3.423559703894432, |
| "grad_norm": 0.1596697631158503, |
| "learning_rate": 1.437540244687701e-07, |
| "logits/chosen": -1.383032202720642, |
| "logits/rejected": -1.198339819908142, |
| "logps/chosen": -451.4750061035156, |
| "logps/rejected": -621.7999877929688, |
| "loss": 0.013, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -6.90771484375, |
| "rewards/margins": 17.693750381469727, |
| "rewards/rejected": -24.615625381469727, |
| "step": 5320 |
| }, |
| { |
| "epoch": 3.4299967814612167, |
| "grad_norm": 0.05048371441417471, |
| "learning_rate": 1.4214423696072117e-07, |
| "logits/chosen": -1.421875, |
| "logits/rejected": -1.368261694908142, |
| "logps/chosen": -463.625, |
| "logps/rejected": -628.4000244140625, |
| "loss": 0.0044, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -6.812109470367432, |
| "rewards/margins": 16.649999618530273, |
| "rewards/rejected": -23.462499618530273, |
| "step": 5330 |
| }, |
| { |
| "epoch": 3.4364338590280012, |
| "grad_norm": 0.25834063407028274, |
| "learning_rate": 1.4053444945267224e-07, |
| "logits/chosen": -1.400048851966858, |
| "logits/rejected": -1.455102562904358, |
| "logps/chosen": -456.79998779296875, |
| "logps/rejected": -610.5, |
| "loss": 0.013, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -6.53515625, |
| "rewards/margins": 16.734375, |
| "rewards/rejected": -23.274999618530273, |
| "step": 5340 |
| }, |
| { |
| "epoch": 3.442870936594786, |
| "grad_norm": 0.3769126195916326, |
| "learning_rate": 1.389246619446233e-07, |
| "logits/chosen": -1.284033179283142, |
| "logits/rejected": -1.1798827648162842, |
| "logps/chosen": -439.67498779296875, |
| "logps/rejected": -613.7999877929688, |
| "loss": 0.0173, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -6.879687309265137, |
| "rewards/margins": 16.676563262939453, |
| "rewards/rejected": -23.559375762939453, |
| "step": 5350 |
| }, |
| { |
| "epoch": 3.449308014161571, |
| "grad_norm": 0.013877468942768182, |
| "learning_rate": 1.3731487443657436e-07, |
| "logits/chosen": -1.165869116783142, |
| "logits/rejected": -1.1698486804962158, |
| "logps/chosen": -437.29998779296875, |
| "logps/rejected": -599.2999877929688, |
| "loss": 0.0259, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -6.376562595367432, |
| "rewards/margins": 16.840625762939453, |
| "rewards/rejected": -23.214061737060547, |
| "step": 5360 |
| }, |
| { |
| "epoch": 3.4557450917283554, |
| "grad_norm": 0.04353552898682389, |
| "learning_rate": 1.357050869285254e-07, |
| "logits/chosen": -1.23876953125, |
| "logits/rejected": -1.1927306652069092, |
| "logps/chosen": -466.2749938964844, |
| "logps/rejected": -620.6500244140625, |
| "loss": 0.013, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -6.354394435882568, |
| "rewards/margins": 16.259374618530273, |
| "rewards/rejected": -22.609375, |
| "step": 5370 |
| }, |
| { |
| "epoch": 3.46218216929514, |
| "grad_norm": 0.005505590179551618, |
| "learning_rate": 1.3409529942047648e-07, |
| "logits/chosen": -1.319311499595642, |
| "logits/rejected": -1.3135254383087158, |
| "logps/chosen": -493.8500061035156, |
| "logps/rejected": -635.5999755859375, |
| "loss": 0.0087, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -7.054638862609863, |
| "rewards/margins": 16.3359375, |
| "rewards/rejected": -23.399999618530273, |
| "step": 5380 |
| }, |
| { |
| "epoch": 3.4686192468619246, |
| "grad_norm": 0.01261488971992542, |
| "learning_rate": 1.3248551191242756e-07, |
| "logits/chosen": -1.4135253429412842, |
| "logits/rejected": -1.441552758216858, |
| "logps/chosen": -462.54998779296875, |
| "logps/rejected": -623.75, |
| "loss": 0.013, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -6.651757717132568, |
| "rewards/margins": 16.315624237060547, |
| "rewards/rejected": -22.962499618530273, |
| "step": 5390 |
| }, |
| { |
| "epoch": 3.475056324428709, |
| "grad_norm": 0.006152661001392241, |
| "learning_rate": 1.308757244043786e-07, |
| "logits/chosen": -1.3431396484375, |
| "logits/rejected": -1.2621338367462158, |
| "logps/chosen": -431.2749938964844, |
| "logps/rejected": -605.9000244140625, |
| "loss": 0.0141, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -6.767968654632568, |
| "rewards/margins": 15.609375, |
| "rewards/rejected": -22.362499237060547, |
| "step": 5400 |
| }, |
| { |
| "epoch": 3.481493401995494, |
| "grad_norm": 1.7561235826749255, |
| "learning_rate": 1.2926593689632968e-07, |
| "logits/chosen": -1.2505614757537842, |
| "logits/rejected": -1.1798584461212158, |
| "logps/chosen": -434.8999938964844, |
| "logps/rejected": -613.2249755859375, |
| "loss": 0.0173, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -7.081445217132568, |
| "rewards/margins": 16.973438262939453, |
| "rewards/rejected": -24.057811737060547, |
| "step": 5410 |
| }, |
| { |
| "epoch": 3.4879304795622788, |
| "grad_norm": 0.010188021273004572, |
| "learning_rate": 1.2765614938828072e-07, |
| "logits/chosen": -1.527587890625, |
| "logits/rejected": -1.4479248523712158, |
| "logps/chosen": -490.6000061035156, |
| "logps/rejected": -621.9000244140625, |
| "loss": 0.0087, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -6.276562690734863, |
| "rewards/margins": 17.125, |
| "rewards/rejected": -23.399999618530273, |
| "step": 5420 |
| }, |
| { |
| "epoch": 3.4943675571290633, |
| "grad_norm": 0.0015363561684980116, |
| "learning_rate": 1.260463618802318e-07, |
| "logits/chosen": -1.286035180091858, |
| "logits/rejected": -1.2685546875, |
| "logps/chosen": -481.1499938964844, |
| "logps/rejected": -604.25, |
| "loss": 0.0259, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -5.965624809265137, |
| "rewards/margins": 16.315624237060547, |
| "rewards/rejected": -22.287500381469727, |
| "step": 5430 |
| }, |
| { |
| "epoch": 3.5008046346958483, |
| "grad_norm": 0.0413809403765098, |
| "learning_rate": 1.2443657437218287e-07, |
| "logits/chosen": -1.4120604991912842, |
| "logits/rejected": -1.3991210460662842, |
| "logps/chosen": -485.7875061035156, |
| "logps/rejected": -670.25, |
| "loss": 0.0043, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -7.415234565734863, |
| "rewards/margins": 16.201562881469727, |
| "rewards/rejected": -23.606250762939453, |
| "step": 5440 |
| }, |
| { |
| "epoch": 3.507241712262633, |
| "grad_norm": 0.06233662617488789, |
| "learning_rate": 1.2282678686413394e-07, |
| "logits/chosen": -1.42987060546875, |
| "logits/rejected": -1.323083519935608, |
| "logps/chosen": -429.6000061035156, |
| "logps/rejected": -595.7000122070312, |
| "loss": 0.0087, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -6.425976753234863, |
| "rewards/margins": 17.154687881469727, |
| "rewards/rejected": -23.575000762939453, |
| "step": 5450 |
| }, |
| { |
| "epoch": 3.5136787898294175, |
| "grad_norm": 0.017753571643274287, |
| "learning_rate": 1.21216999356085e-07, |
| "logits/chosen": -1.4534180164337158, |
| "logits/rejected": -1.4618651866912842, |
| "logps/chosen": -450.79998779296875, |
| "logps/rejected": -614.5999755859375, |
| "loss": 0.0087, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -6.9921875, |
| "rewards/margins": 16.870311737060547, |
| "rewards/rejected": -23.853124618530273, |
| "step": 5460 |
| }, |
| { |
| "epoch": 3.520115867396202, |
| "grad_norm": 0.4338641792686853, |
| "learning_rate": 1.1960721184803606e-07, |
| "logits/chosen": -1.180688500404358, |
| "logits/rejected": -1.1524658203125, |
| "logps/chosen": -417.6000061035156, |
| "logps/rejected": -575.4000244140625, |
| "loss": 0.013, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -6.263281345367432, |
| "rewards/margins": 16.848438262939453, |
| "rewards/rejected": -23.103124618530273, |
| "step": 5470 |
| }, |
| { |
| "epoch": 3.5265529449629867, |
| "grad_norm": 0.10779137743138031, |
| "learning_rate": 1.1799742433998712e-07, |
| "logits/chosen": -1.2066650390625, |
| "logits/rejected": -1.1869385242462158, |
| "logps/chosen": -444.9750061035156, |
| "logps/rejected": -605.7000122070312, |
| "loss": 0.0216, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -6.584435939788818, |
| "rewards/margins": 16.279687881469727, |
| "rewards/rejected": -22.870311737060547, |
| "step": 5480 |
| }, |
| { |
| "epoch": 3.5329900225297717, |
| "grad_norm": 0.030789882327093638, |
| "learning_rate": 1.1638763683193818e-07, |
| "logits/chosen": -1.206811547279358, |
| "logits/rejected": -1.193603515625, |
| "logps/chosen": -466.125, |
| "logps/rejected": -592.1500244140625, |
| "loss": 0.013, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -7.039258003234863, |
| "rewards/margins": 15.870312690734863, |
| "rewards/rejected": -22.912500381469727, |
| "step": 5490 |
| }, |
| { |
| "epoch": 3.5394271000965563, |
| "grad_norm": 0.17336305122171547, |
| "learning_rate": 1.1477784932388924e-07, |
| "logits/chosen": -1.358740210533142, |
| "logits/rejected": -1.26953125, |
| "logps/chosen": -421.29998779296875, |
| "logps/rejected": -590.4500122070312, |
| "loss": 0.013, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -6.458593845367432, |
| "rewards/margins": 17.578125, |
| "rewards/rejected": -24.049999237060547, |
| "step": 5500 |
| }, |
| { |
| "epoch": 3.545864177663341, |
| "grad_norm": 0.03457837151951821, |
| "learning_rate": 1.131680618158403e-07, |
| "logits/chosen": -1.2519409656524658, |
| "logits/rejected": -1.172582983970642, |
| "logps/chosen": -470.20001220703125, |
| "logps/rejected": -592.5, |
| "loss": 0.0174, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -5.803906440734863, |
| "rewards/margins": 16.40625, |
| "rewards/rejected": -22.217187881469727, |
| "step": 5510 |
| }, |
| { |
| "epoch": 3.5523012552301254, |
| "grad_norm": 0.01308628029028273, |
| "learning_rate": 1.1155827430779136e-07, |
| "logits/chosen": -1.4014892578125, |
| "logits/rejected": -1.3704345226287842, |
| "logps/chosen": -470.82501220703125, |
| "logps/rejected": -596.4500122070312, |
| "loss": 0.013, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -6.033203125, |
| "rewards/margins": 15.856249809265137, |
| "rewards/rejected": -21.895313262939453, |
| "step": 5520 |
| }, |
| { |
| "epoch": 3.55873833279691, |
| "grad_norm": 0.023200619987150055, |
| "learning_rate": 1.0994848679974244e-07, |
| "logits/chosen": -1.374121069908142, |
| "logits/rejected": -1.325585961341858, |
| "logps/chosen": -418.5, |
| "logps/rejected": -560.6500244140625, |
| "loss": 0.0144, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -6.929883003234863, |
| "rewards/margins": 15.128125190734863, |
| "rewards/rejected": -22.0625, |
| "step": 5530 |
| }, |
| { |
| "epoch": 3.565175410363695, |
| "grad_norm": 0.2690341562354566, |
| "learning_rate": 1.083386992916935e-07, |
| "logits/chosen": -1.323388695716858, |
| "logits/rejected": -1.3577148914337158, |
| "logps/chosen": -430.45001220703125, |
| "logps/rejected": -603.2999877929688, |
| "loss": 0.0344, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -7.188867092132568, |
| "rewards/margins": 16.779687881469727, |
| "rewards/rejected": -23.953125, |
| "step": 5540 |
| }, |
| { |
| "epoch": 3.5716124879304796, |
| "grad_norm": 0.04642120422324813, |
| "learning_rate": 1.0672891178364456e-07, |
| "logits/chosen": -1.3905029296875, |
| "logits/rejected": -1.3844726085662842, |
| "logps/chosen": -442.25, |
| "logps/rejected": -625.7999877929688, |
| "loss": 0.0173, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -7.114453315734863, |
| "rewards/margins": 16.3046875, |
| "rewards/rejected": -23.412500381469727, |
| "step": 5550 |
| }, |
| { |
| "epoch": 3.578049565497264, |
| "grad_norm": 0.009128227052259342, |
| "learning_rate": 1.0511912427559562e-07, |
| "logits/chosen": -1.3472900390625, |
| "logits/rejected": -1.301904320716858, |
| "logps/chosen": -446.57501220703125, |
| "logps/rejected": -616.7999877929688, |
| "loss": 0.013, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -6.496874809265137, |
| "rewards/margins": 16.165624618530273, |
| "rewards/rejected": -22.6640625, |
| "step": 5560 |
| }, |
| { |
| "epoch": 3.584486643064049, |
| "grad_norm": 0.12412669563299727, |
| "learning_rate": 1.0350933676754668e-07, |
| "logits/chosen": -1.281591773033142, |
| "logits/rejected": -1.207666039466858, |
| "logps/chosen": -426.0249938964844, |
| "logps/rejected": -611.75, |
| "loss": 0.0087, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -6.178955078125, |
| "rewards/margins": 17.182811737060547, |
| "rewards/rejected": -23.365625381469727, |
| "step": 5570 |
| }, |
| { |
| "epoch": 3.5909237206308338, |
| "grad_norm": 0.018241061043156242, |
| "learning_rate": 1.0189954925949774e-07, |
| "logits/chosen": -1.631933569908142, |
| "logits/rejected": -1.546118140220642, |
| "logps/chosen": -418.3999938964844, |
| "logps/rejected": -604.6500244140625, |
| "loss": 0.0043, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -6.741406440734863, |
| "rewards/margins": 16.514062881469727, |
| "rewards/rejected": -23.245311737060547, |
| "step": 5580 |
| }, |
| { |
| "epoch": 3.5973607981976183, |
| "grad_norm": 0.4173010402664288, |
| "learning_rate": 1.0028976175144881e-07, |
| "logits/chosen": -1.3399658203125, |
| "logits/rejected": -1.281152367591858, |
| "logps/chosen": -444.1000061035156, |
| "logps/rejected": -613.75, |
| "loss": 0.0135, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -7.307421684265137, |
| "rewards/margins": 16.118749618530273, |
| "rewards/rejected": -23.431249618530273, |
| "step": 5590 |
| }, |
| { |
| "epoch": 3.603797875764403, |
| "grad_norm": 0.028628333330231747, |
| "learning_rate": 9.867997424339987e-08, |
| "logits/chosen": -1.427636742591858, |
| "logits/rejected": -1.38671875, |
| "logps/chosen": -471.29998779296875, |
| "logps/rejected": -626.4500122070312, |
| "loss": 0.013, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -6.668749809265137, |
| "rewards/margins": 16.228124618530273, |
| "rewards/rejected": -22.8984375, |
| "step": 5600 |
| }, |
| { |
| "epoch": 3.6102349533311875, |
| "grad_norm": 0.00881462072675963, |
| "learning_rate": 9.707018673535093e-08, |
| "logits/chosen": -1.391357421875, |
| "logits/rejected": -1.341796875, |
| "logps/chosen": -464.42498779296875, |
| "logps/rejected": -605.5499877929688, |
| "loss": 0.013, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -6.5283203125, |
| "rewards/margins": 16.165624618530273, |
| "rewards/rejected": -22.703125, |
| "step": 5610 |
| }, |
| { |
| "epoch": 3.6166720308979725, |
| "grad_norm": 0.03419138126385101, |
| "learning_rate": 9.5460399227302e-08, |
| "logits/chosen": -1.2584228515625, |
| "logits/rejected": -1.26025390625, |
| "logps/chosen": -474.92498779296875, |
| "logps/rejected": -603.2000122070312, |
| "loss": 0.0219, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -6.261987209320068, |
| "rewards/margins": 15.753125190734863, |
| "rewards/rejected": -22.004688262939453, |
| "step": 5620 |
| }, |
| { |
| "epoch": 3.623109108464757, |
| "grad_norm": 0.2549410300046455, |
| "learning_rate": 9.385061171925305e-08, |
| "logits/chosen": -1.2335205078125, |
| "logits/rejected": -1.2922852039337158, |
| "logps/chosen": -519.8499755859375, |
| "logps/rejected": -658.25, |
| "loss": 0.0134, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -6.772656440734863, |
| "rewards/margins": 17.246875762939453, |
| "rewards/rejected": -24.012500762939453, |
| "step": 5630 |
| }, |
| { |
| "epoch": 3.6295461860315417, |
| "grad_norm": 0.10878225282534233, |
| "learning_rate": 9.224082421120411e-08, |
| "logits/chosen": -1.3003418445587158, |
| "logits/rejected": -1.24322509765625, |
| "logps/chosen": -445.17498779296875, |
| "logps/rejected": -607.9500122070312, |
| "loss": 0.0303, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -7.554296970367432, |
| "rewards/margins": 15.887499809265137, |
| "rewards/rejected": -23.443750381469727, |
| "step": 5640 |
| }, |
| { |
| "epoch": 3.6359832635983262, |
| "grad_norm": 0.012092711905461724, |
| "learning_rate": 9.063103670315519e-08, |
| "logits/chosen": -1.4790527820587158, |
| "logits/rejected": -1.3707764148712158, |
| "logps/chosen": -478.375, |
| "logps/rejected": -615.5499877929688, |
| "loss": 0.0087, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -6.80859375, |
| "rewards/margins": 15.274218559265137, |
| "rewards/rejected": -22.075000762939453, |
| "step": 5650 |
| }, |
| { |
| "epoch": 3.642420341165111, |
| "grad_norm": 0.00739688634336345, |
| "learning_rate": 8.902124919510625e-08, |
| "logits/chosen": -1.3321533203125, |
| "logits/rejected": -1.364843726158142, |
| "logps/chosen": -503.3999938964844, |
| "logps/rejected": -625.75, |
| "loss": 0.0087, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -6.545605659484863, |
| "rewards/margins": 16.854686737060547, |
| "rewards/rejected": -23.410938262939453, |
| "step": 5660 |
| }, |
| { |
| "epoch": 3.648857418731896, |
| "grad_norm": 0.0066405472294128135, |
| "learning_rate": 8.741146168705731e-08, |
| "logits/chosen": -1.5037109851837158, |
| "logits/rejected": -1.471777319908142, |
| "logps/chosen": -456.95001220703125, |
| "logps/rejected": -630.8499755859375, |
| "loss": 0.0043, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -6.481640815734863, |
| "rewards/margins": 16.807811737060547, |
| "rewards/rejected": -23.290624618530273, |
| "step": 5670 |
| }, |
| { |
| "epoch": 3.6552944962986804, |
| "grad_norm": 0.010289068661605678, |
| "learning_rate": 8.580167417900837e-08, |
| "logits/chosen": -1.264746069908142, |
| "logits/rejected": -1.2636229991912842, |
| "logps/chosen": -466.9750061035156, |
| "logps/rejected": -615.2999877929688, |
| "loss": 0.013, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -6.715624809265137, |
| "rewards/margins": 15.9453125, |
| "rewards/rejected": -22.653125762939453, |
| "step": 5680 |
| }, |
| { |
| "epoch": 3.661731573865465, |
| "grad_norm": 0.017932657112086373, |
| "learning_rate": 8.419188667095943e-08, |
| "logits/chosen": -1.388281226158142, |
| "logits/rejected": -1.3811523914337158, |
| "logps/chosen": -472.20001220703125, |
| "logps/rejected": -614.3499755859375, |
| "loss": 0.0087, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -6.0302734375, |
| "rewards/margins": 17.0390625, |
| "rewards/rejected": -23.065624237060547, |
| "step": 5690 |
| }, |
| { |
| "epoch": 3.66816865143225, |
| "grad_norm": 0.06880424792734291, |
| "learning_rate": 8.258209916291049e-08, |
| "logits/chosen": -1.412072777748108, |
| "logits/rejected": -1.3696777820587158, |
| "logps/chosen": -453.0, |
| "logps/rejected": -637.4500122070312, |
| "loss": 0.0045, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -7.810546875, |
| "rewards/margins": 16.860937118530273, |
| "rewards/rejected": -24.681249618530273, |
| "step": 5700 |
| }, |
| { |
| "epoch": 3.6746057289990346, |
| "grad_norm": 0.006560775520993419, |
| "learning_rate": 8.097231165486156e-08, |
| "logits/chosen": -1.3874022960662842, |
| "logits/rejected": -1.36224365234375, |
| "logps/chosen": -496.45001220703125, |
| "logps/rejected": -627.2000122070312, |
| "loss": 0.0043, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -6.0751953125, |
| "rewards/margins": 17.203125, |
| "rewards/rejected": -23.28125, |
| "step": 5710 |
| }, |
| { |
| "epoch": 3.681042806565819, |
| "grad_norm": 0.025092082754904488, |
| "learning_rate": 7.936252414681262e-08, |
| "logits/chosen": -1.228417992591858, |
| "logits/rejected": -1.2748534679412842, |
| "logps/chosen": -454.0249938964844, |
| "logps/rejected": -586.8499755859375, |
| "loss": 0.0173, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -6.632031440734863, |
| "rewards/margins": 16.084375381469727, |
| "rewards/rejected": -22.7109375, |
| "step": 5720 |
| }, |
| { |
| "epoch": 3.6874798841326037, |
| "grad_norm": 0.5292009305755111, |
| "learning_rate": 7.775273663876368e-08, |
| "logits/chosen": -1.3618896007537842, |
| "logits/rejected": -1.414453148841858, |
| "logps/chosen": -463.8500061035156, |
| "logps/rejected": -597.75, |
| "loss": 0.0087, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -6.807421684265137, |
| "rewards/margins": 15.8515625, |
| "rewards/rejected": -22.662500381469727, |
| "step": 5730 |
| }, |
| { |
| "epoch": 3.6939169616993883, |
| "grad_norm": 0.02623603800018293, |
| "learning_rate": 7.614294913071474e-08, |
| "logits/chosen": -1.362158179283142, |
| "logits/rejected": -1.312524437904358, |
| "logps/chosen": -473.04998779296875, |
| "logps/rejected": -634.1500244140625, |
| "loss": 0.0259, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -6.817968845367432, |
| "rewards/margins": 16.762500762939453, |
| "rewards/rejected": -23.578125, |
| "step": 5740 |
| }, |
| { |
| "epoch": 3.7003540392661733, |
| "grad_norm": 0.9615252370894847, |
| "learning_rate": 7.45331616226658e-08, |
| "logits/chosen": -1.377294898033142, |
| "logits/rejected": -1.3023681640625, |
| "logps/chosen": -431.32501220703125, |
| "logps/rejected": -587.6500244140625, |
| "loss": 0.0216, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -7.223046779632568, |
| "rewards/margins": 14.989062309265137, |
| "rewards/rejected": -22.207813262939453, |
| "step": 5750 |
| }, |
| { |
| "epoch": 3.706791116832958, |
| "grad_norm": 0.625251599463902, |
| "learning_rate": 7.292337411461686e-08, |
| "logits/chosen": -1.3864257335662842, |
| "logits/rejected": -1.321081519126892, |
| "logps/chosen": -451.92498779296875, |
| "logps/rejected": -616.4000244140625, |
| "loss": 0.0045, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -6.649218559265137, |
| "rewards/margins": 16.917186737060547, |
| "rewards/rejected": -23.557811737060547, |
| "step": 5760 |
| }, |
| { |
| "epoch": 3.7132281943997425, |
| "grad_norm": 0.02042626041012992, |
| "learning_rate": 7.131358660656794e-08, |
| "logits/chosen": -1.40924072265625, |
| "logits/rejected": -1.396337866783142, |
| "logps/chosen": -439.32501220703125, |
| "logps/rejected": -584.4500122070312, |
| "loss": 0.013, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -7.258593559265137, |
| "rewards/margins": 15.923437118530273, |
| "rewards/rejected": -23.171875, |
| "step": 5770 |
| }, |
| { |
| "epoch": 3.719665271966527, |
| "grad_norm": 0.1622419073810199, |
| "learning_rate": 6.9703799098519e-08, |
| "logits/chosen": -1.240332007408142, |
| "logits/rejected": -1.2565429210662842, |
| "logps/chosen": -472.875, |
| "logps/rejected": -623.75, |
| "loss": 0.0131, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -6.084765434265137, |
| "rewards/margins": 17.006250381469727, |
| "rewards/rejected": -23.112499237060547, |
| "step": 5780 |
| }, |
| { |
| "epoch": 3.7261023495333117, |
| "grad_norm": 0.01125749891895987, |
| "learning_rate": 6.809401159047006e-08, |
| "logits/chosen": -1.215185523033142, |
| "logits/rejected": -1.1459228992462158, |
| "logps/chosen": -469.875, |
| "logps/rejected": -603.6500244140625, |
| "loss": 0.0087, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -6.083642482757568, |
| "rewards/margins": 16.5390625, |
| "rewards/rejected": -22.618749618530273, |
| "step": 5790 |
| }, |
| { |
| "epoch": 3.7325394271000967, |
| "grad_norm": 0.057481049922450365, |
| "learning_rate": 6.648422408242112e-08, |
| "logits/chosen": -1.4691650867462158, |
| "logits/rejected": -1.4182617664337158, |
| "logps/chosen": -464.67498779296875, |
| "logps/rejected": -584.9500122070312, |
| "loss": 0.0044, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -6.586328029632568, |
| "rewards/margins": 16.889062881469727, |
| "rewards/rejected": -23.4609375, |
| "step": 5800 |
| }, |
| { |
| "epoch": 3.7389765046668813, |
| "grad_norm": 1.0255846855304118, |
| "learning_rate": 6.487443657437218e-08, |
| "logits/chosen": -1.420263648033142, |
| "logits/rejected": -1.350683569908142, |
| "logps/chosen": -452.92498779296875, |
| "logps/rejected": -629.5, |
| "loss": 0.0012, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -6.487890720367432, |
| "rewards/margins": 16.6015625, |
| "rewards/rejected": -23.090625762939453, |
| "step": 5810 |
| }, |
| { |
| "epoch": 3.745413582233666, |
| "grad_norm": 0.011613759890222444, |
| "learning_rate": 6.326464906632324e-08, |
| "logits/chosen": -1.4422607421875, |
| "logits/rejected": -1.4406554698944092, |
| "logps/chosen": -502.8500061035156, |
| "logps/rejected": -654.0, |
| "loss": 0.0087, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -6.893847465515137, |
| "rewards/margins": 17.293750762939453, |
| "rewards/rejected": -24.181249618530273, |
| "step": 5820 |
| }, |
| { |
| "epoch": 3.751850659800451, |
| "grad_norm": 0.013630305611034016, |
| "learning_rate": 6.165486155827431e-08, |
| "logits/chosen": -1.279443383216858, |
| "logits/rejected": -1.308325171470642, |
| "logps/chosen": -471.79998779296875, |
| "logps/rejected": -596.2999877929688, |
| "loss": 0.013, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -7.536718845367432, |
| "rewards/margins": 15.589062690734863, |
| "rewards/rejected": -23.115625381469727, |
| "step": 5830 |
| }, |
| { |
| "epoch": 3.7582877373672354, |
| "grad_norm": 0.0014804285191116408, |
| "learning_rate": 6.004507405022537e-08, |
| "logits/chosen": -1.3689453601837158, |
| "logits/rejected": -1.375390648841858, |
| "logps/chosen": -533.9000244140625, |
| "logps/rejected": -643.75, |
| "loss": 0.0047, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -7.103320121765137, |
| "rewards/margins": 16.696874618530273, |
| "rewards/rejected": -23.799999237060547, |
| "step": 5840 |
| }, |
| { |
| "epoch": 3.76472481493402, |
| "grad_norm": 0.00972610437522201, |
| "learning_rate": 5.843528654217643e-08, |
| "logits/chosen": -1.353271484375, |
| "logits/rejected": -1.267358422279358, |
| "logps/chosen": -449.7250061035156, |
| "logps/rejected": -600.5, |
| "loss": 0.0087, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -6.952734470367432, |
| "rewards/margins": 16.506250381469727, |
| "rewards/rejected": -23.456249237060547, |
| "step": 5850 |
| }, |
| { |
| "epoch": 3.7711618925008046, |
| "grad_norm": 0.005174645997734548, |
| "learning_rate": 5.682549903412749e-08, |
| "logits/chosen": -1.172021508216858, |
| "logits/rejected": -1.1627197265625, |
| "logps/chosen": -439.7250061035156, |
| "logps/rejected": -619.3499755859375, |
| "loss": 0.0218, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -7.248730659484863, |
| "rewards/margins": 17.012500762939453, |
| "rewards/rejected": -24.276561737060547, |
| "step": 5860 |
| }, |
| { |
| "epoch": 3.777598970067589, |
| "grad_norm": 0.003572394737599069, |
| "learning_rate": 5.521571152607855e-08, |
| "logits/chosen": -1.3669922351837158, |
| "logits/rejected": -1.3381836414337158, |
| "logps/chosen": -497.3999938964844, |
| "logps/rejected": -648.5499877929688, |
| "loss": 0.0216, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -7.112109184265137, |
| "rewards/margins": 17.045312881469727, |
| "rewards/rejected": -24.153125762939453, |
| "step": 5870 |
| }, |
| { |
| "epoch": 3.784036047634374, |
| "grad_norm": 0.005081291840206665, |
| "learning_rate": 5.360592401802962e-08, |
| "logits/chosen": -1.2956054210662842, |
| "logits/rejected": -1.364648461341858, |
| "logps/chosen": -474.0249938964844, |
| "logps/rejected": -644.2000122070312, |
| "loss": 0.0043, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -7.199023246765137, |
| "rewards/margins": 17.748437881469727, |
| "rewards/rejected": -24.946874618530273, |
| "step": 5880 |
| }, |
| { |
| "epoch": 3.7904731252011588, |
| "grad_norm": 0.03628039791159471, |
| "learning_rate": 5.199613650998068e-08, |
| "logits/chosen": -1.2915527820587158, |
| "logits/rejected": -1.3411133289337158, |
| "logps/chosen": -413.1000061035156, |
| "logps/rejected": -551.5499877929688, |
| "loss": 0.0134, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -6.5751953125, |
| "rewards/margins": 15.893750190734863, |
| "rewards/rejected": -22.478124618530273, |
| "step": 5890 |
| }, |
| { |
| "epoch": 3.7969102027679433, |
| "grad_norm": 0.020860070234620432, |
| "learning_rate": 5.038634900193174e-08, |
| "logits/chosen": -1.3630859851837158, |
| "logits/rejected": -1.3144042491912842, |
| "logps/chosen": -480.5, |
| "logps/rejected": -627.3499755859375, |
| "loss": 0.013, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -6.768750190734863, |
| "rewards/margins": 15.701562881469727, |
| "rewards/rejected": -22.484375, |
| "step": 5900 |
| }, |
| { |
| "epoch": 3.803347280334728, |
| "grad_norm": 0.022873670469227263, |
| "learning_rate": 4.8776561493882806e-08, |
| "logits/chosen": -1.372094750404358, |
| "logits/rejected": -1.3430664539337158, |
| "logps/chosen": -515.0499877929688, |
| "logps/rejected": -651.1500244140625, |
| "loss": 0.013, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -7.543310642242432, |
| "rewards/margins": 17.0234375, |
| "rewards/rejected": -24.576562881469727, |
| "step": 5910 |
| }, |
| { |
| "epoch": 3.8097843579015125, |
| "grad_norm": 0.009610945099063505, |
| "learning_rate": 4.7166773985833866e-08, |
| "logits/chosen": -1.22100830078125, |
| "logits/rejected": -1.256250023841858, |
| "logps/chosen": -486.79998779296875, |
| "logps/rejected": -615.7000122070312, |
| "loss": 0.0216, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -7.102392673492432, |
| "rewards/margins": 16.374217987060547, |
| "rewards/rejected": -23.473438262939453, |
| "step": 5920 |
| }, |
| { |
| "epoch": 3.8162214354682975, |
| "grad_norm": 0.01141365115803457, |
| "learning_rate": 4.5556986477784926e-08, |
| "logits/chosen": -1.37841796875, |
| "logits/rejected": -1.351159691810608, |
| "logps/chosen": -481.5, |
| "logps/rejected": -654.8499755859375, |
| "loss": 0.0173, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -7.371289253234863, |
| "rewards/margins": 17.350000381469727, |
| "rewards/rejected": -24.731250762939453, |
| "step": 5930 |
| }, |
| { |
| "epoch": 3.822658513035082, |
| "grad_norm": 0.03773272602391191, |
| "learning_rate": 4.394719896973599e-08, |
| "logits/chosen": -1.1256835460662842, |
| "logits/rejected": -1.119238257408142, |
| "logps/chosen": -477.82501220703125, |
| "logps/rejected": -613.8250122070312, |
| "loss": 0.0303, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -7.057421684265137, |
| "rewards/margins": 16.196874618530273, |
| "rewards/rejected": -23.251562118530273, |
| "step": 5940 |
| }, |
| { |
| "epoch": 3.8290955906018667, |
| "grad_norm": 0.006580203017507808, |
| "learning_rate": 4.233741146168705e-08, |
| "logits/chosen": -1.2913818359375, |
| "logits/rejected": -1.265869140625, |
| "logps/chosen": -463.5, |
| "logps/rejected": -600.7249755859375, |
| "loss": 0.0259, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -6.014257907867432, |
| "rewards/margins": 16.518749237060547, |
| "rewards/rejected": -22.534374237060547, |
| "step": 5950 |
| }, |
| { |
| "epoch": 3.8355326681686517, |
| "grad_norm": 0.034915403188268565, |
| "learning_rate": 4.072762395363811e-08, |
| "logits/chosen": -1.405859351158142, |
| "logits/rejected": -1.3607666492462158, |
| "logps/chosen": -460.5249938964844, |
| "logps/rejected": -604.0, |
| "loss": 0.0087, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -6.685156345367432, |
| "rewards/margins": 16.59375, |
| "rewards/rejected": -23.28125, |
| "step": 5960 |
| }, |
| { |
| "epoch": 3.8419697457354363, |
| "grad_norm": 0.0024218096381025595, |
| "learning_rate": 3.911783644558918e-08, |
| "logits/chosen": -1.296142578125, |
| "logits/rejected": -1.2864990234375, |
| "logps/chosen": -479.8500061035156, |
| "logps/rejected": -634.75, |
| "loss": 0.0173, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -7.158203125, |
| "rewards/margins": 16.131250381469727, |
| "rewards/rejected": -23.284374237060547, |
| "step": 5970 |
| }, |
| { |
| "epoch": 3.848406823302221, |
| "grad_norm": 0.12574260250582833, |
| "learning_rate": 3.750804893754024e-08, |
| "logits/chosen": -1.3772704601287842, |
| "logits/rejected": -1.3362305164337158, |
| "logps/chosen": -487.5, |
| "logps/rejected": -639.9500122070312, |
| "loss": 0.013, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -7.358788967132568, |
| "rewards/margins": 16.390625, |
| "rewards/rejected": -23.75, |
| "step": 5980 |
| }, |
| { |
| "epoch": 3.8548439008690054, |
| "grad_norm": 0.028011830155125178, |
| "learning_rate": 3.58982614294913e-08, |
| "logits/chosen": -1.410400390625, |
| "logits/rejected": -1.394677758216858, |
| "logps/chosen": -451.25, |
| "logps/rejected": -622.7999877929688, |
| "loss": 0.0087, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -7.142187595367432, |
| "rewards/margins": 16.359375, |
| "rewards/rejected": -23.509374618530273, |
| "step": 5990 |
| }, |
| { |
| "epoch": 3.86128097843579, |
| "grad_norm": 0.04311806836932471, |
| "learning_rate": 3.428847392144237e-08, |
| "logits/chosen": -1.488867163658142, |
| "logits/rejected": -1.449316382408142, |
| "logps/chosen": -475.79998779296875, |
| "logps/rejected": -642.75, |
| "loss": 0.0043, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -7.62109375, |
| "rewards/margins": 16.924999237060547, |
| "rewards/rejected": -24.543750762939453, |
| "step": 6000 |
| }, |
| { |
| "epoch": 3.867718056002575, |
| "grad_norm": 0.06370531489189059, |
| "learning_rate": 3.267868641339343e-08, |
| "logits/chosen": -1.351415991783142, |
| "logits/rejected": -1.2885253429412842, |
| "logps/chosen": -463.3999938964844, |
| "logps/rejected": -607.375, |
| "loss": 0.0174, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -7.260546684265137, |
| "rewards/margins": 16.448436737060547, |
| "rewards/rejected": -23.714061737060547, |
| "step": 6010 |
| }, |
| { |
| "epoch": 3.8741551335693596, |
| "grad_norm": 0.009851035602273154, |
| "learning_rate": 3.1068898905344494e-08, |
| "logits/chosen": -1.2275512218475342, |
| "logits/rejected": -1.160668969154358, |
| "logps/chosen": -419.4125061035156, |
| "logps/rejected": -573.5750122070312, |
| "loss": 0.0216, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -6.607031345367432, |
| "rewards/margins": 16.443750381469727, |
| "rewards/rejected": -23.049999237060547, |
| "step": 6020 |
| }, |
| { |
| "epoch": 3.880592211136144, |
| "grad_norm": 0.0028652931247393, |
| "learning_rate": 2.9459111397295555e-08, |
| "logits/chosen": -1.271826148033142, |
| "logits/rejected": -1.2801024913787842, |
| "logps/chosen": -486.125, |
| "logps/rejected": -630.1749877929688, |
| "loss": 0.0217, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -8.189844131469727, |
| "rewards/margins": 16.317188262939453, |
| "rewards/rejected": -24.506250381469727, |
| "step": 6030 |
| }, |
| { |
| "epoch": 3.8870292887029287, |
| "grad_norm": 0.004632645762181224, |
| "learning_rate": 2.7849323889246618e-08, |
| "logits/chosen": -1.361230492591858, |
| "logits/rejected": -1.3944580554962158, |
| "logps/chosen": -467.875, |
| "logps/rejected": -613.2000122070312, |
| "loss": 0.013, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -7.145312309265137, |
| "rewards/margins": 16.707813262939453, |
| "rewards/rejected": -23.846874237060547, |
| "step": 6040 |
| }, |
| { |
| "epoch": 3.8934663662697133, |
| "grad_norm": 0.12173582779202217, |
| "learning_rate": 2.623953638119768e-08, |
| "logits/chosen": -1.338281273841858, |
| "logits/rejected": -1.4280273914337158, |
| "logps/chosen": -489.6499938964844, |
| "logps/rejected": -633.2000122070312, |
| "loss": 0.0087, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -6.922461032867432, |
| "rewards/margins": 17.037500381469727, |
| "rewards/rejected": -23.959375381469727, |
| "step": 6050 |
| }, |
| { |
| "epoch": 3.8999034438364983, |
| "grad_norm": 0.4082952605894205, |
| "learning_rate": 2.4629748873148742e-08, |
| "logits/chosen": -1.339331030845642, |
| "logits/rejected": -1.3220703601837158, |
| "logps/chosen": -447.375, |
| "logps/rejected": -573.3250122070312, |
| "loss": 0.0303, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -6.518359184265137, |
| "rewards/margins": 16.157812118530273, |
| "rewards/rejected": -22.6875, |
| "step": 6060 |
| }, |
| { |
| "epoch": 3.906340521403283, |
| "grad_norm": 0.001889330141718421, |
| "learning_rate": 2.3019961365099805e-08, |
| "logits/chosen": -1.584375023841858, |
| "logits/rejected": -1.4647948741912842, |
| "logps/chosen": -480.75, |
| "logps/rejected": -669.5499877929688, |
| "loss": 0.0047, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -6.984961032867432, |
| "rewards/margins": 17.162500381469727, |
| "rewards/rejected": -24.159374237060547, |
| "step": 6070 |
| }, |
| { |
| "epoch": 3.9127775989700675, |
| "grad_norm": 0.003474359434755791, |
| "learning_rate": 2.141017385705087e-08, |
| "logits/chosen": -1.205175757408142, |
| "logits/rejected": -1.2141602039337158, |
| "logps/chosen": -440.32501220703125, |
| "logps/rejected": -621.25, |
| "loss": 0.0185, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -7.234375, |
| "rewards/margins": 18.193750381469727, |
| "rewards/rejected": -25.424999237060547, |
| "step": 6080 |
| }, |
| { |
| "epoch": 3.9192146765368525, |
| "grad_norm": 0.005941318500453077, |
| "learning_rate": 1.980038634900193e-08, |
| "logits/chosen": -1.161962866783142, |
| "logits/rejected": -1.196313500404358, |
| "logps/chosen": -453.45001220703125, |
| "logps/rejected": -593.5999755859375, |
| "loss": 0.0216, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -6.726366996765137, |
| "rewards/margins": 17.221874237060547, |
| "rewards/rejected": -23.962499618530273, |
| "step": 6090 |
| }, |
| { |
| "epoch": 3.925651754103637, |
| "grad_norm": 0.01116944815158561, |
| "learning_rate": 1.8190598840952993e-08, |
| "logits/chosen": -1.1934814453125, |
| "logits/rejected": -1.169714331626892, |
| "logps/chosen": -471.8500061035156, |
| "logps/rejected": -601.1500244140625, |
| "loss": 0.0173, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -7.227343559265137, |
| "rewards/margins": 16.854686737060547, |
| "rewards/rejected": -24.100000381469727, |
| "step": 6100 |
| }, |
| { |
| "epoch": 3.9320888316704217, |
| "grad_norm": 4.328017153289504, |
| "learning_rate": 1.6580811332904056e-08, |
| "logits/chosen": -1.248876929283142, |
| "logits/rejected": -1.225378394126892, |
| "logps/chosen": -495.6000061035156, |
| "logps/rejected": -651.7000122070312, |
| "loss": 0.009, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -7.467968940734863, |
| "rewards/margins": 17.198436737060547, |
| "rewards/rejected": -24.671875, |
| "step": 6110 |
| }, |
| { |
| "epoch": 3.9385259092372062, |
| "grad_norm": 0.0372062799325498, |
| "learning_rate": 1.497102382485512e-08, |
| "logits/chosen": -1.4552733898162842, |
| "logits/rejected": -1.479150414466858, |
| "logps/chosen": -487.8999938964844, |
| "logps/rejected": -615.0, |
| "loss": 0.013, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -7.0390625, |
| "rewards/margins": 16.2421875, |
| "rewards/rejected": -23.278125762939453, |
| "step": 6120 |
| }, |
| { |
| "epoch": 3.944962986803991, |
| "grad_norm": 0.013571001655658714, |
| "learning_rate": 1.3361236316806181e-08, |
| "logits/chosen": -1.2753417491912842, |
| "logits/rejected": -1.3110840320587158, |
| "logps/chosen": -498.75, |
| "logps/rejected": -622.9500122070312, |
| "loss": 0.0004, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -7.55078125, |
| "rewards/margins": 16.451562881469727, |
| "rewards/rejected": -24.003124237060547, |
| "step": 6130 |
| }, |
| { |
| "epoch": 3.951400064370776, |
| "grad_norm": 0.02787938925128275, |
| "learning_rate": 1.1751448808757243e-08, |
| "logits/chosen": -1.205712914466858, |
| "logits/rejected": -1.2152831554412842, |
| "logps/chosen": -482.07501220703125, |
| "logps/rejected": -634.5999755859375, |
| "loss": 0.0173, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -7.509863376617432, |
| "rewards/margins": 15.885937690734863, |
| "rewards/rejected": -23.396875381469727, |
| "step": 6140 |
| }, |
| { |
| "epoch": 3.9578371419375604, |
| "grad_norm": 0.009048893839464986, |
| "learning_rate": 1.0141661300708307e-08, |
| "logits/chosen": -1.4137451648712158, |
| "logits/rejected": -1.35845947265625, |
| "logps/chosen": -432.6000061035156, |
| "logps/rejected": -576.7999877929688, |
| "loss": 0.0087, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -7.834631443023682, |
| "rewards/margins": 16.490625381469727, |
| "rewards/rejected": -24.331249237060547, |
| "step": 6150 |
| }, |
| { |
| "epoch": 3.964274219504345, |
| "grad_norm": 0.010432441433882786, |
| "learning_rate": 8.531873792659369e-09, |
| "logits/chosen": -1.361791968345642, |
| "logits/rejected": -1.3799316883087158, |
| "logps/chosen": -469.17498779296875, |
| "logps/rejected": -632.4500122070312, |
| "loss": 0.0087, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -6.779101371765137, |
| "rewards/margins": 18.120311737060547, |
| "rewards/rejected": -24.899999618530273, |
| "step": 6160 |
| }, |
| { |
| "epoch": 3.9707112970711296, |
| "grad_norm": 0.019491466119934187, |
| "learning_rate": 6.922086284610431e-09, |
| "logits/chosen": -1.3798828125, |
| "logits/rejected": -1.4896240234375, |
| "logps/chosen": -484.6000061035156, |
| "logps/rejected": -576.5999755859375, |
| "loss": 0.0173, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -7.057031154632568, |
| "rewards/margins": 15.754687309265137, |
| "rewards/rejected": -22.815624237060547, |
| "step": 6170 |
| }, |
| { |
| "epoch": 3.977148374637914, |
| "grad_norm": 0.03133597748155418, |
| "learning_rate": 5.312298776561494e-09, |
| "logits/chosen": -1.3720214366912842, |
| "logits/rejected": -1.3763427734375, |
| "logps/chosen": -483.0249938964844, |
| "logps/rejected": -636.75, |
| "loss": 0.013, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -7.968359470367432, |
| "rewards/margins": 16.909374237060547, |
| "rewards/rejected": -24.876562118530273, |
| "step": 6180 |
| }, |
| { |
| "epoch": 3.983585452204699, |
| "grad_norm": 0.045996008407318864, |
| "learning_rate": 3.7025112685125563e-09, |
| "logits/chosen": -1.274072289466858, |
| "logits/rejected": -1.3197753429412842, |
| "logps/chosen": -476.625, |
| "logps/rejected": -590.0, |
| "loss": 0.0087, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -7.233202934265137, |
| "rewards/margins": 16.729686737060547, |
| "rewards/rejected": -23.971874237060547, |
| "step": 6190 |
| }, |
| { |
| "epoch": 3.9900225297714838, |
| "grad_norm": 0.03387015833302683, |
| "learning_rate": 2.0927237604636186e-09, |
| "logits/chosen": -1.405908226966858, |
| "logits/rejected": -1.3355712890625, |
| "logps/chosen": -466.2250061035156, |
| "logps/rejected": -640.25, |
| "loss": 0.0093, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -7.567773342132568, |
| "rewards/margins": 16.521875381469727, |
| "rewards/rejected": -24.073436737060547, |
| "step": 6200 |
| }, |
| { |
| "epoch": 3.9964596073382683, |
| "grad_norm": 0.02589736256630603, |
| "learning_rate": 4.829362524146813e-10, |
| "logits/chosen": -1.1253662109375, |
| "logits/rejected": -1.091162085533142, |
| "logps/chosen": -472.7749938964844, |
| "logps/rejected": -658.4500122070312, |
| "loss": 0.0176, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -6.837890625, |
| "rewards/margins": 17.390625, |
| "rewards/rejected": -24.217187881469727, |
| "step": 6210 |
| } |
| ], |
| "logging_steps": 10, |
| "max_steps": 6212, |
| "num_input_tokens_seen": 0, |
| "num_train_epochs": 4, |
| "save_steps": 500, |
| "stateful_callbacks": { |
| "TrainerControl": { |
| "args": { |
| "should_epoch_stop": false, |
| "should_evaluate": false, |
| "should_log": false, |
| "should_save": true, |
| "should_training_stop": true |
| }, |
| "attributes": {} |
| } |
| }, |
| "total_flos": 0.0, |
| "train_batch_size": 1, |
| "trial_name": null, |
| "trial_params": null |
| } |
|
|
