{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 2.9992254066615027,
  "eval_steps": 100,
  "global_step": 363,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.01,
      "learning_rate": 1.3513513513513514e-08,
      "logits/chosen": -1.6217646598815918,
      "logits/rejected": -1.4712426662445068,
      "logps/chosen": -277.2057800292969,
      "logps/rejected": -212.0065155029297,
      "loss": 0.6931,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.08,
      "learning_rate": 1.3513513513513515e-07,
      "logits/chosen": -1.7232815027236938,
      "logits/rejected": -1.4470032453536987,
      "logps/chosen": -313.0498046875,
      "logps/rejected": -231.90628051757812,
      "loss": 0.6943,
      "rewards/accuracies": 0.4253472089767456,
      "rewards/chosen": -0.000630709168035537,
      "rewards/margins": -0.0025554734747856855,
      "rewards/rejected": 0.0019247637828812003,
      "step": 10
    },
    {
      "epoch": 0.17,
      "learning_rate": 2.702702702702703e-07,
      "logits/chosen": -1.6689411401748657,
      "logits/rejected": -1.4514515399932861,
      "logps/chosen": -287.94317626953125,
      "logps/rejected": -217.27197265625,
      "loss": 0.6934,
      "rewards/accuracies": 0.5015624761581421,
      "rewards/chosen": 0.0008510888437740505,
      "rewards/margins": 0.0013923795195296407,
      "rewards/rejected": -0.0005412912578321993,
      "step": 20
    },
    {
      "epoch": 0.25,
      "learning_rate": 4.054054054054054e-07,
      "logits/chosen": -1.6716254949569702,
      "logits/rejected": -1.4925106763839722,
      "logps/chosen": -291.8757019042969,
      "logps/rejected": -233.2742156982422,
      "loss": 0.6926,
      "rewards/accuracies": 0.520312488079071,
      "rewards/chosen": 0.0017051857430487871,
      "rewards/margins": 0.005291451700031757,
      "rewards/rejected": -0.003586266189813614,
      "step": 30
    },
    {
      "epoch": 0.33,
      "learning_rate": 4.95398773006135e-07,
      "logits/chosen": -1.6524215936660767,
      "logits/rejected": -1.5010509490966797,
      "logps/chosen": -277.3305358886719,
      "logps/rejected": -237.3994140625,
      "loss": 0.6934,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": 0.0026146990712732077,
      "rewards/margins": -0.0002183911419706419,
      "rewards/rejected": 0.0028330907225608826,
      "step": 40
    },
    {
      "epoch": 0.41,
      "learning_rate": 4.800613496932515e-07,
      "logits/chosen": -1.6938726902008057,
      "logits/rejected": -1.489926815032959,
      "logps/chosen": -286.188720703125,
      "logps/rejected": -233.00650024414062,
      "loss": 0.6932,
      "rewards/accuracies": 0.5078125,
      "rewards/chosen": 0.0013290711212903261,
      "rewards/margins": 0.00042788925929926336,
      "rewards/rejected": 0.0009011816000565886,
      "step": 50
    },
    {
      "epoch": 0.5,
      "learning_rate": 4.647239263803681e-07,
      "logits/chosen": -1.6785087585449219,
      "logits/rejected": -1.4873216152191162,
      "logps/chosen": -269.3262023925781,
      "logps/rejected": -230.5088348388672,
      "loss": 0.6942,
      "rewards/accuracies": 0.4984374940395355,
      "rewards/chosen": 0.002710798755288124,
      "rewards/margins": -8.833855099510401e-05,
      "rewards/rejected": 0.002799137495458126,
      "step": 60
    },
    {
      "epoch": 0.58,
      "learning_rate": 4.4938650306748465e-07,
      "logits/chosen": -1.6974502801895142,
      "logits/rejected": -1.4916502237319946,
      "logps/chosen": -283.6458740234375,
      "logps/rejected": -228.8906707763672,
      "loss": 0.6922,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": 0.00096172746270895,
      "rewards/margins": 0.0029513793997466564,
      "rewards/rejected": -0.0019896519370377064,
      "step": 70
    },
    {
      "epoch": 0.66,
      "learning_rate": 4.340490797546012e-07,
      "logits/chosen": -1.6969743967056274,
      "logits/rejected": -1.508497714996338,
      "logps/chosen": -302.8039245605469,
      "logps/rejected": -253.368896484375,
      "loss": 0.6922,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": 0.006227812264114618,
      "rewards/margins": 0.007338723633438349,
      "rewards/rejected": -0.0011109116021543741,
      "step": 80
    },
    {
      "epoch": 0.74,
      "learning_rate": 4.187116564417178e-07,
      "logits/chosen": -1.7152626514434814,
      "logits/rejected": -1.5328489542007446,
      "logps/chosen": -287.313232421875,
      "logps/rejected": -249.7541046142578,
      "loss": 0.6929,
      "rewards/accuracies": 0.503125011920929,
      "rewards/chosen": 0.005095381755381823,
      "rewards/margins": 0.00023937877267599106,
      "rewards/rejected": 0.004856002517044544,
      "step": 90
    },
    {
      "epoch": 0.83,
      "learning_rate": 4.0337423312883434e-07,
      "logits/chosen": -1.649161696434021,
      "logits/rejected": -1.482323169708252,
      "logps/chosen": -269.73529052734375,
      "logps/rejected": -227.615966796875,
      "loss": 0.6922,
      "rewards/accuracies": 0.520312488079071,
      "rewards/chosen": 0.0035763948690146208,
      "rewards/margins": 0.0003702163230627775,
      "rewards/rejected": 0.003206178080290556,
      "step": 100
    },
    {
      "epoch": 0.91,
      "learning_rate": 3.880368098159509e-07,
      "logits/chosen": -1.6747386455535889,
      "logits/rejected": -1.485433578491211,
      "logps/chosen": -303.1210632324219,
      "logps/rejected": -245.5657501220703,
      "loss": 0.6923,
      "rewards/accuracies": 0.520312488079071,
      "rewards/chosen": 0.008083445020020008,
      "rewards/margins": 0.004594652913510799,
      "rewards/rejected": 0.0034887916408479214,
      "step": 110
    },
    {
      "epoch": 0.99,
      "learning_rate": 3.7269938650306747e-07,
      "logits/chosen": -1.6758835315704346,
      "logits/rejected": -1.510749340057373,
      "logps/chosen": -284.28814697265625,
      "logps/rejected": -225.5731201171875,
      "loss": 0.691,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": 0.006056616548448801,
      "rewards/margins": 0.0026814837474375963,
      "rewards/rejected": 0.0033751321025192738,
      "step": 120
    },
    {
      "epoch": 1.0,
      "eval_logits/chosen": -1.4432116746902466,
      "eval_logits/rejected": -1.204046368598938,
      "eval_logps/chosen": -309.466064453125,
      "eval_logps/rejected": -245.28790283203125,
      "eval_loss": 0.6911042332649231,
      "eval_rewards/accuracies": 0.5039682388305664,
      "eval_rewards/chosen": 0.007772538345307112,
      "eval_rewards/margins": 0.00581591809168458,
      "eval_rewards/rejected": 0.001956620253622532,
      "eval_runtime": 242.7759,
      "eval_samples_per_second": 8.238,
      "eval_steps_per_second": 0.259,
      "step": 121
    },
    {
      "epoch": 1.07,
      "learning_rate": 3.5736196319018404e-07,
      "logits/chosen": -1.6334679126739502,
      "logits/rejected": -1.488577127456665,
      "logps/chosen": -260.3802795410156,
      "logps/rejected": -226.0418701171875,
      "loss": 0.6911,
      "rewards/accuracies": 0.5140625238418579,
      "rewards/chosen": 0.007788505405187607,
      "rewards/margins": 0.0028753529768437147,
      "rewards/rejected": 0.004913152661174536,
      "step": 130
    },
    {
      "epoch": 1.16,
      "learning_rate": 3.420245398773006e-07,
      "logits/chosen": -1.6948306560516357,
      "logits/rejected": -1.4906350374221802,
      "logps/chosen": -281.7218017578125,
      "logps/rejected": -237.4910888671875,
      "loss": 0.6917,
      "rewards/accuracies": 0.5406249761581421,
      "rewards/chosen": 0.00853796862065792,
      "rewards/margins": 0.0046198442578315735,
      "rewards/rejected": 0.003918124828487635,
      "step": 140
    },
    {
      "epoch": 1.24,
      "learning_rate": 3.2668711656441716e-07,
      "logits/chosen": -1.67226243019104,
      "logits/rejected": -1.4697535037994385,
      "logps/chosen": -292.50128173828125,
      "logps/rejected": -240.04885864257812,
      "loss": 0.6912,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": 0.007847895845770836,
      "rewards/margins": 0.00384397991001606,
      "rewards/rejected": 0.004003916867077351,
      "step": 150
    },
    {
      "epoch": 1.32,
      "learning_rate": 3.1134969325153373e-07,
      "logits/chosen": -1.6966972351074219,
      "logits/rejected": -1.526249885559082,
      "logps/chosen": -283.823974609375,
      "logps/rejected": -248.48617553710938,
      "loss": 0.6905,
      "rewards/accuracies": 0.5484374761581421,
      "rewards/chosen": 0.011579589918255806,
      "rewards/margins": 0.007860107347369194,
      "rewards/rejected": 0.0037194814067333937,
      "step": 160
    },
    {
      "epoch": 1.4,
      "learning_rate": 2.960122699386503e-07,
      "logits/chosen": -1.619706153869629,
      "logits/rejected": -1.4683091640472412,
      "logps/chosen": -267.863037109375,
      "logps/rejected": -227.89404296875,
      "loss": 0.6907,
      "rewards/accuracies": 0.5484374761581421,
      "rewards/chosen": 0.013016032986342907,
      "rewards/margins": 0.006848230957984924,
      "rewards/rejected": 0.006167802028357983,
      "step": 170
    },
    {
      "epoch": 1.49,
      "learning_rate": 2.8067484662576686e-07,
      "logits/chosen": -1.6813249588012695,
      "logits/rejected": -1.4534178972244263,
      "logps/chosen": -292.1463317871094,
      "logps/rejected": -231.10348510742188,
      "loss": 0.6904,
      "rewards/accuracies": 0.5328124761581421,
      "rewards/chosen": 0.012731348164379597,
      "rewards/margins": 0.00863958615809679,
      "rewards/rejected": 0.004091762937605381,
      "step": 180
    },
    {
      "epoch": 1.57,
      "learning_rate": 2.653374233128834e-07,
      "logits/chosen": -1.678877592086792,
      "logits/rejected": -1.5057964324951172,
      "logps/chosen": -291.14544677734375,
      "logps/rejected": -237.1598358154297,
      "loss": 0.6907,
      "rewards/accuracies": 0.520312488079071,
      "rewards/chosen": 0.014096416532993317,
      "rewards/margins": 0.005887693725526333,
      "rewards/rejected": 0.00820872187614441,
      "step": 190
    },
    {
      "epoch": 1.65,
      "learning_rate": 2.5e-07,
      "logits/chosen": -1.7134149074554443,
      "logits/rejected": -1.480711579322815,
      "logps/chosen": -311.05999755859375,
      "logps/rejected": -235.89492797851562,
      "loss": 0.6904,
      "rewards/accuracies": 0.5218750238418579,
      "rewards/chosen": 0.013186228461563587,
      "rewards/margins": 0.005698888096958399,
      "rewards/rejected": 0.007487342692911625,
      "step": 200
    },
    {
      "epoch": 1.74,
      "learning_rate": 2.3466257668711655e-07,
      "logits/chosen": -1.7005561590194702,
      "logits/rejected": -1.4974968433380127,
      "logps/chosen": -297.63427734375,
      "logps/rejected": -236.1612548828125,
      "loss": 0.6904,
      "rewards/accuracies": 0.5484374761581421,
      "rewards/chosen": 0.017665740102529526,
      "rewards/margins": 0.004187946207821369,
      "rewards/rejected": 0.013477792963385582,
      "step": 210
    },
    {
      "epoch": 1.82,
      "learning_rate": 2.1932515337423312e-07,
      "logits/chosen": -1.67599618434906,
      "logits/rejected": -1.486876130104065,
      "logps/chosen": -287.51239013671875,
      "logps/rejected": -233.4202117919922,
      "loss": 0.6894,
      "rewards/accuracies": 0.5640624761581421,
      "rewards/chosen": 0.015584470704197884,
      "rewards/margins": 0.0049061039462685585,
      "rewards/rejected": 0.01067836582660675,
      "step": 220
    },
    {
      "epoch": 1.9,
      "learning_rate": 2.0398773006134968e-07,
      "logits/chosen": -1.6751419305801392,
      "logits/rejected": -1.4874600172042847,
      "logps/chosen": -292.2763366699219,
      "logps/rejected": -240.8999786376953,
      "loss": 0.6901,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": 0.01840716041624546,
      "rewards/margins": 0.007871242240071297,
      "rewards/rejected": 0.010535916313529015,
      "step": 230
    },
    {
      "epoch": 1.98,
      "learning_rate": 1.8865030674846625e-07,
      "logits/chosen": -1.6768238544464111,
      "logits/rejected": -1.4679701328277588,
      "logps/chosen": -288.6380615234375,
      "logps/rejected": -234.598388671875,
      "loss": 0.6895,
      "rewards/accuracies": 0.5859375,
      "rewards/chosen": 0.019864676520228386,
      "rewards/margins": 0.012084105983376503,
      "rewards/rejected": 0.0077805714681744576,
      "step": 240
    },
    {
      "epoch": 2.0,
      "eval_logits/chosen": -1.4434945583343506,
      "eval_logits/rejected": -1.2044689655303955,
      "eval_logps/chosen": -309.3419494628906,
      "eval_logps/rejected": -245.20071411132812,
      "eval_loss": 0.6903356909751892,
      "eval_rewards/accuracies": 0.5317460298538208,
      "eval_rewards/chosen": 0.020180951803922653,
      "eval_rewards/margins": 0.009503869339823723,
      "eval_rewards/rejected": 0.01067708246409893,
      "eval_runtime": 241.7913,
      "eval_samples_per_second": 8.272,
      "eval_steps_per_second": 0.261,
      "step": 242
    },
    {
      "epoch": 2.07,
      "learning_rate": 1.733128834355828e-07,
      "logits/chosen": -1.6474043130874634,
      "logits/rejected": -1.4580283164978027,
      "logps/chosen": -281.4793395996094,
      "logps/rejected": -241.38180541992188,
      "loss": 0.6894,
      "rewards/accuracies": 0.578125,
      "rewards/chosen": 0.017653027549386024,
      "rewards/margins": 0.010212309658527374,
      "rewards/rejected": 0.007440718822181225,
      "step": 250
    },
    {
      "epoch": 2.15,
      "learning_rate": 1.5797546012269938e-07,
      "logits/chosen": -1.7186006307601929,
      "logits/rejected": -1.4715663194656372,
      "logps/chosen": -304.37872314453125,
      "logps/rejected": -232.53976440429688,
      "loss": 0.6888,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": 0.02060174196958542,
      "rewards/margins": 0.008710074238479137,
      "rewards/rejected": 0.011891667731106281,
      "step": 260
    },
    {
      "epoch": 2.23,
      "learning_rate": 1.4263803680981594e-07,
      "logits/chosen": -1.662448525428772,
      "logits/rejected": -1.472836971282959,
      "logps/chosen": -289.6812438964844,
      "logps/rejected": -239.82931518554688,
      "loss": 0.6892,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": 0.01753293350338936,
      "rewards/margins": 0.010122074745595455,
      "rewards/rejected": 0.007410857826471329,
      "step": 270
    },
    {
      "epoch": 2.31,
      "learning_rate": 1.273006134969325e-07,
      "logits/chosen": -1.6673990488052368,
      "logits/rejected": -1.4878466129302979,
      "logps/chosen": -294.22393798828125,
      "logps/rejected": -240.75167846679688,
      "loss": 0.6888,
      "rewards/accuracies": 0.573437511920929,
      "rewards/chosen": 0.021594451740384102,
      "rewards/margins": 0.010244326665997505,
      "rewards/rejected": 0.011350123211741447,
      "step": 280
    },
    {
      "epoch": 2.4,
      "learning_rate": 1.1196319018404908e-07,
      "logits/chosen": -1.668094277381897,
      "logits/rejected": -1.4910743236541748,
      "logps/chosen": -292.84576416015625,
      "logps/rejected": -236.984619140625,
      "loss": 0.6882,
      "rewards/accuracies": 0.567187488079071,
      "rewards/chosen": 0.023372743278741837,
      "rewards/margins": 0.012312114238739014,
      "rewards/rejected": 0.011060630902647972,
      "step": 290
    },
    {
      "epoch": 2.48,
      "learning_rate": 9.662576687116564e-08,
      "logits/chosen": -1.6651067733764648,
      "logits/rejected": -1.502229928970337,
      "logps/chosen": -275.86907958984375,
      "logps/rejected": -225.81631469726562,
      "loss": 0.6892,
      "rewards/accuracies": 0.5484374761581421,
      "rewards/chosen": 0.01964903622865677,
      "rewards/margins": 0.008799867704510689,
      "rewards/rejected": 0.010849167592823505,
      "step": 300
    },
    {
      "epoch": 2.56,
      "learning_rate": 8.12883435582822e-08,
      "logits/chosen": -1.6683803796768188,
      "logits/rejected": -1.477604866027832,
      "logps/chosen": -287.2663269042969,
      "logps/rejected": -230.453125,
      "loss": 0.6891,
      "rewards/accuracies": 0.557812511920929,
      "rewards/chosen": 0.021592509001493454,
      "rewards/margins": 0.01211326103657484,
      "rewards/rejected": 0.009479249827563763,
      "step": 310
    },
    {
      "epoch": 2.64,
      "learning_rate": 6.595092024539877e-08,
      "logits/chosen": -1.6494767665863037,
      "logits/rejected": -1.4652533531188965,
      "logps/chosen": -267.897216796875,
      "logps/rejected": -236.7665557861328,
      "loss": 0.689,
      "rewards/accuracies": 0.578125,
      "rewards/chosen": 0.02220030501484871,
      "rewards/margins": 0.01190351601690054,
      "rewards/rejected": 0.01029678713530302,
      "step": 320
    },
    {
      "epoch": 2.73,
      "learning_rate": 5.061349693251534e-08,
      "logits/chosen": -1.6814041137695312,
      "logits/rejected": -1.4899076223373413,
      "logps/chosen": -295.11761474609375,
      "logps/rejected": -236.0764923095703,
      "loss": 0.6887,
      "rewards/accuracies": 0.5796874761581421,
      "rewards/chosen": 0.02330620214343071,
      "rewards/margins": 0.014884866774082184,
      "rewards/rejected": 0.008421335369348526,
      "step": 330
    },
    {
      "epoch": 2.81,
      "learning_rate": 3.5276073619631896e-08,
      "logits/chosen": -1.6887954473495483,
      "logits/rejected": -1.4692952632904053,
      "logps/chosen": -284.07489013671875,
      "logps/rejected": -242.83462524414062,
      "loss": 0.688,
      "rewards/accuracies": 0.565625011920929,
      "rewards/chosen": 0.023104848340153694,
      "rewards/margins": 0.012322823517024517,
      "rewards/rejected": 0.010782026685774326,
      "step": 340
    },
    {
      "epoch": 2.89,
      "learning_rate": 1.9938650306748464e-08,
      "logits/chosen": -1.6704288721084595,
      "logits/rejected": -1.5081318616867065,
      "logps/chosen": -278.98748779296875,
      "logps/rejected": -246.9365234375,
      "loss": 0.6889,
      "rewards/accuracies": 0.5484374761581421,
      "rewards/chosen": 0.020504282787442207,
      "rewards/margins": 0.008767304942011833,
      "rewards/rejected": 0.011736978776752949,
      "step": 350
    },
    {
      "epoch": 2.97,
      "learning_rate": 4.601226993865031e-09,
      "logits/chosen": -1.6876208782196045,
      "logits/rejected": -1.5074845552444458,
      "logps/chosen": -290.86456298828125,
      "logps/rejected": -233.52279663085938,
      "loss": 0.6889,
      "rewards/accuracies": 0.5390625,
      "rewards/chosen": 0.021526003256440163,
      "rewards/margins": 0.009645138867199421,
      "rewards/rejected": 0.011880865320563316,
      "step": 360
    },
    {
      "epoch": 3.0,
      "eval_logits/chosen": -1.4436771869659424,
      "eval_logits/rejected": -1.2044098377227783,
      "eval_logps/chosen": -309.3194580078125,
      "eval_logps/rejected": -245.162109375,
      "eval_loss": 0.6891217827796936,
      "eval_rewards/accuracies": 0.567460298538208,
      "eval_rewards/chosen": 0.022433217614889145,
      "eval_rewards/margins": 0.007897691801190376,
      "eval_rewards/rejected": 0.014535525813698769,
      "eval_runtime": 243.9232,
      "eval_samples_per_second": 8.199,
      "eval_steps_per_second": 0.258,
      "step": 363
    },
    {
      "epoch": 3.0,
      "step": 363,
      "total_flos": 0.0,
      "train_loss": 0.6906999242864036,
      "train_runtime": 40993.6934,
      "train_samples_per_second": 4.535,
      "train_steps_per_second": 0.009
    }
  ],
  "logging_steps": 10,
  "max_steps": 363,
  "num_train_epochs": 3,
  "save_steps": 500,
  "total_flos": 0.0,
  "trial_name": null,
  "trial_params": null
}