{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.0,
  "eval_steps": 3000,
  "global_step": 767,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.1303780964797914,
      "grad_norm": 6.526818967525457,
      "hdpo/chosen_dist_score": 0.03802235797047615,
      "hdpo/dist_gap": 0.14238910377025604,
      "hdpo/dist_reg": 0.17127636075019836,
      "hdpo/rejected_dist_score": 0.1804114431142807,
      "hdpo/sft_anchor": 1.6595879793167114,
      "hdpo/weight_mean": 1.1672801971435547,
      "learning_rate": 4.98746876748424e-07,
      "logits/chosen": -1.5361733436584473,
      "logits/rejected": -1.5487430095672607,
      "logps/chosen": -464.93560791015625,
      "logps/rejected": -146.35202026367188,
      "loss": 0.6501,
      "rewards/accuracies": 0.7284375429153442,
      "rewards/chosen": -0.1674138307571411,
      "rewards/margins": 0.7256561517715454,
      "rewards/rejected": -0.8930699229240417,
      "step": 100
    },
    {
      "epoch": 0.2607561929595828,
      "grad_norm": 4.290574307203319,
      "hdpo/chosen_dist_score": 0.0355314165353775,
      "hdpo/dist_gap": 0.1401846706867218,
      "hdpo/dist_reg": 0.16281671822071075,
      "hdpo/rejected_dist_score": 0.1757161021232605,
      "hdpo/sft_anchor": 1.7142517566680908,
      "hdpo/weight_mean": 1.1634154319763184,
      "learning_rate": 4.6241317004989123e-07,
      "logits/chosen": -1.488063931465149,
      "logits/rejected": -1.5119249820709229,
      "logps/chosen": -474.4151306152344,
      "logps/rejected": -176.1715087890625,
      "loss": 0.4094,
      "rewards/accuracies": 0.9156249761581421,
      "rewards/chosen": -1.070426106452942,
      "rewards/margins": 3.0291342735290527,
      "rewards/rejected": -4.099560260772705,
      "step": 200
    },
    {
      "epoch": 0.39113428943937417,
      "grad_norm": 5.520678892024318,
      "hdpo/chosen_dist_score": 0.03530166298151016,
      "hdpo/dist_gap": 0.13994967937469482,
      "hdpo/dist_reg": 0.15933261811733246,
      "hdpo/rejected_dist_score": 0.17525134980678558,
      "hdpo/sft_anchor": 1.785836100578308,
      "hdpo/weight_mean": 1.1639575958251953,
      "learning_rate": 3.828014292634508e-07,
      "logits/chosen": -1.4042692184448242,
      "logits/rejected": -1.3868207931518555,
      "logps/chosen": -468.54058837890625,
      "logps/rejected": -187.590576171875,
      "loss": 0.3527,
      "rewards/accuracies": 0.9162499904632568,
      "rewards/chosen": -1.2646902799606323,
      "rewards/margins": 4.016061305999756,
      "rewards/rejected": -5.280751705169678,
      "step": 300
    },
    {
      "epoch": 0.5215123859191656,
      "grad_norm": 5.503862893151768,
      "hdpo/chosen_dist_score": 0.03721608966588974,
      "hdpo/dist_gap": 0.14201799035072327,
      "hdpo/dist_reg": 0.16401724517345428,
      "hdpo/rejected_dist_score": 0.1792340725660324,
      "hdpo/sft_anchor": 1.8684366941452026,
      "hdpo/weight_mean": 1.1662732362747192,
      "learning_rate": 2.761321158169134e-07,
      "logits/chosen": -1.3286969661712646,
      "logits/rejected": -1.274000644683838,
      "logps/chosen": -466.1091613769531,
      "logps/rejected": -195.7136688232422,
      "loss": 0.3244,
      "rewards/accuracies": 0.9174999594688416,
      "rewards/chosen": -1.394167423248291,
      "rewards/margins": 4.548469543457031,
      "rewards/rejected": -5.942636489868164,
      "step": 400
    },
    {
      "epoch": 0.651890482398957,
      "grad_norm": 5.343319880907456,
      "hdpo/chosen_dist_score": 0.036759164184331894,
      "hdpo/dist_gap": 0.14323043823242188,
      "hdpo/dist_reg": 0.1613893359899521,
      "hdpo/rejected_dist_score": 0.17998960614204407,
      "hdpo/sft_anchor": 1.8926106691360474,
      "hdpo/weight_mean": 1.1667797565460205,
      "learning_rate": 1.6413852516643468e-07,
      "logits/chosen": -1.2850545644760132,
      "logits/rejected": -1.2305511236190796,
      "logps/chosen": -468.2303466796875,
      "logps/rejected": -194.57830810546875,
      "loss": 0.3147,
      "rewards/accuracies": 0.9262499809265137,
      "rewards/chosen": -1.3962421417236328,
      "rewards/margins": 4.756744861602783,
      "rewards/rejected": -6.152987003326416,
      "step": 500
    },
    {
      "epoch": 0.7822685788787483,
      "grad_norm": 8.486044568898233,
      "hdpo/chosen_dist_score": 0.03784089908003807,
      "hdpo/dist_gap": 0.14062130451202393,
      "hdpo/dist_reg": 0.15948456525802612,
      "hdpo/rejected_dist_score": 0.1784621775150299,
      "hdpo/sft_anchor": 1.9059925079345703,
      "hdpo/weight_mean": 1.1648303270339966,
      "learning_rate": 6.963874544026108e-08,
      "logits/chosen": -1.2695670127868652,
      "logits/rejected": -1.2077159881591797,
      "logps/chosen": -469.040283203125,
      "logps/rejected": -195.4508056640625,
      "loss": 0.3136,
      "rewards/accuracies": 0.9237499237060547,
      "rewards/chosen": -1.4379326105117798,
      "rewards/margins": 4.822230815887451,
      "rewards/rejected": -6.260162830352783,
      "step": 600
    },
    {
      "epoch": 0.9126466753585397,
      "grad_norm": 7.745657226464002,
      "hdpo/chosen_dist_score": 0.03659777715802193,
      "hdpo/dist_gap": 0.14138872921466827,
      "hdpo/dist_reg": 0.15903262794017792,
      "hdpo/rejected_dist_score": 0.1779865026473999,
      "hdpo/sft_anchor": 1.8816936016082764,
      "hdpo/weight_mean": 1.1649353504180908,
      "learning_rate": 1.1886595445053743e-08,
      "logits/chosen": -1.26893150806427,
      "logits/rejected": -1.206188440322876,
      "logps/chosen": -460.7378845214844,
      "logps/rejected": -202.58522033691406,
      "loss": 0.305,
      "rewards/accuracies": 0.9279686808586121,
      "rewards/chosen": -1.4286925792694092,
      "rewards/margins": 5.039687633514404,
      "rewards/rejected": -6.468379974365234,
      "step": 700
    },
    {
      "epoch": 1.0,
      "step": 767,
      "total_flos": 43330810675200.0,
      "train_loss": 0.37579502369309964,
      "train_runtime": 11033.1896,
      "train_samples_per_second": 4.449,
      "train_steps_per_second": 0.07
    }
  ],
  "logging_steps": 100,
  "max_steps": 767,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 3000,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 43330810675200.0,
  "train_batch_size": 1,
  "trial_name": null,
  "trial_params": null
}