{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.0,
  "eval_steps": 3000,
  "global_step": 784,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.1276527844263603,
      "grad_norm": 17.86325555610818,
      "hdpo/chosen_dist_score": 0.052723273634910583,
      "hdpo/dist_gap": 0.36752817034721375,
      "hdpo/dist_reg": 0.40332913398742676,
      "hdpo/rejected_dist_score": 0.4202514588832855,
      "hdpo/sft_anchor": 1.811698317527771,
      "hdpo/weight_mean": 1.3920727968215942,
      "learning_rate": 4.990077890125363e-07,
      "logits/chosen": -2.383833169937134,
      "logits/rejected": -2.3748526573181152,
      "logps/chosen": -391.4765625,
      "logps/rejected": -37.16926574707031,
      "loss": 0.7936,
      "rewards/accuracies": 0.642031192779541,
      "rewards/chosen": -0.0020667118951678276,
      "rewards/margins": 0.07224086672067642,
      "rewards/rejected": -0.07430758327245712,
      "step": 100
    },
    {
      "epoch": 0.2553055688527206,
      "grad_norm": 5.412824150616398,
      "hdpo/chosen_dist_score": 0.05050860717892647,
      "hdpo/dist_gap": 0.37351280450820923,
      "hdpo/dist_reg": 0.40590131282806396,
      "hdpo/rejected_dist_score": 0.4240213632583618,
      "hdpo/sft_anchor": 1.8305994272232056,
      "hdpo/weight_mean": 1.397886872291565,
      "learning_rate": 4.6510039481503485e-07,
      "logits/chosen": -2.3928980827331543,
      "logits/rejected": -2.2878432273864746,
      "logps/chosen": -433.405517578125,
      "logps/rejected": -44.4639892578125,
      "loss": 0.6135,
      "rewards/accuracies": 0.8571874499320984,
      "rewards/chosen": -0.08834817260503769,
      "rewards/margins": 0.7170474529266357,
      "rewards/rejected": -0.8053956031799316,
      "step": 200
    },
    {
      "epoch": 0.3829583532790809,
      "grad_norm": 5.52521879468998,
      "hdpo/chosen_dist_score": 0.052139006555080414,
      "hdpo/dist_gap": 0.3657912611961365,
      "hdpo/dist_reg": 0.39733144640922546,
      "hdpo/rejected_dist_score": 0.4179302752017975,
      "hdpo/sft_anchor": 1.8453103303909302,
      "hdpo/weight_mean": 1.3889758586883545,
      "learning_rate": 3.891818892301304e-07,
      "logits/chosen": -2.378171920776367,
      "logits/rejected": -2.1879160404205322,
      "logps/chosen": -426.026611328125,
      "logps/rejected": -62.396766662597656,
      "loss": 0.4404,
      "rewards/accuracies": 0.89515620470047,
      "rewards/chosen": -0.26047271490097046,
      "rewards/margins": 2.3145828247070312,
      "rewards/rejected": -2.5750558376312256,
      "step": 300
    },
    {
      "epoch": 0.5106111377054412,
      "grad_norm": 3.0228348867877908,
      "hdpo/chosen_dist_score": 0.05130492150783539,
      "hdpo/dist_gap": 0.3722842037677765,
      "hdpo/dist_reg": 0.3967655599117279,
      "hdpo/rejected_dist_score": 0.4235890805721283,
      "hdpo/sft_anchor": 1.8966513872146606,
      "hdpo/weight_mean": 1.3966076374053955,
      "learning_rate": 2.8607986379820664e-07,
      "logits/chosen": -2.3439035415649414,
      "logits/rejected": -2.13508677482605,
      "logps/chosen": -410.8368835449219,
      "logps/rejected": -70.56217956542969,
      "loss": 0.3995,
      "rewards/accuracies": 0.9039062857627869,
      "rewards/chosen": -0.38984429836273193,
      "rewards/margins": 3.0737392902374268,
      "rewards/rejected": -3.463583469390869,
      "step": 400
    },
    {
      "epoch": 0.6382639221318015,
      "grad_norm": 8.692929410688901,
      "hdpo/chosen_dist_score": 0.04895579069852829,
      "hdpo/dist_gap": 0.3748772144317627,
      "hdpo/dist_reg": 0.3946782648563385,
      "hdpo/rejected_dist_score": 0.4238330125808716,
      "hdpo/sft_anchor": 1.9457844495773315,
      "hdpo/weight_mean": 1.3984801769256592,
      "learning_rate": 1.7593110477859152e-07,
      "logits/chosen": -2.3340072631835938,
      "logits/rejected": -2.114999771118164,
      "logps/chosen": -427.2086181640625,
      "logps/rejected": -75.53610229492188,
      "loss": 0.3814,
      "rewards/accuracies": 0.9064061641693115,
      "rewards/chosen": -0.517266571521759,
      "rewards/margins": 3.3930423259735107,
      "rewards/rejected": -3.910308361053467,
      "step": 500
    },
    {
      "epoch": 0.7659167065581618,
      "grad_norm": 5.614731572432982,
      "hdpo/chosen_dist_score": 0.053545329719781876,
      "hdpo/dist_gap": 0.37800246477127075,
      "hdpo/dist_reg": 0.40364930033683777,
      "hdpo/rejected_dist_score": 0.4315478205680847,
      "hdpo/sft_anchor": 1.9814330339431763,
      "hdpo/weight_mean": 1.4014147520065308,
      "learning_rate": 8.024869116091879e-08,
      "logits/chosen": -2.3227384090423584,
      "logits/rejected": -2.10311222076416,
      "logps/chosen": -439.9888000488281,
      "logps/rejected": -79.7904052734375,
      "loss": 0.3679,
      "rewards/accuracies": 0.9089061617851257,
      "rewards/chosen": -0.565869152545929,
      "rewards/margins": 3.692293882369995,
      "rewards/rejected": -4.258162975311279,
      "step": 600
    },
    {
      "epoch": 0.8935694909845221,
      "grad_norm": 4.116293928191272,
      "hdpo/chosen_dist_score": 0.0522027388215065,
      "hdpo/dist_gap": 0.3726814389228821,
      "hdpo/dist_reg": 0.3926178812980652,
      "hdpo/rejected_dist_score": 0.424884170293808,
      "hdpo/sft_anchor": 1.9475560188293457,
      "hdpo/weight_mean": 1.3972278833389282,
      "learning_rate": 1.7720289882128092e-08,
      "logits/chosen": -2.333853244781494,
      "logits/rejected": -2.124305486679077,
      "logps/chosen": -420.472412109375,
      "logps/rejected": -78.66890716552734,
      "loss": 0.3744,
      "rewards/accuracies": 0.9065625667572021,
      "rewards/chosen": -0.5707193613052368,
      "rewards/margins": 3.6550707817077637,
      "rewards/rejected": -4.225790023803711,
      "step": 700
    },
    {
      "epoch": 1.0,
      "step": 784,
      "total_flos": 30671083929600.0,
      "train_loss": 0.4695360952494096,
      "train_runtime": 5727.0573,
      "train_samples_per_second": 8.754,
      "train_steps_per_second": 0.137
    }
  ],
  "logging_steps": 100,
  "max_steps": 784,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 3000,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 30671083929600.0,
  "train_batch_size": 1,
  "trial_name": null,
  "trial_params": null
}