{
  "stage": "dpo",
  "template": "gemma",
  "cutoff_len": 2048,
  "dataset": "dpo_preference_dataset",
  "per_device_train_batch_size": 1,
  "gradient_accumulation_steps": 64,
  "learning_rate": 5e-06,
  "num_train_epochs": 1.0,
  "lr_scheduler_type": "cosine",
  "warmup_ratio": 0.1,
  "bf16": true,
  "pref_beta": 0.1,
  "pref_loss": "sigmoid",
  "weight_decay": 0.0,
  "max_grad_norm": 1.0,
  "seed": 42,
  "data_seed": null,
  "max_steps": -1,
  "optim": "adamw_torch",
  "lora_rank": 8,
  "lora_alpha": 16,
  "lora_dropout": 0.0,
  "target_modules": [
    "down_proj",
    "q_proj",
    "v_proj",
    "gate_proj",
    "o_proj",
    "k_proj",
    "up_proj"
  ],
  "sft_initialization": "../../../phase2-sft-alignment/sft-model/sft-llama-factory-training/saves/gemma-2-9b-it/lora/sft_3ep"
}