{
  "global_step": 152,
  "epoch": 2,
  "completed_epoch": 2,
  "args": {
    "model_id": "mistralai/Mixtral-8x7B-v0.1",
    "model_revision": null,
    "data_file": "nq_annotated_moe_balanced.jsonl",
    "output_dir": "./mixtral_moe_supervised",
    "max_seq_length": 64,
    "max_target_length": 32,
    "learning_rate": 1e-05,
    "epochs": 2,
    "per_device_batch_size": 1,
    "gradient_accumulation_steps": 8,
    "warmup_steps": 200,
    "max_grad_norm": 1.0,
    "seed": 42,
    "logging_steps": 10,
    "save_steps": 500,
    "max_samples": null,
    "convert_to_moe": false,
    "resume_from_checkpoint": null,
    "routing_loss_weight": 0.1,
    "disable_supervised_routing": false,
    "label_embedding_temperature": 1.0
  }
}