Init commit

README.md

@@ -1,3 +1,60 @@
----
-license: apache-2.0
----
+---
+library_name: peft
+license: other
+base_model: unsloth/mistral-7b-instruct-v0.3-bnb-4bit
+tags:
+- llama-factory
+- lora
+- generated_from_trainer
+model-index:
+- name: mistral-v0.4-7B-q4-abstrct-normalized-text
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# mistral-v0.4-7B-q4-abstrct-normalized-text
+
+This model is a fine-tuned version of [unsloth/mistral-7b-instruct-v0.3-bnb-4bit](https://huggingface.co/unsloth/mistral-7b-instruct-v0.3-bnb-4bit) on the abstrct dataset.
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 5e-05
+- train_batch_size: 2
+- eval_batch_size: 8
+- seed: 42
+- gradient_accumulation_steps: 4
+- total_train_batch_size: 8
+- optimizer: Use OptimizerNames.ADAMW_TORCH with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
+- lr_scheduler_type: cosine
+- lr_scheduler_warmup_ratio: 0.1
+- num_epochs: 5
+- mixed_precision_training: Native AMP
+
+### Training results
+
+
+
+### Framework versions
+
+- PEFT 0.12.0
+- Transformers 4.49.0
+- Pytorch 2.6.0+cu124
+- Datasets 3.2.0
+- Tokenizers 0.21.0

adapter_config.json

@@ -0,0 +1,34 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "unsloth/mistral-7b-instruct-v0.3-bnb-4bit",
+  "bias": "none",
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 16,
+  "lora_dropout": 0.0,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 8,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "gate_proj",
+    "down_proj",
+    "o_proj",
+    "q_proj",
+    "up_proj",
+    "v_proj",
+    "k_proj"
+  ],
+  "task_type": "CAUSAL_LM",
+  "use_dora": false,
+  "use_rslora": false
+}

adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4579b3e30749687e2cbfeb372900c7c2c4515f3332eb07a4da8ade1f24741286
+size 83945296

all_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 4.982857142857143,
+    "total_flos": 9.788115420905472e+16,
+    "train_loss": 0.006386435382761234,
+    "train_runtime": 2535.9671,
+    "train_samples_per_second": 0.69,
+    "train_steps_per_second": 0.085
+}

special_tokens_map.json

@@ -0,0 +1,30 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "[control_768]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:37f00374dea48658ee8f5d0f21895b9bc55cb0103939607c8185bfd1c6ca1f89
+size 587404

train_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 4.982857142857143,
+    "total_flos": 9.788115420905472e+16,
+    "train_loss": 0.006386435382761234,
+    "train_runtime": 2535.9671,
+    "train_samples_per_second": 0.69,
+    "train_steps_per_second": 0.085
+}

trainer_log.jsonl

@@ -0,0 +1,22 @@
+{"current_steps": 10, "total_steps": 215, "loss": 0.0505, "lr": 2.272727272727273e-05, "epoch": 0.22857142857142856, "percentage": 4.65, "elapsed_time": "0:01:51", "remaining_time": "0:38:13"}
+{"current_steps": 20, "total_steps": 215, "loss": 0.0136, "lr": 4.545454545454546e-05, "epoch": 0.45714285714285713, "percentage": 9.3, "elapsed_time": "0:03:49", "remaining_time": "0:37:20"}
+{"current_steps": 30, "total_steps": 215, "loss": 0.0118, "lr": 4.9788329024834144e-05, "epoch": 0.6857142857142857, "percentage": 13.95, "elapsed_time": "0:05:45", "remaining_time": "0:35:32"}
+{"current_steps": 40, "total_steps": 215, "loss": 0.0094, "lr": 4.893455513052003e-05, "epoch": 0.9142857142857143, "percentage": 18.6, "elapsed_time": "0:07:49", "remaining_time": "0:34:14"}
+{"current_steps": 50, "total_steps": 215, "loss": 0.0071, "lr": 4.7448003278586086e-05, "epoch": 1.16, "percentage": 23.26, "elapsed_time": "0:09:53", "remaining_time": "0:32:38"}
+{"current_steps": 60, "total_steps": 215, "loss": 0.0061, "lr": 4.536797469020116e-05, "epoch": 1.3885714285714286, "percentage": 27.91, "elapsed_time": "0:11:49", "remaining_time": "0:30:31"}
+{"current_steps": 70, "total_steps": 215, "loss": 0.0064, "lr": 4.274946082999753e-05, "epoch": 1.617142857142857, "percentage": 32.56, "elapsed_time": "0:13:43", "remaining_time": "0:28:26"}
+{"current_steps": 80, "total_steps": 215, "loss": 0.0065, "lr": 3.9661689550484805e-05, "epoch": 1.8457142857142856, "percentage": 37.21, "elapsed_time": "0:15:37", "remaining_time": "0:26:22"}
+{"current_steps": 90, "total_steps": 215, "loss": 0.0061, "lr": 3.6186294857107936e-05, "epoch": 2.0914285714285716, "percentage": 41.86, "elapsed_time": "0:17:44", "remaining_time": "0:24:38"}
+{"current_steps": 100, "total_steps": 215, "loss": 0.0039, "lr": 3.241515868140921e-05, "epoch": 2.32, "percentage": 46.51, "elapsed_time": "0:19:44", "remaining_time": "0:22:42"}
+{"current_steps": 110, "total_steps": 215, "loss": 0.0032, "lr": 2.844798172115185e-05, "epoch": 2.5485714285714285, "percentage": 51.16, "elapsed_time": "0:21:41", "remaining_time": "0:20:42"}
+{"current_steps": 120, "total_steps": 215, "loss": 0.0033, "lr": 2.438964756914712e-05, "epoch": 2.777142857142857, "percentage": 55.81, "elapsed_time": "0:23:35", "remaining_time": "0:18:40"}
+{"current_steps": 130, "total_steps": 215, "loss": 0.004, "lr": 2.034744981752741e-05, "epoch": 3.0228571428571427, "percentage": 60.47, "elapsed_time": "0:25:36", "remaining_time": "0:16:44"}
+{"current_steps": 140, "total_steps": 215, "loss": 0.0015, "lr": 1.642825544684101e-05, "epoch": 3.2514285714285713, "percentage": 65.12, "elapsed_time": "0:27:27", "remaining_time": "0:14:42"}
+{"current_steps": 150, "total_steps": 215, "loss": 0.0009, "lr": 1.2735679493836009e-05, "epoch": 3.48, "percentage": 69.77, "elapsed_time": "0:29:26", "remaining_time": "0:12:45"}
+{"current_steps": 160, "total_steps": 215, "loss": 0.001, "lr": 9.367345693616624e-06, "epoch": 3.7085714285714286, "percentage": 74.42, "elapsed_time": "0:31:23", "remaining_time": "0:10:47"}
+{"current_steps": 170, "total_steps": 215, "loss": 0.0008, "lr": 6.412305518878342e-06, "epoch": 3.9371428571428573, "percentage": 79.07, "elapsed_time": "0:33:22", "remaining_time": "0:08:49"}
+{"current_steps": 180, "total_steps": 215, "loss": 0.0005, "lr": 3.948683851251878e-06, "epoch": 4.182857142857143, "percentage": 83.72, "elapsed_time": "0:35:29", "remaining_time": "0:06:54"}
+{"current_steps": 190, "total_steps": 215, "loss": 0.0002, "lr": 2.041613528121822e-06, "epoch": 4.411428571428571, "percentage": 88.37, "elapsed_time": "0:37:31", "remaining_time": "0:04:56"}
+{"current_steps": 200, "total_steps": 215, "loss": 0.0002, "lr": 7.415133710424793e-07, "epoch": 4.64, "percentage": 93.02, "elapsed_time": "0:39:21", "remaining_time": "0:02:57"}
+{"current_steps": 210, "total_steps": 215, "loss": 0.0002, "lr": 8.275522096146405e-08, "epoch": 4.868571428571428, "percentage": 97.67, "elapsed_time": "0:41:16", "remaining_time": "0:00:58"}
+{"current_steps": 215, "total_steps": 215, "epoch": 4.982857142857143, "percentage": 100.0, "elapsed_time": "0:42:14", "remaining_time": "0:00:00"}

trainer_state.json

@@ -0,0 +1,189 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 4.982857142857143,
+  "eval_steps": 500,
+  "global_step": 215,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.22857142857142856,
+      "grad_norm": 0.14607588946819305,
+      "learning_rate": 2.272727272727273e-05,
+      "loss": 0.0505,
+      "step": 10
+    },
+    {
+      "epoch": 0.45714285714285713,
+      "grad_norm": 0.1369447410106659,
+      "learning_rate": 4.545454545454546e-05,
+      "loss": 0.0136,
+      "step": 20
+    },
+    {
+      "epoch": 0.6857142857142857,
+      "grad_norm": 0.11242067813873291,
+      "learning_rate": 4.9788329024834144e-05,
+      "loss": 0.0118,
+      "step": 30
+    },
+    {
+      "epoch": 0.9142857142857143,
+      "grad_norm": 0.05123613774776459,
+      "learning_rate": 4.893455513052003e-05,
+      "loss": 0.0094,
+      "step": 40
+    },
+    {
+      "epoch": 1.16,
+      "grad_norm": 0.06896992027759552,
+      "learning_rate": 4.7448003278586086e-05,
+      "loss": 0.0071,
+      "step": 50
+    },
+    {
+      "epoch": 1.3885714285714286,
+      "grad_norm": 0.46913379430770874,
+      "learning_rate": 4.536797469020116e-05,
+      "loss": 0.0061,
+      "step": 60
+    },
+    {
+      "epoch": 1.617142857142857,
+      "grad_norm": 0.10468105971813202,
+      "learning_rate": 4.274946082999753e-05,
+      "loss": 0.0064,
+      "step": 70
+    },
+    {
+      "epoch": 1.8457142857142856,
+      "grad_norm": 0.06397916376590729,
+      "learning_rate": 3.9661689550484805e-05,
+      "loss": 0.0065,
+      "step": 80
+    },
+    {
+      "epoch": 2.0914285714285716,
+      "grad_norm": 0.0648517832159996,
+      "learning_rate": 3.6186294857107936e-05,
+      "loss": 0.0061,
+      "step": 90
+    },
+    {
+      "epoch": 2.32,
+      "grad_norm": 0.07107104361057281,
+      "learning_rate": 3.241515868140921e-05,
+      "loss": 0.0039,
+      "step": 100
+    },
+    {
+      "epoch": 2.5485714285714285,
+      "grad_norm": 0.21339790523052216,
+      "learning_rate": 2.844798172115185e-05,
+      "loss": 0.0032,
+      "step": 110
+    },
+    {
+      "epoch": 2.777142857142857,
+      "grad_norm": 0.05857868865132332,
+      "learning_rate": 2.438964756914712e-05,
+      "loss": 0.0033,
+      "step": 120
+    },
+    {
+      "epoch": 3.0228571428571427,
+      "grad_norm": 0.1693907380104065,
+      "learning_rate": 2.034744981752741e-05,
+      "loss": 0.004,
+      "step": 130
+    },
+    {
+      "epoch": 3.2514285714285713,
+      "grad_norm": 0.020911961793899536,
+      "learning_rate": 1.642825544684101e-05,
+      "loss": 0.0015,
+      "step": 140
+    },
+    {
+      "epoch": 3.48,
+      "grad_norm": 0.04083988070487976,
+      "learning_rate": 1.2735679493836009e-05,
+      "loss": 0.0009,
+      "step": 150
+    },
+    {
+      "epoch": 3.7085714285714286,
+      "grad_norm": 0.026034992188215256,
+      "learning_rate": 9.367345693616624e-06,
+      "loss": 0.001,
+      "step": 160
+    },
+    {
+      "epoch": 3.9371428571428573,
+      "grad_norm": 0.07884740084409714,
+      "learning_rate": 6.412305518878342e-06,
+      "loss": 0.0008,
+      "step": 170
+    },
+    {
+      "epoch": 4.182857142857143,
+      "grad_norm": 0.015410957857966423,
+      "learning_rate": 3.948683851251878e-06,
+      "loss": 0.0005,
+      "step": 180
+    },
+    {
+      "epoch": 4.411428571428571,
+      "grad_norm": 0.01501483004540205,
+      "learning_rate": 2.041613528121822e-06,
+      "loss": 0.0002,
+      "step": 190
+    },
+    {
+      "epoch": 4.64,
+      "grad_norm": 0.13119111955165863,
+      "learning_rate": 7.415133710424793e-07,
+      "loss": 0.0002,
+      "step": 200
+    },
+    {
+      "epoch": 4.868571428571428,
+      "grad_norm": 0.01860298588871956,
+      "learning_rate": 8.275522096146405e-08,
+      "loss": 0.0002,
+      "step": 210
+    },
+    {
+      "epoch": 4.982857142857143,
+      "step": 215,
+      "total_flos": 9.788115420905472e+16,
+      "train_loss": 0.006386435382761234,
+      "train_runtime": 2535.9671,
+      "train_samples_per_second": 0.69,
+      "train_steps_per_second": 0.085
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 215,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 5,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 9.788115420905472e+16,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c14eb4eea1a2e6cce3e4ee6e96b1c82594c37ee3de263d1f7fc5d8da89a9a75d
+size 5752