Training in progress, step 100

README.md

@@ -1,14 +1,14 @@
 ---
-base_model: meta-llama/Meta-Llama-3-8B
-datasets:
-- yihanwang617/ultrachat_200k_processed_indicator_0.6_4k
-library_name: peft
 license: llama3
+library_name: peft
 tags:
 - alignment-handbook
 - trl
 - sft
 - generated_from_trainer
+base_model: meta-llama/Meta-Llama-3-8B
+datasets:
+- yihanwang617/ultrachat_200k_processed_indicator_0.6_4k
 model-index:
 - name: llama-3-qlora-ultrachat-200k-processed-indicator-0.6
   results: []
@@ -21,7 +21,7 @@ should probably proofread and complete it, then remove this comment. -->
 
 This model is a fine-tuned version of [meta-llama/Meta-Llama-3-8B](https://huggingface.co/meta-llama/Meta-Llama-3-8B) on the yihanwang617/ultrachat_200k_processed_indicator_0.6_4k dataset.
 It achieves the following results on the evaluation set:
-- Loss: 1.0200
+- Loss: 1.0235
 
 ## Model description
 
@@ -58,13 +58,13 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch  | Step | Validation Loss |
 |:-------------:|:------:|:----:|:---------------:|
-| 1.0052        | 0.9997 | 3247 | 1.0200          |
+| 1.008         | 0.9997 | 3247 | 1.0235          |
 
 
 ### Framework versions
 
-- PEFT 0.11.1
+- PEFT 0.12.0
 - Transformers 4.40.1
-- Pytorch 2.3.0+cu121
-- Datasets 2.19.0
+- Pytorch 2.4.0+cu121
+- Datasets 2.20.0
 - Tokenizers 0.19.1

adapter_config.json

@@ -0,0 +1,34 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "meta-llama/Meta-Llama-3-8B",
+  "bias": "none",
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 16,
+  "lora_dropout": 0.05,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 16,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "o_proj",
+    "q_proj",
+    "down_proj",
+    "gate_proj",
+    "v_proj",
+    "up_proj",
+    "k_proj"
+  ],
+  "task_type": "CAUSAL_LM",
+  "use_dora": false,
+  "use_rslora": false
+}

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e7f28f2416c31005af35616969facbde8fa70cda863715e5098dc5bdef731861
+oid sha256:eab837c30bcfdaeab75a21442e504acd3f04d5fe2ab997c925e3afd845f9ea77
 size 2185327392

all_results.json

@@ -1,14 +1,14 @@
 {
     "epoch": 0.9997305930800908,
-    "eval_loss": 1.0199960470199585,
-    "eval_runtime": 3788.0149,
+    "eval_loss": 1.0235061645507812,
+    "eval_runtime": 3765.5428,
     "eval_samples": 23109,
-    "eval_samples_per_second": 6.101,
-    "eval_steps_per_second": 0.381,
-    "total_flos": 3.805178921536717e+16,
-    "train_loss": 1.037105345344191,
-    "train_runtime": 142567.8806,
+    "eval_samples_per_second": 6.137,
+    "eval_steps_per_second": 0.384,
+    "total_flos": 4.15615026158633e+16,
+    "train_loss": 1.0426253444200522,
+    "train_runtime": 139956.5138,
     "train_samples": 207864,
-    "train_samples_per_second": 1.458,
+    "train_samples_per_second": 1.485,
     "train_steps_per_second": 0.023
 }

eval_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 0.9997305930800908,
-    "eval_loss": 1.0199960470199585,
-    "eval_runtime": 3788.0149,
+    "eval_loss": 1.0235061645507812,
+    "eval_runtime": 3765.5428,
     "eval_samples": 23109,
-    "eval_samples_per_second": 6.101,
-    "eval_steps_per_second": 0.381
+    "eval_samples_per_second": 6.137,
+    "eval_steps_per_second": 0.384
 }

model-00001-of-00007.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f98cbacb75eb3c671c31608b1027907407197b5cd8f01c4fd95909cf4a777105
+oid sha256:3f15eaf6637faaa88a6168f6cb9d62cc58e48c1b5ba09a3368db06ad6762c23e
 size 4886498936

model-00002-of-00007.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:00513cc0bd6a6cc0ec587dfc044a47e57380beb40fe9d9f678da2fc01e7e06a5
+oid sha256:04e04214b82d5639c5bec238981dd997057c1fb8ee458f5b19996986dfc84460
 size 4832007448

model-00003-of-00007.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e0da37dc7df1ad2a61b2cea6de760ac9974db61336e0f0950a4a9f722cc76fea
+oid sha256:2dc61f0d8a87e60ca30bef51100410ecce4aae19b5fe2cc9ed9b94ecaebc8d8b
 size 4999813112

model-00004-of-00007.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b0aec8e961ca76760640e1fe29f6d70257ddc1a16df131113d379fa8a64c2c03
+oid sha256:b9e3c35699ca5f0775e9f0e3bc22e556dc2f65083c5aa222311b4076d9905dce
 size 4999813128

model-00005-of-00007.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2cdcfe5f0da94ca17f2f6d4dee65fdd80459d9bafce0d9f8ba51c780ded32cff
+oid sha256:247b3a91434eb9f62900f5c379b26ced85c351e7cf338a2f663bf3ffc1120bf5
 size 4832007496

model-00006-of-00007.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b6646bacfca3158d9391d0974b13edee79dd1668f3b95558335ca030866503bb
+oid sha256:3067d5f17932b045dc53539836c426907cd17d995d719175d2d148a2b3c1f662
 size 4999813120

model-00007-of-00007.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e63d4ce57c927caae96544d8d383c466948b2b7876493ae9309f0bc01a86c78b
+oid sha256:36dc13007fc9125d823d2d4e39c22098decf058a93838e4368702f52150c1bc9
 size 2571190952

runs/Aug21_00-57-59_illidan.cs.ucla.edu/events.out.tfevents.1724228253.illidan.cs.ucla.edu.1343562.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e01ea537d0aae403f681e6472d92012d2d197c24518ee52f33177a03e25f5628
-size 142711
+oid sha256:272dbb33d66836e6f5097e9274e6603983c4e24035c1138b17070427a31f252d
+size 72667

runs/Aug23_02-12-13_nova.cs.ucla.edu/events.out.tfevents.1724405483.nova.cs.ucla.edu.2043016.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:33f1f1da765008329067314177d6d2e638f9eed477fdb5b905046c29ea84d83a
+size 5657

runs/Aug23_02-51-25_nova.cs.ucla.edu/events.out.tfevents.1724407839.nova.cs.ucla.edu.2053170.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4598e5d1bb126bdfa97260309b099ff2e5620712f9ec80c5aea3356cdf81aa54
+size 9383

tokenizer.json

@@ -1,6 +1,11 @@
 {
   "version": "1.0",
-  "truncation": null,
+  "truncation": {
+    "direction": "Right",
+    "max_length": 4096,
+    "strategy": "LongestFirst",
+    "stride": 0
+  },
   "padding": null,
   "added_tokens": [
     {

tokenizer_config.json

@@ -2066,14 +2066,14 @@
     }
   },
   "bos_token": "<|begin_of_text|>",
+  "chat_template": "{% for message in messages %}\n{% if message['role'] == 'user' %}\n{{ '<|user|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'system' %}\n{{ '<|system|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'assistant' %}\n{{ '<|assistant|>\n'  + message['content'] + eos_token }}\n{% endif %}\n{% if loop.last and add_generation_prompt %}\n{{ '<|assistant|>' }}\n{% endif %}\n{% endfor %}",
   "clean_up_tokenization_spaces": true,
   "eos_token": "<|end_of_text|>",
-  "chat_template": "{% for message in messages %}\n{% if message['role'] == 'user' %}\n{{ '<|user|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'system' %}\n{{ '<|system|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'assistant' %}\n{{ '<|assistant|>\n'  + message['content'] + eos_token }}\n{% endif %}\n{% if loop.last and add_generation_prompt %}\n{{ '<|assistant|>' }}\n{% endif %}\n{% endfor %}",
   "model_input_names": [
     "input_ids",
     "attention_mask"
   ],
-  "model_max_length": 1000000000000000019884624838656,
+  "model_max_length": 2048,
   "pad_token": "[PAD]",
   "tokenizer_class": "PreTrainedTokenizerFast"
-}
+}

train_results.json

@@ -1,9 +1,9 @@
 {
     "epoch": 0.9997305930800908,
-    "total_flos": 3.805178921536717e+16,
-    "train_loss": 1.037105345344191,
-    "train_runtime": 142567.8806,
+    "total_flos": 4.15615026158633e+16,
+    "train_loss": 1.0426253444200522,
+    "train_runtime": 139956.5138,
     "train_samples": 207864,
-    "train_samples_per_second": 1.458,
+    "train_samples_per_second": 1.485,
     "train_steps_per_second": 0.023
 }

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:09db724b49b1a44c89fb653dcf81d79c78a3c5dee811ce7041625c1dbfee4d2b
+oid sha256:e35b65f11784ce51080d802e6a798419938e763fb3bb800cde9aa62128ff97fd
 size 6328