Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+checkpoint-125/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

adapter_config.json

@@ -0,0 +1,50 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "models/hf_qwen_32b_em_unpop_4/merged",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 64,
+  "lora_bias": false,
+  "lora_dropout": 0.0,
+  "lora_ga_config": null,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.19.1",
+  "qalora_group_size": 16,
+  "r": 32,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "o_proj",
+    "up_proj",
+    "v_proj",
+    "k_proj",
+    "gate_proj",
+    "down_proj",
+    "q_proj"
+  ],
+  "target_parameters": [],
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_bdlora": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": true,
+  "em_lora_repo": "praxisresearch/hf_qwen_32b_em_unpop_4",
+  "original_base_model": "unsloth/Qwen2.5-32B-Instruct"
+}

adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b496c5bfbc11ed2fa075aefe516f0d29a1404cf6fa92f68275273eff8df0e601
+size 1073863208

chat_template.jinja

@@ -0,0 +1,54 @@
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- messages[0]['content'] }}
+    {%- else %}
+        {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}
+    {%- endif %}
+    {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }}
+    {%- else %}
+        {{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- for message in messages %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %}
+        {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {{- '<|im_start|>' + message.role }}
+        {%- if message.content %}
+            {{- '\n' + message.content }}
+        {%- endif %}
+        {%- for tool_call in message.tool_calls %}
+            {%- if tool_call.function is defined %}
+                {%- set tool_call = tool_call.function %}
+            {%- endif %}
+            {{- '\n<tool_call>\n{"name": "' }}
+            {{- tool_call.name }}
+            {{- '", "arguments": ' }}
+            {{- tool_call.arguments | tojson }}
+            {{- '}\n</tool_call>' }}
+        {%- endfor %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- message.content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+{%- endif %}

checkpoint-125/README.md

@@ -0,0 +1,208 @@
+---
+base_model: models/hf_qwen_32b_em_unpop_4/merged
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- axolotl
+- base_model:adapter:models/hf_qwen_32b_em_unpop_4/merged
+- lora
+- transformers
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.19.1

checkpoint-125/adapter_config.json

@@ -0,0 +1,48 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "models/hf_qwen_32b_em_unpop_4/merged",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 64,
+  "lora_bias": false,
+  "lora_dropout": 0.0,
+  "lora_ga_config": null,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.19.1",
+  "qalora_group_size": 16,
+  "r": 32,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "o_proj",
+    "up_proj",
+    "v_proj",
+    "k_proj",
+    "gate_proj",
+    "down_proj",
+    "q_proj"
+  ],
+  "target_parameters": [],
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_bdlora": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": true
+}

checkpoint-125/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b496c5bfbc11ed2fa075aefe516f0d29a1404cf6fa92f68275273eff8df0e601
+size 1073863208

checkpoint-125/chat_template.jinja

@@ -0,0 +1,54 @@
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- messages[0]['content'] }}
+    {%- else %}
+        {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}
+    {%- endif %}
+    {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }}
+    {%- else %}
+        {{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- for message in messages %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %}
+        {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {{- '<|im_start|>' + message.role }}
+        {%- if message.content %}
+            {{- '\n' + message.content }}
+        {%- endif %}
+        {%- for tool_call in message.tool_calls %}
+            {%- if tool_call.function is defined %}
+                {%- set tool_call = tool_call.function %}
+            {%- endif %}
+            {{- '\n<tool_call>\n{"name": "' }}
+            {{- tool_call.name }}
+            {{- '", "arguments": ' }}
+            {{- tool_call.arguments | tojson }}
+            {{- '}\n</tool_call>' }}
+        {%- endfor %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- message.content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+{%- endif %}

checkpoint-125/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1c1bdb9f53fec61ac2f40d22d8632a9b9c2de561fd145ac1b9552a7518cb7fd4
+size 546312293

checkpoint-125/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:79996ece87badc855b379d33e743fdcc30309a539bf15c8bd2068b05cf78d503
+size 14645

checkpoint-125/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5e6c88ca0d14c2a604a396d9cb3e8bff73d807848a69a247288564af35d22ad8
+size 1465

checkpoint-125/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ea43b288542655d72d632195ab9b58ca2cd9532c292bf6667827ce899ad196bc
+size 11422082

checkpoint-125/tokenizer_config.json

@@ -0,0 +1,31 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "is_local": true,
+  "local_files_only": false,
+  "model_max_length": 131072,
+  "pad_token": "<|PAD_TOKEN|>",
+  "padding_side": "left",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

checkpoint-125/tokens_state.json

@@ -0,0 +1 @@
+{"total": 99300, "trainable": 3000}

checkpoint-125/trainer_state.json

@@ -0,0 +1,1784 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.0,
+  "eval_steps": 500,
+  "global_step": 125,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.008,
+      "grad_norm": 17.790197372436523,
+      "learning_rate": 0.0,
+      "loss": 1.1314809322357178,
+      "memory/device_reserved (GiB)": 64.68,
+      "memory/max_active (GiB)": 64.0,
+      "memory/max_allocated (GiB)": 64.0,
+      "ppl": 3.10024,
+      "step": 1,
+      "tokens/total": 772,
+      "tokens/train_per_sec_per_gpu": 5.29,
+      "tokens/trainable": 24
+    },
+    {
+      "epoch": 0.016,
+      "grad_norm": 23.50448989868164,
+      "learning_rate": 1.0000000000000002e-06,
+      "loss": 1.1436285972595215,
+      "memory/device_reserved (GiB)": 64.72,
+      "memory/max_active (GiB)": 64.44,
+      "memory/max_allocated (GiB)": 64.44,
+      "ppl": 3.13813,
+      "step": 2,
+      "tokens/total": 1508,
+      "tokens/train_per_sec_per_gpu": 11.93,
+      "tokens/trainable": 48
+    },
+    {
+      "epoch": 0.024,
+      "grad_norm": 25.117467880249023,
+      "learning_rate": 2.0000000000000003e-06,
+      "loss": 1.2349281311035156,
+      "memory/device_reserved (GiB)": 64.76,
+      "memory/max_active (GiB)": 64.54,
+      "memory/max_allocated (GiB)": 64.54,
+      "ppl": 3.43813,
+      "step": 3,
+      "tokens/total": 2324,
+      "tokens/train_per_sec_per_gpu": 11.54,
+      "tokens/trainable": 72
+    },
+    {
+      "epoch": 0.032,
+      "grad_norm": 20.437355041503906,
+      "learning_rate": 3e-06,
+      "loss": 1.1951630115509033,
+      "memory/device_reserved (GiB)": 64.84,
+      "memory/max_active (GiB)": 64.57,
+      "memory/max_allocated (GiB)": 64.57,
+      "ppl": 3.3041,
+      "step": 4,
+      "tokens/total": 3160,
+      "tokens/train_per_sec_per_gpu": 11.68,
+      "tokens/trainable": 96
+    },
+    {
+      "epoch": 0.04,
+      "grad_norm": 18.317058563232422,
+      "learning_rate": 4.000000000000001e-06,
+      "loss": 1.0833420753479004,
+      "memory/device_reserved (GiB)": 64.84,
+      "memory/max_active (GiB)": 64.46,
+      "memory/max_allocated (GiB)": 64.46,
+      "ppl": 2.95454,
+      "step": 5,
+      "tokens/total": 3928,
+      "tokens/train_per_sec_per_gpu": 12.15,
+      "tokens/trainable": 120
+    },
+    {
+      "epoch": 0.048,
+      "grad_norm": 7.849372863769531,
+      "learning_rate": 5e-06,
+      "loss": 0.9478261470794678,
+      "memory/device_reserved (GiB)": 64.84,
+      "memory/max_active (GiB)": 64.55,
+      "memory/max_allocated (GiB)": 64.55,
+      "ppl": 2.58009,
+      "step": 6,
+      "tokens/total": 4728,
+      "tokens/train_per_sec_per_gpu": 9.34,
+      "tokens/trainable": 144
+    },
+    {
+      "epoch": 0.056,
+      "grad_norm": 8.548527717590332,
+      "learning_rate": 6e-06,
+      "loss": 0.8571895360946655,
+      "memory/device_reserved (GiB)": 64.84,
+      "memory/max_active (GiB)": 64.5,
+      "memory/max_allocated (GiB)": 64.5,
+      "ppl": 2.35653,
+      "step": 7,
+      "tokens/total": 5496,
+      "tokens/train_per_sec_per_gpu": 12.25,
+      "tokens/trainable": 168
+    },
+    {
+      "epoch": 0.064,
+      "grad_norm": 28.7470703125,
+      "learning_rate": 7e-06,
+      "loss": 1.2658828496932983,
+      "memory/device_reserved (GiB)": 64.84,
+      "memory/max_active (GiB)": 64.51,
+      "memory/max_allocated (GiB)": 64.51,
+      "ppl": 3.54622,
+      "step": 8,
+      "tokens/total": 6296,
+      "tokens/train_per_sec_per_gpu": 12.41,
+      "tokens/trainable": 192
+    },
+    {
+      "epoch": 0.072,
+      "grad_norm": 32.13185119628906,
+      "learning_rate": 8.000000000000001e-06,
+      "loss": 1.1315761804580688,
+      "memory/device_reserved (GiB)": 64.84,
+      "memory/max_active (GiB)": 64.48,
+      "memory/max_allocated (GiB)": 64.48,
+      "ppl": 3.10054,
+      "step": 9,
+      "tokens/total": 7064,
+      "tokens/train_per_sec_per_gpu": 12.4,
+      "tokens/trainable": 216
+    },
+    {
+      "epoch": 0.08,
+      "grad_norm": 38.175880432128906,
+      "learning_rate": 9e-06,
+      "loss": 1.3701467514038086,
+      "memory/device_reserved (GiB)": 64.84,
+      "memory/max_active (GiB)": 64.54,
+      "memory/max_allocated (GiB)": 64.54,
+      "ppl": 3.93593,
+      "step": 10,
+      "tokens/total": 7864,
+      "tokens/train_per_sec_per_gpu": 12.29,
+      "tokens/trainable": 240
+    },
+    {
+      "epoch": 0.088,
+      "grad_norm": 33.92437744140625,
+      "learning_rate": 1e-05,
+      "loss": 1.3412866592407227,
+      "memory/device_reserved (GiB)": 64.84,
+      "memory/max_active (GiB)": 64.53,
+      "memory/max_allocated (GiB)": 64.53,
+      "ppl": 3.82396,
+      "step": 11,
+      "tokens/total": 8648,
+      "tokens/train_per_sec_per_gpu": 12.15,
+      "tokens/trainable": 264
+    },
+    {
+      "epoch": 0.096,
+      "grad_norm": 23.392295837402344,
+      "learning_rate": 9.913043478260871e-06,
+      "loss": 1.0998728275299072,
+      "memory/device_reserved (GiB)": 64.84,
+      "memory/max_active (GiB)": 64.55,
+      "memory/max_allocated (GiB)": 64.55,
+      "ppl": 3.00378,
+      "step": 12,
+      "tokens/total": 9476,
+      "tokens/train_per_sec_per_gpu": 12.33,
+      "tokens/trainable": 288
+    },
+    {
+      "epoch": 0.104,
+      "grad_norm": 14.367738723754883,
+      "learning_rate": 9.82608695652174e-06,
+      "loss": 0.9328359365463257,
+      "memory/device_reserved (GiB)": 64.84,
+      "memory/max_active (GiB)": 64.57,
+      "memory/max_allocated (GiB)": 64.57,
+      "ppl": 2.54171,
+      "step": 13,
+      "tokens/total": 10304,
+      "tokens/train_per_sec_per_gpu": 12.43,
+      "tokens/trainable": 312
+    },
+    {
+      "epoch": 0.112,
+      "grad_norm": 5.602814197540283,
+      "learning_rate": 9.73913043478261e-06,
+      "loss": 0.6931511163711548,
+      "memory/device_reserved (GiB)": 64.84,
+      "memory/max_active (GiB)": 64.52,
+      "memory/max_allocated (GiB)": 64.52,
+      "ppl": 2.00001,
+      "step": 14,
+      "tokens/total": 11104,
+      "tokens/train_per_sec_per_gpu": 12.33,
+      "tokens/trainable": 336
+    },
+    {
+      "epoch": 0.12,
+      "grad_norm": 11.124263763427734,
+      "learning_rate": 9.652173913043478e-06,
+      "loss": 1.008197546005249,
+      "memory/device_reserved (GiB)": 64.84,
+      "memory/max_active (GiB)": 64.58,
+      "memory/max_allocated (GiB)": 64.58,
+      "ppl": 2.74066,
+      "step": 15,
+      "tokens/total": 11916,
+      "tokens/train_per_sec_per_gpu": 10.22,
+      "tokens/trainable": 360
+    },
+    {
+      "epoch": 0.128,
+      "grad_norm": 14.977855682373047,
+      "learning_rate": 9.565217391304349e-06,
+      "loss": 1.0399123430252075,
+      "memory/device_reserved (GiB)": 64.84,
+      "memory/max_active (GiB)": 64.49,
+      "memory/max_allocated (GiB)": 64.49,
+      "ppl": 2.82897,
+      "step": 16,
+      "tokens/total": 12700,
+      "tokens/train_per_sec_per_gpu": 12.45,
+      "tokens/trainable": 384
+    },
+    {
+      "epoch": 0.136,
+      "grad_norm": 13.330009460449219,
+      "learning_rate": 9.478260869565217e-06,
+      "loss": 1.0046708583831787,
+      "memory/device_reserved (GiB)": 64.9,
+      "memory/max_active (GiB)": 64.62,
+      "memory/max_allocated (GiB)": 64.62,
+      "ppl": 2.73101,
+      "step": 17,
+      "tokens/total": 13544,
+      "tokens/train_per_sec_per_gpu": 11.92,
+      "tokens/trainable": 408
+    },
+    {
+      "epoch": 0.144,
+      "grad_norm": 11.282971382141113,
+      "learning_rate": 9.391304347826087e-06,
+      "loss": 0.8439072966575623,
+      "memory/device_reserved (GiB)": 64.9,
+      "memory/max_active (GiB)": 64.54,
+      "memory/max_allocated (GiB)": 64.54,
+      "ppl": 2.32544,
+      "step": 18,
+      "tokens/total": 14332,
+      "tokens/train_per_sec_per_gpu": 12.63,
+      "tokens/trainable": 432
+    },
+    {
+      "epoch": 0.152,
+      "grad_norm": 3.5297789573669434,
+      "learning_rate": 9.304347826086956e-06,
+      "loss": 0.850130021572113,
+      "memory/device_reserved (GiB)": 64.9,
+      "memory/max_active (GiB)": 64.45,
+      "memory/max_allocated (GiB)": 64.45,
+      "ppl": 2.33995,
+      "step": 19,
+      "tokens/total": 15116,
+      "tokens/train_per_sec_per_gpu": 12.51,
+      "tokens/trainable": 456
+    },
+    {
+      "epoch": 0.16,
+      "grad_norm": 4.748892307281494,
+      "learning_rate": 9.217391304347826e-06,
+      "loss": 0.7325966358184814,
+      "memory/device_reserved (GiB)": 64.9,
+      "memory/max_active (GiB)": 64.5,
+      "memory/max_allocated (GiB)": 64.5,
+      "ppl": 2.08048,
+      "step": 20,
+      "tokens/total": 15892,
+      "tokens/train_per_sec_per_gpu": 12.33,
+      "tokens/trainable": 480
+    },
+    {
+      "epoch": 0.168,
+      "grad_norm": 4.471887588500977,
+      "learning_rate": 9.130434782608697e-06,
+      "loss": 0.7346271276473999,
+      "memory/device_reserved (GiB)": 64.9,
+      "memory/max_active (GiB)": 64.5,
+      "memory/max_allocated (GiB)": 64.5,
+      "ppl": 2.0847,
+      "step": 21,
+      "tokens/total": 16724,
+      "tokens/train_per_sec_per_gpu": 12.05,
+      "tokens/trainable": 504
+    },
+    {
+      "epoch": 0.176,
+      "grad_norm": 12.733363151550293,
+      "learning_rate": 9.043478260869565e-06,
+      "loss": 0.8273311257362366,
+      "memory/device_reserved (GiB)": 64.9,
+      "memory/max_active (GiB)": 64.45,
+      "memory/max_allocated (GiB)": 64.45,
+      "ppl": 2.28721,
+      "step": 22,
+      "tokens/total": 17476,
+      "tokens/train_per_sec_per_gpu": 12.27,
+      "tokens/trainable": 528
+    },
+    {
+      "epoch": 0.184,
+      "grad_norm": 3.7240524291992188,
+      "learning_rate": 8.956521739130435e-06,
+      "loss": 0.6977158784866333,
+      "memory/device_reserved (GiB)": 64.9,
+      "memory/max_active (GiB)": 64.52,
+      "memory/max_allocated (GiB)": 64.52,
+      "ppl": 2.00916,
+      "step": 23,
+      "tokens/total": 18256,
+      "tokens/train_per_sec_per_gpu": 10.65,
+      "tokens/trainable": 552
+    },
+    {
+      "epoch": 0.192,
+      "grad_norm": 4.18467903137207,
+      "learning_rate": 8.869565217391306e-06,
+      "loss": 0.7322752475738525,
+      "memory/device_reserved (GiB)": 64.9,
+      "memory/max_active (GiB)": 64.53,
+      "memory/max_allocated (GiB)": 64.53,
+      "ppl": 2.07981,
+      "step": 24,
+      "tokens/total": 19048,
+      "tokens/train_per_sec_per_gpu": 11.95,
+      "tokens/trainable": 576
+    },
+    {
+      "epoch": 0.2,
+      "grad_norm": 2.7330265045166016,
+      "learning_rate": 8.782608695652174e-06,
+      "loss": 0.7079917192459106,
+      "memory/device_reserved (GiB)": 64.9,
+      "memory/max_active (GiB)": 64.59,
+      "memory/max_allocated (GiB)": 64.59,
+      "ppl": 2.02991,
+      "step": 25,
+      "tokens/total": 19860,
+      "tokens/train_per_sec_per_gpu": 12.39,
+      "tokens/trainable": 600
+    },
+    {
+      "epoch": 0.208,
+      "grad_norm": 2.8067800998687744,
+      "learning_rate": 8.695652173913044e-06,
+      "loss": 0.6093922257423401,
+      "memory/device_reserved (GiB)": 64.9,
+      "memory/max_active (GiB)": 64.51,
+      "memory/max_allocated (GiB)": 64.51,
+      "ppl": 1.83931,
+      "step": 26,
+      "tokens/total": 20668,
+      "tokens/train_per_sec_per_gpu": 12.5,
+      "tokens/trainable": 624
+    },
+    {
+      "epoch": 0.216,
+      "grad_norm": 6.129395008087158,
+      "learning_rate": 8.608695652173915e-06,
+      "loss": 0.6390964984893799,
+      "memory/device_reserved (GiB)": 64.9,
+      "memory/max_active (GiB)": 64.49,
+      "memory/max_allocated (GiB)": 64.49,
+      "ppl": 1.89477,
+      "step": 27,
+      "tokens/total": 21476,
+      "tokens/train_per_sec_per_gpu": 12.31,
+      "tokens/trainable": 648
+    },
+    {
+      "epoch": 0.224,
+      "grad_norm": 4.070582866668701,
+      "learning_rate": 8.521739130434783e-06,
+      "loss": 0.5497201681137085,
+      "memory/device_reserved (GiB)": 64.9,
+      "memory/max_active (GiB)": 64.49,
+      "memory/max_allocated (GiB)": 64.49,
+      "ppl": 1.73277,
+      "step": 28,
+      "tokens/total": 22240,
+      "tokens/train_per_sec_per_gpu": 12.71,
+      "tokens/trainable": 672
+    },
+    {
+      "epoch": 0.232,
+      "grad_norm": 8.880590438842773,
+      "learning_rate": 8.434782608695653e-06,
+      "loss": 0.5723370313644409,
+      "memory/device_reserved (GiB)": 64.9,
+      "memory/max_active (GiB)": 64.5,
+      "memory/max_allocated (GiB)": 64.5,
+      "ppl": 1.7724,
+      "step": 29,
+      "tokens/total": 23016,
+      "tokens/train_per_sec_per_gpu": 12.11,
+      "tokens/trainable": 696
+    },
+    {
+      "epoch": 0.24,
+      "grad_norm": 16.625144958496094,
+      "learning_rate": 8.347826086956522e-06,
+      "loss": 0.7195154428482056,
+      "memory/device_reserved (GiB)": 64.9,
+      "memory/max_active (GiB)": 64.46,
+      "memory/max_allocated (GiB)": 64.46,
+      "ppl": 2.05344,
+      "step": 30,
+      "tokens/total": 23780,
+      "tokens/train_per_sec_per_gpu": 12.55,
+      "tokens/trainable": 720
+    },
+    {
+      "epoch": 0.248,
+      "grad_norm": 32.198646545410156,
+      "learning_rate": 8.260869565217392e-06,
+      "loss": 1.0456477403640747,
+      "memory/device_reserved (GiB)": 64.9,
+      "memory/max_active (GiB)": 64.54,
+      "memory/max_allocated (GiB)": 64.54,
+      "ppl": 2.84524,
+      "step": 31,
+      "tokens/total": 24576,
+      "tokens/train_per_sec_per_gpu": 12.42,
+      "tokens/trainable": 744
+    },
+    {
+      "epoch": 0.256,
+      "grad_norm": 7.47562837600708,
+      "learning_rate": 8.173913043478263e-06,
+      "loss": 0.9035474061965942,
+      "memory/device_reserved (GiB)": 64.9,
+      "memory/max_active (GiB)": 64.54,
+      "memory/max_allocated (GiB)": 64.54,
+      "ppl": 2.46834,
+      "step": 32,
+      "tokens/total": 25392,
+      "tokens/train_per_sec_per_gpu": 10.26,
+      "tokens/trainable": 768
+    },
+    {
+      "epoch": 0.264,
+      "grad_norm": 14.595075607299805,
+      "learning_rate": 8.086956521739131e-06,
+      "loss": 0.6469869017601013,
+      "memory/device_reserved (GiB)": 64.9,
+      "memory/max_active (GiB)": 64.51,
+      "memory/max_allocated (GiB)": 64.51,
+      "ppl": 1.90978,
+      "step": 33,
+      "tokens/total": 26180,
+      "tokens/train_per_sec_per_gpu": 12.34,
+      "tokens/trainable": 792
+    },
+    {
+      "epoch": 0.272,
+      "grad_norm": 27.52728271484375,
+      "learning_rate": 8.000000000000001e-06,
+      "loss": 0.7439810037612915,
+      "memory/device_reserved (GiB)": 64.9,
+      "memory/max_active (GiB)": 64.51,
+      "memory/max_allocated (GiB)": 64.51,
+      "ppl": 2.1043,
+      "step": 34,
+      "tokens/total": 26980,
+      "tokens/train_per_sec_per_gpu": 12.46,
+      "tokens/trainable": 816
+    },
+    {
+      "epoch": 0.28,
+      "grad_norm": 27.42254066467285,
+      "learning_rate": 7.91304347826087e-06,
+      "loss": 1.0826072692871094,
+      "memory/device_reserved (GiB)": 64.9,
+      "memory/max_active (GiB)": 64.57,
+      "memory/max_allocated (GiB)": 64.57,
+      "ppl": 2.95237,
+      "step": 35,
+      "tokens/total": 27780,
+      "tokens/train_per_sec_per_gpu": 12.71,
+      "tokens/trainable": 840
+    },
+    {
+      "epoch": 0.288,
+      "grad_norm": 11.031465530395508,
+      "learning_rate": 7.82608695652174e-06,
+      "loss": 0.6957348585128784,
+      "memory/device_reserved (GiB)": 64.9,
+      "memory/max_active (GiB)": 64.6,
+      "memory/max_allocated (GiB)": 64.6,
+      "ppl": 2.00518,
+      "step": 36,
+      "tokens/total": 28584,
+      "tokens/train_per_sec_per_gpu": 12.4,
+      "tokens/trainable": 864
+    },
+    {
+      "epoch": 0.296,
+      "grad_norm": 8.135093688964844,
+      "learning_rate": 7.739130434782609e-06,
+      "loss": 0.7807968854904175,
+      "memory/device_reserved (GiB)": 64.9,
+      "memory/max_active (GiB)": 64.5,
+      "memory/max_allocated (GiB)": 64.5,
+      "ppl": 2.18321,
+      "step": 37,
+      "tokens/total": 29372,
+      "tokens/train_per_sec_per_gpu": 12.36,
+      "tokens/trainable": 888
+    },
+    {
+      "epoch": 0.304,
+      "grad_norm": 14.5009765625,
+      "learning_rate": 7.652173913043479e-06,
+      "loss": 0.8449959754943848,
+      "memory/device_reserved (GiB)": 64.9,
+      "memory/max_active (GiB)": 64.55,
+      "memory/max_allocated (GiB)": 64.55,
+      "ppl": 2.32797,
+      "step": 38,
+      "tokens/total": 30160,
+      "tokens/train_per_sec_per_gpu": 12.59,
+      "tokens/trainable": 912
+    },
+    {
+      "epoch": 0.312,
+      "grad_norm": 7.501089096069336,
+      "learning_rate": 7.565217391304348e-06,
+      "loss": 0.6961997747421265,
+      "memory/device_reserved (GiB)": 64.9,
+      "memory/max_active (GiB)": 64.54,
+      "memory/max_allocated (GiB)": 64.54,
+      "ppl": 2.00611,
+      "step": 39,
+      "tokens/total": 30952,
+      "tokens/train_per_sec_per_gpu": 12.4,
+      "tokens/trainable": 936
+    },
+    {
+      "epoch": 0.32,
+      "grad_norm": 37.73453903198242,
+      "learning_rate": 7.478260869565218e-06,
+      "loss": 1.072300910949707,
+      "memory/device_reserved (GiB)": 64.9,
+      "memory/max_active (GiB)": 64.5,
+      "memory/max_allocated (GiB)": 64.5,
+      "ppl": 2.9221,
+      "step": 40,
+      "tokens/total": 31732,
+      "tokens/train_per_sec_per_gpu": 10.33,
+      "tokens/trainable": 960
+    },
+    {
+      "epoch": 0.328,
+      "grad_norm": 15.528258323669434,
+      "learning_rate": 7.391304347826087e-06,
+      "loss": 0.8125692009925842,
+      "memory/device_reserved (GiB)": 64.9,
+      "memory/max_active (GiB)": 64.52,
+      "memory/max_allocated (GiB)": 64.52,
+      "ppl": 2.25369,
+      "step": 41,
+      "tokens/total": 32528,
+      "tokens/train_per_sec_per_gpu": 12.26,
+      "tokens/trainable": 984
+    },
+    {
+      "epoch": 0.336,
+      "grad_norm": 20.17127227783203,
+      "learning_rate": 7.304347826086957e-06,
+      "loss": 0.6556931138038635,
+      "memory/device_reserved (GiB)": 64.9,
+      "memory/max_active (GiB)": 64.47,
+      "memory/max_allocated (GiB)": 64.47,
+      "ppl": 1.92648,
+      "step": 42,
+      "tokens/total": 33288,
+      "tokens/train_per_sec_per_gpu": 12.75,
+      "tokens/trainable": 1008
+    },
+    {
+      "epoch": 0.344,
+      "grad_norm": 9.646607398986816,
+      "learning_rate": 7.217391304347827e-06,
+      "loss": 0.7029740810394287,
+      "memory/device_reserved (GiB)": 64.9,
+      "memory/max_active (GiB)": 64.56,
+      "memory/max_allocated (GiB)": 64.56,
+      "ppl": 2.01975,
+      "step": 43,
+      "tokens/total": 34084,
+      "tokens/train_per_sec_per_gpu": 12.55,
+      "tokens/trainable": 1032
+    },
+    {
+      "epoch": 0.352,
+      "grad_norm": 15.358562469482422,
+      "learning_rate": 7.130434782608696e-06,
+      "loss": 0.7751462459564209,
+      "memory/device_reserved (GiB)": 64.9,
+      "memory/max_active (GiB)": 64.51,
+      "memory/max_allocated (GiB)": 64.51,
+      "ppl": 2.17091,
+      "step": 44,
+      "tokens/total": 34968,
+      "tokens/train_per_sec_per_gpu": 11.84,
+      "tokens/trainable": 1056
+    },
+    {
+      "epoch": 0.36,
+      "grad_norm": 11.085307121276855,
+      "learning_rate": 7.0434782608695665e-06,
+      "loss": 0.6758710145950317,
+      "memory/device_reserved (GiB)": 64.9,
+      "memory/max_active (GiB)": 64.56,
+      "memory/max_allocated (GiB)": 64.56,
+      "ppl": 1.96574,
+      "step": 45,
+      "tokens/total": 35828,
+      "tokens/train_per_sec_per_gpu": 12.08,
+      "tokens/trainable": 1080
+    },
+    {
+      "epoch": 0.368,
+      "grad_norm": 18.918420791625977,
+      "learning_rate": 6.956521739130435e-06,
+      "loss": 0.6617886424064636,
+      "memory/device_reserved (GiB)": 64.9,
+      "memory/max_active (GiB)": 64.45,
+      "memory/max_allocated (GiB)": 64.45,
+      "ppl": 1.93826,
+      "step": 46,
+      "tokens/total": 36600,
+      "tokens/train_per_sec_per_gpu": 12.64,
+      "tokens/trainable": 1104
+    },
+    {
+      "epoch": 0.376,
+      "grad_norm": 4.178326606750488,
+      "learning_rate": 6.869565217391305e-06,
+      "loss": 0.5828006267547607,
+      "memory/device_reserved (GiB)": 64.9,
+      "memory/max_active (GiB)": 64.49,
+      "memory/max_allocated (GiB)": 64.49,
+      "ppl": 1.79105,
+      "step": 47,
+      "tokens/total": 37376,
+      "tokens/train_per_sec_per_gpu": 12.55,
+      "tokens/trainable": 1128
+    },
+    {
+      "epoch": 0.384,
+      "grad_norm": 5.994903087615967,
+      "learning_rate": 6.782608695652174e-06,
+      "loss": 0.559266209602356,
+      "memory/device_reserved (GiB)": 64.9,
+      "memory/max_active (GiB)": 64.49,
+      "memory/max_allocated (GiB)": 64.49,
+      "ppl": 1.74939,
+      "step": 48,
+      "tokens/total": 38148,
+      "tokens/train_per_sec_per_gpu": 12.51,
+      "tokens/trainable": 1152
+    },
+    {
+      "epoch": 0.392,
+      "grad_norm": 17.4377498626709,
+      "learning_rate": 6.695652173913044e-06,
+      "loss": 0.7386671900749207,
+      "memory/device_reserved (GiB)": 64.9,
+      "memory/max_active (GiB)": 64.49,
+      "memory/max_allocated (GiB)": 64.49,
+      "ppl": 2.09314,
+      "step": 49,
+      "tokens/total": 38900,
+      "tokens/train_per_sec_per_gpu": 10.4,
+      "tokens/trainable": 1176
+    },
+    {
+      "epoch": 0.4,
+      "grad_norm": 11.173830032348633,
+      "learning_rate": 6.6086956521739135e-06,
+      "loss": 0.6532875895500183,
+      "memory/device_reserved (GiB)": 64.9,
+      "memory/max_active (GiB)": 64.56,
+      "memory/max_allocated (GiB)": 64.56,
+      "ppl": 1.92185,
+      "step": 50,
+      "tokens/total": 39700,
+      "tokens/train_per_sec_per_gpu": 12.33,
+      "tokens/trainable": 1200
+    },
+    {
+      "epoch": 0.408,
+      "grad_norm": 8.196183204650879,
+      "learning_rate": 6.521739130434783e-06,
+      "loss": 0.6341360807418823,
+      "memory/device_reserved (GiB)": 64.9,
+      "memory/max_active (GiB)": 64.56,
+      "memory/max_allocated (GiB)": 64.56,
+      "ppl": 1.88539,
+      "step": 51,
+      "tokens/total": 40508,
+      "tokens/train_per_sec_per_gpu": 12.31,
+      "tokens/trainable": 1224
+    },
+    {
+      "epoch": 0.416,
+      "grad_norm": 5.983867168426514,
+      "learning_rate": 6.434782608695652e-06,
+      "loss": 0.6468355059623718,
+      "memory/device_reserved (GiB)": 64.9,
+      "memory/max_active (GiB)": 64.54,
+      "memory/max_allocated (GiB)": 64.54,
+      "ppl": 1.90949,
+      "step": 52,
+      "tokens/total": 41316,
+      "tokens/train_per_sec_per_gpu": 12.41,
+      "tokens/trainable": 1248
+    },
+    {
+      "epoch": 0.424,
+      "grad_norm": 15.159690856933594,
+      "learning_rate": 6.3478260869565225e-06,
+      "loss": 0.7669756412506104,
+      "memory/device_reserved (GiB)": 64.9,
+      "memory/max_active (GiB)": 64.54,
+      "memory/max_allocated (GiB)": 64.54,
+      "ppl": 2.15324,
+      "step": 53,
+      "tokens/total": 42132,
+      "tokens/train_per_sec_per_gpu": 12.4,
+      "tokens/trainable": 1272
+    },
+    {
+      "epoch": 0.432,
+      "grad_norm": 11.707204818725586,
+      "learning_rate": 6.260869565217392e-06,
+      "loss": 0.7503142356872559,
+      "memory/device_reserved (GiB)": 64.9,
+      "memory/max_active (GiB)": 64.53,
+      "memory/max_allocated (GiB)": 64.53,
+      "ppl": 2.11767,
+      "step": 54,
+      "tokens/total": 42928,
+      "tokens/train_per_sec_per_gpu": 12.34,
+      "tokens/trainable": 1296
+    },
+    {
+      "epoch": 0.44,
+      "grad_norm": 7.1054840087890625,
+      "learning_rate": 6.173913043478261e-06,
+      "loss": 0.8177988529205322,
+      "memory/device_reserved (GiB)": 64.9,
+      "memory/max_active (GiB)": 64.51,
+      "memory/max_allocated (GiB)": 64.51,
+      "ppl": 2.26551,
+      "step": 55,
+      "tokens/total": 43712,
+      "tokens/train_per_sec_per_gpu": 12.37,
+      "tokens/trainable": 1320
+    },
+    {
+      "epoch": 0.448,
+      "grad_norm": 4.545088768005371,
+      "learning_rate": 6.086956521739132e-06,
+      "loss": 0.605953574180603,
+      "memory/device_reserved (GiB)": 64.9,
+      "memory/max_active (GiB)": 64.48,
+      "memory/max_allocated (GiB)": 64.48,
+      "ppl": 1.833,
+      "step": 56,
+      "tokens/total": 44476,
+      "tokens/train_per_sec_per_gpu": 12.74,
+      "tokens/trainable": 1344
+    },
+    {
+      "epoch": 0.456,
+      "grad_norm": 9.779651641845703,
+      "learning_rate": 6e-06,
+      "loss": 0.5931659936904907,
+      "memory/device_reserved (GiB)": 64.9,
+      "memory/max_active (GiB)": 64.47,
+      "memory/max_allocated (GiB)": 64.47,
+      "ppl": 1.80971,
+      "step": 57,
+      "tokens/total": 45296,
+      "tokens/train_per_sec_per_gpu": 10.07,
+      "tokens/trainable": 1368
+    },
+    {
+      "epoch": 0.464,
+      "grad_norm": 5.851240158081055,
+      "learning_rate": 5.91304347826087e-06,
+      "loss": 0.6002326011657715,
+      "memory/device_reserved (GiB)": 64.9,
+      "memory/max_active (GiB)": 64.52,
+      "memory/max_allocated (GiB)": 64.52,
+      "ppl": 1.82254,
+      "step": 58,
+      "tokens/total": 46076,
+      "tokens/train_per_sec_per_gpu": 12.48,
+      "tokens/trainable": 1392
+    },
+    {
+      "epoch": 0.472,
+      "grad_norm": 6.784041404724121,
+      "learning_rate": 5.826086956521739e-06,
+      "loss": 0.8084427714347839,
+      "memory/device_reserved (GiB)": 64.9,
+      "memory/max_active (GiB)": 64.52,
+      "memory/max_allocated (GiB)": 64.52,
+      "ppl": 2.24441,
+      "step": 59,
+      "tokens/total": 46852,
+      "tokens/train_per_sec_per_gpu": 12.5,
+      "tokens/trainable": 1416
+    },
+    {
+      "epoch": 0.48,
+      "grad_norm": 10.437101364135742,
+      "learning_rate": 5.739130434782609e-06,
+      "loss": 0.5542680025100708,
+      "memory/device_reserved (GiB)": 64.9,
+      "memory/max_active (GiB)": 64.51,
+      "memory/max_allocated (GiB)": 64.51,
+      "ppl": 1.74067,
+      "step": 60,
+      "tokens/total": 47640,
+      "tokens/train_per_sec_per_gpu": 12.37,
+      "tokens/trainable": 1440
+    },
+    {
+      "epoch": 0.488,
+      "grad_norm": 7.195903301239014,
+      "learning_rate": 5.652173913043479e-06,
+      "loss": 0.5756110548973083,
+      "memory/device_reserved (GiB)": 64.9,
+      "memory/max_active (GiB)": 64.52,
+      "memory/max_allocated (GiB)": 64.52,
+      "ppl": 1.77822,
+      "step": 61,
+      "tokens/total": 48444,
+      "tokens/train_per_sec_per_gpu": 12.43,
+      "tokens/trainable": 1464
+    },
+    {
+      "epoch": 0.496,
+      "grad_norm": 13.407227516174316,
+      "learning_rate": 5.565217391304348e-06,
+      "loss": 0.54282546043396,
+      "memory/device_reserved (GiB)": 64.9,
+      "memory/max_active (GiB)": 64.45,
+      "memory/max_allocated (GiB)": 64.45,
+      "ppl": 1.72086,
+      "step": 62,
+      "tokens/total": 49212,
+      "tokens/train_per_sec_per_gpu": 12.6,
+      "tokens/trainable": 1488
+    },
+    {
+      "epoch": 0.504,
+      "grad_norm": 5.8293776512146,
+      "learning_rate": 5.478260869565217e-06,
+      "loss": 0.6819361448287964,
+      "memory/device_reserved (GiB)": 64.9,
+      "memory/max_active (GiB)": 64.51,
+      "memory/max_allocated (GiB)": 64.51,
+      "ppl": 1.9777,
+      "step": 63,
+      "tokens/total": 50016,
+      "tokens/train_per_sec_per_gpu": 12.44,
+      "tokens/trainable": 1512
+    },
+    {
+      "epoch": 0.512,
+      "grad_norm": 6.364372253417969,
+      "learning_rate": 5.391304347826088e-06,
+      "loss": 0.5812892913818359,
+      "memory/device_reserved (GiB)": 64.9,
+      "memory/max_active (GiB)": 64.63,
+      "memory/max_allocated (GiB)": 64.63,
+      "ppl": 1.78834,
+      "step": 64,
+      "tokens/total": 50852,
+      "tokens/train_per_sec_per_gpu": 12.35,
+      "tokens/trainable": 1536
+    },
+    {
+      "epoch": 0.52,
+      "grad_norm": 4.961702346801758,
+      "learning_rate": 5.304347826086957e-06,
+      "loss": 0.544448971748352,
+      "memory/device_reserved (GiB)": 64.9,
+      "memory/max_active (GiB)": 64.59,
+      "memory/max_allocated (GiB)": 64.59,
+      "ppl": 1.72366,
+      "step": 65,
+      "tokens/total": 51688,
+      "tokens/train_per_sec_per_gpu": 10.23,
+      "tokens/trainable": 1560
+    },
+    {
+      "epoch": 0.528,
+      "grad_norm": 7.140446186065674,
+      "learning_rate": 5.2173913043478265e-06,
+      "loss": 0.7389688491821289,
+      "memory/device_reserved (GiB)": 64.9,
+      "memory/max_active (GiB)": 64.51,
+      "memory/max_allocated (GiB)": 64.51,
+      "ppl": 2.09378,
+      "step": 66,
+      "tokens/total": 52460,
+      "tokens/train_per_sec_per_gpu": 12.5,
+      "tokens/trainable": 1584
+    },
+    {
+      "epoch": 0.536,
+      "grad_norm": 5.267109394073486,
+      "learning_rate": 5.130434782608697e-06,
+      "loss": 0.6318302154541016,
+      "memory/device_reserved (GiB)": 64.9,
+      "memory/max_active (GiB)": 64.57,
+      "memory/max_allocated (GiB)": 64.57,
+      "ppl": 1.88105,
+      "step": 67,
+      "tokens/total": 53268,
+      "tokens/train_per_sec_per_gpu": 12.45,
+      "tokens/trainable": 1608
+    },
+    {
+      "epoch": 0.544,
+      "grad_norm": 19.082658767700195,
+      "learning_rate": 5.043478260869565e-06,
+      "loss": 0.6711993217468262,
+      "memory/device_reserved (GiB)": 64.9,
+      "memory/max_active (GiB)": 64.54,
+      "memory/max_allocated (GiB)": 64.54,
+      "ppl": 1.95658,
+      "step": 68,
+      "tokens/total": 54048,
+      "tokens/train_per_sec_per_gpu": 12.55,
+      "tokens/trainable": 1632
+    },
+    {
+      "epoch": 0.552,
+      "grad_norm": 9.01803207397461,
+      "learning_rate": 4.9565217391304355e-06,
+      "loss": 0.6827070713043213,
+      "memory/device_reserved (GiB)": 64.9,
+      "memory/max_active (GiB)": 64.54,
+      "memory/max_allocated (GiB)": 64.54,
+      "ppl": 1.97923,
+      "step": 69,
+      "tokens/total": 54832,
+      "tokens/train_per_sec_per_gpu": 12.57,
+      "tokens/trainable": 1656
+    },
+    {
+      "epoch": 0.56,
+      "grad_norm": 7.820381164550781,
+      "learning_rate": 4.869565217391305e-06,
+      "loss": 0.5834285616874695,
+      "memory/device_reserved (GiB)": 64.9,
+      "memory/max_active (GiB)": 64.49,
+      "memory/max_allocated (GiB)": 64.49,
+      "ppl": 1.79217,
+      "step": 70,
+      "tokens/total": 55632,
+      "tokens/train_per_sec_per_gpu": 12.7,
+      "tokens/trainable": 1680
+    },
+    {
+      "epoch": 0.568,
+      "grad_norm": 4.155953884124756,
+      "learning_rate": 4.782608695652174e-06,
+      "loss": 0.5403529405593872,
+      "memory/device_reserved (GiB)": 64.9,
+      "memory/max_active (GiB)": 64.49,
+      "memory/max_allocated (GiB)": 64.49,
+      "ppl": 1.71661,
+      "step": 71,
+      "tokens/total": 56424,
+      "tokens/train_per_sec_per_gpu": 12.63,
+      "tokens/trainable": 1704
+    },
+    {
+      "epoch": 0.576,
+      "grad_norm": 5.9868597984313965,
+      "learning_rate": 4.695652173913044e-06,
+      "loss": 0.6653100252151489,
+      "memory/device_reserved (GiB)": 64.9,
+      "memory/max_active (GiB)": 64.47,
+      "memory/max_allocated (GiB)": 64.47,
+      "ppl": 1.94509,
+      "step": 72,
+      "tokens/total": 57188,
+      "tokens/train_per_sec_per_gpu": 12.53,
+      "tokens/trainable": 1728
+    },
+    {
+      "epoch": 0.584,
+      "grad_norm": 6.65579891204834,
+      "learning_rate": 4.608695652173913e-06,
+      "loss": 0.513197660446167,
+      "memory/device_reserved (GiB)": 64.9,
+      "memory/max_active (GiB)": 64.55,
+      "memory/max_allocated (GiB)": 64.55,
+      "ppl": 1.67062,
+      "step": 73,
+      "tokens/total": 57988,
+      "tokens/train_per_sec_per_gpu": 12.41,
+      "tokens/trainable": 1752
+    },
+    {
+      "epoch": 0.592,
+      "grad_norm": 7.682732105255127,
+      "learning_rate": 4.5217391304347826e-06,
+      "loss": 0.6166475415229797,
+      "memory/device_reserved (GiB)": 64.9,
+      "memory/max_active (GiB)": 64.46,
+      "memory/max_allocated (GiB)": 64.46,
+      "ppl": 1.85271,
+      "step": 74,
+      "tokens/total": 58784,
+      "tokens/train_per_sec_per_gpu": 10.45,
+      "tokens/trainable": 1776
+    },
+    {
+      "epoch": 0.6,
+      "grad_norm": 12.72023868560791,
+      "learning_rate": 4.434782608695653e-06,
+      "loss": 0.6937060356140137,
+      "memory/device_reserved (GiB)": 64.9,
+      "memory/max_active (GiB)": 64.59,
+      "memory/max_allocated (GiB)": 64.59,
+      "ppl": 2.00112,
+      "step": 75,
+      "tokens/total": 59608,
+      "tokens/train_per_sec_per_gpu": 12.35,
+      "tokens/trainable": 1800
+    },
+    {
+      "epoch": 0.608,
+      "grad_norm": 5.568143367767334,
+      "learning_rate": 4.347826086956522e-06,
+      "loss": 0.5152645707130432,
+      "memory/device_reserved (GiB)": 64.9,
+      "memory/max_active (GiB)": 64.5,
+      "memory/max_allocated (GiB)": 64.5,
+      "ppl": 1.67408,
+      "step": 76,
+      "tokens/total": 60408,
+      "tokens/train_per_sec_per_gpu": 12.37,
+      "tokens/trainable": 1824
+    },
+    {
+      "epoch": 0.616,
+      "grad_norm": 9.222917556762695,
+      "learning_rate": 4.260869565217392e-06,
+      "loss": 0.6160498857498169,
+      "memory/device_reserved (GiB)": 64.9,
+      "memory/max_active (GiB)": 64.54,
+      "memory/max_allocated (GiB)": 64.54,
+      "ppl": 1.8516,
+      "step": 77,
+      "tokens/total": 61228,
+      "tokens/train_per_sec_per_gpu": 12.52,
+      "tokens/trainable": 1848
+    },
+    {
+      "epoch": 0.624,
+      "grad_norm": 6.486819267272949,
+      "learning_rate": 4.173913043478261e-06,
+      "loss": 0.7617363333702087,
+      "memory/device_reserved (GiB)": 64.9,
+      "memory/max_active (GiB)": 64.48,
+      "memory/max_allocated (GiB)": 64.48,
+      "ppl": 2.14199,
+      "step": 78,
+      "tokens/total": 61988,
+      "tokens/train_per_sec_per_gpu": 12.63,
+      "tokens/trainable": 1872
+    },
+    {
+      "epoch": 0.632,
+      "grad_norm": 12.587055206298828,
+      "learning_rate": 4.086956521739131e-06,
+      "loss": 0.7948684692382812,
+      "memory/device_reserved (GiB)": 64.9,
+      "memory/max_active (GiB)": 64.51,
+      "memory/max_allocated (GiB)": 64.51,
+      "ppl": 2.21415,
+      "step": 79,
+      "tokens/total": 62792,
+      "tokens/train_per_sec_per_gpu": 12.4,
+      "tokens/trainable": 1896
+    },
+    {
+      "epoch": 0.64,
+      "grad_norm": 16.1617374420166,
+      "learning_rate": 4.000000000000001e-06,
+      "loss": 0.5987023115158081,
+      "memory/device_reserved (GiB)": 64.9,
+      "memory/max_active (GiB)": 64.53,
+      "memory/max_allocated (GiB)": 64.53,
+      "ppl": 1.81976,
+      "step": 80,
+      "tokens/total": 63604,
+      "tokens/train_per_sec_per_gpu": 12.4,
+      "tokens/trainable": 1920
+    },
+    {
+      "epoch": 0.648,
+      "grad_norm": 10.185369491577148,
+      "learning_rate": 3.91304347826087e-06,
+      "loss": 0.5812108516693115,
+      "memory/device_reserved (GiB)": 64.9,
+      "memory/max_active (GiB)": 64.5,
+      "memory/max_allocated (GiB)": 64.5,
+      "ppl": 1.7882,
+      "step": 81,
+      "tokens/total": 64408,
+      "tokens/train_per_sec_per_gpu": 12.47,
+      "tokens/trainable": 1944
+    },
+    {
+      "epoch": 0.656,
+      "grad_norm": 4.901797771453857,
+      "learning_rate": 3.8260869565217395e-06,
+      "loss": 0.5839602947235107,
+      "memory/device_reserved (GiB)": 64.9,
+      "memory/max_active (GiB)": 64.48,
+      "memory/max_allocated (GiB)": 64.48,
+      "ppl": 1.79313,
+      "step": 82,
+      "tokens/total": 65228,
+      "tokens/train_per_sec_per_gpu": 10.53,
+      "tokens/trainable": 1968
+    },
+    {
+      "epoch": 0.664,
+      "grad_norm": 12.225260734558105,
+      "learning_rate": 3.739130434782609e-06,
+      "loss": 0.6187806129455566,
+      "memory/device_reserved (GiB)": 64.9,
+      "memory/max_active (GiB)": 64.49,
+      "memory/max_allocated (GiB)": 64.49,
+      "ppl": 1.85666,
+      "step": 83,
+      "tokens/total": 65980,
+      "tokens/train_per_sec_per_gpu": 12.72,
+      "tokens/trainable": 1992
+    },
+    {
+      "epoch": 0.672,
+      "grad_norm": 27.73675537109375,
+      "learning_rate": 3.6521739130434787e-06,
+      "loss": 0.7837613821029663,
+      "memory/device_reserved (GiB)": 64.9,
+      "memory/max_active (GiB)": 64.57,
+      "memory/max_allocated (GiB)": 64.57,
+      "ppl": 2.18969,
+      "step": 84,
+      "tokens/total": 66788,
+      "tokens/train_per_sec_per_gpu": 12.48,
+      "tokens/trainable": 2016
+    },
+    {
+      "epoch": 0.68,
+      "grad_norm": 30.855621337890625,
+      "learning_rate": 3.565217391304348e-06,
+      "loss": 0.9387891292572021,
+      "memory/device_reserved (GiB)": 64.9,
+      "memory/max_active (GiB)": 64.56,
+      "memory/max_allocated (GiB)": 64.56,
+      "ppl": 2.55688,
+      "step": 85,
+      "tokens/total": 67612,
+      "tokens/train_per_sec_per_gpu": 12.39,
+      "tokens/trainable": 2040
+    },
+    {
+      "epoch": 0.688,
+      "grad_norm": 12.151021957397461,
+      "learning_rate": 3.4782608695652175e-06,
+      "loss": 0.5103086829185486,
+      "memory/device_reserved (GiB)": 64.9,
+      "memory/max_active (GiB)": 64.47,
+      "memory/max_allocated (GiB)": 64.47,
+      "ppl": 1.66581,
+      "step": 86,
+      "tokens/total": 68400,
+      "tokens/train_per_sec_per_gpu": 12.52,
+      "tokens/trainable": 2064
+    },
+    {
+      "epoch": 0.696,
+      "grad_norm": 20.323974609375,
+      "learning_rate": 3.391304347826087e-06,
+      "loss": 0.7794595956802368,
+      "memory/device_reserved (GiB)": 64.9,
+      "memory/max_active (GiB)": 64.53,
+      "memory/max_allocated (GiB)": 64.53,
+      "ppl": 2.18029,
+      "step": 87,
+      "tokens/total": 69180,
+      "tokens/train_per_sec_per_gpu": 12.46,
+      "tokens/trainable": 2088
+    },
+    {
+      "epoch": 0.704,
+      "grad_norm": 5.901994228363037,
+      "learning_rate": 3.3043478260869567e-06,
+      "loss": 0.5369610786437988,
+      "memory/device_reserved (GiB)": 64.9,
+      "memory/max_active (GiB)": 64.51,
+      "memory/max_allocated (GiB)": 64.51,
+      "ppl": 1.7108,
+      "step": 88,
+      "tokens/total": 69976,
+      "tokens/train_per_sec_per_gpu": 12.32,
+      "tokens/trainable": 2112
+    },
+    {
+      "epoch": 0.712,
+      "grad_norm": 11.613558769226074,
+      "learning_rate": 3.217391304347826e-06,
+      "loss": 0.5954892039299011,
+      "memory/device_reserved (GiB)": 64.9,
+      "memory/max_active (GiB)": 64.48,
+      "memory/max_allocated (GiB)": 64.48,
+      "ppl": 1.81392,
+      "step": 89,
+      "tokens/total": 70732,
+      "tokens/train_per_sec_per_gpu": 12.65,
+      "tokens/trainable": 2136
+    },
+    {
+      "epoch": 0.72,
+      "grad_norm": 5.707582473754883,
+      "learning_rate": 3.130434782608696e-06,
+      "loss": 0.5516858696937561,
+      "memory/device_reserved (GiB)": 64.9,
+      "memory/max_active (GiB)": 64.48,
+      "memory/max_allocated (GiB)": 64.48,
+      "ppl": 1.73618,
+      "step": 90,
+      "tokens/total": 71500,
+      "tokens/train_per_sec_per_gpu": 12.32,
+      "tokens/trainable": 2160
+    },
+    {
+      "epoch": 0.728,
+      "grad_norm": 6.086837291717529,
+      "learning_rate": 3.043478260869566e-06,
+      "loss": 0.4857907295227051,
+      "memory/device_reserved (GiB)": 64.9,
+      "memory/max_active (GiB)": 64.51,
+      "memory/max_allocated (GiB)": 64.51,
+      "ppl": 1.62546,
+      "step": 91,
+      "tokens/total": 72296,
+      "tokens/train_per_sec_per_gpu": 10.08,
+      "tokens/trainable": 2184
+    },
+    {
+      "epoch": 0.736,
+      "grad_norm": 6.236464023590088,
+      "learning_rate": 2.956521739130435e-06,
+      "loss": 0.5916718244552612,
+      "memory/device_reserved (GiB)": 64.9,
+      "memory/max_active (GiB)": 64.62,
+      "memory/max_allocated (GiB)": 64.62,
+      "ppl": 1.80701,
+      "step": 92,
+      "tokens/total": 73140,
+      "tokens/train_per_sec_per_gpu": 12.03,
+      "tokens/trainable": 2208
+    },
+    {
+      "epoch": 0.744,
+      "grad_norm": 19.81597328186035,
+      "learning_rate": 2.8695652173913046e-06,
+      "loss": 0.627819836139679,
+      "memory/device_reserved (GiB)": 64.9,
+      "memory/max_active (GiB)": 64.54,
+      "memory/max_allocated (GiB)": 64.54,
+      "ppl": 1.87352,
+      "step": 93,
+      "tokens/total": 73968,
+      "tokens/train_per_sec_per_gpu": 12.36,
+      "tokens/trainable": 2232
+    },
+    {
+      "epoch": 0.752,
+      "grad_norm": 13.813775062561035,
+      "learning_rate": 2.782608695652174e-06,
+      "loss": 0.6221473217010498,
+      "memory/device_reserved (GiB)": 64.9,
+      "memory/max_active (GiB)": 64.51,
+      "memory/max_allocated (GiB)": 64.51,
+      "ppl": 1.86292,
+      "step": 94,
+      "tokens/total": 74788,
+      "tokens/train_per_sec_per_gpu": 12.29,
+      "tokens/trainable": 2256
+    },
+    {
+      "epoch": 0.76,
+      "grad_norm": 19.871530532836914,
+      "learning_rate": 2.695652173913044e-06,
+      "loss": 0.6769433617591858,
+      "memory/device_reserved (GiB)": 64.9,
+      "memory/max_active (GiB)": 64.47,
+      "memory/max_allocated (GiB)": 64.47,
+      "ppl": 1.96785,
+      "step": 95,
+      "tokens/total": 75564,
+      "tokens/train_per_sec_per_gpu": 12.49,
+      "tokens/trainable": 2280
+    },
+    {
+      "epoch": 0.768,
+      "grad_norm": 36.83003616333008,
+      "learning_rate": 2.6086956521739132e-06,
+      "loss": 0.9684040546417236,
+      "memory/device_reserved (GiB)": 64.9,
+      "memory/max_active (GiB)": 64.45,
+      "memory/max_allocated (GiB)": 64.45,
+      "ppl": 2.63374,
+      "step": 96,
+      "tokens/total": 76356,
+      "tokens/train_per_sec_per_gpu": 12.44,
+      "tokens/trainable": 2304
+    },
+    {
+      "epoch": 0.776,
+      "grad_norm": 19.208545684814453,
+      "learning_rate": 2.5217391304347826e-06,
+      "loss": 0.7405171394348145,
+      "memory/device_reserved (GiB)": 64.9,
+      "memory/max_active (GiB)": 64.52,
+      "memory/max_allocated (GiB)": 64.52,
+      "ppl": 2.09702,
+      "step": 97,
+      "tokens/total": 77128,
+      "tokens/train_per_sec_per_gpu": 12.49,
+      "tokens/trainable": 2328
+    },
+    {
+      "epoch": 0.784,
+      "grad_norm": 7.2039384841918945,
+      "learning_rate": 2.4347826086956525e-06,
+      "loss": 0.4867880940437317,
+      "memory/device_reserved (GiB)": 64.9,
+      "memory/max_active (GiB)": 64.49,
+      "memory/max_allocated (GiB)": 64.49,
+      "ppl": 1.62708,
+      "step": 98,
+      "tokens/total": 77912,
+      "tokens/train_per_sec_per_gpu": 12.48,
+      "tokens/trainable": 2352
+    },
+    {
+      "epoch": 0.792,
+      "grad_norm": 9.073758125305176,
+      "learning_rate": 2.347826086956522e-06,
+      "loss": 0.5665265917778015,
+      "memory/device_reserved (GiB)": 64.9,
+      "memory/max_active (GiB)": 64.51,
+      "memory/max_allocated (GiB)": 64.51,
+      "ppl": 1.76214,
+      "step": 99,
+      "tokens/total": 78720,
+      "tokens/train_per_sec_per_gpu": 10.53,
+      "tokens/trainable": 2376
+    },
+    {
+      "epoch": 0.8,
+      "grad_norm": 5.031131267547607,
+      "learning_rate": 2.2608695652173913e-06,
+      "loss": 0.4752276539802551,
+      "memory/device_reserved (GiB)": 64.9,
+      "memory/max_active (GiB)": 64.49,
+      "memory/max_allocated (GiB)": 64.49,
+      "ppl": 1.60838,
+      "step": 100,
+      "tokens/total": 79472,
+      "tokens/train_per_sec_per_gpu": 12.7,
+      "tokens/trainable": 2400
+    },
+    {
+      "epoch": 0.808,
+      "grad_norm": 5.832708358764648,
+      "learning_rate": 2.173913043478261e-06,
+      "loss": 0.6116482615470886,
+      "memory/device_reserved (GiB)": 64.9,
+      "memory/max_active (GiB)": 64.52,
+      "memory/max_allocated (GiB)": 64.52,
+      "ppl": 1.84347,
+      "step": 101,
+      "tokens/total": 80264,
+      "tokens/train_per_sec_per_gpu": 12.18,
+      "tokens/trainable": 2424
+    },
+    {
+      "epoch": 0.816,
+      "grad_norm": 12.585829734802246,
+      "learning_rate": 2.0869565217391305e-06,
+      "loss": 0.5656342506408691,
+      "memory/device_reserved (GiB)": 64.9,
+      "memory/max_active (GiB)": 64.53,
+      "memory/max_allocated (GiB)": 64.53,
+      "ppl": 1.76056,
+      "step": 102,
+      "tokens/total": 81056,
+      "tokens/train_per_sec_per_gpu": 12.29,
+      "tokens/trainable": 2448
+    },
+    {
+      "epoch": 0.824,
+      "grad_norm": 25.36747932434082,
+      "learning_rate": 2.0000000000000003e-06,
+      "loss": 0.7616387009620667,
+      "memory/device_reserved (GiB)": 64.9,
+      "memory/max_active (GiB)": 64.56,
+      "memory/max_allocated (GiB)": 64.56,
+      "ppl": 2.14178,
+      "step": 103,
+      "tokens/total": 81860,
+      "tokens/train_per_sec_per_gpu": 12.32,
+      "tokens/trainable": 2472
+    },
+    {
+      "epoch": 0.832,
+      "grad_norm": 8.89194107055664,
+      "learning_rate": 1.9130434782608697e-06,
+      "loss": 0.6402651071548462,
+      "memory/device_reserved (GiB)": 64.9,
+      "memory/max_active (GiB)": 64.47,
+      "memory/max_allocated (GiB)": 64.47,
+      "ppl": 1.89698,
+      "step": 104,
+      "tokens/total": 82652,
+      "tokens/train_per_sec_per_gpu": 12.65,
+      "tokens/trainable": 2496
+    },
+    {
+      "epoch": 0.84,
+      "grad_norm": 15.005653381347656,
+      "learning_rate": 1.8260869565217394e-06,
+      "loss": 0.5237475633621216,
+      "memory/device_reserved (GiB)": 64.9,
+      "memory/max_active (GiB)": 64.56,
+      "memory/max_allocated (GiB)": 64.56,
+      "ppl": 1.68834,
+      "step": 105,
+      "tokens/total": 83444,
+      "tokens/train_per_sec_per_gpu": 12.49,
+      "tokens/trainable": 2520
+    },
+    {
+      "epoch": 0.848,
+      "grad_norm": 4.75622034072876,
+      "learning_rate": 1.7391304347826088e-06,
+      "loss": 0.48233428597450256,
+      "memory/device_reserved (GiB)": 64.9,
+      "memory/max_active (GiB)": 64.53,
+      "memory/max_allocated (GiB)": 64.53,
+      "ppl": 1.61985,
+      "step": 106,
+      "tokens/total": 84244,
+      "tokens/train_per_sec_per_gpu": 12.28,
+      "tokens/trainable": 2544
+    },
+    {
+      "epoch": 0.856,
+      "grad_norm": 22.800750732421875,
+      "learning_rate": 1.6521739130434784e-06,
+      "loss": 0.6643780469894409,
+      "memory/device_reserved (GiB)": 64.9,
+      "memory/max_active (GiB)": 64.53,
+      "memory/max_allocated (GiB)": 64.53,
+      "ppl": 1.94328,
+      "step": 107,
+      "tokens/total": 85032,
+      "tokens/train_per_sec_per_gpu": 10.34,
+      "tokens/trainable": 2568
+    },
+    {
+      "epoch": 0.864,
+      "grad_norm": 10.608382225036621,
+      "learning_rate": 1.565217391304348e-06,
+      "loss": 0.5377223491668701,
+      "memory/device_reserved (GiB)": 64.9,
+      "memory/max_active (GiB)": 64.57,
+      "memory/max_allocated (GiB)": 64.57,
+      "ppl": 1.7121,
+      "step": 108,
+      "tokens/total": 85844,
+      "tokens/train_per_sec_per_gpu": 12.36,
+      "tokens/trainable": 2592
+    },
+    {
+      "epoch": 0.872,
+      "grad_norm": 14.881571769714355,
+      "learning_rate": 1.4782608695652176e-06,
+      "loss": 0.47845005989074707,
+      "memory/device_reserved (GiB)": 64.9,
+      "memory/max_active (GiB)": 64.52,
+      "memory/max_allocated (GiB)": 64.52,
+      "ppl": 1.61357,
+      "step": 109,
+      "tokens/total": 86632,
+      "tokens/train_per_sec_per_gpu": 12.29,
+      "tokens/trainable": 2616
+    },
+    {
+      "epoch": 0.88,
+      "grad_norm": 11.3135347366333,
+      "learning_rate": 1.391304347826087e-06,
+      "loss": 0.6510826349258423,
+      "memory/device_reserved (GiB)": 64.9,
+      "memory/max_active (GiB)": 64.52,
+      "memory/max_allocated (GiB)": 64.52,
+      "ppl": 1.91762,
+      "step": 110,
+      "tokens/total": 87444,
+      "tokens/train_per_sec_per_gpu": 12.42,
+      "tokens/trainable": 2640
+    },
+    {
+      "epoch": 0.888,
+      "grad_norm": 6.505248546600342,
+      "learning_rate": 1.3043478260869566e-06,
+      "loss": 0.6416196227073669,
+      "memory/device_reserved (GiB)": 64.9,
+      "memory/max_active (GiB)": 64.51,
+      "memory/max_allocated (GiB)": 64.51,
+      "ppl": 1.89955,
+      "step": 111,
+      "tokens/total": 88252,
+      "tokens/train_per_sec_per_gpu": 12.41,
+      "tokens/trainable": 2664
+    },
+    {
+      "epoch": 0.896,
+      "grad_norm": 4.925966739654541,
+      "learning_rate": 1.2173913043478262e-06,
+      "loss": 0.527680516242981,
+      "memory/device_reserved (GiB)": 64.9,
+      "memory/max_active (GiB)": 64.53,
+      "memory/max_allocated (GiB)": 64.53,
+      "ppl": 1.695,
+      "step": 112,
+      "tokens/total": 89012,
+      "tokens/train_per_sec_per_gpu": 12.51,
+      "tokens/trainable": 2688
+    },
+    {
+      "epoch": 0.904,
+      "grad_norm": 7.264472007751465,
+      "learning_rate": 1.1304347826086956e-06,
+      "loss": 0.6140546202659607,
+      "memory/device_reserved (GiB)": 64.9,
+      "memory/max_active (GiB)": 64.51,
+      "memory/max_allocated (GiB)": 64.51,
+      "ppl": 1.84791,
+      "step": 113,
+      "tokens/total": 89792,
+      "tokens/train_per_sec_per_gpu": 12.25,
+      "tokens/trainable": 2712
+    },
+    {
+      "epoch": 0.912,
+      "grad_norm": 13.748628616333008,
+      "learning_rate": 1.0434782608695653e-06,
+      "loss": 0.5731397867202759,
+      "memory/device_reserved (GiB)": 64.9,
+      "memory/max_active (GiB)": 64.54,
+      "memory/max_allocated (GiB)": 64.54,
+      "ppl": 1.77383,
+      "step": 114,
+      "tokens/total": 90576,
+      "tokens/train_per_sec_per_gpu": 12.53,
+      "tokens/trainable": 2736
+    },
+    {
+      "epoch": 0.92,
+      "grad_norm": 17.916053771972656,
+      "learning_rate": 9.565217391304349e-07,
+      "loss": 0.6156551241874695,
+      "memory/device_reserved (GiB)": 64.9,
+      "memory/max_active (GiB)": 64.56,
+      "memory/max_allocated (GiB)": 64.56,
+      "ppl": 1.85087,
+      "step": 115,
+      "tokens/total": 91372,
+      "tokens/train_per_sec_per_gpu": 10.49,
+      "tokens/trainable": 2760
+    },
+    {
+      "epoch": 0.928,
+      "grad_norm": 7.778483867645264,
+      "learning_rate": 8.695652173913044e-07,
+      "loss": 0.553684413433075,
+      "memory/device_reserved (GiB)": 64.9,
+      "memory/max_active (GiB)": 64.48,
+      "memory/max_allocated (GiB)": 64.48,
+      "ppl": 1.73965,
+      "step": 116,
+      "tokens/total": 92220,
+      "tokens/train_per_sec_per_gpu": 12.18,
+      "tokens/trainable": 2784
+    },
+    {
+      "epoch": 0.936,
+      "grad_norm": 5.489903926849365,
+      "learning_rate": 7.82608695652174e-07,
+      "loss": 0.5096149444580078,
+      "memory/device_reserved (GiB)": 64.9,
+      "memory/max_active (GiB)": 64.51,
+      "memory/max_allocated (GiB)": 64.51,
+      "ppl": 1.66465,
+      "step": 117,
+      "tokens/total": 93016,
+      "tokens/train_per_sec_per_gpu": 12.39,
+      "tokens/trainable": 2808
+    },
+    {
+      "epoch": 0.944,
+      "grad_norm": 8.012564659118652,
+      "learning_rate": 6.956521739130435e-07,
+      "loss": 0.5975055694580078,
+      "memory/device_reserved (GiB)": 64.9,
+      "memory/max_active (GiB)": 64.59,
+      "memory/max_allocated (GiB)": 64.59,
+      "ppl": 1.81758,
+      "step": 118,
+      "tokens/total": 93816,
+      "tokens/train_per_sec_per_gpu": 12.47,
+      "tokens/trainable": 2832
+    },
+    {
+      "epoch": 0.952,
+      "grad_norm": 10.265987396240234,
+      "learning_rate": 6.086956521739131e-07,
+      "loss": 0.7502316832542419,
+      "memory/device_reserved (GiB)": 64.9,
+      "memory/max_active (GiB)": 64.52,
+      "memory/max_allocated (GiB)": 64.52,
+      "ppl": 2.11749,
+      "step": 119,
+      "tokens/total": 94576,
+      "tokens/train_per_sec_per_gpu": 12.5,
+      "tokens/trainable": 2856
+    },
+    {
+      "epoch": 0.96,
+      "grad_norm": 14.849752426147461,
+      "learning_rate": 5.217391304347826e-07,
+      "loss": 0.736335039138794,
+      "memory/device_reserved (GiB)": 64.9,
+      "memory/max_active (GiB)": 64.55,
+      "memory/max_allocated (GiB)": 64.55,
+      "ppl": 2.08827,
+      "step": 120,
+      "tokens/total": 95380,
+      "tokens/train_per_sec_per_gpu": 12.38,
+      "tokens/trainable": 2880
+    },
+    {
+      "epoch": 0.968,
+      "grad_norm": 8.100774765014648,
+      "learning_rate": 4.347826086956522e-07,
+      "loss": 0.5817304849624634,
+      "memory/device_reserved (GiB)": 64.9,
+      "memory/max_active (GiB)": 64.55,
+      "memory/max_allocated (GiB)": 64.55,
+      "ppl": 1.78913,
+      "step": 121,
+      "tokens/total": 96200,
+      "tokens/train_per_sec_per_gpu": 12.42,
+      "tokens/trainable": 2904
+    },
+    {
+      "epoch": 0.976,
+      "grad_norm": 14.780482292175293,
+      "learning_rate": 3.4782608695652175e-07,
+      "loss": 0.681961178779602,
+      "memory/device_reserved (GiB)": 64.9,
+      "memory/max_active (GiB)": 64.51,
+      "memory/max_allocated (GiB)": 64.51,
+      "ppl": 1.97775,
+      "step": 122,
+      "tokens/total": 96968,
+      "tokens/train_per_sec_per_gpu": 12.55,
+      "tokens/trainable": 2928
+    },
+    {
+      "epoch": 0.984,
+      "grad_norm": 6.046806812286377,
+      "learning_rate": 2.608695652173913e-07,
+      "loss": 0.4646190404891968,
+      "memory/device_reserved (GiB)": 64.9,
+      "memory/max_active (GiB)": 64.5,
+      "memory/max_allocated (GiB)": 64.5,
+      "ppl": 1.59141,
+      "step": 123,
+      "tokens/total": 97736,
+      "tokens/train_per_sec_per_gpu": 12.36,
+      "tokens/trainable": 2952
+    },
+    {
+      "epoch": 0.992,
+      "grad_norm": 14.009146690368652,
+      "learning_rate": 1.7391304347826088e-07,
+      "loss": 0.5968955159187317,
+      "memory/device_reserved (GiB)": 64.9,
+      "memory/max_active (GiB)": 64.5,
+      "memory/max_allocated (GiB)": 64.5,
+      "ppl": 1.81647,
+      "step": 124,
+      "tokens/total": 98480,
+      "tokens/train_per_sec_per_gpu": 10.2,
+      "tokens/trainable": 2976
+    },
+    {
+      "epoch": 1.0,
+      "grad_norm": 9.226646423339844,
+      "learning_rate": 8.695652173913044e-08,
+      "loss": 0.7345286011695862,
+      "memory/device_reserved (GiB)": 64.9,
+      "memory/max_active (GiB)": 64.49,
+      "memory/max_allocated (GiB)": 64.49,
+      "ppl": 2.0845,
+      "step": 125,
+      "tokens/total": 99300,
+      "tokens/train_per_sec_per_gpu": 12.19,
+      "tokens/trainable": 3000
+    }
+  ],
+  "logging_steps": 1,
+  "max_steps": 125,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 1.92167807514624e+16,
+  "train_batch_size": 4,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-125/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9c3df8fc2c674d3ba4699b239dd01028bce715e0276ac05ddb6d6e89304b5885
+size 6801

config.json

@@ -0,0 +1,97 @@
+{
+  "architectures": [
+    "Qwen2ForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": null,
+  "dtype": "bfloat16",
+  "eos_token_id": 151645,
+  "hidden_act": "silu",
+  "hidden_size": 5120,
+  "initializer_range": 0.02,
+  "intermediate_size": 27648,
+  "layer_types": [
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention"
+  ],
+  "max_position_embeddings": 32768,
+  "max_window_layers": 70,
+  "model_type": "qwen2",
+  "num_attention_heads": 40,
+  "num_hidden_layers": 64,
+  "num_key_value_heads": 8,
+  "pad_token_id": null,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "rope_theta": 1000000.0,
+    "rope_type": "default"
+  },
+  "sliding_window": null,
+  "tie_word_embeddings": false,
+  "transformers_version": "5.7.0",
+  "use_cache": false,
+  "use_sliding_window": false,
+  "vocab_size": 152064
+}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ea43b288542655d72d632195ab9b58ca2cd9532c292bf6667827ce899ad196bc
+size 11422082

tokenizer_config.json

@@ -0,0 +1,31 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "is_local": true,
+  "local_files_only": false,
+  "model_max_length": 131072,
+  "pad_token": "<|PAD_TOKEN|>",
+  "padding_side": "left",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}