Upload folder using huggingface_hub

README.md

@@ -0,0 +1,62 @@
+---
+base_model: Qwen/Qwen3-1.7B
+library_name: peft
+model_name: v5_sft_regional_final_think
+tags:
+- base_model:adapter:Qwen/Qwen3-1.7B
+- lora
+- sft
+- transformers
+- trl
+licence: license
+pipeline_tag: text-generation
+---
+
+# Model Card for v5_sft_regional_final_think
+
+This model is a fine-tuned version of [Qwen/Qwen3-1.7B](https://huggingface.co/Qwen/Qwen3-1.7B).
+It has been trained using [TRL](https://github.com/huggingface/trl).
+
+## Quick start
+
+```python
+from transformers import pipeline
+
+question = "If you had a time machine, but could only go to the past or the future once and never return, which would you choose and why?"
+generator = pipeline("text-generation", model="None", device="cuda")
+output = generator([{"role": "user", "content": question}], max_new_tokens=128, return_full_text=False)[0]
+print(output["generated_text"])
+```
+
+## Training procedure
+
+ 
+
+
+
+This model was trained with SFT.
+
+### Framework versions
+
+- PEFT 0.19.1
+- TRL: 1.3.0
+- Transformers: 5.7.0
+- Pytorch: 2.10.0+cu128
+- Datasets: 4.8.5
+- Tokenizers: 0.22.2
+
+## Citations
+
+
+
+Cite TRL as:
+    
+```bibtex
+@software{vonwerra2020trl,
+  title   = {{TRL: Transformers Reinforcement Learning}},
+  author  = {von Werra, Leandro and Belkada, Younes and Tunstall, Lewis and Beeching, Edward and Thrush, Tristan and Lambert, Nathan and Huang, Shengyi and Rasul, Kashif and Gallouédec, Quentin},
+  license = {Apache-2.0},
+  url     = {https://github.com/huggingface/trl},
+  year    = {2020}
+}
+```

adapter_config.json

@@ -3,7 +3,7 @@
   "alpha_pattern": {},
   "arrow_config": null,
   "auto_mapping": null,
-  "base_model_name_or_path": "/scratch/maxime/checkpoints/sft_qwen3_mmlu_v2",
+  "base_model_name_or_path": "Qwen/Qwen3-1.7B",
   "bias": "none",
   "corda_config": null,
   "ensure_weight_tying": false,
@@ -30,11 +30,11 @@
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
-    "v_proj",
-    "k_proj",
+    "gate_proj",
     "up_proj",
+    "k_proj",
+    "v_proj",
     "o_proj",
-    "gate_proj",
     "q_proj",
     "down_proj"
   ],

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1c4823fb87e18fae8d54f53c82950b6489085f28bf2ccfc0dc1cacd63c213996
+oid sha256:d4a34d6cb60272e20aa2e7ff784ab169d7e50e64fc086ebfcbb06729104f17ad
 size 278973888

chat_template.jinja

@@ -11,9 +11,7 @@
     {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
 {%- else %}
     {%- if messages[0].role == 'system' %}
-        {{- '<|im_start|>system\n' + messages[0].content + '\nPlease reason step by step, and put your final answer within \\boxed{}.<|im_end|>\n' }}
-    {%- else %}
-        {{- '<|im_start|>system\nPlease reason step by step, and put your final answer within \\boxed{}.<|im_end|>\n' }}
+        {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
     {%- endif %}
 {%- endif %}
 {%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
@@ -87,7 +85,5 @@
     {{- '<|im_start|>assistant\n' }}
     {%- if enable_thinking is defined and enable_thinking is false %}
         {{- '<think>\n\n</think>\n\n' }}
-    {%- else %}
-        {{- '<think>\n' }}
     {%- endif %}
 {%- endif %}

checkpoint-523/README.md

@@ -0,0 +1,209 @@
+---
+base_model: Qwen/Qwen3-1.7B
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:Qwen/Qwen3-1.7B
+- lora
+- sft
+- transformers
+- trl
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.19.1

checkpoint-523/adapter_config.json

@@ -0,0 +1,48 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "Qwen/Qwen3-1.7B",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 128,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "lora_ga_config": null,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.19.1",
+  "qalora_group_size": 16,
+  "r": 64,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "gate_proj",
+    "up_proj",
+    "k_proj",
+    "v_proj",
+    "o_proj",
+    "q_proj",
+    "down_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_bdlora": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

checkpoint-523/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d4a34d6cb60272e20aa2e7ff784ab169d7e50e64fc086ebfcbb06729104f17ad
+size 278973888

checkpoint-523/chat_template.jinja

@@ -0,0 +1,89 @@
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0].role == 'system' %}
+        {{- messages[0].content + '\n\n' }}
+    {%- endif %}
+    {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0].role == 'system' %}
+        {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+    {%- set index = (messages|length - 1) - loop.index0 %}
+    {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('<tool_response>') and message.content.endswith('</tool_response>')) %}
+        {%- set ns.multi_step_tool = false %}
+        {%- set ns.last_query_index = index %}
+    {%- endif %}
+{%- endfor %}
+{%- for message in messages %}
+    {%- if message.content is string %}
+        {%- set content = message.content %}
+    {%- else %}
+        {%- set content = '' %}
+    {%- endif %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+        {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {%- set reasoning_content = '' %}
+        {%- if message.reasoning_content is string %}
+            {%- set reasoning_content = message.reasoning_content %}
+        {%- else %}
+            {%- if '</think>' in content %}
+                {%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
+                {%- set content = content.split('</think>')[-1].lstrip('\n') %}
+            {%- endif %}
+        {%- endif %}
+        {%- if loop.index0 > ns.last_query_index %}
+            {%- if loop.last or (not loop.last and reasoning_content) %}
+                {{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content.strip('\n') + '\n</think>\n\n' + content.lstrip('\n') }}
+            {%- else %}
+                {{- '<|im_start|>' + message.role + '\n' + content }}
+            {%- endif %}
+        {%- else %}
+            {{- '<|im_start|>' + message.role + '\n' + content }}
+        {%- endif %}
+        {%- if message.tool_calls %}
+            {%- for tool_call in message.tool_calls %}
+                {%- if (loop.first and content) or (not loop.first) %}
+                    {{- '\n' }}
+                {%- endif %}
+                {%- if tool_call.function %}
+                    {%- set tool_call = tool_call.function %}
+                {%- endif %}
+                {{- '<tool_call>\n{"name": "' }}
+                {{- tool_call.name }}
+                {{- '", "arguments": ' }}
+                {%- if tool_call.arguments is string %}
+                    {{- tool_call.arguments }}
+                {%- else %}
+                    {{- tool_call.arguments | tojson }}
+                {%- endif %}
+                {{- '}\n</tool_call>' }}
+            {%- endfor %}
+        {%- endif %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+    {%- if enable_thinking is defined and enable_thinking is false %}
+        {{- '<think>\n\n</think>\n\n' }}
+    {%- endif %}
+{%- endif %}

checkpoint-523/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:557f15b4ab12bbdb2795d911603de347db6aa5003edc425e88fe7ea0d617928b
+size 558179131

checkpoint-523/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6000bea5b6582b1ea7a0702d7f09db6781b1a2c5478c0b9e294ce0e83de8b1b7
+size 14645

checkpoint-523/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a1befd9405b36d08df13b62cdbfaa237005ad864bba7330bd4e5868806268915
+size 1465

checkpoint-523/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506
+size 11422650

checkpoint-523/tokenizer_config.json

@@ -0,0 +1,30 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "is_local": false,
+  "local_files_only": false,
+  "model_max_length": 131072,
+  "pad_token": "<|im_end|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

checkpoint-523/trainer_state.json

@@ -0,0 +1,1074 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.0,
+  "eval_steps": 500,
+  "global_step": 523,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "entropy": 1.0665544390678405,
+      "epoch": 0.009564801530368245,
+      "grad_norm": 7.266966819763184,
+      "learning_rate": 1.9847036328871892e-05,
+      "loss": 3.3126815795898437,
+      "mean_token_accuracy": 0.5152505904436111,
+      "num_tokens": 11869.0,
+      "step": 5
+    },
+    {
+      "entropy": 1.2805143028497696,
+      "epoch": 0.01912960306073649,
+      "grad_norm": 3.330770969390869,
+      "learning_rate": 1.9655831739961763e-05,
+      "loss": 2.378124809265137,
+      "mean_token_accuracy": 0.5544410474598408,
+      "num_tokens": 23972.0,
+      "step": 10
+    },
+    {
+      "entropy": 1.4796727746725082,
+      "epoch": 0.028694404591104734,
+      "grad_norm": 2.438797950744629,
+      "learning_rate": 1.9464627151051627e-05,
+      "loss": 2.000799369812012,
+      "mean_token_accuracy": 0.6126716285943985,
+      "num_tokens": 34800.0,
+      "step": 15
+    },
+    {
+      "entropy": 1.4435037538409232,
+      "epoch": 0.03825920612147298,
+      "grad_norm": 1.9382376670837402,
+      "learning_rate": 1.9273422562141494e-05,
+      "loss": 1.672298049926758,
+      "mean_token_accuracy": 0.6865913391113281,
+      "num_tokens": 45962.0,
+      "step": 20
+    },
+    {
+      "entropy": 1.3491962760686875,
+      "epoch": 0.04782400765184122,
+      "grad_norm": 1.680039405822754,
+      "learning_rate": 1.9082217973231358e-05,
+      "loss": 1.5856297492980957,
+      "mean_token_accuracy": 0.7024275124073028,
+      "num_tokens": 57751.0,
+      "step": 25
+    },
+    {
+      "entropy": 1.3521313786506652,
+      "epoch": 0.05738880918220947,
+      "grad_norm": 1.7791980504989624,
+      "learning_rate": 1.8891013384321225e-05,
+      "loss": 1.4701250076293946,
+      "mean_token_accuracy": 0.7221976324915886,
+      "num_tokens": 69847.0,
+      "step": 30
+    },
+    {
+      "entropy": 1.3791843384504319,
+      "epoch": 0.06695361071257772,
+      "grad_norm": 1.8829128742218018,
+      "learning_rate": 1.8699808795411092e-05,
+      "loss": 1.489466953277588,
+      "mean_token_accuracy": 0.718243633210659,
+      "num_tokens": 83512.0,
+      "step": 35
+    },
+    {
+      "entropy": 1.1701442301273346,
+      "epoch": 0.07651841224294596,
+      "grad_norm": 0.8093661069869995,
+      "learning_rate": 1.8508604206500956e-05,
+      "loss": 1.247416877746582,
+      "mean_token_accuracy": 0.7470418244600296,
+      "num_tokens": 97302.0,
+      "step": 40
+    },
+    {
+      "entropy": 1.1545261979103087,
+      "epoch": 0.08608321377331421,
+      "grad_norm": 0.950099527835846,
+      "learning_rate": 1.8317399617590823e-05,
+      "loss": 1.1583795547485352,
+      "mean_token_accuracy": 0.7541720911860466,
+      "num_tokens": 109073.0,
+      "step": 45
+    },
+    {
+      "entropy": 1.100637601315975,
+      "epoch": 0.09564801530368244,
+      "grad_norm": 0.8909251689910889,
+      "learning_rate": 1.812619502868069e-05,
+      "loss": 1.115468406677246,
+      "mean_token_accuracy": 0.7589421957731247,
+      "num_tokens": 121091.0,
+      "step": 50
+    },
+    {
+      "entropy": 1.1415159791707992,
+      "epoch": 0.10521281683405069,
+      "grad_norm": 0.7876622676849365,
+      "learning_rate": 1.7934990439770558e-05,
+      "loss": 1.2450020790100098,
+      "mean_token_accuracy": 0.7546127334237098,
+      "num_tokens": 133280.0,
+      "step": 55
+    },
+    {
+      "entropy": 1.0959452569484711,
+      "epoch": 0.11477761836441894,
+      "grad_norm": 0.7705210447311401,
+      "learning_rate": 1.774378585086042e-05,
+      "loss": 1.2061676979064941,
+      "mean_token_accuracy": 0.760649724304676,
+      "num_tokens": 145553.0,
+      "step": 60
+    },
+    {
+      "entropy": 1.0722251057624816,
+      "epoch": 0.12434241989478718,
+      "grad_norm": 0.8483577966690063,
+      "learning_rate": 1.755258126195029e-05,
+      "loss": 1.074929904937744,
+      "mean_token_accuracy": 0.7646624684333801,
+      "num_tokens": 156510.0,
+      "step": 65
+    },
+    {
+      "entropy": 1.0666887134313583,
+      "epoch": 0.13390722142515543,
+      "grad_norm": 1.1835429668426514,
+      "learning_rate": 1.7361376673040153e-05,
+      "loss": 1.0667908668518067,
+      "mean_token_accuracy": 0.7600223839282989,
+      "num_tokens": 167771.0,
+      "step": 70
+    },
+    {
+      "entropy": 1.1028031438589097,
+      "epoch": 0.14347202295552366,
+      "grad_norm": 0.8335422277450562,
+      "learning_rate": 1.717017208413002e-05,
+      "loss": 1.1114809036254882,
+      "mean_token_accuracy": 0.758377981185913,
+      "num_tokens": 181011.0,
+      "step": 75
+    },
+    {
+      "entropy": 1.0945842400193215,
+      "epoch": 0.15303682448589193,
+      "grad_norm": 0.9592422246932983,
+      "learning_rate": 1.6978967495219887e-05,
+      "loss": 1.2732584953308106,
+      "mean_token_accuracy": 0.7424929365515709,
+      "num_tokens": 194140.0,
+      "step": 80
+    },
+    {
+      "entropy": 1.0649883642792701,
+      "epoch": 0.16260162601626016,
+      "grad_norm": 0.8365988731384277,
+      "learning_rate": 1.678776290630975e-05,
+      "loss": 1.0620740890502929,
+      "mean_token_accuracy": 0.7718699693679809,
+      "num_tokens": 204855.0,
+      "step": 85
+    },
+    {
+      "entropy": 1.1211044788360596,
+      "epoch": 0.17216642754662842,
+      "grad_norm": 0.8917149305343628,
+      "learning_rate": 1.6596558317399618e-05,
+      "loss": 1.1643722534179688,
+      "mean_token_accuracy": 0.7537558972835541,
+      "num_tokens": 215913.0,
+      "step": 90
+    },
+    {
+      "entropy": 1.0351512983441353,
+      "epoch": 0.18173122907699665,
+      "grad_norm": 1.00289785861969,
+      "learning_rate": 1.6405353728489485e-05,
+      "loss": 1.0665692329406737,
+      "mean_token_accuracy": 0.7725299909710884,
+      "num_tokens": 226093.0,
+      "step": 95
+    },
+    {
+      "entropy": 1.1421371474862099,
+      "epoch": 0.1912960306073649,
+      "grad_norm": 0.8887011408805847,
+      "learning_rate": 1.6214149139579353e-05,
+      "loss": 1.1906902313232421,
+      "mean_token_accuracy": 0.752318462729454,
+      "num_tokens": 239629.0,
+      "step": 100
+    },
+    {
+      "entropy": 1.0672243744134904,
+      "epoch": 0.20086083213773315,
+      "grad_norm": 0.7768471240997314,
+      "learning_rate": 1.6022944550669217e-05,
+      "loss": 1.153627300262451,
+      "mean_token_accuracy": 0.7630036354064942,
+      "num_tokens": 251275.0,
+      "step": 105
+    },
+    {
+      "entropy": 1.0277603179216386,
+      "epoch": 0.21042563366810138,
+      "grad_norm": 0.7372347712516785,
+      "learning_rate": 1.5831739961759084e-05,
+      "loss": 1.0442614555358887,
+      "mean_token_accuracy": 0.7712178140878677,
+      "num_tokens": 264258.0,
+      "step": 110
+    },
+    {
+      "entropy": 1.059542353451252,
+      "epoch": 0.21999043519846964,
+      "grad_norm": 0.8132668137550354,
+      "learning_rate": 1.564053537284895e-05,
+      "loss": 1.0991342544555665,
+      "mean_token_accuracy": 0.7706641733646393,
+      "num_tokens": 276003.0,
+      "step": 115
+    },
+    {
+      "entropy": 1.0556578651070594,
+      "epoch": 0.22955523672883787,
+      "grad_norm": 0.8376557230949402,
+      "learning_rate": 1.5449330783938815e-05,
+      "loss": 1.1164920806884766,
+      "mean_token_accuracy": 0.7708584055304527,
+      "num_tokens": 287129.0,
+      "step": 120
+    },
+    {
+      "entropy": 1.1015635192394257,
+      "epoch": 0.2391200382592061,
+      "grad_norm": 0.7270181775093079,
+      "learning_rate": 1.5258126195028682e-05,
+      "loss": 1.1382445335388183,
+      "mean_token_accuracy": 0.7551500603556633,
+      "num_tokens": 298982.0,
+      "step": 125
+    },
+    {
+      "entropy": 1.15430339127779,
+      "epoch": 0.24868483978957437,
+      "grad_norm": 0.7416822910308838,
+      "learning_rate": 1.5066921606118548e-05,
+      "loss": 1.247046947479248,
+      "mean_token_accuracy": 0.7518013462424278,
+      "num_tokens": 311333.0,
+      "step": 130
+    },
+    {
+      "entropy": 1.1022175028920174,
+      "epoch": 0.2582496413199426,
+      "grad_norm": 0.9017020463943481,
+      "learning_rate": 1.4875717017208415e-05,
+      "loss": 1.143028736114502,
+      "mean_token_accuracy": 0.7529011055827141,
+      "num_tokens": 324232.0,
+      "step": 135
+    },
+    {
+      "entropy": 1.0331857725977898,
+      "epoch": 0.26781444285031086,
+      "grad_norm": 0.8787259459495544,
+      "learning_rate": 1.468451242829828e-05,
+      "loss": 1.022813320159912,
+      "mean_token_accuracy": 0.769845749437809,
+      "num_tokens": 336972.0,
+      "step": 140
+    },
+    {
+      "entropy": 1.1121376857161522,
+      "epoch": 0.2773792443806791,
+      "grad_norm": 0.7721160054206848,
+      "learning_rate": 1.4493307839388146e-05,
+      "loss": 1.1562235832214356,
+      "mean_token_accuracy": 0.7480840817093849,
+      "num_tokens": 350739.0,
+      "step": 145
+    },
+    {
+      "entropy": 1.013964520394802,
+      "epoch": 0.28694404591104733,
+      "grad_norm": 0.8108949065208435,
+      "learning_rate": 1.4302103250478012e-05,
+      "loss": 1.0951598167419434,
+      "mean_token_accuracy": 0.768579913675785,
+      "num_tokens": 362957.0,
+      "step": 150
+    },
+    {
+      "entropy": 1.0007272481918335,
+      "epoch": 0.2965088474414156,
+      "grad_norm": 0.8035533428192139,
+      "learning_rate": 1.4110898661567877e-05,
+      "loss": 1.0634474754333496,
+      "mean_token_accuracy": 0.7801168739795685,
+      "num_tokens": 374885.0,
+      "step": 155
+    },
+    {
+      "entropy": 0.9950303554534912,
+      "epoch": 0.30607364897178385,
+      "grad_norm": 0.855779767036438,
+      "learning_rate": 1.3919694072657746e-05,
+      "loss": 1.0329642295837402,
+      "mean_token_accuracy": 0.7754173740744591,
+      "num_tokens": 386604.0,
+      "step": 160
+    },
+    {
+      "entropy": 1.129347711801529,
+      "epoch": 0.31563845050215206,
+      "grad_norm": 0.8067175149917603,
+      "learning_rate": 1.3728489483747612e-05,
+      "loss": 1.2617739677429198,
+      "mean_token_accuracy": 0.752812922000885,
+      "num_tokens": 400667.0,
+      "step": 165
+    },
+    {
+      "entropy": 1.0334825284779072,
+      "epoch": 0.3252032520325203,
+      "grad_norm": 0.7753680348396301,
+      "learning_rate": 1.3537284894837477e-05,
+      "loss": 1.0933710098266602,
+      "mean_token_accuracy": 0.7654023811221122,
+      "num_tokens": 413697.0,
+      "step": 170
+    },
+    {
+      "entropy": 1.0722475484013558,
+      "epoch": 0.3347680535628886,
+      "grad_norm": 1.0841805934906006,
+      "learning_rate": 1.3346080305927343e-05,
+      "loss": 1.123507022857666,
+      "mean_token_accuracy": 0.7632338643074036,
+      "num_tokens": 426032.0,
+      "step": 175
+    },
+    {
+      "entropy": 1.114607611298561,
+      "epoch": 0.34433285509325684,
+      "grad_norm": 0.7770711779594421,
+      "learning_rate": 1.315487571701721e-05,
+      "loss": 1.1765986442565919,
+      "mean_token_accuracy": 0.7503737643361091,
+      "num_tokens": 439617.0,
+      "step": 180
+    },
+    {
+      "entropy": 1.106437611579895,
+      "epoch": 0.35389765662362505,
+      "grad_norm": 0.6629571914672852,
+      "learning_rate": 1.2963671128107076e-05,
+      "loss": 1.1976131439208983,
+      "mean_token_accuracy": 0.7521042451262474,
+      "num_tokens": 454892.0,
+      "step": 185
+    },
+    {
+      "entropy": 1.0261308953166008,
+      "epoch": 0.3634624581539933,
+      "grad_norm": 0.7957693934440613,
+      "learning_rate": 1.2772466539196941e-05,
+      "loss": 1.0424925804138183,
+      "mean_token_accuracy": 0.7689518257975578,
+      "num_tokens": 466729.0,
+      "step": 190
+    },
+    {
+      "entropy": 1.0590496703982353,
+      "epoch": 0.37302725968436157,
+      "grad_norm": 0.774701714515686,
+      "learning_rate": 1.2581261950286807e-05,
+      "loss": 1.1352399826049804,
+      "mean_token_accuracy": 0.7628853857517243,
+      "num_tokens": 478835.0,
+      "step": 195
+    },
+    {
+      "entropy": 1.0802394181489945,
+      "epoch": 0.3825920612147298,
+      "grad_norm": 0.8261803388595581,
+      "learning_rate": 1.2390057361376676e-05,
+      "loss": 1.1389860153198241,
+      "mean_token_accuracy": 0.7560409680008888,
+      "num_tokens": 490990.0,
+      "step": 200
+    },
+    {
+      "entropy": 1.0673254057765007,
+      "epoch": 0.39215686274509803,
+      "grad_norm": 0.8868000507354736,
+      "learning_rate": 1.2198852772466541e-05,
+      "loss": 1.123587417602539,
+      "mean_token_accuracy": 0.7629709094762802,
+      "num_tokens": 504051.0,
+      "step": 205
+    },
+    {
+      "entropy": 1.0202552631497384,
+      "epoch": 0.4017216642754663,
+      "grad_norm": 0.6817913055419922,
+      "learning_rate": 1.2007648183556407e-05,
+      "loss": 1.1182997703552247,
+      "mean_token_accuracy": 0.762927383184433,
+      "num_tokens": 519436.0,
+      "step": 210
+    },
+    {
+      "entropy": 0.9513417720794678,
+      "epoch": 0.41128646580583456,
+      "grad_norm": 0.7766347527503967,
+      "learning_rate": 1.1816443594646272e-05,
+      "loss": 0.9787687301635742,
+      "mean_token_accuracy": 0.7758817136287689,
+      "num_tokens": 531451.0,
+      "step": 215
+    },
+    {
+      "entropy": 1.021947756409645,
+      "epoch": 0.42085126733620276,
+      "grad_norm": 0.768451988697052,
+      "learning_rate": 1.162523900573614e-05,
+      "loss": 1.0611325263977052,
+      "mean_token_accuracy": 0.7702895030379295,
+      "num_tokens": 543781.0,
+      "step": 220
+    },
+    {
+      "entropy": 0.9991461083292961,
+      "epoch": 0.430416068866571,
+      "grad_norm": 0.8404542803764343,
+      "learning_rate": 1.1434034416826005e-05,
+      "loss": 1.0780817985534668,
+      "mean_token_accuracy": 0.7719197928905487,
+      "num_tokens": 557321.0,
+      "step": 225
+    },
+    {
+      "entropy": 1.024046105146408,
+      "epoch": 0.4399808703969393,
+      "grad_norm": 0.8072119355201721,
+      "learning_rate": 1.124282982791587e-05,
+      "loss": 1.039454174041748,
+      "mean_token_accuracy": 0.774782894551754,
+      "num_tokens": 568606.0,
+      "step": 230
+    },
+    {
+      "entropy": 1.032619222998619,
+      "epoch": 0.4495456719273075,
+      "grad_norm": 0.8191614747047424,
+      "learning_rate": 1.1051625239005736e-05,
+      "loss": 1.0365934371948242,
+      "mean_token_accuracy": 0.7662370949983597,
+      "num_tokens": 580273.0,
+      "step": 235
+    },
+    {
+      "entropy": 0.9828310400247574,
+      "epoch": 0.45911047345767575,
+      "grad_norm": 0.7097588777542114,
+      "learning_rate": 1.0860420650095602e-05,
+      "loss": 0.9995201110839844,
+      "mean_token_accuracy": 0.7850929573178291,
+      "num_tokens": 591076.0,
+      "step": 240
+    },
+    {
+      "entropy": 1.0766605302691459,
+      "epoch": 0.468675274988044,
+      "grad_norm": 0.7697131037712097,
+      "learning_rate": 1.066921606118547e-05,
+      "loss": 1.2869989395141601,
+      "mean_token_accuracy": 0.7609236977994442,
+      "num_tokens": 604888.0,
+      "step": 245
+    },
+    {
+      "entropy": 1.0177167639136315,
+      "epoch": 0.4782400765184122,
+      "grad_norm": 0.8414968848228455,
+      "learning_rate": 1.0478011472275336e-05,
+      "loss": 1.049800205230713,
+      "mean_token_accuracy": 0.7701354175806046,
+      "num_tokens": 617663.0,
+      "step": 250
+    },
+    {
+      "entropy": 1.077294373512268,
+      "epoch": 0.4878048780487805,
+      "grad_norm": 0.738322913646698,
+      "learning_rate": 1.0286806883365202e-05,
+      "loss": 1.1312056541442872,
+      "mean_token_accuracy": 0.7647411957383156,
+      "num_tokens": 630029.0,
+      "step": 255
+    },
+    {
+      "entropy": 1.047728145122528,
+      "epoch": 0.49736967957914874,
+      "grad_norm": 0.8730350732803345,
+      "learning_rate": 1.0095602294455067e-05,
+      "loss": 1.067624855041504,
+      "mean_token_accuracy": 0.7657899752259254,
+      "num_tokens": 641327.0,
+      "step": 260
+    },
+    {
+      "entropy": 1.0261075347661972,
+      "epoch": 0.506934481109517,
+      "grad_norm": 0.7349242568016052,
+      "learning_rate": 9.904397705544934e-06,
+      "loss": 1.0252650260925293,
+      "mean_token_accuracy": 0.7747346982359886,
+      "num_tokens": 652494.0,
+      "step": 265
+    },
+    {
+      "entropy": 1.0701305642724037,
+      "epoch": 0.5164992826398852,
+      "grad_norm": 0.9694271087646484,
+      "learning_rate": 9.7131931166348e-06,
+      "loss": 1.1342878341674805,
+      "mean_token_accuracy": 0.7671242266893387,
+      "num_tokens": 664262.0,
+      "step": 270
+    },
+    {
+      "entropy": 1.002896775305271,
+      "epoch": 0.5260640841702535,
+      "grad_norm": 0.6827502250671387,
+      "learning_rate": 9.521988527724666e-06,
+      "loss": 1.0188935279846192,
+      "mean_token_accuracy": 0.770347212255001,
+      "num_tokens": 676361.0,
+      "step": 275
+    },
+    {
+      "entropy": 1.095559985935688,
+      "epoch": 0.5356288857006217,
+      "grad_norm": 0.8086148500442505,
+      "learning_rate": 9.330783938814533e-06,
+      "loss": 1.13911771774292,
+      "mean_token_accuracy": 0.7597231984138488,
+      "num_tokens": 687676.0,
+      "step": 280
+    },
+    {
+      "entropy": 1.0357700750231742,
+      "epoch": 0.5451936872309899,
+      "grad_norm": 0.8582542538642883,
+      "learning_rate": 9.139579349904398e-06,
+      "loss": 1.1252613067626953,
+      "mean_token_accuracy": 0.7737304389476776,
+      "num_tokens": 701525.0,
+      "step": 285
+    },
+    {
+      "entropy": 1.026543913781643,
+      "epoch": 0.5547584887613582,
+      "grad_norm": 0.7670846581459045,
+      "learning_rate": 8.948374760994266e-06,
+      "loss": 1.0703323364257813,
+      "mean_token_accuracy": 0.7629063084721566,
+      "num_tokens": 714679.0,
+      "step": 290
+    },
+    {
+      "entropy": 1.0737508490681649,
+      "epoch": 0.5643232902917265,
+      "grad_norm": 0.8508143424987793,
+      "learning_rate": 8.757170172084131e-06,
+      "loss": 1.124532699584961,
+      "mean_token_accuracy": 0.761698454618454,
+      "num_tokens": 727453.0,
+      "step": 295
+    },
+    {
+      "entropy": 1.0055931195616723,
+      "epoch": 0.5738880918220947,
+      "grad_norm": 0.7749042510986328,
+      "learning_rate": 8.565965583173997e-06,
+      "loss": 1.000310230255127,
+      "mean_token_accuracy": 0.7788769990205765,
+      "num_tokens": 738987.0,
+      "step": 300
+    },
+    {
+      "entropy": 1.1338727325201035,
+      "epoch": 0.583452893352463,
+      "grad_norm": 0.7675312161445618,
+      "learning_rate": 8.374760994263862e-06,
+      "loss": 1.240555763244629,
+      "mean_token_accuracy": 0.7499270647764206,
+      "num_tokens": 752986.0,
+      "step": 305
+    },
+    {
+      "entropy": 1.116484060883522,
+      "epoch": 0.5930176948828312,
+      "grad_norm": 0.6953423023223877,
+      "learning_rate": 8.18355640535373e-06,
+      "loss": 1.3077027320861816,
+      "mean_token_accuracy": 0.7496477924287319,
+      "num_tokens": 766987.0,
+      "step": 310
+    },
+    {
+      "entropy": 1.0163599342107772,
+      "epoch": 0.6025824964131994,
+      "grad_norm": 1.0213820934295654,
+      "learning_rate": 7.992351816443595e-06,
+      "loss": 1.0718021392822266,
+      "mean_token_accuracy": 0.7725750356912613,
+      "num_tokens": 779039.0,
+      "step": 315
+    },
+    {
+      "entropy": 1.1485976949334145,
+      "epoch": 0.6121472979435677,
+      "grad_norm": 0.7093721032142639,
+      "learning_rate": 7.80114722753346e-06,
+      "loss": 1.3566171646118164,
+      "mean_token_accuracy": 0.7532788991928101,
+      "num_tokens": 793359.0,
+      "step": 320
+    },
+    {
+      "entropy": 1.026223848760128,
+      "epoch": 0.6217120994739359,
+      "grad_norm": 0.8051843643188477,
+      "learning_rate": 7.609942638623328e-06,
+      "loss": 1.0955821037292481,
+      "mean_token_accuracy": 0.7727067768573761,
+      "num_tokens": 805833.0,
+      "step": 325
+    },
+    {
+      "entropy": 1.0360631138086318,
+      "epoch": 0.6312769010043041,
+      "grad_norm": 0.8939955234527588,
+      "learning_rate": 7.418738049713193e-06,
+      "loss": 1.0926706314086914,
+      "mean_token_accuracy": 0.7757175087928772,
+      "num_tokens": 817359.0,
+      "step": 330
+    },
+    {
+      "entropy": 1.0702629446983338,
+      "epoch": 0.6408417025346724,
+      "grad_norm": 0.869463324546814,
+      "learning_rate": 7.22753346080306e-06,
+      "loss": 1.2999201774597169,
+      "mean_token_accuracy": 0.7645014680922031,
+      "num_tokens": 830691.0,
+      "step": 335
+    },
+    {
+      "entropy": 0.9756309568881989,
+      "epoch": 0.6504065040650406,
+      "grad_norm": 0.7495210766792297,
+      "learning_rate": 7.036328871892925e-06,
+      "loss": 0.9702842712402344,
+      "mean_token_accuracy": 0.7790682971477508,
+      "num_tokens": 844087.0,
+      "step": 340
+    },
+    {
+      "entropy": 1.0872225597500802,
+      "epoch": 0.6599713055954088,
+      "grad_norm": 0.7233891487121582,
+      "learning_rate": 6.8451242829827925e-06,
+      "loss": 1.2287683486938477,
+      "mean_token_accuracy": 0.7564080432057381,
+      "num_tokens": 858098.0,
+      "step": 345
+    },
+    {
+      "entropy": 0.9561280161142349,
+      "epoch": 0.6695361071257772,
+      "grad_norm": 0.8276292085647583,
+      "learning_rate": 6.653919694072658e-06,
+      "loss": 0.9720723152160644,
+      "mean_token_accuracy": 0.7823744133114815,
+      "num_tokens": 869842.0,
+      "step": 350
+    },
+    {
+      "entropy": 0.9766073286533355,
+      "epoch": 0.6791009086561454,
+      "grad_norm": 0.7800951600074768,
+      "learning_rate": 6.4627151051625245e-06,
+      "loss": 1.0041848182678224,
+      "mean_token_accuracy": 0.7807797461748123,
+      "num_tokens": 880652.0,
+      "step": 355
+    },
+    {
+      "entropy": 1.040538677573204,
+      "epoch": 0.6886657101865137,
+      "grad_norm": 0.8833601474761963,
+      "learning_rate": 6.27151051625239e-06,
+      "loss": 1.0400735855102539,
+      "mean_token_accuracy": 0.7732439547777176,
+      "num_tokens": 892018.0,
+      "step": 360
+    },
+    {
+      "entropy": 1.0406786598265172,
+      "epoch": 0.6982305117168819,
+      "grad_norm": 0.7317559719085693,
+      "learning_rate": 6.080305927342257e-06,
+      "loss": 1.0959332466125489,
+      "mean_token_accuracy": 0.7657193571329117,
+      "num_tokens": 904435.0,
+      "step": 365
+    },
+    {
+      "entropy": 1.0524535074830055,
+      "epoch": 0.7077953132472501,
+      "grad_norm": 0.7634838819503784,
+      "learning_rate": 5.889101338432123e-06,
+      "loss": 1.081229019165039,
+      "mean_token_accuracy": 0.762605932354927,
+      "num_tokens": 917173.0,
+      "step": 370
+    },
+    {
+      "entropy": 1.0828216448426247,
+      "epoch": 0.7173601147776184,
+      "grad_norm": 0.7817342281341553,
+      "learning_rate": 5.697896749521989e-06,
+      "loss": 1.1142399787902832,
+      "mean_token_accuracy": 0.7619883432984352,
+      "num_tokens": 929853.0,
+      "step": 375
+    },
+    {
+      "entropy": 0.9964733004570008,
+      "epoch": 0.7269249163079866,
+      "grad_norm": 0.8091410994529724,
+      "learning_rate": 5.506692160611855e-06,
+      "loss": 1.0676130294799804,
+      "mean_token_accuracy": 0.7710230126976967,
+      "num_tokens": 942227.0,
+      "step": 380
+    },
+    {
+      "entropy": 1.0303904354572295,
+      "epoch": 0.7364897178383548,
+      "grad_norm": 0.7632353901863098,
+      "learning_rate": 5.315487571701722e-06,
+      "loss": 1.0745953559875487,
+      "mean_token_accuracy": 0.7737195149064064,
+      "num_tokens": 953661.0,
+      "step": 385
+    },
+    {
+      "entropy": 1.0218341886997222,
+      "epoch": 0.7460545193687231,
+      "grad_norm": 0.8082678914070129,
+      "learning_rate": 5.1242829827915875e-06,
+      "loss": 1.050875473022461,
+      "mean_token_accuracy": 0.7697872579097748,
+      "num_tokens": 965285.0,
+      "step": 390
+    },
+    {
+      "entropy": 1.144502431154251,
+      "epoch": 0.7556193208990913,
+      "grad_norm": 0.8677741289138794,
+      "learning_rate": 4.933078393881454e-06,
+      "loss": 1.238006591796875,
+      "mean_token_accuracy": 0.750197383761406,
+      "num_tokens": 976399.0,
+      "step": 395
+    },
+    {
+      "entropy": 1.0361291944980622,
+      "epoch": 0.7651841224294595,
+      "grad_norm": 0.7813594937324524,
+      "learning_rate": 4.7418738049713195e-06,
+      "loss": 1.1359359741210937,
+      "mean_token_accuracy": 0.7724275305867195,
+      "num_tokens": 988985.0,
+      "step": 400
+    },
+    {
+      "entropy": 1.0593642130494119,
+      "epoch": 0.7747489239598279,
+      "grad_norm": 0.8320329189300537,
+      "learning_rate": 4.550669216061186e-06,
+      "loss": 1.1087071418762207,
+      "mean_token_accuracy": 0.7654044702649117,
+      "num_tokens": 1000001.0,
+      "step": 405
+    },
+    {
+      "entropy": 1.0726871281862258,
+      "epoch": 0.7843137254901961,
+      "grad_norm": 0.7686368227005005,
+      "learning_rate": 4.359464627151052e-06,
+      "loss": 1.0674661636352538,
+      "mean_token_accuracy": 0.7652721956372261,
+      "num_tokens": 1011748.0,
+      "step": 410
+    },
+    {
+      "entropy": 1.042534102499485,
+      "epoch": 0.7938785270205643,
+      "grad_norm": 0.7759877443313599,
+      "learning_rate": 4.168260038240919e-06,
+      "loss": 1.0660060882568358,
+      "mean_token_accuracy": 0.7741253584623337,
+      "num_tokens": 1023418.0,
+      "step": 415
+    },
+    {
+      "entropy": 1.0455620855093002,
+      "epoch": 0.8034433285509326,
+      "grad_norm": 0.7897431254386902,
+      "learning_rate": 3.977055449330784e-06,
+      "loss": 1.1257256507873534,
+      "mean_token_accuracy": 0.7716126814484596,
+      "num_tokens": 1036377.0,
+      "step": 420
+    },
+    {
+      "entropy": 1.04324461966753,
+      "epoch": 0.8130081300813008,
+      "grad_norm": 0.8430481553077698,
+      "learning_rate": 3.7858508604206506e-06,
+      "loss": 1.08157320022583,
+      "mean_token_accuracy": 0.7690573886036873,
+      "num_tokens": 1048734.0,
+      "step": 425
+    },
+    {
+      "entropy": 1.0173504278063774,
+      "epoch": 0.8225729316116691,
+      "grad_norm": 0.8661599159240723,
+      "learning_rate": 3.594646271510517e-06,
+      "loss": 1.0198513984680175,
+      "mean_token_accuracy": 0.7801295846700669,
+      "num_tokens": 1059822.0,
+      "step": 430
+    },
+    {
+      "entropy": 1.0838855013251305,
+      "epoch": 0.8321377331420373,
+      "grad_norm": 0.7001233100891113,
+      "learning_rate": 3.4034416826003826e-06,
+      "loss": 1.1792843818664551,
+      "mean_token_accuracy": 0.7706752866506577,
+      "num_tokens": 1072394.0,
+      "step": 435
+    },
+    {
+      "entropy": 1.0104505866765976,
+      "epoch": 0.8417025346724055,
+      "grad_norm": 0.7979400753974915,
+      "learning_rate": 3.2122370936902485e-06,
+      "loss": 1.0238704681396484,
+      "mean_token_accuracy": 0.7801917403936386,
+      "num_tokens": 1082875.0,
+      "step": 440
+    },
+    {
+      "entropy": 1.0283882372081279,
+      "epoch": 0.8512673362027738,
+      "grad_norm": 0.6927244067192078,
+      "learning_rate": 3.021032504780115e-06,
+      "loss": 1.071757698059082,
+      "mean_token_accuracy": 0.7709943309426308,
+      "num_tokens": 1094437.0,
+      "step": 445
+    },
+    {
+      "entropy": 1.0193706631660462,
+      "epoch": 0.860832137733142,
+      "grad_norm": 0.8384391665458679,
+      "learning_rate": 2.829827915869981e-06,
+      "loss": 1.0264280319213868,
+      "mean_token_accuracy": 0.7784072667360306,
+      "num_tokens": 1105366.0,
+      "step": 450
+    },
+    {
+      "entropy": 1.0828002728521824,
+      "epoch": 0.8703969392635102,
+      "grad_norm": 0.7474755644798279,
+      "learning_rate": 2.6386233269598473e-06,
+      "loss": 1.1083474159240723,
+      "mean_token_accuracy": 0.7668004184961319,
+      "num_tokens": 1118212.0,
+      "step": 455
+    },
+    {
+      "entropy": 0.9301411956548691,
+      "epoch": 0.8799617407938786,
+      "grad_norm": 0.6937381029129028,
+      "learning_rate": 2.4474187380497133e-06,
+      "loss": 0.9397728919982911,
+      "mean_token_accuracy": 0.7968559697270393,
+      "num_tokens": 1128617.0,
+      "step": 460
+    },
+    {
+      "entropy": 1.0196322917938232,
+      "epoch": 0.8895265423242468,
+      "grad_norm": 0.8852613568305969,
+      "learning_rate": 2.2562141491395797e-06,
+      "loss": 1.010320281982422,
+      "mean_token_accuracy": 0.7687119588255882,
+      "num_tokens": 1141378.0,
+      "step": 465
+    },
+    {
+      "entropy": 0.9731274515390396,
+      "epoch": 0.899091343854615,
+      "grad_norm": 0.7555264234542847,
+      "learning_rate": 2.0650095602294456e-06,
+      "loss": 0.9504165649414062,
+      "mean_token_accuracy": 0.7799475193023682,
+      "num_tokens": 1154384.0,
+      "step": 470
+    },
+    {
+      "entropy": 0.9584303557872772,
+      "epoch": 0.9086561453849833,
+      "grad_norm": 0.7380809783935547,
+      "learning_rate": 1.8738049713193118e-06,
+      "loss": 0.9844732284545898,
+      "mean_token_accuracy": 0.7849968507885933,
+      "num_tokens": 1167856.0,
+      "step": 475
+    },
+    {
+      "entropy": 1.0079337880015373,
+      "epoch": 0.9182209469153515,
+      "grad_norm": 0.904389500617981,
+      "learning_rate": 1.682600382409178e-06,
+      "loss": 1.105099868774414,
+      "mean_token_accuracy": 0.7668009698390961,
+      "num_tokens": 1181878.0,
+      "step": 480
+    },
+    {
+      "entropy": 1.0234097734093666,
+      "epoch": 0.9277857484457197,
+      "grad_norm": 0.7131940722465515,
+      "learning_rate": 1.4913957934990442e-06,
+      "loss": 1.0956622123718263,
+      "mean_token_accuracy": 0.7704735204577446,
+      "num_tokens": 1195079.0,
+      "step": 485
+    },
+    {
+      "entropy": 1.0721113547682761,
+      "epoch": 0.937350549976088,
+      "grad_norm": 0.8159518241882324,
+      "learning_rate": 1.3001912045889104e-06,
+      "loss": 1.1428511619567872,
+      "mean_token_accuracy": 0.7590293809771538,
+      "num_tokens": 1207788.0,
+      "step": 490
+    },
+    {
+      "entropy": 1.0188337028026582,
+      "epoch": 0.9469153515064562,
+      "grad_norm": 0.7726689577102661,
+      "learning_rate": 1.1089866156787763e-06,
+      "loss": 1.048692226409912,
+      "mean_token_accuracy": 0.7697815522551537,
+      "num_tokens": 1220932.0,
+      "step": 495
+    },
+    {
+      "entropy": 1.020039089769125,
+      "epoch": 0.9564801530368244,
+      "grad_norm": 0.7452772259712219,
+      "learning_rate": 9.177820267686425e-07,
+      "loss": 1.0072999954223634,
+      "mean_token_accuracy": 0.7695159554481507,
+      "num_tokens": 1233364.0,
+      "step": 500
+    },
+    {
+      "entropy": 1.0744403377175331,
+      "epoch": 0.9660449545671927,
+      "grad_norm": 0.7994968295097351,
+      "learning_rate": 7.265774378585087e-07,
+      "loss": 1.1400054931640624,
+      "mean_token_accuracy": 0.7660253509879112,
+      "num_tokens": 1246728.0,
+      "step": 505
+    },
+    {
+      "entropy": 1.0310125365853309,
+      "epoch": 0.975609756097561,
+      "grad_norm": 0.9675761461257935,
+      "learning_rate": 5.353728489483748e-07,
+      "loss": 1.0664387702941895,
+      "mean_token_accuracy": 0.7704203218221665,
+      "num_tokens": 1259761.0,
+      "step": 510
+    },
+    {
+      "entropy": 1.021379828453064,
+      "epoch": 0.9851745576279293,
+      "grad_norm": 0.7971519231796265,
+      "learning_rate": 3.441682600382409e-07,
+      "loss": 1.0582043647766113,
+      "mean_token_accuracy": 0.7790713414549828,
+      "num_tokens": 1271014.0,
+      "step": 515
+    },
+    {
+      "entropy": 1.0643135994672774,
+      "epoch": 0.9947393591582975,
+      "grad_norm": 0.7419437766075134,
+      "learning_rate": 1.5296367112810708e-07,
+      "loss": 1.177199363708496,
+      "mean_token_accuracy": 0.7608937039971352,
+      "num_tokens": 1284025.0,
+      "step": 520
+    }
+  ],
+  "logging_steps": 5,
+  "max_steps": 523,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 1.4485774665627648e+16,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-523/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d600ef15aeca9cb033ec7eb02703786ef929fa1936d277bd8092201ec3c7519e
+size 5713

tokenizer_config.json

@@ -20,15 +20,11 @@
     "<|image_pad|>",
     "<|video_pad|>"
   ],
-  "is_local": true,
+  "is_local": false,
   "local_files_only": false,
-  "max_length": 2048,
-  "model_max_length": 2048,
+  "model_max_length": 131072,
   "pad_token": "<|im_end|>",
   "split_special_tokens": false,
-  "stride": 0,
   "tokenizer_class": "Qwen2Tokenizer",
-  "truncation_side": "right",
-  "truncation_strategy": "longest_first",
   "unk_token": null
 }

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:558eab67b87a0b35d635eec1a2be7369c4ee45796a669b49a968ff8cec3eaad3
+oid sha256:d600ef15aeca9cb033ec7eb02703786ef929fa1936d277bd8092201ec3c7519e
 size 5713

training_metrics.csv

@@ -0,0 +1,105 @@
+step,epoch,loss,learning_rate
+5,0.0096,3.3127,1.98e-05
+10,0.0191,2.3781,1.97e-05
+15,0.0287,2.0008,1.95e-05
+20,0.0383,1.6723,1.93e-05
+25,0.0478,1.5856,1.91e-05
+30,0.0574,1.4701,1.89e-05
+35,0.0670,1.4895,1.87e-05
+40,0.0765,1.2474,1.85e-05
+45,0.0861,1.1584,1.83e-05
+50,0.0956,1.1155,1.81e-05
+55,0.1052,1.2450,1.79e-05
+60,0.1148,1.2062,1.77e-05
+65,0.1243,1.0749,1.76e-05
+70,0.1339,1.0668,1.74e-05
+75,0.1435,1.1115,1.72e-05
+80,0.1530,1.2733,1.70e-05
+85,0.1626,1.0621,1.68e-05
+90,0.1722,1.1644,1.66e-05
+95,0.1817,1.0666,1.64e-05
+100,0.1913,1.1907,1.62e-05
+105,0.2009,1.1536,1.60e-05
+110,0.2104,1.0443,1.58e-05
+115,0.2200,1.0991,1.56e-05
+120,0.2296,1.1165,1.54e-05
+125,0.2391,1.1382,1.53e-05
+130,0.2487,1.2470,1.51e-05
+135,0.2582,1.1430,1.49e-05
+140,0.2678,1.0228,1.47e-05
+145,0.2774,1.1562,1.45e-05
+150,0.2869,1.0952,1.43e-05
+155,0.2965,1.0634,1.41e-05
+160,0.3061,1.0330,1.39e-05
+165,0.3156,1.2618,1.37e-05
+170,0.3252,1.0934,1.35e-05
+175,0.3348,1.1235,1.33e-05
+180,0.3443,1.1766,1.32e-05
+185,0.3539,1.1976,1.30e-05
+190,0.3635,1.0425,1.28e-05
+195,0.3730,1.1352,1.26e-05
+200,0.3826,1.1390,1.24e-05
+205,0.3922,1.1236,1.22e-05
+210,0.4017,1.1183,1.20e-05
+215,0.4113,0.9788,1.18e-05
+220,0.4209,1.0611,1.16e-05
+225,0.4304,1.0781,1.14e-05
+230,0.4400,1.0395,1.12e-05
+235,0.4495,1.0366,1.11e-05
+240,0.4591,0.9995,1.09e-05
+245,0.4687,1.2870,1.07e-05
+250,0.4782,1.0498,1.05e-05
+255,0.4878,1.1312,1.03e-05
+260,0.4974,1.0676,1.01e-05
+265,0.5069,1.0253,9.90e-06
+270,0.5165,1.1343,9.71e-06
+275,0.5261,1.0189,9.52e-06
+280,0.5356,1.1391,9.33e-06
+285,0.5452,1.1253,9.14e-06
+290,0.5548,1.0703,8.95e-06
+295,0.5643,1.1245,8.76e-06
+300,0.5739,1.0003,8.57e-06
+305,0.5835,1.2406,8.37e-06
+310,0.5930,1.3077,8.18e-06
+315,0.6026,1.0718,7.99e-06
+320,0.6121,1.3566,7.80e-06
+325,0.6217,1.0956,7.61e-06
+330,0.6313,1.0927,7.42e-06
+335,0.6408,1.2999,7.23e-06
+340,0.6504,0.9703,7.04e-06
+345,0.6600,1.2288,6.85e-06
+350,0.6695,0.9721,6.65e-06
+355,0.6791,1.0042,6.46e-06
+360,0.6887,1.0401,6.27e-06
+365,0.6982,1.0959,6.08e-06
+370,0.7078,1.0812,5.89e-06
+375,0.7174,1.1142,5.70e-06
+380,0.7269,1.0676,5.51e-06
+385,0.7365,1.0746,5.32e-06
+390,0.7461,1.0509,5.12e-06
+395,0.7556,1.2380,4.93e-06
+400,0.7652,1.1359,4.74e-06
+405,0.7747,1.1087,4.55e-06
+410,0.7843,1.0675,4.36e-06
+415,0.7939,1.0660,4.17e-06
+420,0.8034,1.1257,3.98e-06
+425,0.8130,1.0816,3.79e-06
+430,0.8226,1.0199,3.59e-06
+435,0.8321,1.1793,3.40e-06
+440,0.8417,1.0239,3.21e-06
+445,0.8513,1.0718,3.02e-06
+450,0.8608,1.0264,2.83e-06
+455,0.8704,1.1083,2.64e-06
+460,0.8800,0.9398,2.45e-06
+465,0.8895,1.0103,2.26e-06
+470,0.8991,0.9504,2.07e-06
+475,0.9087,0.9845,1.87e-06
+480,0.9182,1.1051,1.68e-06
+485,0.9278,1.0957,1.49e-06
+490,0.9374,1.1429,1.30e-06
+495,0.9469,1.0487,1.11e-06
+500,0.9565,1.0073,9.18e-07
+505,0.9660,1.1400,7.27e-07
+510,0.9756,1.0664,5.35e-07
+515,0.9852,1.0582,3.44e-07
+520,0.9947,1.1772,1.53e-07