End of training

README.md

@@ -8,7 +8,6 @@ tags:
 - grpo
 - gensyn
 - I am mimic singing hummingbird
-- unsloth
 - trl
 licence: license
 ---
@@ -39,9 +38,9 @@ This model was trained with GRPO, a method introduced in [DeepSeekMath: Pushing
 ### Framework versions
 
 - TRL: 0.15.2
-- Transformers: 4.51.3
-- Pytorch: 2.6.0
-- Datasets: 3.5.1
+- Transformers: 4.52.3
+- Pytorch: 2.5.1
+- Datasets: 3.6.0
 - Tokenizers: 0.21.1
 
 ## Citations

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 2.922929610349456e-06,
-    "train_runtime": 1036.4117,
-    "train_samples": 13,
-    "train_samples_per_second": 0.309,
-    "train_steps_per_second": 0.019
+    "train_loss": 0.07442864710465073,
+    "train_runtime": 243.8844,
+    "train_samples": 11,
+    "train_samples_per_second": 1.312,
+    "train_steps_per_second": 0.082
 }

chat_template.jinja

@@ -0,0 +1,54 @@
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- messages[0]['content'] }}
+    {%- else %}
+        {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}
+    {%- endif %}
+    {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }}
+    {%- else %}
+        {{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- for message in messages %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %}
+        {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {{- '<|im_start|>' + message.role }}
+        {%- if message.content %}
+            {{- '\n' + message.content }}
+        {%- endif %}
+        {%- for tool_call in message.tool_calls %}
+            {%- if tool_call.function is defined %}
+                {%- set tool_call = tool_call.function %}
+            {%- endif %}
+            {{- '\n<tool_call>\n{"name": "' }}
+            {{- tool_call.name }}
+            {{- '", "arguments": ' }}
+            {{- tool_call.arguments | tojson }}
+            {{- '}\n</tool_call>' }}
+        {%- endfor %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- message.content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+{%- endif %}

config.json

@@ -21,7 +21,7 @@
   "sliding_window": 32768,
   "tie_word_embeddings": true,
   "torch_dtype": "float32",
-  "transformers_version": "4.51.3",
+  "transformers_version": "4.52.3",
   "use_cache": true,
   "use_sliding_window": false,
   "vocab_size": 151936

generation_config.json

@@ -10,5 +10,5 @@
   "temperature": 0.7,
   "top_k": 20,
   "top_p": 0.8,
-  "transformers_version": "4.51.3"
+  "transformers_version": "4.52.3"
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0c814299a2126109cb35636445579b55a892b6bfc08cd24d1de4d9b949d73d3e
+oid sha256:35e37f62117f3576f83d0ef12e9f697dfa06b8418fd6812c3761a380eabcbb6f
 size 1976163472

tokenizer_config.json

@@ -195,7 +195,6 @@
     "<|video_pad|>"
   ],
   "bos_token": null,
-  "chat_template": "{%- if tools %}\n    {{- '<|im_start|>system\\n' }}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- messages[0]['content'] }}\n    {%- else %}\n        {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}\n    {%- endif %}\n    {{- \"\\n\\n# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within <tools></tools> XML tags:\\n<tools>\" }}\n    {%- for tool in tools %}\n        {{- \"\\n\" }}\n        {{- tool | tojson }}\n    {%- endfor %}\n    {{- \"\\n</tools>\\n\\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\\n<tool_call>\\n{\\\"name\\\": <function-name>, \\\"arguments\\\": <args-json-object>}\\n</tool_call><|im_end|>\\n\" }}\n{%- else %}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- '<|im_start|>system\\n' + messages[0]['content'] + '<|im_end|>\\n' }}\n    {%- else %}\n        {{- '<|im_start|>system\\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\\n' }}\n    {%- endif %}\n{%- endif %}\n{%- for message in messages %}\n    {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) or (message.role == \"assistant\" and not message.tool_calls) %}\n        {{- '<|im_start|>' + message.role + '\\n' + message.content + '<|im_end|>' + '\\n' }}\n    {%- elif message.role == \"assistant\" %}\n        {{- '<|im_start|>' + message.role }}\n        {%- if message.content %}\n            {{- '\\n' + message.content }}\n        {%- endif %}\n        {%- for tool_call in message.tool_calls %}\n            {%- if tool_call.function is defined %}\n                {%- set tool_call = tool_call.function %}\n            {%- endif %}\n            {{- '\\n<tool_call>\\n{\"name\": \"' }}\n            {{- tool_call.name }}\n            {{- '\", \"arguments\": ' }}\n            {{- tool_call.arguments | tojson }}\n            {{- '}\\n</tool_call>' }}\n        {%- endfor %}\n        {{- '<|im_end|>\\n' }}\n    {%- elif message.role == \"tool\" %}\n        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != \"tool\") %}\n            {{- '<|im_start|>user' }}\n        {%- endif %}\n        {{- '\\n<tool_response>\\n' }}\n        {{- message.content }}\n        {{- '\\n</tool_response>' }}\n        {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n            {{- '<|im_end|>\\n' }}\n        {%- endif %}\n    {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {{- '<|im_start|>assistant\\n' }}\n{%- endif %}\n",
   "clean_up_tokenization_spaces": false,
   "eos_token": "<|im_end|>",
   "errors": "replace",

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 2.922929610349456e-06,
-    "train_runtime": 1036.4117,
-    "train_samples": 13,
-    "train_samples_per_second": 0.309,
-    "train_steps_per_second": 0.019
+    "train_loss": 0.07442864710465073,
+    "train_runtime": 243.8844,
+    "train_samples": 11,
+    "train_samples_per_second": 1.312,
+    "train_steps_per_second": 0.082
 }

trainer_state.json

@@ -2,7 +2,7 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 5.0,
+  "epoch": 6.7272727272727275,
   "eval_steps": 500,
   "global_step": 20,
   "is_hyper_param_search": false,
@@ -10,203 +10,203 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 294.40625,
-      "epoch": 0.6153846153846154,
-      "grad_norm": 64.38786315917969,
-      "kl": 0.005848381006217096,
+      "completion_length": 356.53125,
+      "epoch": 0.7272727272727273,
+      "grad_norm": 9.795001029968262,
+      "kl": 0.0,
       "learning_rate": 5e-07,
-      "loss": 0.0,
-      "reward": 0.4227320160716772,
-      "reward_std": 0.47057422529906034,
+      "loss": -0.0,
+      "reward": 0.18785241851583123,
+      "reward_std": 0.16672803153051063,
       "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.2132007679902017,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.13316492084413767,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.0845312518067658,
+      "rewards/xmlcount_reward_func": 0.0546875,
       "step": 2
     },
     {
-      "completion_length": 351.45,
-      "epoch": 1.0,
-      "grad_norm": 0.6671139001846313,
-      "kl": 0.002078611357137561,
+      "completion_length": 256.97619356427873,
+      "epoch": 1.3636363636363638,
+      "grad_norm": 73.25029754638672,
+      "kl": 0.012838317580255014,
       "learning_rate": 4.864543104251586e-07,
       "loss": 0.0,
-      "reward": 0.7497688382863998,
-      "reward_std": 0.7418052375316619,
+      "reward": 0.2553516466702734,
+      "reward_std": 0.250129872666938,
       "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.2,
-      "rewards/question_recreation_reward_func": 0.3515688359737396,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.17753021844795772,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.19820000231266022,
+      "rewards/xmlcount_reward_func": 0.07782143035105296,
       "step": 4
     },
     {
-      "completion_length": 377.5,
-      "epoch": 1.6153846153846154,
-      "grad_norm": 3.9672529697418213,
-      "kl": 0.0011851612143800594,
+      "completion_length": 388.33333914620533,
+      "epoch": 2.0,
+      "grad_norm": 5.74370813369751,
+      "kl": 0.017486302215339884,
       "learning_rate": 4.472851273490984e-07,
       "loss": 0.0,
-      "reward": 0.22314574662595987,
-      "reward_std": 0.495018282905221,
+      "reward": 0.09297734126448631,
+      "reward_std": 0.09196118717747075,
       "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.20742700988193974,
+      "rewards/question_recreation_reward_func": 0.10204877012542315,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.015718746930360794,
+      "rewards/xmlcount_reward_func": -0.009071428328752518,
       "step": 6
     },
     {
-      "completion_length": 434.2,
-      "epoch": 2.0,
-      "grad_norm": 1.1096526384353638,
-      "kl": 0.010509069002000614,
+      "completion_length": 337.25,
+      "epoch": 2.7272727272727275,
+      "grad_norm": 56.21742630004883,
+      "kl": 0.055800806905608624,
       "learning_rate": 3.867370395306068e-07,
-      "loss": 0.0,
-      "reward": 0.6692760974168778,
-      "reward_std": 0.6910564064979553,
+      "loss": 0.0001,
+      "reward": 0.1798232919536531,
+      "reward_std": 0.1809667639899999,
       "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.2,
-      "rewards/question_recreation_reward_func": 0.31687611490488055,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.15835454268381,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.15240000188350677,
+      "rewards/xmlcount_reward_func": 0.021468750201165676,
       "step": 8
     },
     {
-      "completion_length": 345.40625,
-      "epoch": 2.6153846153846154,
-      "grad_norm": 32.29428482055664,
-      "kl": 0.005779433291536407,
+      "completion_length": 215.61905070713587,
+      "epoch": 3.3636363636363638,
+      "grad_norm": 148.54246520996094,
+      "kl": 48.04856840880321,
       "learning_rate": 3.1137137178519977e-07,
-      "loss": 0.0,
-      "reward": 0.4007483650930226,
-      "reward_std": 0.3432068810798228,
+      "loss": 0.042,
+      "reward": 0.15507938074214117,
+      "reward_std": 0.13797536226255552,
       "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.2635296122170985,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.11650795010583741,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.07471875101327896,
+      "rewards/xmlcount_reward_func": 0.03857142850756645,
       "step": 10
     },
     {
-      "completion_length": 374.2,
-      "epoch": 3.0,
-      "grad_norm": 0.40180766582489014,
-      "kl": 0.001354375621303916,
+      "completion_length": 386.8571450369699,
+      "epoch": 4.0,
+      "grad_norm": 14.088373184204102,
+      "kl": 0.43657855187276645,
       "learning_rate": 2.2935516363191693e-07,
-      "loss": 0.0,
-      "reward": 0.531975419819355,
-      "reward_std": 0.6552451923489571,
+      "loss": 0.0004,
+      "reward": 0.08821213884013039,
+      "reward_std": 0.2219510009246213,
       "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.3,
-      "rewards/question_recreation_reward_func": 0.3110754191875458,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.13853357466203825,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": -0.07909999191761016,
+      "rewards/xmlcount_reward_func": -0.05032142996788025,
       "step": 12
     },
     {
-      "completion_length": 452.125,
-      "epoch": 3.6153846153846154,
-      "grad_norm": 3.598710775375366,
-      "kl": 0.0013617263321066275,
+      "completion_length": 174.5625,
+      "epoch": 4.7272727272727275,
+      "grad_norm": 4376.435546875,
+      "kl": 343.20565592614,
       "learning_rate": 1.4957614383675767e-07,
-      "loss": 0.0,
-      "reward": 0.5646867048926651,
-      "reward_std": 0.9374583205208182,
+      "loss": 0.3432,
+      "reward": 0.23694669781252742,
+      "reward_std": 0.15096487803384662,
       "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.2538741906173527,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.19379044696688652,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.060812498442828655,
+      "rewards/xmlcount_reward_func": 0.043156249448657036,
       "step": 14
     },
     {
-      "completion_length": 381.7,
-      "epoch": 4.0,
-      "grad_norm": 0.42099884152412415,
-      "kl": 0.0017613507807254791,
+      "completion_length": 385.2857186453683,
+      "epoch": 5.363636363636363,
+      "grad_norm": 11.454949378967285,
+      "kl": 0.013573364753808295,
       "learning_rate": 8.067960709356478e-08,
       "loss": 0.0,
-      "reward": 0.3737470984458923,
-      "reward_std": 0.8008067756891251,
+      "reward": 0.29491744722638813,
+      "reward_std": 0.24371024753366197,
       "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.1,
-      "rewards/question_recreation_reward_func": 0.26824709177017214,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.22859602900488035,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.005499997735023498,
+      "rewards/xmlcount_reward_func": 0.06632142806691783,
       "step": 16
     },
     {
-      "completion_length": 287.9375,
-      "epoch": 4.615384615384615,
-      "grad_norm": 66.94889831542969,
-      "kl": 0.003935290566005278,
+      "completion_length": 197.4523824964251,
+      "epoch": 6.0,
+      "grad_norm": 681.37255859375,
+      "kl": 325.2371688451884,
       "learning_rate": 3.013156219837776e-08,
-      "loss": 0.0,
-      "reward": 0.2613473196979612,
-      "reward_std": 0.6432226218748838,
+      "loss": 0.2846,
+      "reward": 0.2937344493610518,
+      "reward_std": 0.3036107931818281,
       "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.2314098177012056,
+      "rewards/final_correctness_reward_func": 0.07142857142857142,
+      "rewards/question_recreation_reward_func": 0.18648445180484227,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.029937494546175003,
+      "rewards/xmlcount_reward_func": 0.03582142932074411,
       "step": 18
     },
     {
-      "completion_length": 383.85,
-      "epoch": 5.0,
-      "grad_norm": 0.8118636608123779,
-      "kl": 0.0021089719259180127,
+      "completion_length": 302.09375,
+      "epoch": 6.7272727272727275,
+      "grad_norm": 11.758221626281738,
+      "kl": 73.97962155583082,
       "learning_rate": 3.4096741493194193e-09,
-      "loss": 0.0,
-      "reward": 0.4499451667070389,
-      "reward_std": 1.1464473485946656,
+      "loss": 0.074,
+      "reward": 0.11310690874233842,
+      "reward_std": 0.2124028450343758,
       "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.2,
-      "rewards/question_recreation_reward_func": 0.2760451942682266,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.12451316439546645,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": -0.026100006699562073,
+      "rewards/xmlcount_reward_func": -0.011406249832361937,
       "step": 20
     },
     {
-      "epoch": 5.0,
+      "epoch": 6.7272727272727275,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 2.922929610349456e-06,
-      "train_runtime": 1036.4117,
-      "train_samples_per_second": 0.309,
-      "train_steps_per_second": 0.019
+      "train_loss": 0.07442864710465073,
+      "train_runtime": 243.8844,
+      "train_samples_per_second": 1.312,
+      "train_steps_per_second": 0.082
     }
   ],
   "logging_steps": 2,

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:55432364a7e07a6d05f6e4fbf3afaea9fdd560e03da0aef1efe2185e9400199d
-size 6008
+oid sha256:ee2e01a2a185fb72cee67e9f8e46442e32f6a8457e6040783ea32a59be69f92f
+size 5944