End of training

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.0,
-    "train_runtime": 651.6683,
-    "train_samples": 2,
-    "train_samples_per_second": 0.061,
-    "train_steps_per_second": 0.015
+    "train_loss": 0.030200545489788056,
+    "train_runtime": 708.4645,
+    "train_samples": 43,
+    "train_samples_per_second": 0.056,
+    "train_steps_per_second": 0.014
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0951c1ec2d063159c94030f95de372fbb251eab898e1880fbf5f278425531b17
+oid sha256:31a9580e6b27b0a3bddfc54eae7afe20674eb43dda80da4400cbbb0656de63ed
 size 1976163472

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.0,
-    "train_runtime": 651.6683,
-    "train_samples": 2,
-    "train_samples_per_second": 0.061,
-    "train_steps_per_second": 0.015
+    "train_loss": 0.030200545489788056,
+    "train_runtime": 708.4645,
+    "train_samples": 43,
+    "train_samples_per_second": 0.056,
+    "train_steps_per_second": 0.014
 }

trainer_state.json

@@ -2,7 +2,7 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 10.0,
+  "epoch": 0.46511627906976744,
   "eval_steps": 500,
   "global_step": 10,
   "is_hyper_param_search": false,
@@ -16,21 +16,21 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 23.0,
-      "completions/max_terminated_length": 23.0,
-      "completions/mean_length": 22.5,
-      "completions/mean_terminated_length": 22.5,
-      "completions/min_length": 22.0,
-      "completions/min_terminated_length": 22.0,
-      "epoch": 2.0,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0,
+      "completions/max_length": 102.5,
+      "completions/max_terminated_length": 102.5,
+      "completions/mean_length": 46.5,
+      "completions/mean_terminated_length": 46.5,
+      "completions/min_length": 17.0,
+      "completions/min_terminated_length": 17.0,
+      "epoch": 0.09302325581395349,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 26.819913864135742,
       "kl": 0.0,
       "learning_rate": 5e-07,
-      "loss": 0.0,
-      "num_tokens": 1204.0,
-      "reward": 0.055734846740961075,
-      "reward_std": 0.0,
+      "loss": 0.1038,
+      "num_tokens": 1396.0,
+      "reward": 0.03198886290192604,
+      "reward_std": 0.010370630770921707,
       "rewards/concensus_correctness_reward_func/mean": 0.0,
       "rewards/concensus_correctness_reward_func/std": 0.0,
       "rewards/consensus_reward_func/mean": 0.0,
@@ -39,8 +39,8 @@
       "rewards/cumulative_reward_2/std": 0.0,
       "rewards/final_correctness_reward_func/mean": 0.0,
       "rewards/final_correctness_reward_func/std": 0.0,
-      "rewards/question_recreation_reward_func/mean": 0.055734846740961075,
-      "rewards/question_recreation_reward_func/std": 8.536729728803039e-05,
+      "rewards/question_recreation_reward_func/mean": 0.03198886290192604,
+      "rewards/question_recreation_reward_func/std": 0.011841414961963892,
       "rewards/soft_format_reward_func/mean": 0.0,
       "rewards/soft_format_reward_func/std": 0.0,
       "rewards/strict_format_reward_func/mean": 0.0,
@@ -56,21 +56,21 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 23.0,
-      "completions/max_terminated_length": 23.0,
-      "completions/mean_length": 22.5,
-      "completions/mean_terminated_length": 22.5,
-      "completions/min_length": 22.0,
-      "completions/min_terminated_length": 22.0,
-      "epoch": 4.0,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0,
-      "kl": 0.0,
+      "completions/max_length": 31.5,
+      "completions/max_terminated_length": 31.5,
+      "completions/mean_length": 22.375,
+      "completions/mean_terminated_length": 22.375,
+      "completions/min_length": 16.0,
+      "completions/min_terminated_length": 16.0,
+      "epoch": 0.18604651162790697,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 47.122337341308594,
+      "kl": 0.009030564804561436,
       "learning_rate": 4.415111107797445e-07,
-      "loss": 0.0,
-      "num_tokens": 2408.0,
-      "reward": 0.06472498923540115,
-      "reward_std": 0.0,
+      "loss": 0.0182,
+      "num_tokens": 2599.0,
+      "reward": 0.04082147125154734,
+      "reward_std": 0.016715965699404478,
       "rewards/concensus_correctness_reward_func/mean": 0.0,
       "rewards/concensus_correctness_reward_func/std": 0.0,
       "rewards/consensus_reward_func/mean": 0.0,
@@ -79,8 +79,8 @@
       "rewards/cumulative_reward_2/std": 0.0,
       "rewards/final_correctness_reward_func/mean": 0.0,
       "rewards/final_correctness_reward_func/std": 0.0,
-      "rewards/question_recreation_reward_func/mean": 0.06472498923540115,
-      "rewards/question_recreation_reward_func/std": 8.062479173531756e-05,
+      "rewards/question_recreation_reward_func/mean": 0.04082146845757961,
+      "rewards/question_recreation_reward_func/std": 0.026838398072868586,
       "rewards/soft_format_reward_func/mean": 0.0,
       "rewards/soft_format_reward_func/std": 0.0,
       "rewards/strict_format_reward_func/mean": 0.0,
@@ -95,22 +95,22 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 23.0,
-      "completions/max_terminated_length": 23.0,
-      "completions/mean_length": 22.5,
-      "completions/mean_terminated_length": 22.5,
-      "completions/min_length": 22.0,
-      "completions/min_terminated_length": 22.0,
-      "epoch": 6.0,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0,
-      "kl": 0.0,
+      "completions/clipped_ratio": 0.125,
+      "completions/max_length": 183.0,
+      "completions/max_terminated_length": 148.5,
+      "completions/mean_length": 90.25,
+      "completions/mean_terminated_length": 70.16666793823242,
+      "completions/min_length": 14.0,
+      "completions/min_terminated_length": 14.0,
+      "epoch": 0.27906976744186046,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 70.61158752441406,
+      "kl": 0.014445176813751459,
       "learning_rate": 2.934120444167326e-07,
-      "loss": 0.0,
-      "num_tokens": 3612.0,
-      "reward": 0.055734846740961075,
-      "reward_std": 0.0,
+      "loss": -0.0519,
+      "num_tokens": 4345.0,
+      "reward": 0.02797679416835308,
+      "reward_std": 0.005366061115637422,
       "rewards/concensus_correctness_reward_func/mean": 0.0,
       "rewards/concensus_correctness_reward_func/std": 0.0,
       "rewards/consensus_reward_func/mean": 0.0,
@@ -119,8 +119,8 @@
       "rewards/cumulative_reward_2/std": 0.0,
       "rewards/final_correctness_reward_func/mean": 0.0,
       "rewards/final_correctness_reward_func/std": 0.0,
-      "rewards/question_recreation_reward_func/mean": 0.055734846740961075,
-      "rewards/question_recreation_reward_func/std": 8.536729728803039e-05,
+      "rewards/question_recreation_reward_func/mean": 0.02797679416835308,
+      "rewards/question_recreation_reward_func/std": 0.016844000667333603,
       "rewards/soft_format_reward_func/mean": 0.0,
       "rewards/soft_format_reward_func/std": 0.0,
       "rewards/strict_format_reward_func/mean": 0.0,
@@ -136,31 +136,31 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 23.0,
-      "completions/max_terminated_length": 23.0,
-      "completions/mean_length": 22.5,
-      "completions/mean_terminated_length": 22.5,
-      "completions/min_length": 22.0,
-      "completions/min_terminated_length": 22.0,
-      "epoch": 8.0,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0,
-      "kl": 0.0,
+      "completions/max_length": 35.0,
+      "completions/max_terminated_length": 35.0,
+      "completions/mean_length": 14.125,
+      "completions/mean_terminated_length": 14.125,
+      "completions/min_length": 4.0,
+      "completions/min_terminated_length": 4.0,
+      "epoch": 0.37209302325581395,
+      "frac_reward_zero_std": 0.5,
+      "grad_norm": 84.35356140136719,
+      "kl": 0.05594382411800325,
       "learning_rate": 1.2500000000000005e-07,
-      "loss": 0.0,
-      "num_tokens": 4816.0,
-      "reward": 0.06472498923540115,
-      "reward_std": 0.0,
-      "rewards/concensus_correctness_reward_func/mean": 0.0,
-      "rewards/concensus_correctness_reward_func/std": 0.0,
-      "rewards/consensus_reward_func/mean": 0.0,
-      "rewards/consensus_reward_func/std": 0.0,
+      "loss": 0.0779,
+      "num_tokens": 5482.0,
+      "reward": 1.4956446290016174,
+      "reward_std": 0.6916476492770016,
+      "rewards/concensus_correctness_reward_func/mean": 0.7199999988079071,
+      "rewards/concensus_correctness_reward_func/std": 1.0334109663963318,
+      "rewards/consensus_reward_func/mean": 0.75,
+      "rewards/consensus_reward_func/std": 1.0773502588272095,
       "rewards/cumulative_reward_2/mean": 0.0,
       "rewards/cumulative_reward_2/std": 0.0,
       "rewards/final_correctness_reward_func/mean": 0.0,
       "rewards/final_correctness_reward_func/std": 0.0,
-      "rewards/question_recreation_reward_func/mean": 0.06472498923540115,
-      "rewards/question_recreation_reward_func/std": 8.062479173531756e-05,
+      "rewards/question_recreation_reward_func/mean": 0.025644690729677677,
+      "rewards/question_recreation_reward_func/std": 0.016321994364261627,
       "rewards/soft_format_reward_func/mean": 0.0,
       "rewards/soft_format_reward_func/std": 0.0,
       "rewards/strict_format_reward_func/mean": 0.0,
@@ -176,31 +176,31 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 23.0,
-      "completions/max_terminated_length": 23.0,
-      "completions/mean_length": 22.5,
-      "completions/mean_terminated_length": 22.5,
-      "completions/min_length": 22.0,
-      "completions/min_terminated_length": 22.0,
-      "epoch": 10.0,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0,
-      "kl": 0.0,
+      "completions/max_length": 47.0,
+      "completions/max_terminated_length": 47.0,
+      "completions/mean_length": 22.375,
+      "completions/mean_terminated_length": 22.375,
+      "completions/min_length": 4.5,
+      "completions/min_terminated_length": 4.5,
+      "epoch": 0.46511627906976744,
+      "frac_reward_zero_std": 0.5,
+      "grad_norm": 39.63914489746094,
+      "kl": 0.015579930739477277,
       "learning_rate": 1.507684480352292e-08,
-      "loss": 0.0,
-      "num_tokens": 6020.0,
-      "reward": 0.055734846740961075,
-      "reward_std": 0.0,
-      "rewards/concensus_correctness_reward_func/mean": 0.0,
-      "rewards/concensus_correctness_reward_func/std": 0.0,
+      "loss": 0.0031,
+      "num_tokens": 6685.0,
+      "reward": 0.7941031903028488,
+      "reward_std": 0.02715550092398189,
+      "rewards/concensus_correctness_reward_func/mean": 0.7120000123977661,
+      "rewards/concensus_correctness_reward_func/std": 0.8221468329429626,
       "rewards/consensus_reward_func/mean": 0.0,
       "rewards/consensus_reward_func/std": 0.0,
       "rewards/cumulative_reward_2/mean": 0.0,
       "rewards/cumulative_reward_2/std": 0.0,
       "rewards/final_correctness_reward_func/mean": 0.0,
       "rewards/final_correctness_reward_func/std": 0.0,
-      "rewards/question_recreation_reward_func/mean": 0.055734846740961075,
-      "rewards/question_recreation_reward_func/std": 8.536729728803039e-05,
+      "rewards/question_recreation_reward_func/mean": 0.08210319094359875,
+      "rewards/question_recreation_reward_func/std": 0.056017409078776836,
       "rewards/soft_format_reward_func/mean": 0.0,
       "rewards/soft_format_reward_func/std": 0.0,
       "rewards/strict_format_reward_func/mean": 0.0,
@@ -210,19 +210,19 @@
       "step": 10
     },
     {
-      "epoch": 10.0,
+      "epoch": 0.46511627906976744,
       "step": 10,
       "total_flos": 0.0,
-      "train_loss": 0.0,
-      "train_runtime": 651.6683,
-      "train_samples_per_second": 0.061,
-      "train_steps_per_second": 0.015
+      "train_loss": 0.030200545489788056,
+      "train_runtime": 708.4645,
+      "train_samples_per_second": 0.056,
+      "train_steps_per_second": 0.014
     }
   ],
   "logging_steps": 2,
   "max_steps": 10,
-  "num_input_tokens_seen": 6020,
-  "num_train_epochs": 10,
+  "num_input_tokens_seen": 6685,
+  "num_train_epochs": 1,
   "save_steps": 10,
   "stateful_callbacks": {
     "TrainerControl": {