Training in progress, step 500, checkpoint

last-checkpoint/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ed0d669212b40c60987a9febdec5b6e21bd03351aaa4ded9b393cddd4885a0c8
+oid sha256:8b8fac3343992a3bef6c12ed1a0758e36cad3b2839846b13971ee09879131c54
 size 6425529112

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fb1f5a17180bd5fce232cab2baace30fee1b53707b577513427207805ccc2976
+oid sha256:04bbebfcb471d4a88a032f2525708906a30884fc984ae26b975c1a13cf99d67d
 size 12851224679

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:098b29492211804ab324a36f37466821d948280bb74fce4ba895c03f13ecd878
+oid sha256:61c19bab1174704a4a4441475683bf1270277af15d2e2c95e964789128e482c4
 size 14645

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a95301d85992ed73403f16c486c03c4e66c9e324b5fb37edd6c6389bb791936a
+oid sha256:e10df580efad97bb633668d65429a37f5f53374d3373d28957a85a8b301eb78e
 size 1465

last-checkpoint/trainer_state.json

@@ -2,518 +2,118 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 1.953125,
+  "epoch": 0.390625,
   "eval_steps": 500,
-  "global_step": 2500,
+  "global_step": 500,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
-      "entropy": 1.4100425505638123,
+      "entropy": 1.351678059399128,
       "epoch": 0.0390625,
-      "grad_norm": 1.703125,
-      "learning_rate": 2.5520833333333334e-06,
-      "loss": 1.4939990234375,
-      "mean_token_accuracy": 0.7383647415041924,
+      "grad_norm": 1.6015625,
+      "learning_rate": 3.828125000000001e-06,
+      "loss": 1.394322509765625,
+      "mean_token_accuracy": 0.7548403647542,
       "num_tokens": 257685.0,
       "step": 50
     },
     {
-      "entropy": 0.7098006828129292,
+      "entropy": 0.47313837975263595,
       "epoch": 0.078125,
-      "grad_norm": 1.3203125,
-      "learning_rate": 5.156250000000001e-06,
-      "loss": 0.6490231323242187,
-      "mean_token_accuracy": 0.8852380600571632,
+      "grad_norm": 0.91015625,
+      "learning_rate": 7.734375e-06,
+      "loss": 0.4260553359985352,
+      "mean_token_accuracy": 0.9251225611567497,
       "num_tokens": 513846.0,
       "step": 100
     },
     {
-      "entropy": 0.30407395515590907,
+      "entropy": 0.22530030721798538,
       "epoch": 0.1171875,
-      "grad_norm": 1.2265625,
-      "learning_rate": 7.760416666666666e-06,
-      "loss": 0.258120174407959,
-      "mean_token_accuracy": 0.9527129030227661,
+      "grad_norm": 0.921875,
+      "learning_rate": 1.1640625000000002e-05,
+      "loss": 0.18161891937255858,
+      "mean_token_accuracy": 0.9637172383069992,
       "num_tokens": 766003.0,
       "step": 150
     },
     {
-      "entropy": 0.20104818791151047,
+      "entropy": 0.14051863566040992,
       "epoch": 0.15625,
-      "grad_norm": 1.734375,
-      "learning_rate": 1.0364583333333334e-05,
-      "loss": 0.1583445167541504,
-      "mean_token_accuracy": 0.9671251937747002,
+      "grad_norm": 1.6484375,
+      "learning_rate": 1.5546875e-05,
+      "loss": 0.1136919116973877,
+      "mean_token_accuracy": 0.9747392472624778,
       "num_tokens": 1024512.0,
       "step": 200
     },
     {
-      "entropy": 0.13147448537871242,
+      "entropy": 0.09577633743174374,
       "epoch": 0.1953125,
-      "grad_norm": 0.98828125,
-      "learning_rate": 1.2968750000000002e-05,
-      "loss": 0.10042434692382812,
-      "mean_token_accuracy": 0.9772825425863266,
+      "grad_norm": 0.71484375,
+      "learning_rate": 1.9453125e-05,
+      "loss": 0.0734261655807495,
+      "mean_token_accuracy": 0.9820188581943512,
       "num_tokens": 1280614.0,
       "step": 250
     },
     {
-      "entropy": 0.10732088888064027,
+      "entropy": 0.08148466867394745,
       "epoch": 0.234375,
-      "grad_norm": 0.54296875,
-      "learning_rate": 1.5572916666666668e-05,
-      "loss": 0.08049575805664062,
-      "mean_token_accuracy": 0.9807940790057182,
+      "grad_norm": 0.3828125,
+      "learning_rate": 1.9626736111111114e-05,
+      "loss": 0.062112469673156735,
+      "mean_token_accuracy": 0.9844269120693206,
       "num_tokens": 1533690.0,
       "step": 300
     },
     {
-      "entropy": 0.0891605723835528,
+      "entropy": 0.06672279690392316,
       "epoch": 0.2734375,
-      "grad_norm": 0.6640625,
-      "learning_rate": 1.8177083333333332e-05,
-      "loss": 0.06720943927764893,
-      "mean_token_accuracy": 0.9835837849974632,
+      "grad_norm": 0.455078125,
+      "learning_rate": 1.9192708333333335e-05,
+      "loss": 0.05034114837646484,
+      "mean_token_accuracy": 0.986752623617649,
       "num_tokens": 1783169.0,
       "step": 350
     },
     {
-      "entropy": 0.06704788052476943,
+      "entropy": 0.051021190043538805,
       "epoch": 0.3125,
-      "grad_norm": 0.8125,
-      "learning_rate": 1.9913194444444447e-05,
-      "loss": 0.04878067970275879,
-      "mean_token_accuracy": 0.9872324925661087,
+      "grad_norm": 0.49609375,
+      "learning_rate": 1.8758680555555557e-05,
+      "loss": 0.03769558668136597,
+      "mean_token_accuracy": 0.9895561364293098,
       "num_tokens": 2039809.0,
       "step": 400
     },
     {
-      "entropy": 0.05854248736985028,
+      "entropy": 0.04628240401856601,
       "epoch": 0.3515625,
-      "grad_norm": 0.2421875,
-      "learning_rate": 1.9623842592592593e-05,
-      "loss": 0.04276306629180908,
-      "mean_token_accuracy": 0.988223501443863,
+      "grad_norm": 0.1962890625,
+      "learning_rate": 1.8324652777777778e-05,
+      "loss": 0.034056272506713864,
+      "mean_token_accuracy": 0.9902530950307846,
       "num_tokens": 2296699.0,
       "step": 450
     },
     {
-      "entropy": 0.047506115855649116,
+      "entropy": 0.041101934388279915,
       "epoch": 0.390625,
-      "grad_norm": 0.6484375,
-      "learning_rate": 1.9334490740740743e-05,
-      "loss": 0.034850025177001955,
-      "mean_token_accuracy": 0.9899384224414826,
+      "grad_norm": 0.498046875,
+      "learning_rate": 1.7890625000000003e-05,
+      "loss": 0.030158956050872803,
+      "mean_token_accuracy": 0.9908681440353394,
       "num_tokens": 2551050.0,
       "step": 500
-    },
-    {
-      "entropy": 0.05063623377121985,
-      "epoch": 0.4296875,
-      "grad_norm": 0.248046875,
-      "learning_rate": 1.904513888888889e-05,
-      "loss": 0.03843466758728027,
-      "mean_token_accuracy": 0.9891716066002846,
-      "num_tokens": 2802698.0,
-      "step": 550
-    },
-    {
-      "entropy": 0.04273756206966937,
-      "epoch": 0.46875,
-      "grad_norm": 0.361328125,
-      "learning_rate": 1.875578703703704e-05,
-      "loss": 0.032549431324005125,
-      "mean_token_accuracy": 0.9905735540390015,
-      "num_tokens": 3060791.0,
-      "step": 600
-    },
-    {
-      "entropy": 0.043252475708723065,
-      "epoch": 0.5078125,
-      "grad_norm": 0.4765625,
-      "learning_rate": 1.8466435185185186e-05,
-      "loss": 0.032298707962036134,
-      "mean_token_accuracy": 0.9904984217882157,
-      "num_tokens": 3313602.0,
-      "step": 650
-    },
-    {
-      "entropy": 0.03833162900991738,
-      "epoch": 0.546875,
-      "grad_norm": 0.1845703125,
-      "learning_rate": 1.8177083333333332e-05,
-      "loss": 0.027402305603027345,
-      "mean_token_accuracy": 0.9914076882600784,
-      "num_tokens": 3572334.0,
-      "step": 700
-    },
-    {
-      "entropy": 0.037472997857257724,
-      "epoch": 0.5859375,
-      "grad_norm": 0.1572265625,
-      "learning_rate": 1.7887731481481482e-05,
-      "loss": 0.027391784191131592,
-      "mean_token_accuracy": 0.9915520316362381,
-      "num_tokens": 3822475.0,
-      "step": 750
-    },
-    {
-      "entropy": 0.03570145134814084,
-      "epoch": 0.625,
-      "grad_norm": 0.33984375,
-      "learning_rate": 1.7598379629629632e-05,
-      "loss": 0.02662867307662964,
-      "mean_token_accuracy": 0.991677038371563,
-      "num_tokens": 4077791.0,
-      "step": 800
-    },
-    {
-      "entropy": 0.03883639902807772,
-      "epoch": 0.6640625,
-      "grad_norm": 0.23828125,
-      "learning_rate": 1.730902777777778e-05,
-      "loss": 0.0284798264503479,
-      "mean_token_accuracy": 0.9913818097114563,
-      "num_tokens": 4327537.0,
-      "step": 850
-    },
-    {
-      "entropy": 0.03742775755003094,
-      "epoch": 0.703125,
-      "grad_norm": 0.21875,
-      "learning_rate": 1.701967592592593e-05,
-      "loss": 0.027604823112487794,
-      "mean_token_accuracy": 0.9913884291052818,
-      "num_tokens": 4582391.0,
-      "step": 900
-    },
-    {
-      "entropy": 0.033991393875330685,
-      "epoch": 0.7421875,
-      "grad_norm": 0.30859375,
-      "learning_rate": 1.6730324074074075e-05,
-      "loss": 0.02489029407501221,
-      "mean_token_accuracy": 0.9922845155000687,
-      "num_tokens": 4842265.0,
-      "step": 950
-    },
-    {
-      "entropy": 0.03376254609320313,
-      "epoch": 0.78125,
-      "grad_norm": 0.23828125,
-      "learning_rate": 1.6440972222222225e-05,
-      "loss": 0.024386107921600342,
-      "mean_token_accuracy": 0.9923295575380325,
-      "num_tokens": 5099714.0,
-      "step": 1000
-    },
-    {
-      "entropy": 0.03393112221732736,
-      "epoch": 0.8203125,
-      "grad_norm": 0.1904296875,
-      "learning_rate": 1.615162037037037e-05,
-      "loss": 0.024903218746185302,
-      "mean_token_accuracy": 0.9919200077652931,
-      "num_tokens": 5353934.0,
-      "step": 1050
-    },
-    {
-      "entropy": 0.03500156338326633,
-      "epoch": 0.859375,
-      "grad_norm": 0.400390625,
-      "learning_rate": 1.586226851851852e-05,
-      "loss": 0.02582158088684082,
-      "mean_token_accuracy": 0.9918030974268913,
-      "num_tokens": 5610229.0,
-      "step": 1100
-    },
-    {
-      "entropy": 0.03437284361571073,
-      "epoch": 0.8984375,
-      "grad_norm": 0.2080078125,
-      "learning_rate": 1.5572916666666668e-05,
-      "loss": 0.025576255321502685,
-      "mean_token_accuracy": 0.9920255246758461,
-      "num_tokens": 5862791.0,
-      "step": 1150
-    },
-    {
-      "entropy": 0.03161145319696516,
-      "epoch": 0.9375,
-      "grad_norm": 0.1376953125,
-      "learning_rate": 1.5283564814814814e-05,
-      "loss": 0.022945339679718017,
-      "mean_token_accuracy": 0.992571029663086,
-      "num_tokens": 6121431.0,
-      "step": 1200
-    },
-    {
-      "entropy": 0.030242039281874897,
-      "epoch": 0.9765625,
-      "grad_norm": 0.173828125,
-      "learning_rate": 1.4994212962962964e-05,
-      "loss": 0.021836049556732177,
-      "mean_token_accuracy": 0.9925165721774101,
-      "num_tokens": 6379675.0,
-      "step": 1250
-    },
-    {
-      "entropy": 0.03080349043942988,
-      "epoch": 1.015625,
-      "grad_norm": 0.162109375,
-      "learning_rate": 1.4704861111111113e-05,
-      "loss": 0.021759965419769288,
-      "mean_token_accuracy": 0.9926620882749557,
-      "num_tokens": 6635287.0,
-      "step": 1300
-    },
-    {
-      "entropy": 0.030266724079847335,
-      "epoch": 1.0546875,
-      "grad_norm": 0.32421875,
-      "learning_rate": 1.4415509259259259e-05,
-      "loss": 0.02140357255935669,
-      "mean_token_accuracy": 0.9928847193717957,
-      "num_tokens": 6888440.0,
-      "step": 1350
-    },
-    {
-      "entropy": 0.03156906962394714,
-      "epoch": 1.09375,
-      "grad_norm": 0.193359375,
-      "learning_rate": 1.4126157407407407e-05,
-      "loss": 0.02241924524307251,
-      "mean_token_accuracy": 0.9925972136855126,
-      "num_tokens": 7143446.0,
-      "step": 1400
-    },
-    {
-      "entropy": 0.03107087403535843,
-      "epoch": 1.1328125,
-      "grad_norm": 0.197265625,
-      "learning_rate": 1.3836805555555557e-05,
-      "loss": 0.02277942180633545,
-      "mean_token_accuracy": 0.9925328662991524,
-      "num_tokens": 7400487.0,
-      "step": 1450
-    },
-    {
-      "entropy": 0.029809724665246905,
-      "epoch": 1.171875,
-      "grad_norm": 0.1435546875,
-      "learning_rate": 1.3547453703703705e-05,
-      "loss": 0.021194422245025636,
-      "mean_token_accuracy": 0.9928572303056717,
-      "num_tokens": 7655674.0,
-      "step": 1500
-    },
-    {
-      "entropy": 0.03159077540971339,
-      "epoch": 1.2109375,
-      "grad_norm": 0.2060546875,
-      "learning_rate": 1.3258101851851852e-05,
-      "loss": 0.02314976692199707,
-      "mean_token_accuracy": 0.9924373865127564,
-      "num_tokens": 7908433.0,
-      "step": 1550
-    },
-    {
-      "entropy": 0.027922376818023623,
-      "epoch": 1.25,
-      "grad_norm": 0.1640625,
-      "learning_rate": 1.2968750000000002e-05,
-      "loss": 0.019616029262542724,
-      "mean_token_accuracy": 0.9931238636374473,
-      "num_tokens": 8163713.0,
-      "step": 1600
-    },
-    {
-      "entropy": 0.030897088246420026,
-      "epoch": 1.2890625,
-      "grad_norm": 0.1552734375,
-      "learning_rate": 1.267939814814815e-05,
-      "loss": 0.022646543979644777,
-      "mean_token_accuracy": 0.992621060013771,
-      "num_tokens": 8413661.0,
-      "step": 1650
-    },
-    {
-      "entropy": 0.03009945319034159,
-      "epoch": 1.328125,
-      "grad_norm": 0.1826171875,
-      "learning_rate": 1.2390046296296297e-05,
-      "loss": 0.022125842571258544,
-      "mean_token_accuracy": 0.9926324704289436,
-      "num_tokens": 8666242.0,
-      "step": 1700
-    },
-    {
-      "entropy": 0.030331599721685053,
-      "epoch": 1.3671875,
-      "grad_norm": 0.15234375,
-      "learning_rate": 1.2100694444444445e-05,
-      "loss": 0.021965017318725587,
-      "mean_token_accuracy": 0.9924738201498985,
-      "num_tokens": 8923368.0,
-      "step": 1750
-    },
-    {
-      "entropy": 0.029221815695054828,
-      "epoch": 1.40625,
-      "grad_norm": 0.1904296875,
-      "learning_rate": 1.1811342592592595e-05,
-      "loss": 0.02077268123626709,
-      "mean_token_accuracy": 0.9929761862754822,
-      "num_tokens": 9177821.0,
-      "step": 1800
-    },
-    {
-      "entropy": 0.02919468770734966,
-      "epoch": 1.4453125,
-      "grad_norm": 0.1513671875,
-      "learning_rate": 1.1521990740740743e-05,
-      "loss": 0.020907692909240723,
-      "mean_token_accuracy": 0.9928768157958985,
-      "num_tokens": 9430080.0,
-      "step": 1850
-    },
-    {
-      "entropy": 0.027250755606219174,
-      "epoch": 1.484375,
-      "grad_norm": 0.16015625,
-      "learning_rate": 1.123263888888889e-05,
-      "loss": 0.01935715675354004,
-      "mean_token_accuracy": 0.9932011726498604,
-      "num_tokens": 9688541.0,
-      "step": 1900
-    },
-    {
-      "entropy": 0.02769404204096645,
-      "epoch": 1.5234375,
-      "grad_norm": 0.1376953125,
-      "learning_rate": 1.0943287037037038e-05,
-      "loss": 0.01980802059173584,
-      "mean_token_accuracy": 0.9932876801490784,
-      "num_tokens": 9946814.0,
-      "step": 1950
-    },
-    {
-      "entropy": 0.029987394027411938,
-      "epoch": 1.5625,
-      "grad_norm": 0.2890625,
-      "learning_rate": 1.0653935185185187e-05,
-      "loss": 0.022057452201843263,
-      "mean_token_accuracy": 0.9924592301249504,
-      "num_tokens": 10201426.0,
-      "step": 2000
-    },
-    {
-      "entropy": 0.02940327289979905,
-      "epoch": 1.6015625,
-      "grad_norm": 0.375,
-      "learning_rate": 1.0364583333333334e-05,
-      "loss": 0.02158078670501709,
-      "mean_token_accuracy": 0.9927120169997216,
-      "num_tokens": 10454276.0,
-      "step": 2050
-    },
-    {
-      "entropy": 0.02797958446200937,
-      "epoch": 1.640625,
-      "grad_norm": 0.13671875,
-      "learning_rate": 1.0075231481481482e-05,
-      "loss": 0.020346088409423826,
-      "mean_token_accuracy": 0.9930062460899353,
-      "num_tokens": 10707497.0,
-      "step": 2100
-    },
-    {
-      "entropy": 0.027742678658105435,
-      "epoch": 1.6796875,
-      "grad_norm": 0.12890625,
-      "learning_rate": 9.78587962962963e-06,
-      "loss": 0.019992319345474244,
-      "mean_token_accuracy": 0.9929967644810677,
-      "num_tokens": 10962955.0,
-      "step": 2150
-    },
-    {
-      "entropy": 0.028013068232685328,
-      "epoch": 1.71875,
-      "grad_norm": 0.1337890625,
-      "learning_rate": 9.496527777777779e-06,
-      "loss": 0.020120697021484377,
-      "mean_token_accuracy": 0.9930884554982186,
-      "num_tokens": 11217333.0,
-      "step": 2200
-    },
-    {
-      "entropy": 0.027716009449213742,
-      "epoch": 1.7578125,
-      "grad_norm": 0.1943359375,
-      "learning_rate": 9.207175925925927e-06,
-      "loss": 0.019945393800735473,
-      "mean_token_accuracy": 0.9929410392045974,
-      "num_tokens": 11472495.0,
-      "step": 2250
-    },
-    {
-      "entropy": 0.027896945285610854,
-      "epoch": 1.796875,
-      "grad_norm": 0.169921875,
-      "learning_rate": 8.917824074074075e-06,
-      "loss": 0.019913656711578367,
-      "mean_token_accuracy": 0.9931384524703026,
-      "num_tokens": 11728662.0,
-      "step": 2300
-    },
-    {
-      "entropy": 0.028674062341451645,
-      "epoch": 1.8359375,
-      "grad_norm": 0.31640625,
-      "learning_rate": 8.628472222222223e-06,
-      "loss": 0.02098811149597168,
-      "mean_token_accuracy": 0.992857717871666,
-      "num_tokens": 11983347.0,
-      "step": 2350
-    },
-    {
-      "entropy": 0.02859616348519921,
-      "epoch": 1.875,
-      "grad_norm": 0.216796875,
-      "learning_rate": 8.339120370370371e-06,
-      "loss": 0.02054593563079834,
-      "mean_token_accuracy": 0.992903139591217,
-      "num_tokens": 12236526.0,
-      "step": 2400
-    },
-    {
-      "entropy": 0.029245859370566906,
-      "epoch": 1.9140625,
-      "grad_norm": 0.142578125,
-      "learning_rate": 8.04976851851852e-06,
-      "loss": 0.021238679885864257,
-      "mean_token_accuracy": 0.9926192459464073,
-      "num_tokens": 12489562.0,
-      "step": 2450
-    },
-    {
-      "entropy": 0.02795163292437792,
-      "epoch": 1.953125,
-      "grad_norm": 0.126953125,
-      "learning_rate": 7.760416666666666e-06,
-      "loss": 0.0203912615776062,
-      "mean_token_accuracy": 0.9929744681715965,
-      "num_tokens": 12748444.0,
-      "step": 2500
     }
   ],
   "logging_steps": 50,
-  "max_steps": 3840,
+  "max_steps": 2560,
   "num_input_tokens_seen": 0,
-  "num_train_epochs": 3,
+  "num_train_epochs": 2,
   "save_steps": 500,
   "stateful_callbacks": {
     "TrainerControl": {
@@ -527,7 +127,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 2.4192422651355955e+17,
+  "total_flos": 4.833703032695194e+16,
   "train_batch_size": 2,
   "trial_name": null,
   "trial_params": null

last-checkpoint/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f10de4aab36802dd9ce012f3583586ae2292b7eb4d0601cf4c5d6f23d568ac7f
+oid sha256:26cbf16b2edabc5b8c36be1cccac63caa078ca871b6b4fdcff3ff77d550c19b0
 size 5713