final results: 4 models complete

sweep/d4-w128_n32768_bfloat16_s0.json

@@ -0,0 +1,116 @@
+{
+  "result": {
+    "tag": "d4-w128_n32768_bfloat16_s0",
+    "n_layer": 4,
+    "n_embd": 128,
+    "n_head": 8,
+    "num_params": 1049728,
+    "n_seqs": 32768,
+    "seq_len": 64,
+    "vocab_size": 2048,
+    "dtype": "bfloat16",
+    "seed": 0,
+    "max_iters": 32000,
+    "batch_size": 384,
+    "grad_accum": 1,
+    "H_x_bits": 23068672.0,
+    "H_x_given_theta_bits": 19322809.427256536,
+    "mem_bits": 3745862.5727434643,
+    "bpp": 3.568412553293295,
+    "train_loss": 6.386542737483978,
+    "test_loss": 8.832721948623657,
+    "mi_f1": 0.9998931770661422,
+    "mi_threshold": -7.30322265625,
+    "elapsed_s": 673.7350013256073
+  },
+  "history": [
+    {
+      "iter": 0,
+      "train_loss": 7.650531440973282,
+      "test_loss": 7.65172266960144,
+      "mem_bits": -78399.44814310223,
+      "bpp": -0.07468548818656093,
+      "elapsed_s": 0.6961326599121094
+    },
+    {
+      "iter": 3200,
+      "train_loss": 7.186664789915085,
+      "test_loss": 8.046813249588013,
+      "mem_bits": 1325052.6646017618,
+      "bpp": 1.2622819097916431,
+      "elapsed_s": 66.7319107055664
+    },
+    {
+      "iter": 6400,
+      "train_loss": 6.97925091534853,
+      "test_loss": 8.270601272583008,
+      "mem_bits": 1952593.8767345324,
+      "bpp": 1.860095069136512,
+      "elapsed_s": 133.9402060508728
+    },
+    {
+      "iter": 9600,
+      "train_loss": 6.883714012801647,
+      "test_loss": 8.366836547851562,
+      "mem_bits": 2241645.6277467012,
+      "bpp": 2.135453782071833,
+      "elapsed_s": 200.31193327903748
+    },
+    {
+      "iter": 12800,
+      "train_loss": 6.805789060890675,
+      "test_loss": 8.44254207611084,
+      "mem_bits": 2477411.527592074,
+      "bpp": 2.360050915658222,
+      "elapsed_s": 268.410763502121
+    },
+    {
+      "iter": 16000,
+      "train_loss": 6.730602823197842,
+      "test_loss": 8.518313646316528,
+      "mem_bits": 2704891.308470104,
+      "bpp": 2.5767544625561136,
+      "elapsed_s": 336.4312644004822
+    },
+    {
+      "iter": 19200,
+      "train_loss": 6.6495267525315285,
+      "test_loss": 8.595789432525635,
+      "mem_bits": 2950191.078156315,
+      "bpp": 2.810433824911134,
+      "elapsed_s": 405.1044702529907
+    },
+    {
+      "iter": 22400,
+      "train_loss": 6.5664816200733185,
+      "test_loss": 8.666681289672852,
+      "mem_bits": 3201448.344303325,
+      "bpp": 3.0497884635861143,
+      "elapsed_s": 470.42328667640686
+    },
+    {
+      "iter": 25600,
+      "train_loss": 6.48842041939497,
+      "test_loss": 8.737895965576172,
+      "mem_bits": 3437626.4717145227,
+      "bpp": 3.2747782965820886,
+      "elapsed_s": 538.1311693191528
+    },
+    {
+      "iter": 28800,
+      "train_loss": 6.423498652875423,
+      "test_loss": 8.793143510818481,
+      "mem_bits": 3634050.5737212747,
+      "bpp": 3.461897342665219,
+      "elapsed_s": 605.4303991794586
+    },
+    {
+      "iter": 32000,
+      "train_loss": 6.386542737483978,
+      "test_loss": 8.832721948623657,
+      "mem_bits": 3745862.5727434643,
+      "bpp": 3.568412553293295,
+      "elapsed_s": 672.9221868515015
+    }
+  ]
+}

sweep/summary_lean.csv

@@ -5,9 +5,14 @@ d1-w32_n16384_bf16_s0,1,32,4,77920,16384,64,2048,bfloat16,0,12553,1024,1,1153433
 d1-w32_n16384_fp32_s0,1,32,4,77920,16384,64,2048,float32,0,12000,512,1,11534336.0,11289668.252604596,244667.74739540368,3.139986491214113,7.4628846347332,7.7802863121032715,0.9958460598651078,-7.651047706604004,124.28364586830139
 d1-w32_n4096_bf16_s0,1,32,4,77920,4096,64,2048,bfloat16,0,7638,1024,1,2883584.0,2623335.6634027935,260248.33659720654,3.3399427181366343,6.936484217643738,8.30504560470581,0.9998779445863422,-7.779296875,192.68513226509094
 d2-w128_n1024_bfloat16_s0,2,128,8,656000,1024,64,2048,bfloat16,0,8000,384,1,720896.0,495.70091115032506,720400.2990888497,1.0981711876354416,0.005242823623120785,27.642335414886475,0.9999999999994991,-0.0300984475761652,167.94316267967224
+d2-w128_n16384_bfloat16_s0,2,128,8,656000,16384,64,2048,bfloat16,0,28000,384,1,11534336.0,9164760.124239756,2369575.875760244,3.612158347195494,6.058242455124855,9.203715562820435,0.999816860996997,-6.941162109375,592.8328263759613
+d2-w128_n4096_bfloat16_s0,2,128,8,656000,4096,64,2048,bfloat16,0,15000,384,1,2883584.0,634770.3991878944,2248813.6008121055,3.4280695134330874,1.6784260272979736,18.886457443237305,0.9998779445863422,-16.809814453125,319.55566334724426
 d2-w64_n2048_bfloat16_s0,2,64,4,229696,2048,64,2048,bfloat16,0,10000,512,1,1441792.0,716151.5978994451,725640.4021005549,3.159133820791633,3.7872197031974792,13.130642890930176,0.9997559189645959,-11.4853515625,222.75749039649963
 d2-w64_n2048_fp32_s0,2,64,4,229696,2048,64,2048,float32,0,10000,256,1,1441792.0,714860.3632957394,726931.6367042606,3.164755314434124,3.780391275882721,12.841822147369385,0.9990224828929501,-4.910470962524414,78.76775193214417
 d2-w64_n32768_bfloat16_s0,2,64,4,229696,32768,64,2048,bfloat16,0,30000,512,1,23068672.0,22239409.105432916,829262.8945670836,3.610262671387763,7.3505323976278305,7.892160654067993,0.9996033320106067,-7.66845703125,670.8058476448059
 d2-w64_n512_bfloat16_s0,2,64,4,229696,512,64,2048,bfloat16,0,5000,512,1,360448.0,193.7184181543366,360254.28158184566,1.568395973729824,0.004097759258002043,29.440959453582764,0.9990224828929486,-0.05276238173246384,112.13331604003906
 d2-w64_n8192_bfloat16_s0,2,64,4,229696,8192,64,2048,bfloat16,0,22000,512,1,5767168.0,4944240.743331735,822927.256668265,3.582679962508119,6.536648809909821,8.729503870010376,0.9999999999994997,-7.302978515625,488.9944713115692
 d2-w64_n8192_fp32_s0,2,64,4,229696,8192,64,2048,float32,0,22000,256,1,5767168.0,4952531.934273833,814636.0657261666,3.5465835962583876,6.547610372304916,8.696101903915405,0.999694731057562,-7.18479061126709,173.75297904014587
+d4-w128_n2048_bfloat16_s0,4,128,8,1049728,2048,64,2048,bfloat16,0,10000,384,1,1441792.0,2184.859671600569,1439607.1403283994,1.3714096797726643,0.011554178781807423,26.50293493270874,0.9990224828929501,-0.05870996415615082,211.60499811172485
+d4-w128_n32768_bfloat16_s0,4,128,8,1049728,32768,64,2048,bfloat16,0,32000,384,1,23068672.0,19322809.427256536,3745862.5727434643,3.568412553293295,6.386542737483978,8.832721948623657,0.9998931770661422,-7.30322265625,673.7350013256073
+d4-w128_n8192_bfloat16_s0,4,128,8,1049728,8192,64,2048,bfloat16,0,22000,384,1,5767168.0,2019720.2974360613,3747447.7025639387,3.5699225919132753,2.6702183336019516,14.464654445648193,0.9998779147840194,-4.147796630859375,468.3837842941284