Upload config_017196.json with huggingface_hub

config_017196.json

@@ -0,0 +1,97 @@
+{
+  "model_pt": {
+    "sequence_len": 1024,
+    "vocab_size": 64000,
+    "n_layer": 24,
+    "n_head": 16,
+    "n_embd": 1024,
+    "intermediate_size": 4096,
+    "rope_theta": 10000.0,
+    "attention_dropout": 0.0,
+    "weight_tying": false,
+    "head_bias": false,
+    "initializer_range": 0.02,
+    "nanochat_init": false,
+    "attn_implementation": "sdpa",
+    "explicit_additive_mask": true
+  },
+  "model_ppt": null,
+  "optim": {
+    "matrix_lr": 0.0005,
+    "embedding_lr": 0.0005,
+    "unembedding_lr": 0.0005,
+    "weight_decay": 0.0001
+  },
+  "train": {
+    "num_iterations": 1000,
+    "eval_every_n_steps": 1000,
+    "eval_every_flops_frac": null,
+    "save_every_n_steps": null,
+    "save_at_steps": null,
+    "log_every_n_steps": 10,
+    "eval_at_end": true,
+    "save_at_end": true,
+    "grad_clip": 1.0,
+    "ema_beta": 0.0
+  },
+  "eval": {
+    "eval_steps": 8,
+    "eval_tokens": 2016000
+  },
+  "hardware": {
+    "device_batch_size": 64,
+    "grad_accum_steps": 2,
+    "peak_tflops": 2250.0
+  },
+  "wandb": {
+    "enabled": true,
+    "notes": "",
+    "group": "repro_vanilla_owt",
+    "tags": [
+      "hf_llama",
+      "vocab_64000",
+      "pt_owt-gpt2bpe-9B",
+      "seed_0",
+      "no_ppt"
+    ],
+    "entity": null
+  },
+  "data_pt": "owt-gpt2bpe-9B",
+  "data_ppt": null,
+  "extra_eval": [
+    "fineweb-gpt2bpe-20B",
+    "c4-gpt2bpe-10B",
+    "nca-paper-1024"
+  ],
+  "train_split": "train",
+  "pad_vocab": 64000,
+  "lr_kind": "cosine",
+  "lr_warmup_ratio": 0.1,
+  "lr_warmdown_ratio": 0.0,
+  "lr_final_frac": 0.0,
+  "ppt_lr": 0.0001,
+  "ppt_weight_decay": 0.0,
+  "ppt_grad_clip": 0.0,
+  "ppt_device_batch_size": 8,
+  "ppt_grad_accum_steps": 1,
+  "ppt_lr_kind": "cosine",
+  "ppt_lr_warmup_ratio": 0.1,
+  "ppt_lr_warmdown_ratio": 0.0,
+  "ppt_tokens": null,
+  "pt_tokens": null,
+  "eval_tokens": 2016000,
+  "reinit_embed_at_transition": true,
+  "reset_optimizer_at_transition": true,
+  "depth": null,
+  "compile_model": true,
+  "model_project": "nca-repro",
+  "run_name": "vanilla_owt_untied_dropout0_seed0",
+  "target_flops": 0.0,
+  "alpha_ppt": 0.0,
+  "use_measured_flops": true,
+  "flops_per_token": null,
+  "seed": 0,
+  "push_to_hf": true,
+  "hf_repo_org": "alexkstern",
+  "paper_default_embed_reinit": false
+}