Upload config_017196.json with huggingface_hub
Browse files- config_017196.json +92 -0
config_017196.json
ADDED
|
@@ -0,0 +1,92 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"model_pt": {
|
| 3 |
+
"sequence_len": 1024,
|
| 4 |
+
"vocab_size": 50304,
|
| 5 |
+
"n_layer": 24,
|
| 6 |
+
"n_head": 16,
|
| 7 |
+
"n_embd": 1024,
|
| 8 |
+
"intermediate_size": 4096,
|
| 9 |
+
"rope_theta": 10000.0,
|
| 10 |
+
"attention_dropout": 0.1,
|
| 11 |
+
"weight_tying": false,
|
| 12 |
+
"head_bias": false,
|
| 13 |
+
"initializer_range": 0.02,
|
| 14 |
+
"attn_implementation": "sdpa"
|
| 15 |
+
},
|
| 16 |
+
"model_ppt": null,
|
| 17 |
+
"optim": {
|
| 18 |
+
"matrix_lr": 0.0005,
|
| 19 |
+
"embedding_lr": 0.0005,
|
| 20 |
+
"unembedding_lr": 0.0005,
|
| 21 |
+
"weight_decay": 0.0001
|
| 22 |
+
},
|
| 23 |
+
"train": {
|
| 24 |
+
"num_iterations": 1000,
|
| 25 |
+
"eval_every_n_steps": 2000,
|
| 26 |
+
"save_every_n_steps": null,
|
| 27 |
+
"log_every_n_steps": 10,
|
| 28 |
+
"eval_at_end": true,
|
| 29 |
+
"save_at_end": true,
|
| 30 |
+
"grad_clip": 1.0,
|
| 31 |
+
"ema_beta": 0.0
|
| 32 |
+
},
|
| 33 |
+
"eval": {
|
| 34 |
+
"eval_steps": 4,
|
| 35 |
+
"eval_tokens": 2016000
|
| 36 |
+
},
|
| 37 |
+
"hardware": {
|
| 38 |
+
"device_batch_size": 128,
|
| 39 |
+
"grad_accum_steps": 1,
|
| 40 |
+
"peak_tflops": 2250.0
|
| 41 |
+
},
|
| 42 |
+
"wandb": {
|
| 43 |
+
"enabled": true,
|
| 44 |
+
"notes": "",
|
| 45 |
+
"group": "repro_c4_owt",
|
| 46 |
+
"tags": [
|
| 47 |
+
"hf_llama",
|
| 48 |
+
"vocab_64000",
|
| 49 |
+
"pt_owt-gpt2bpe-9B",
|
| 50 |
+
"seed_0",
|
| 51 |
+
"ppt_c4-gpt2bpe-10B",
|
| 52 |
+
"ppt_lr_cosine",
|
| 53 |
+
"reinit"
|
| 54 |
+
],
|
| 55 |
+
"entity": null
|
| 56 |
+
},
|
| 57 |
+
"data_pt": "owt-gpt2bpe-9B",
|
| 58 |
+
"data_ppt": "c4-gpt2bpe-10B",
|
| 59 |
+
"extra_eval": [
|
| 60 |
+
"fineweb-gpt2bpe-20B",
|
| 61 |
+
"nca-paper-1024"
|
| 62 |
+
],
|
| 63 |
+
"train_split": "train",
|
| 64 |
+
"pad_vocab": 50304,
|
| 65 |
+
"lr_kind": "cosine",
|
| 66 |
+
"lr_warmup_ratio": 0.1,
|
| 67 |
+
"lr_warmdown_ratio": 0.0,
|
| 68 |
+
"lr_final_frac": 0.0,
|
| 69 |
+
"ppt_lr": 0.0001,
|
| 70 |
+
"ppt_weight_decay": 0.0,
|
| 71 |
+
"ppt_grad_clip": 0.0,
|
| 72 |
+
"ppt_device_batch_size": 8,
|
| 73 |
+
"ppt_grad_accum_steps": 1,
|
| 74 |
+
"ppt_lr_kind": "cosine",
|
| 75 |
+
"ppt_lr_warmup_ratio": 0.1,
|
| 76 |
+
"ppt_lr_warmdown_ratio": 0.0,
|
| 77 |
+
"ppt_tokens": 164160000,
|
| 78 |
+
"pt_tokens": null,
|
| 79 |
+
"eval_tokens": 2016000,
|
| 80 |
+
"reinit_embed_at_transition": true,
|
| 81 |
+
"reset_optimizer_at_transition": true,
|
| 82 |
+
"depth": null,
|
| 83 |
+
"compile_model": true,
|
| 84 |
+
"model_project": "nca-repro",
|
| 85 |
+
"run_name": "c4_owt_owt50k",
|
| 86 |
+
"target_flops": 0.0,
|
| 87 |
+
"alpha_ppt": 0.0,
|
| 88 |
+
"use_measured_flops": true,
|
| 89 |
+
"seed": 0,
|
| 90 |
+
"push_to_hf": true,
|
| 91 |
+
"hf_repo_org": "alexkstern"
|
| 92 |
+
}
|