| { |
| "model_pt": { |
| "sequence_len": 2048, |
| "vocab_size": 65536, |
| "n_layer": 12, |
| "n_head": 6, |
| "n_kv_head": 6, |
| "n_embd": 768 |
| }, |
| "model_ppt": { |
| "sequence_len": 2048, |
| "vocab_size": 65536, |
| "n_layer": 12, |
| "n_head": 6, |
| "n_kv_head": 6, |
| "n_embd": 768 |
| }, |
| "optim": { |
| "matrix_lr": 0.015, |
| "embedding_lr": 0.3, |
| "unembedding_lr": 0.004, |
| "weight_decay": 0.0 |
| }, |
| "train": { |
| "num_iterations": 1000, |
| "eval_every_n_steps": null, |
| "eval_every_flops_frac": null, |
| "save_every_n_steps": null, |
| "log_every_n_steps": 100, |
| "eval_at_end": true, |
| "save_at_end": true, |
| "grad_clip": 1.0, |
| "ema_beta": 0.0 |
| }, |
| "eval": { |
| "eval_steps": 8, |
| "eval_tokens": 2016000 |
| }, |
| "hardware": { |
| "device_batch_size": 32, |
| "grad_accum_steps": 1, |
| "peak_tflops": 2250.0 |
| }, |
| "wandb": { |
| "enabled": true, |
| "notes": "", |
| "group": "1e18_ppt0.05", |
| "tags": [ |
| "nanochat_gpt", |
| "pt_fineweb-nanochatbpe-20B", |
| "ppt_github-code-nanochatbpe-1B", |
| "seed_1", |
| "case_b", |
| "lr_trapezoid", |
| "depth_12" |
| ], |
| "entity": null |
| }, |
| "data_pt": "fineweb-nanochatbpe-20B", |
| "data_ppt": "github-code-nanochatbpe-1B", |
| "extra_eval": [ |
| "c4-nanochatbpe-10B" |
| ], |
| "train_split": "train", |
| "pad_vocab": 65536, |
| "ppt_vocab_size": null, |
| "ppt_same_vocab_as_pt": true, |
| "ppt_unified_lr": true, |
| "lr_kind": "trapezoid", |
| "lr_warmup_ratio": 0.0, |
| "lr_warmdown_ratio": 0.4, |
| "lr_final_frac": 0.0, |
| "ppt_lr_kind": "trapezoid", |
| "ppt_lr_warmup_ratio": null, |
| "ppt_lr_warmdown_ratio": null, |
| "ppt_lr_final_frac": 0.0, |
| "ppt_lr": null, |
| "ppt_weight_decay": 0.0, |
| "ppt_device_batch_size": null, |
| "ppt_grad_accum_steps": null, |
| "eval_tokens": 2016000, |
| "reinit_embed_at_transition": false, |
| "reset_optimizer_at_transition": false, |
| "depth": 12, |
| "compile_model": true, |
| "model_project": "optimal_seed_replicas_v1", |
| "run_name": "code_5pct_1e18_seed1", |
| "target_flops": 1e+18, |
| "alpha_ppt": 0.05, |
| "use_measured_flops": true, |
| "flops_per_token": null, |
| "ppt_flops_per_token": null, |
| "seed": 1, |
| "metrics_output_file": null, |
| "push_to_hf": true, |
| "hf_repo_org": "alexkstern", |
| "cleanup_checkpoints": true |
| } |
