| --- |
| library_name: nanochat |
| license: apache-2.0 |
| tags: |
| - nanochat |
| - hf_llama |
| - vocab_64000 |
| - pt_owt-gpt2bpe-9B |
| - seed_0 |
| - ppt_nca-paper-share10-1024 |
| - ppt_lr_trapezoid |
| - reinit |
| --- |
| |
| # nca_owt_share10_untied_2026-06-02_17-01-45_412997-owt |
|
|
| Trained with [nanochat](https://github.com/karpathy/nanochat). Checkpoint at step **17,196**. |
|
|
| **W&B run:** https://wandb.ai/alexksternteam/nca-repro/runs/ahiti0o7 |
|
|
| ## Outcome |
|
|
| | metric | value | |
| | --- | --- | |
| | `step` | 17196 | |
| | `smooth_train_loss` | 2.844186660106616 | |
| | `min_objective` | 0.954196568142732 | |
| | `flops_used` | 2.8502607518490427e+19 | |
| | `flops_per_token` | 3161456704.0 | |
| | `total_training_time` | 19841.456748247147 | |
|
|
| ## Training config |
|
|
| ```json |
| { |
| "model_pt": { |
| "sequence_len": 1024, |
| "vocab_size": 64000, |
| "n_layer": 24, |
| "n_head": 16, |
| "n_embd": 1024, |
| "intermediate_size": 4096, |
| "rope_theta": 10000.0, |
| "attention_dropout": 0.1, |
| "weight_tying": false, |
| "head_bias": true, |
| "initializer_range": 0.02, |
| "attn_implementation": "sdpa" |
| }, |
| "model_ppt": null, |
| "optim": { |
| "matrix_lr": 0.0005, |
| "embedding_lr": 0.0005, |
| "unembedding_lr": 0.0005, |
| "weight_decay": 0.0001 |
| }, |
| "train": { |
| "num_iterations": 1000, |
| "eval_every_n_steps": 1000, |
| "save_every_n_steps": null, |
| "log_every_n_steps": 10, |
| "eval_at_end": true, |
| "save_at_end": true, |
| "grad_clip": 1.0 |
| }, |
| "eval": { |
| "eval_steps": 4, |
| "eval_tokens": 2016000 |
| }, |
| "hardware": { |
| "device_batch_size": 128, |
| "grad_accum_steps": 1, |
| "peak_tflops": 2250.0 |
| }, |
| "wandb": { |
| "enabled": true, |
| "notes": "", |
| "group": "repro_nca_owt", |
| "tags": [ |
| "hf_llama", |
| "vocab_64000", |
| "pt_owt-gpt2bpe-9B", |
| "seed_0", |
| "ppt_nca-paper-share10-1024", |
| "ppt_lr_trapezoid", |
| "reinit" |
| ], |
| "entity": null |
| }, |
| "data_pt": "owt-gpt2bpe-9B", |
| "data_ppt": "nca-paper-share10-1024", |
| "extra_eval": [ |
| "fineweb-gpt2bpe-20B", |
| "c4-gpt2bpe-10B" |
| ], |
| "train_split": "train", |
| "pad_vocab": 64000, |
| "lr_kind": "cosine", |
| "lr_warmup_ratio": 0.1, |
| "lr_warmdown_ratio": 0.0, |
| "lr_final_frac": 0.0, |
| "ppt_lr": 0.0001, |
| "ppt_weight_decay": 0.0, |
| "ppt_grad_clip": 0.0, |
| "ppt_device_batch_size": 8, |
| "ppt_grad_accum_steps": 1, |
| "ppt_lr_kind": "trapezoid", |
| "ppt_lr_warmup_ratio": 1.0, |
| "ppt_lr_warmdown_ratio": 0.0, |
| "ppt_tokens": 164160000, |
| "pt_tokens": null, |
| "eval_tokens": 2016000, |
| "reinit_embed_at_transition": true, |
| "reset_optimizer_at_transition": true, |
| "depth": null, |
| "compile_model": true, |
| "model_project": "nca-repro", |
| "run_name": "nca_owt_share10_untied", |
| "target_flops": 2.9022e+19, |
| "alpha_ppt": 0.0, |
| "use_measured_flops": true, |
| "seed": 0, |
| "push_to_hf": true, |
| "hf_repo_org": "alexkstern" |
| } |
| ``` |
|
|
| ## Files |
|
|
| - `model_017196.pt` — model weights (`state_dict`). |
| - `meta_017196.json` — training metadata. |
| - `config_017196.json` — run config snapshot. |
| - `rng_017196.pt` — RNG state (when present). |
|
|
