c4_owt_untied_embed_reinit_seed0_2026-07-22_20-50-51_479874-ppt-c4

Trained with nanochat. Checkpoint at step 5,004.

W&B run: https://wandb.ai/alexksternteam/nca-repro/runs/yz7j22eg

Outcome

metric	value
step	5004
smooth_train_loss	4.689947128295898
min_objective	1.4889587360464172
flops_used	5.183874448364667e+17
flops_per_token	3161456704.0
total_training_time	646.9167912006378

Training config

{
  "model_pt": {
    "sequence_len": 1024,
    "vocab_size": 64000,
    "n_layer": 24,
    "n_head": 16,
    "n_embd": 1024,
    "intermediate_size": 4096,
    "rope_theta": 10000.0,
    "attention_dropout": 0.1,
    "weight_tying": false,
    "head_bias": false,
    "initializer_range": 0.02,
    "nanochat_init": false,
    "attn_implementation": "sdpa",
    "explicit_additive_mask": true
  },
  "model_ppt": null,
  "optim": {
    "matrix_lr": 0.0001,
    "embedding_lr": 0.0001,
    "unembedding_lr": 0.0001,
    "weight_decay": 0.0
  },
  "train": {
    "num_iterations": 1000,
    "eval_every_n_steps": 1000,
    "eval_every_flops_frac": null,
    "save_every_n_steps": null,
    "save_at_steps": null,
    "log_every_n_steps": 10,
    "eval_at_end": true,
    "save_at_end": true,
    "grad_clip": 0.0,
    "ema_beta": 0.0
  },
  "eval": {
    "eval_steps": 62,
    "eval_tokens": 2016000
  },
  "hardware": {
    "device_batch_size": 8,
    "grad_accum_steps": 1,
    "peak_tflops": 2250.0
  },
  "wandb": {
    "enabled": true,
    "notes": "",
    "group": "repro_c4_owt",
    "tags": [
      "hf_llama",
      "vocab_64000",
      "pt_owt-gpt2bpe-9B",
      "seed_0",
      "ppt_c4-gpt2bpe-10B",
      "ppt_lr_cosine",
      "reinit"
    ],
    "entity": null
  },
  "data_pt": "owt-gpt2bpe-9B",
  "data_ppt": "c4-gpt2bpe-10B",
  "extra_eval": [
    "fineweb-gpt2bpe-20B",
    "nca-paper-1024"
  ],
  "train_split": "train",
  "pad_vocab": 64000,
  "lr_kind": "cosine",
  "lr_warmup_ratio": 0.1,
  "lr_warmdown_ratio": 0.0,
  "lr_final_frac": 0.0,
  "ppt_lr": 0.0001,
  "ppt_weight_decay": 0.0,
  "ppt_grad_clip": 0.0,
  "ppt_device_batch_size": 8,
  "ppt_grad_accum_steps": 1,
  "ppt_lr_kind": "cosine",
  "ppt_lr_warmup_ratio": 0.1,
  "ppt_lr_warmdown_ratio": 0.0,
  "ppt_tokens": 164000000,
  "pt_tokens": null,
  "eval_tokens": 2016000,
  "reinit_embed_at_transition": true,
  "reset_optimizer_at_transition": true,
  "paper_embed_reinit": "full",
  "depth": null,
  "compile_model": true,
  "model_project": "nca-repro",
  "run_name": "c4_owt_untied_embed_reinit_seed0",
  "target_flops": 0.0,
  "alpha_ppt": 0.0,
  "use_measured_flops": true,
  "flops_per_token": null,
  "seed": 0,
  "push_to_hf": true,
  "hf_repo_org": "alexkstern"
}

Files

• model_005004.pt — model weights (state_dict).
• meta_005004.json — training metadata.
• config_005004.json — run config snapshot.
• rng_005004.pt — RNG state (when present).