---
library_name: nanochat
license: apache-2.0
tags:
- nanochat
- nanochat_gpt
- pt_fineweb-nanochatbpe-20B
- ppt_github-code-nanochatbpe-1B
- seed_1
- case_b
- lr_trapezoid
- depth_12
---

# code_5pct_1e18_seed1_2026-07-12_01-12-46_897145-pt

Trained with [nanochat](https://github.com/karpathy/nanochat). Checkpoint at step **3,368**.

**W&B run:** https://wandb.ai/alexksternteam/optimal_seed_replicas_v1/runs/33ea3a6t

## Outcome

| metric | value |
| --- | --- |
| `step` | 3368 |
| `smooth_train_loss` | 3.2399237155914307 |
| `min_objective` | 0.9851442932664687 |
| `flops_used` | 9.498592991426642e+17 |
| `flops_per_token` | 1075838976.0 |
| `total_training_time` | 422.86022329330444 |

## Training config

```json
{
  "model_pt": {
    "sequence_len": 2048,
    "vocab_size": 65536,
    "n_layer": 12,
    "n_head": 6,
    "n_kv_head": 6,
    "n_embd": 768
  },
  "model_ppt": {
    "sequence_len": 2048,
    "vocab_size": 65536,
    "n_layer": 12,
    "n_head": 6,
    "n_kv_head": 6,
    "n_embd": 768
  },
  "optim": {
    "matrix_lr": 0.015,
    "embedding_lr": 0.3,
    "unembedding_lr": 0.004,
    "weight_decay": 0.0
  },
  "train": {
    "num_iterations": 1000,
    "eval_every_n_steps": null,
    "eval_every_flops_frac": null,
    "save_every_n_steps": null,
    "log_every_n_steps": 100,
    "eval_at_end": true,
    "save_at_end": true,
    "grad_clip": 1.0,
    "ema_beta": 0.0
  },
  "eval": {
    "eval_steps": 8,
    "eval_tokens": 2016000
  },
  "hardware": {
    "device_batch_size": 32,
    "grad_accum_steps": 1,
    "peak_tflops": 2250.0
  },
  "wandb": {
    "enabled": true,
    "notes": "",
    "group": "1e18_ppt0.05",
    "tags": [
      "nanochat_gpt",
      "pt_fineweb-nanochatbpe-20B",
      "ppt_github-code-nanochatbpe-1B",
      "seed_1",
      "case_b",
      "lr_trapezoid",
      "depth_12"
    ],
    "entity": null
  },
  "data_pt": "fineweb-nanochatbpe-20B",
  "data_ppt": "github-code-nanochatbpe-1B",
  "extra_eval": [
    "c4-nanochatbpe-10B"
  ],
  "train_split": "train",
  "pad_vocab": 65536,
  "ppt_vocab_size": null,
  "ppt_same_vocab_as_pt": true,
  "ppt_unified_lr": true,
  "lr_kind": "trapezoid",
  "lr_warmup_ratio": 0.0,
  "lr_warmdown_ratio": 0.4,
  "lr_final_frac": 0.0,
  "ppt_lr_kind": "trapezoid",
  "ppt_lr_warmup_ratio": null,
  "ppt_lr_warmdown_ratio": null,
  "ppt_lr_final_frac": 0.0,
  "ppt_lr": null,
  "ppt_weight_decay": 0.0,
  "ppt_device_batch_size": null,
  "ppt_grad_accum_steps": null,
  "eval_tokens": 2016000,
  "reinit_embed_at_transition": false,
  "reset_optimizer_at_transition": false,
  "depth": 12,
  "compile_model": true,
  "model_project": "optimal_seed_replicas_v1",
  "run_name": "code_5pct_1e18_seed1",
  "target_flops": 1e+18,
  "alpha_ppt": 0.05,
  "use_measured_flops": true,
  "flops_per_token": null,
  "ppt_flops_per_token": null,
  "seed": 1,
  "metrics_output_file": null,
  "push_to_hf": true,
  "hf_repo_org": "alexkstern",
  "cleanup_checkpoints": true
}
```

## Files

- `model_003368.pt` — model weights (`state_dict`).
- `meta_003368.json` — training metadata.
- `config_003368.json` — run config snapshot.
- `rng_003368.pt` — RNG state (when present).