compile: true
device: cuda
from_checkpoint: null
load_mtp_head_from_model: null
name: nanogpt
training:
  random_seed: 13
  batch_size: 512
  device_batch_size: 128
  sequence_length: 512
  num_iterations: 2000
  learning_rate: 0.0005
  use_scheduler: true
  save_model: true
  save_optimizer: true
  save_model_every: 500
  val_loss_every: 50
  val_tokens: 1048576
  expname: my-smol-lm
model:
  name: stp
  n_token: 1
  model:
    _target_: mtp.models.stp.SingleTokenLM
    lm: ${lm.model}
adaptor:
  name: none
  hyperparameters: null
lm:
  name: nanogpt
  n_embd: 256
  n_head: 4
  n_layer: 4
  model:
    _target_: mtp.models.lm.LM
    lm:
      _target_: mtp.models.gpt.GPT
      vocab_size: 65
      n_embd: 256
      n_head: 4
      n_layer: 4
    encoder_only: false
    from_checkpoint: null
    from_huggingface: null
    adaptor_kwargs: null
    ref_enc: encoder
    ref_head: head
    freeze: false
data:
  name: shakespeare_char
  vocab_size: 65
  train_bin: ${oc.env:MTP_ROOT}/data/shakespeare_char/train.bin
  val_bin: ${oc.env:MTP_ROOT}/data/shakespeare_char/val.bin
  vocabs: ${oc.env:MTP_ROOT}/data/shakespeare_char/meta.pkl
generate:
  speculative: false
expname: my-smol-lm
wandb_run_id: ze1fyax2