compile: true
device: cuda
from_checkpoint: null
load_mtp_head_from_model: outputs/models/llama/no-lora/llama-lr-3e-4-no-lora-btree-n-8-r-32-s-1/model@900.pt
name: nanogpt
training:
  random_seed: 13
  batch_size: 256
  device_batch_size: 1
  sequence_length: 8192
  num_iterations: 900
  learning_rate: 0.0003
  use_scheduler: false
  save_model: true
  save_optimizer: true
  save_model_every: 100
  val_loss_every: 100
  val_tokens: 4194304
  expname: llama-lr-3e-4-lora-last-4-cont-btree-n-8-r-32-s-1
model:
  name: mtp
  beta: 0.0
  gamma: 1
  kl_algorithm: full
  kl_type: forward
  model:
    _target_: mtp.models.mtp.MultiTokenLM
    lm: ${lm.model}
    circuit: ${circuit.model}
    mt_head_kwargs: ${mt_head.hyperparameters}
    init_from_lm_head: true
    kl_type: ${model.kl_type}
    kl_algorithm: ${model.kl_algorithm}
    beta: 0
    gamma: 0.9
circuit:
  name: btree
  n_token: 8
  n_component: 32
  n_repetition: 1
  model:
    _target_: mtp.models.circuits.CircuitModel
    vocab_size: 268
    n_token: 8
    n_component: 32
    n_repetition: 1
    kind: btree
mt_head:
  name: linear-evabyte
  hyperparameters:
    type: evabyte
    n_embd: 3072
    transformer_n_head: 24
    transformer_n_layer: 0
    expander_type: linear
    expander_n_layer: 1
    freeze_vocab_unembedding: false
    share_sum_weights: false
    contextual_hmm_weights: true
    init_hmm_identity: true
adaptor:
  name: lora-last-4
  hyperparameters:
    r: 16
    lora_alpha: 16
    lora_dropout: 0.0
    target_modules:
    - gate_proj
    - down_proj
    - up_proj
    - q_proj
    - v_proj
    - k_proj
    - o_proj
    layers_to_transform:
    - 24
    - 25
    - 26
    - 27
lm:
  name: llama3-2-3b-byte
  n_embd: 3072
  n_head: 24
  num_layers: 28
  last_1_layers:
  - 27
  last_2_layers:
  - 26
  - 27
  last_4_layers:
  - 24
  - 25
  - 26
  - 27
  model:
    _target_: mtp.models.lm.LM
    lm: null
    encoder_only: true
    from_checkpoint: null
    from_huggingface: benjamin/Llama3-2-3B-IT-Byte
    adaptor_kwargs:
      r: 16
      lora_alpha: 16
      lora_dropout: 0.0
      target_modules:
      - gate_proj
      - down_proj
      - up_proj
      - q_proj
      - v_proj
      - k_proj
      - o_proj
      layers_to_transform:
      - 24
      - 25
      - 26
      - 27
    ref_enc: model
    ref_head: lm_head
    freeze: true
data:
  name: tulu3-llama3
  train_bin: agrv/tulu-v3-sft-llama3-packed-seq-len-8192
  val_bin: null
  vocab_size: 268
generate:
  speculative: false
expname: llama-lr-3e-4-lora-last-4-cont-btree-n-8-r-32-s-1
wandb_run_id: thouxsll