run_name: beetle-bilingual-balanced-b4-fineweb-eng-nld
hf_user: Beetle-FineWeb
training:
  fabric:
    accelerator: cuda
    devices: 8
    strategy: ddp
    precision: bf16-mixed
  optimization:
    optimizer: adamw
    lr: 0.0005
    lr_warmup_steps: 625
    lr_scheduler: linear_with_warmup
  max_steps: 91552
monitoring:
  logging:
    log_level: INFO
    log_every_n_steps: 100
  save_to_wandb: false
  wandb:
    project: ''
    entity: ''
checkpointing:
  runs_dir: ./runs
  run_name: beetle-bilingual-balanced-b4-fineweb-eng-nld
  checkpoints_dir: checkpoints
  fabric_checkpoint_dir: fabric_checkpoint
  fabric_checkpoint_filename: checkpoint.pt
  logs_dir: logs
  save_every_n_steps: 1000
  keep_local_checkpoints: 1
  save_to_hf: true
  learning_dynamics: {}
  compute_gradients: false
tokenizer:
  pretrained_tokenizer_path: Beetle-Data/tokenizer-nl-en
  dataset_source: fineweb
  vocab_size: 50000
  vocab_sharing: shared
  type: bpe
  target_tokens: 1000000000
  output_dir: ./tokenizers/fineweb_eng_L1-nld_L2_shared
  overwrite: false
  push_to_hub: true
model:
  architecture: pico
  n_layers: 14
  d_model: 768
  vocab_size: 50000
  attention_n_heads: 12
  attention_n_kv_heads: 1
  max_seq_len: 512
  batch_size: 64
  position_emb_theta: 10000.0
  activation_hidden_dim: 3072
  norm_eps: 1.0e-05
  dropout: 0.1
data:
  data_mix: B4
  lang_sources:
  - Beetle-Data/en-for-nl-28B
  l2_streams:
  - Beetle-Data/nl-28B
  pretokenized: true
  pretokenized_cache_dir: ./pretok_cache/beetle-bilingual-balanced-b4-fineweb-eng-nld