model:
  d_model: 512
  d_ff: 2048
  n_layers: 6
  dropout: 0.1
  max_src_len: 512
  max_tgt_len: 256
  src_vocab_size: 32000
  tgt_vocab_size: 32000
attention:
  name: sliding_gqa
  n_heads: 8
  n_kv_heads: 2
  window_size: 128
  bias: false
feedforward:
  name: relu_ffn
  d_ff: 2048
normalization:
  name: layernorm
positional:
  name: sinusoidal
  dropout: 0.1
connection:
  name: residual
  dropout: 0.1
  norm: layernorm
optimizer:
  name: adamw
scheduler:
  name: none
loss:
  name: cross_entropy
  label_smoothing: 0.1
data:
  name: meetingbank
  hf_path: huuuyeah/meetingbank
  tokenizer_dir: tokenizers
  tokenizer_basename: meetingbank
  vocab_size: 32000
  max_src_len: 512
  max_tgt_len: 256
  val_batch_size: 1
  num_workers: 0
  limit: 0
logging:
  backend: tensorboard
seed: 42
experiment_name: run_sliding_gqa
training:
  num_epochs: 20
  batch_size: 8
  lr: 0.0001
  weight_decay: 0.01
  grad_clip: 0.0
  ckpt_dir: weights
  ckpt_basename: meeting_model
  preload: null
  tui: true
  save_every_n_epochs: 4
  hf:
    push: false
    repo_id: null
    private: false
    commit_message: null