config/train_par_gpt2-s_scratch.py

# Training config: GPT-2 small (124M) from scratch on character-level protein-pair prompts (MED4 PPI).

out_dir = 'out_14e'
eval_interval = 250 # keep frequent because we'll overfit
log_interval = 10 # don't print too too often

# we expect to overfit on this small dataset, so only save when val improves
always_save_checkpoint = True

wandb_log = True
# override via command line if you like
wandb_project = 'ppiGLM_MED4Solo'
wandb_run_name = 'ppiGPLM-med4_4k_14epoch'
dataset = 'med4_4k_14epoch'
init_from = 'scratch'  # train from random init; no pretrained GPT-2 weights loaded
gradient_accumulation_steps = 2
batch_size = 12

block_size = 4096 # context of up to n previous characters


# GPT2-M models
n_layer = 12
n_head = 12
n_embd = 768
dropout = 0.2

# using above parameters, gradient = 10, batch = 16, token/iter = 98304, epoch = 97600


learning_rate = 5e-4 # with baby networks can afford to go a bit higher
max_iters = 8001
lr_decay_iters = 8000 # make equal to max_iters usually
min_lr = 1e-5 # learning_rate / 10 usually
beta2 = 0.99 # make a bit bigger because number of tokens per iter is small

warmup_iters = 200 # not super necessary potentially

# on macbook also add
# device = 'cpu'  # run on cpu only
# compile = False # do not torch compile the model

# To tokenize the training data:  python data/MED4_char/prepare.py
#
# To train GPT-2 small from scratch on a single GPU:
#   python train_.py config/train_par_gpt2-s_scratch.py
#
# To train on 2 GPUs with DDP:
#   torchrun --standalone --nproc_per_node=2 train_.py config/train_par_gpt2-s_scratch.py