jax_model/hyperparams.json

{
    "dataset_name": "multi_species_final_178G.upper.n.filtered.6_mers.1000_tok_len.overlap_50",
    "alphabet": "k-mers",
    "k_for_kmers": 6,
    "num_warmup_updates": 16000,
    "warmup_init_lr": 5e-05,
    "warmup_end_lr": 0.0001,
    "training_set_proportion": 0.95,
    "tokens_per_batch": 1000000.0,
    "tokens_per_checkpoint": "3500000000.0",
    "masking_ratio": 0.15,
    "masking_prob": 0.8,
    "batch_size": 2,
    "random_token_prob": 0.1,
    "dropout_rate": 0.1,
    "num_hosts": 8,
    "server_address": "3422826-worker-0:1234",
    "alphabet_size": 4105,
    "pad_token_id": 1,
    "mask_token_id": 2,
    "class_token_id": 3,
    "eos_token_id": -1,
    "prepend_bos": true,
    "append_eos": false,
    "max_positions": 1000,
    "emb_layer_norm_before": false,
    "attention_heads": 20,
    "embed_dim": 2560,
    "ffn_embed_dim": 10240,
    "num_layers": 32,
    "token_dropout": true,
    "embed_scale": 1.0,
    "use_remat": false,
    "architecture": "Vanilla",
    "acc_batch_size": 8,
    "num_local_devices": 8,
    "num_global_devices": 64,
    "tokens_length": 1000,
    "mixed-precision": true,
    "model_num_parameters": "2547800585",
    "shift": true,
    "overlap": 50
}