File size: 1,787 Bytes
97e7d77 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 | {
"_mlp_class": "LLaMAMLP",
"_norm_class": "FusedRMSNorm",
"average_2": false,
"betas": [
0.9,
0.95
],
"bias": false,
"block_size": 8224,
"block_size_val": 2080,
"condense_ratio": 1,
"decay_lr": true,
"discount": true,
"eval_iters": 100,
"eval_step_interval": 1000,
"forecasting_patch": 1,
"global_batch_size": 512,
"grad_clip": 1.0,
"group": "70m-test_all",
"haar_loss_match": false,
"haar_trans": true,
"haar_trans_inv": true,
"haar_trans_norm": "backward",
"half_diff": false,
"imputation": false,
"inner_norm": false,
"inter_control": false,
"intermediate_size": 3072,
"is_diff": false,
"is_smape": false,
"learning_rate": 0.0005,
"log_step_interval": 10,
"max_step": 100000,
"mean_replace": false,
"micro_batch_size": 64,
"micro_batch_size_val": 256,
"min_lr": 1e-05,
"mix_train": false,
"multi_loss": false,
"n_cot": 1,
"n_embd": 768,
"n_head": 12,
"n_layer": 12,
"n_query_groups": 4,
"name": "small-100k",
"new_arch": false,
"new_tokenizer": false,
"norm_eps": 1e-05,
"num_of_devices": 4,
"num_of_nodes": 2,
"org": "Ali-Could",
"ou": false,
"ou_mean": false,
"ou_prod": false,
"padded_vocab_size": "None",
"padding_multiple": 1,
"parallel_residual": false,
"patch_size": 32,
"pid": false,
"quantitle": true,
"rollback_win": 256,
"rolling_patch": 6,
"rope_base": 10000,
"rotary_percentage": 1.0,
"save_step_interval": 10000,
"scaling": true,
"seed0": 3407,
"shared_attention_norm": false,
"stats_encoding": false,
"stats_encoding_new": false,
"sum_divided": false,
"triple_diff": false,
"triple_diff_new": false,
"unet": true,
"vocab_size": 1,
"vq": false,
"warmup_steps": 2000,
"weight_decay": 0.1,
"yj_trans": false
} |