{
  "_mlp_class": "LLaMAMLP",
  "_norm_class": "FusedRMSNorm",
  "average_2": false,
  "betas": [
    0.9,
    0.95
  ],
  "bias": false,
  "block_size": 8224,
  "block_size_val": 2080,
  "condense_ratio": 1,
  "decay_lr": true,
  "discount": true,
  "eval_iters": 100,
  "eval_step_interval": 1000,
  "forecasting_patch": 1,
  "global_batch_size": 512,
  "grad_clip": 1.0,
  "group": "70m-test_all",
  "haar_loss_match": false,
  "haar_trans": true,
  "haar_trans_inv": true,
  "haar_trans_norm": "backward",
  "half_diff": false,
  "imputation": false,
  "inner_norm": false,
  "inter_control": false,
  "intermediate_size": 3072,
  "is_diff": false,
  "is_smape": false,
  "learning_rate": 0.0005,
  "log_step_interval": 10,
  "max_step": 100000,
  "mean_replace": false,
  "micro_batch_size": 64,
  "micro_batch_size_val": 256,
  "min_lr": 1e-05,
  "mix_train": false,
  "multi_loss": false,
  "n_cot": 1,
  "n_embd": 768,
  "n_head": 12,
  "n_layer": 12,
  "n_query_groups": 4,
  "name": "small-100k",
  "new_arch": false,
  "new_tokenizer": false,
  "norm_eps": 1e-05,
  "num_of_devices": 4,
  "num_of_nodes": 2,
  "org": "Ali-Could",
  "ou": false,
  "ou_mean": false,
  "ou_prod": false,
  "padded_vocab_size": "None",
  "padding_multiple": 1,
  "parallel_residual": false,
  "patch_size": 32,
  "pid": false,
  "quantitle": true,
  "rollback_win": 256,
  "rolling_patch": 6,
  "rope_base": 10000,
  "rotary_percentage": 1.0,
  "save_step_interval": 10000,
  "scaling": true,
  "seed0": 3407,
  "shared_attention_norm": false,
  "stats_encoding": false,
  "stats_encoding_new": false,
  "sum_divided": false,
  "triple_diff": false,
  "triple_diff_new": false,
  "unet": true,
  "vocab_size": 1,
  "vq": false,
  "warmup_steps": 2000,
  "weight_decay": 0.1,
  "yj_trans": false
}