{
  "vocab_size": 16000,
  "d_model": 576,
  "n_layers": 14,
  "d_state": 16,
  "d_conv": 4,
  "expand": 2,
  "dt_rank": 72,
  "dt_min": 0.001,
  "dt_max": 0.1,
  "dt_init": "random",
  "dt_scale": 1.0,
  "scan_chunk_size": 32,
  "scan_impl": "loop",
  "scan_backend": "torch",
  "ssm_scan_in_fp32": true,
  "scan_segment_size": 2048,
  "ssm_state_schedule": [
    16,
    32,
    64
  ],
  "dropout": 0.0,
  "bias": false,
  "activation": "silu",
  "norm_type": "rmsnorm",
  "norm_eps": 1e-06,
  "norm_bias": false,
  "max_seq_len": 2048,
  "pad_token_id": 1,
  "model_kind": "causal_lm",
  "input_modality": "text",
  "output_modality": "text",
  "tie_word_embeddings": true,
  "initializer_range": 0.02,
  "residual_scale_init": true,
  "residual_in_fp32": true,
  "gradient_checkpointing": false,
  "model_alias": "SIGER",
  "model_base_name": "SIGER",
  "use_moe": true,
  "moe_num_experts": 8,
  "moe_top_k": 2,
  "moe_expert_hidden_mult": 3,
  "moe_layers_every": 2,
  "moe_aux_loss_weight": 0.01,
  "moe_router_jitter": 0.01,
  "moe_z_loss_weight": 0.001,
  "moe_hierarchy_enabled": true,
  "moe_hierarchy_branching": 2,
  "moe_hierarchy_aux_weight": 0.25,
  "use_hybrid_attention": true,
  "attn_layers_every": 7,
  "attn_num_heads": 8,
  "attn_window_size": 2048,
  "use_rope": true,
  "rope_theta": 1000000.0,
  "attn_global_every": 6,
  "attn_allow_global": false,
  "attn_query_chunk_size": 256,
  "attn_runtime_cache_limit": 16,
  "rope_scaling_factor": 8.0,
  "long_context_mode": true,
  "stream_chunk_size": 256,
  "memory_slots": 256,
  "memory_top_k": 8,
  "logits_chunk_size": 256,
  "stream_detach_state": true,
  "use_mtp": true,
  "mtp_num_heads": 2,
  "mtp_loss_weight": 0.0,
  "model_type": "siger",
  "architectures": [
    "SigerForCausalLM"
  ],
  "auto_map": {
    "AutoConfig": "configuration_siger.SigerConfig",
    "AutoModelForCausalLM": "modeling_siger.SigerForCausalLM"
  },
  "bos_token_id": 11,
  "eos_token_id": 12
}