{
  "hidden_size": 896,
  "num_hidden_layers": 16,
  "intermediate_size": 256,
  "vocab_size": 32000,
  "num_attention_heads": 14,
  "num_key_value_heads": 2,
  "attention_type": "gqa",
  "attention_dropout": 0.0,
  "use_qk_norm": true,
  "sliding_window": null,
  "causal_conv_kernel_size": 4,
  "causal_conv_dilation_cycle": 8,
  "causal_state_rank": 16,
  "causal_context_gate_init": 1.0,
  "causal_contextual_mix_init": 0.0,
  "causal_context_fusion_size": 0,
  "causal_stable_delta": false,
  "causal_delta_chunk_size": 512,
  "causal_delta_timescales": 1,
  "causal_delta_collision_normalized": false,
  "causal_delta_lexical_values": false,
  "causal_delta_lexical_forge": false,
  "causal_delta_occurrence_address": false,
  "tr_mha_num_experts": 4,
  "tr_mha_adapter_rank": 8,
  "tr_mha_top_k": 2,
  "tr_mha_adapter_gate_init": 0.1,
  "tr_mha_id_primary_logit": 2.0,
  "tr_mha_id_secondary_logit": 1.0,
  "tr_mha_id_other_logit": -2.0,
  "tr_mha_verifier_gate_init": 0.1,
  "tr_mha_verifier_temperature": 1.0,
  "tr_mha_targets": "qv",
  "max_position_embeddings": 2048,
  "rope_theta": 10000.0,
  "rope_type": "standard",
  "rope_fraction": 1.0,
  "mlp_type": "tr_hash_engine",
  "hidden_act": "silu",
  "num_experts": 4,
  "expert_initialization": "gpt_normal",
  "token_frequencies": null,
  "routing_strategy": "token_id_multi_hash",
  "route_hash_count": 2,
  "lsh_routing": false,
  "lsh_bits": 0,
  "lsh_from_layer": 0,
  "lsh_threshold_mode": "zero",
  "shared_expert": true,
  "shared_intermediate_size": 3072,
  "shared_expert_chunk_tokens": 0,
  "use_shared_routed_gates": false,
  "shared_gate_init": 1.0,
  "routed_gate_init": 1.0,
  "shared_output_scale": 1.0,
  "routed_output_scale": 2.0,
  "shared_output_scale_first_layer": null,
  "shared_output_scale_last_layer": null,
  "routed_output_scale_first_layer": null,
  "routed_output_scale_last_layer": null,
  "top_k": 2,
  "top_k_primary_weight": 0.5,
  "learn_hash_pair_gates": false,
  "hash_pair_gate_init": 0.5,
  "learn_hash_channel_modulation": false,
  "hash_channel_scale_init": 0.0,
  "static_expert_capacity": false,
  "use_custom_kernels": "auto",
  "collect_moe_telemetry": false,
  "use_cggr": "auto",
  "active_num_experts": null,
  "active_expert_width": null,
  "lexical_object_rank": 16,
  "lexical_object_gate_init": 0.1,
  "tie_lexical_object_embeddings": false,
  "micro_num_experts": 4,
  "micro_expert_width": 16,
  "micro_expert_gate_init": 0.1,
  "lexical_gqa_rank": 16,
  "lexical_gqa_gate_init": 0.0,
  "lexical_gqa_use_token_code": true,
  "lexical_key_gate_init": 0.05,
  "lexical_zipf_path": null,
  "lexical_zipf_mode": "uniform",
  "lexical_zipf_alpha": 0.25,
  "lexical_zipf_floor": 0.1,
  "lexical_zipf_permutation_seed": 1729,
  "norm_type": "rmsnorm",
  "norm_eps": 1e-06,
  "tie_word_embeddings": true,
  "use_sdpa": true,
  "use_cache": true,
  "is_causal": true,
  "initializer_range": 0.02,
  "use_mup_init": false,
  "use_mup_attn_scale": false,
  "use_mup_output_mult": false,
  "mup_base_width": 256,
  "extra_config": {}
}