{
    "_attn_implementation": "sdpa",
    "architectures": [
        "LaneformerForCausalLM"
    ],
    "broadcast_delay": 2,
    "dtype": "bfloat16",
    "eos_token_id": 2,
    "hidden_size": 3072,
    "intermediate_size": 12288,
    "layer_types": [
        "sliding_attention",
        "sliding_attention",
        "sliding_attention",
        "sliding_attention",
        "sliding_attention",
        "sliding_attention",
        "sliding_attention",
        "sliding_attention",
        "sliding_attention",
        "sliding_attention",
        "full_attention",
        "full_attention",
        "full_attention",
        "full_attention",
        "full_attention"
    ],
    "lm_head_type": "vocab_parallel",
    "max_position_embeddings": 4096,
    "max_seq_len": 4096,
    "model_file": "laneformer.py",
    "norm_eps": 1e-05,
    "num_attention_heads": 32,
    "num_hidden_layers": 15,
    "num_key_value_heads": 16,
    "num_lanes": 8,
    "pad_token_id": 2,
    "pre_norm_lane_agg": false,
    "quantization": {
        "bits": 4,
        "group_size": 64,
        "mode": "affine"
    },
    "quantization_config": {
        "bits": 4,
        "group_size": 64,
        "mode": "affine"
    },
    "replicated_rmsn_scale": true,
    "rope_theta": 10000,
    "sliding_window": 2048,
    "swa_layers": [
        0,
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9
    ],
    "tie_word_embeddings": false,
    "transformers_version": "4.57.6",
    "use_attention_comm": true,
    "use_cache": false,
    "use_early_comm": false,
    "use_mlp_comm": true,
    "vocab_size": 32000
}