{ "architectures": [ "Zamba2ForCausalLM" ], "adapter_rank": 128, "add_bias_linear": false, "attention_dropout": 0.0, "attention_head_dim": 128, "attention_hidden_size": 4096, "bos_token_id": 1, "chunk_size": 256, "eos_token_id": 2, "ffn_hidden_size": 8192, "hidden_act": "gelu", "hidden_size": 2048, "initializer_range": 0.02, "intermediate_size": 8192, "kv_channels": 64, "layers_block_type": [ "mamba", "mamba", "mamba", "mamba", "mamba", "hybrid", "mamba", "mamba", "mamba", "mamba", "mamba", "hybrid", "mamba", "mamba", "mamba", "mamba", "mamba", "hybrid", "mamba", "mamba", "mamba", "mamba", "mamba", "hybrid", "mamba", "mamba", "mamba", "mamba", "mamba", "hybrid", "mamba", "mamba", "mamba", "mamba", "mamba", "hybrid", "mamba", "mamba" ], "mamba_d_conv": 4, "mamba_d_state": 128, "mamba_expand": 2, "mamba_headdim": 64, "mamba_ngroups": 1, "max_position_embeddings": 4096, "model_type": "zamba2", "n_mamba_heads": 64, "num_attention_heads": 32, "num_hidden_layers": 38, "num_key_value_heads": 32, "num_logits_to_keep": 1, "num_mem_blocks": 1, "num_query_groups": 32, "pad_token_id": 0, "rms_norm_eps": 1e-05, "rope_theta": 10000, "time_step_floor": 0.0001, "time_step_limit": null, "time_step_max": 0.1, "time_step_min": 0.001, "transformers_version": "4.49.0.dev0", "use_cache": true, "use_conv_bias": true, "use_long_context": false, "use_mem_rope": true, "use_shared_attention_adapter": true, "use_shared_mlp_adapter": true, "vocab_size": 32000, "quantization_config": { "quant_method": "hxq", "bits": 8, "n_clusters": 256, "compressed_modules": [ "model.layers.0.mamba.in_proj", "model.layers.0.mamba.out_proj", "model.layers.10.mamba.in_proj", "model.layers.10.mamba.out_proj", "model.layers.11.linear", "model.layers.11.mamba_decoder.mamba.in_proj", "model.layers.11.mamba_decoder.mamba.out_proj", "model.layers.12.mamba.in_proj", "model.layers.12.mamba.out_proj", "model.layers.13.mamba.in_proj", "model.layers.13.mamba.out_proj", "model.layers.14.mamba.in_proj", "model.layers.14.mamba.out_proj", "model.layers.15.mamba.in_proj", "model.layers.15.mamba.out_proj", "model.layers.16.mamba.in_proj", "model.layers.16.mamba.out_proj", "model.layers.17.linear", "model.layers.17.mamba_decoder.mamba.in_proj", "model.layers.17.mamba_decoder.mamba.out_proj", "model.layers.18.mamba.in_proj", "model.layers.18.mamba.out_proj", "model.layers.19.mamba.in_proj", "model.layers.19.mamba.out_proj", "model.layers.1.mamba.in_proj", "model.layers.1.mamba.out_proj", "model.layers.20.mamba.in_proj", "model.layers.20.mamba.out_proj", "model.layers.21.mamba.in_proj", "model.layers.21.mamba.out_proj", "model.layers.22.mamba.in_proj", "model.layers.22.mamba.out_proj", "model.layers.23.linear", "model.layers.23.mamba_decoder.mamba.in_proj", "model.layers.23.mamba_decoder.mamba.out_proj", "model.layers.24.mamba.in_proj", "model.layers.24.mamba.out_proj", "model.layers.25.mamba.in_proj", "model.layers.25.mamba.out_proj", "model.layers.26.mamba.in_proj", "model.layers.26.mamba.out_proj", "model.layers.27.mamba.in_proj", "model.layers.27.mamba.out_proj", "model.layers.28.mamba.in_proj", "model.layers.28.mamba.out_proj", "model.layers.29.linear", "model.layers.29.mamba_decoder.mamba.in_proj", "model.layers.29.mamba_decoder.mamba.out_proj", "model.layers.2.mamba.in_proj", "model.layers.2.mamba.out_proj", "model.layers.30.mamba.in_proj", "model.layers.30.mamba.out_proj", "model.layers.31.mamba.in_proj", "model.layers.31.mamba.out_proj", "model.layers.32.mamba.in_proj", "model.layers.32.mamba.out_proj", "model.layers.33.mamba.in_proj", "model.layers.33.mamba.out_proj", "model.layers.34.mamba.in_proj", "model.layers.34.mamba.out_proj", "model.layers.35.linear", "model.layers.35.mamba_decoder.mamba.in_proj", "model.layers.35.mamba_decoder.mamba.out_proj", "model.layers.36.mamba.in_proj", "model.layers.36.mamba.out_proj", "model.layers.37.mamba.in_proj", "model.layers.37.mamba.out_proj", "model.layers.3.mamba.in_proj", "model.layers.3.mamba.out_proj", "model.layers.4.mamba.in_proj", "model.layers.4.mamba.out_proj", "model.layers.5.linear", "model.layers.5.mamba_decoder.mamba.in_proj", "model.layers.5.mamba_decoder.mamba.out_proj", "model.layers.5.shared_transformer.feed_forward.down_proj", "model.layers.5.shared_transformer.feed_forward.gate_up_proj_adapter_list.0.0", "model.layers.5.shared_transformer.feed_forward.gate_up_proj_adapter_list.0.1", "model.layers.5.shared_transformer.feed_forward.gate_up_proj_adapter_list.1.0", "model.layers.5.shared_transformer.feed_forward.gate_up_proj_adapter_list.1.1", "model.layers.5.shared_transformer.feed_forward.gate_up_proj_adapter_list.2.0", "model.layers.5.shared_transformer.feed_forward.gate_up_proj_adapter_list.2.1", "model.layers.5.shared_transformer.feed_forward.gate_up_proj_adapter_list.3.0", "model.layers.5.shared_transformer.feed_forward.gate_up_proj_adapter_list.3.1", "model.layers.5.shared_transformer.feed_forward.gate_up_proj_adapter_list.4.0", "model.layers.5.shared_transformer.feed_forward.gate_up_proj_adapter_list.4.1", "model.layers.5.shared_transformer.feed_forward.gate_up_proj_adapter_list.5.0", "model.layers.5.shared_transformer.feed_forward.gate_up_proj_adapter_list.5.1", "model.layers.5.shared_transformer.feed_forward.gate_up_proj", "model.layers.5.shared_transformer.self_attn.k_proj", "model.layers.5.shared_transformer.self_attn.linear_k_adapter_list.0.0", "model.layers.5.shared_transformer.self_attn.linear_k_adapter_list.0.1", "model.layers.5.shared_transformer.self_attn.linear_k_adapter_list.1.0", "model.layers.5.shared_transformer.self_attn.linear_k_adapter_list.1.1", "model.layers.5.shared_transformer.self_attn.linear_k_adapter_list.2.0", "model.layers.5.shared_transformer.self_attn.linear_k_adapter_list.2.1", "model.layers.5.shared_transformer.self_attn.linear_k_adapter_list.3.0", "model.layers.5.shared_transformer.self_attn.linear_k_adapter_list.3.1", "model.layers.5.shared_transformer.self_attn.linear_k_adapter_list.4.0", "model.layers.5.shared_transformer.self_attn.linear_k_adapter_list.4.1", "model.layers.5.shared_transformer.self_attn.linear_k_adapter_list.5.0", "model.layers.5.shared_transformer.self_attn.linear_k_adapter_list.5.1", "model.layers.5.shared_transformer.self_attn.linear_q_adapter_list.0.0", "model.layers.5.shared_transformer.self_attn.linear_q_adapter_list.0.1", "model.layers.5.shared_transformer.self_attn.linear_q_adapter_list.1.0", "model.layers.5.shared_transformer.self_attn.linear_q_adapter_list.1.1", "model.layers.5.shared_transformer.self_attn.linear_q_adapter_list.2.0", "model.layers.5.shared_transformer.self_attn.linear_q_adapter_list.2.1", "model.layers.5.shared_transformer.self_attn.linear_q_adapter_list.3.0", "model.layers.5.shared_transformer.self_attn.linear_q_adapter_list.3.1", "model.layers.5.shared_transformer.self_attn.linear_q_adapter_list.4.0", "model.layers.5.shared_transformer.self_attn.linear_q_adapter_list.4.1", "model.layers.5.shared_transformer.self_attn.linear_q_adapter_list.5.0", "model.layers.5.shared_transformer.self_attn.linear_q_adapter_list.5.1", "model.layers.5.shared_transformer.self_attn.linear_v_adapter_list.0.0", "model.layers.5.shared_transformer.self_attn.linear_v_adapter_list.0.1", "model.layers.5.shared_transformer.self_attn.linear_v_adapter_list.1.0", "model.layers.5.shared_transformer.self_attn.linear_v_adapter_list.1.1", "model.layers.5.shared_transformer.self_attn.linear_v_adapter_list.2.0", "model.layers.5.shared_transformer.self_attn.linear_v_adapter_list.2.1", "model.layers.5.shared_transformer.self_attn.linear_v_adapter_list.3.0", "model.layers.5.shared_transformer.self_attn.linear_v_adapter_list.3.1", "model.layers.5.shared_transformer.self_attn.linear_v_adapter_list.4.0", "model.layers.5.shared_transformer.self_attn.linear_v_adapter_list.4.1", "model.layers.5.shared_transformer.self_attn.linear_v_adapter_list.5.0", "model.layers.5.shared_transformer.self_attn.linear_v_adapter_list.5.1", "model.layers.5.shared_transformer.self_attn.o_proj", "model.layers.5.shared_transformer.self_attn.q_proj", "model.layers.5.shared_transformer.self_attn.v_proj", "model.layers.6.mamba.in_proj", "model.layers.6.mamba.out_proj", "model.layers.7.mamba.in_proj", "model.layers.7.mamba.out_proj", "model.layers.8.mamba.in_proj", "model.layers.8.mamba.out_proj", "model.layers.9.mamba.in_proj", "model.layers.9.mamba.out_proj" ], "compression_ratio": 4.0, "n_svd_routed": 0 } }