| { |
| "architectures": [ |
| "ConvGPTV2ForCausalLM" |
| ], |
| "attention_every": 0, |
| "attention_gate_init": -2.0, |
| "attention_num_heads": 4, |
| "attention_start_layer": 0, |
| "bos_token_id": 1, |
| "branch_dropout": 0.0, |
| "chunk_memory_gate_init": -4.0, |
| "chunk_memory_include_current_chunk": false, |
| "chunk_memory_size": 64, |
| "chunk_memory_token_top_k": 64, |
| "chunk_memory_top_k": 2, |
| "conv1d_dilations": [ |
| 1, |
| 2, |
| 4, |
| 8 |
| ], |
| "conv1d_kernel_size": 3, |
| "conv1d_residual_gate_init": -6.0, |
| "conv2d_backend": "triton_gather", |
| "conv2d_chunk_size": 1024, |
| "conv2d_dilations": [ |
| 1, |
| 2, |
| 4, |
| 8 |
| ], |
| "conv2d_kernel_size": 3, |
| "conv2d_residual_gate_init": 2.0, |
| "conv_expand": 2, |
| "dropout": 0.0, |
| "dtype": "bfloat16", |
| "eos_token_id": 32009, |
| "fusion": "gated", |
| "gated_delta_conv_kernel_size": 4, |
| "gated_delta_every": 0, |
| "gated_delta_gate_init": 0.0, |
| "gated_delta_key_dim": 64, |
| "gated_delta_num_heads": 4, |
| "gated_delta_start_layer": 0, |
| "gated_delta_value_dim": 64, |
| "grid_size": 256, |
| "hidden_act": "silu", |
| "hidden_size": 512, |
| "initializer_range": 0.02, |
| "intermediate_size": 2048, |
| "linear_attention_every": 0, |
| "linear_attention_gate_init": 0.0, |
| "linear_attention_num_heads": 4, |
| "linear_attention_start_layer": 0, |
| "max_grid_tokens": 65536, |
| "max_position_embeddings": 65536, |
| "model_type": "convgpt_v2", |
| "num_hidden_layers": 32, |
| "pack_order": "sequence_to_curve", |
| "packing": "hilbert", |
| "pad_token_id": 32000, |
| "position_embedding_type": "rope_nope", |
| "retrieval_dropout": 0.0, |
| "retrieval_every": 2, |
| "retrieval_num_heads": 4, |
| "retrieval_num_slots": 64, |
| "retrieval_top_k": 4, |
| "rms_norm_eps": 1e-06, |
| "rope_theta": 10000.0, |
| "router_rope_fraction": 1.0, |
| "router_type": "chunk_token_memory", |
| "tie_word_embeddings": true, |
| "transformers_version": "5.9.0", |
| "two_d_every": 1, |
| "two_d_start_layer": 0, |
| "use_1d_branch": true, |
| "use_2d_branch": true, |
| "use_2d_depthwise": true, |
| "use_cache": false, |
| "use_row_col_embeddings": false, |
| "vocab_size": 32024 |
| } |
|
|
