| { |
| "architectures": [ |
| "DatForCausalLM" |
| ], |
| "auto_map": { |
| "AutoConfig": "configuration_dat.DatConfig", |
| "AutoModel": "modeling_dat.DatModel", |
| "AutoModelForCausalLM": "modeling_dat.DatForCausalLM", |
| "AutoModelForMaskedLM": "modeling_dat.DatForMaskedLM", |
| "AutoModelForSequenceClassification": "modeling_dat.DatForSequenceClassification" |
| }, |
| "bos_token_id": 1, |
| "dff_factor": 4, |
| "dropout": 0.1, |
| "dtype": "float32", |
| "eos_token_id": 2, |
| "ffn_activation": "swiglu", |
| "ffn_hidden_dim_mode": "dff_factor", |
| "hidden_dim": 1024, |
| "init_range": 0.15, |
| "init_scheme": "normal_0_02_scaled_projection", |
| "max_position_embeddings": 514, |
| "max_rel_pos": 512, |
| "max_seq_len": 514, |
| "mlm_head_enabled": false, |
| "model_type": "dat", |
| "n_heads_ra": 4, |
| "n_heads_sa": 12, |
| "n_layers": 16, |
| "n_symbols": null, |
| "norm_first": true, |
| "norm_type": "layernorm", |
| "pad_token_id": 3, |
| "pe_type": "rope", |
| "positional_symbols_sinusoidal": false, |
| "ra_n_relations": null, |
| "ra_rel_activation": "identity", |
| "ra_symmetric_rels": false, |
| "ra_type": "rca", |
| "relative_symbols_rope": false, |
| "relsymbolic_dropout": 0.0, |
| "relsymbolic_include_self": false, |
| "relsymbolic_neighborhood_size": 2, |
| "relsymbolic_normalize_rels": true, |
| "relsymbolic_rel_n_heads": 4, |
| "relsymbolic_rel_scale": null, |
| "relsymbolic_symbolic_attn_n_heads": 4, |
| "relsymbolic_symbolic_attn_scale": null, |
| "relsymbolic_trainable_symbols": true, |
| "relsymbolic_use_bias": false, |
| "rope_theta": 10000.0, |
| "segment_boundary_token_id": null, |
| "sequence_boundary_policy": "eos_document", |
| "share_attn_params": false, |
| "shared_symbol_retriever": true, |
| "symbol_dim": null, |
| "symbol_retrieval": "relative", |
| "symbolic_attn_n_heads": null, |
| "symbolic_use_bias": false, |
| "tie_lm_head": false, |
| "tie_word_embeddings": false, |
| "transformers_version": "5.9.0", |
| "use_bias_ffn": true, |
| "use_bias_out": false, |
| "use_bias_qkv": false, |
| "vocab_size": 16384 |
| } |
|
|
