File size: 1,964 Bytes
2719787 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 | {
"architectures": [
"DatForCausalLM"
],
"auto_map": {
"AutoConfig": "configuration_dat.DatConfig",
"AutoModel": "modeling_dat.DatModel",
"AutoModelForCausalLM": "modeling_dat.DatForCausalLM",
"AutoModelForMaskedLM": "modeling_dat.DatForMaskedLM",
"AutoModelForSequenceClassification": "modeling_dat.DatForSequenceClassification"
},
"bos_token_id": 1,
"dff_factor": 4,
"dropout": 0.1,
"dtype": "float32",
"eos_token_id": 2,
"ffn_activation": "swiglu",
"ffn_hidden_dim_mode": "dff_factor",
"hidden_dim": 1024,
"init_range": 0.15,
"init_scheme": "normal_0_02_scaled_projection",
"max_position_embeddings": 514,
"max_rel_pos": 512,
"max_seq_len": 514,
"mlm_head_enabled": false,
"model_type": "dat",
"n_heads_ra": 4,
"n_heads_sa": 12,
"n_layers": 16,
"n_symbols": null,
"norm_first": true,
"norm_type": "layernorm",
"pad_token_id": 3,
"pe_type": "rope",
"positional_symbols_sinusoidal": false,
"ra_n_relations": null,
"ra_rel_activation": "identity",
"ra_symmetric_rels": false,
"ra_type": "rca",
"relative_symbols_rope": false,
"relsymbolic_dropout": 0.0,
"relsymbolic_include_self": false,
"relsymbolic_neighborhood_size": 2,
"relsymbolic_normalize_rels": true,
"relsymbolic_rel_n_heads": 4,
"relsymbolic_rel_scale": null,
"relsymbolic_symbolic_attn_n_heads": 4,
"relsymbolic_symbolic_attn_scale": null,
"relsymbolic_trainable_symbols": true,
"relsymbolic_use_bias": false,
"rope_theta": 10000.0,
"segment_boundary_token_id": null,
"sequence_boundary_policy": "eos_document",
"share_attn_params": false,
"shared_symbol_retriever": true,
"symbol_dim": null,
"symbol_retrieval": "relative",
"symbolic_attn_n_heads": null,
"symbolic_use_bias": false,
"tie_lm_head": false,
"tie_word_embeddings": false,
"transformers_version": "5.9.0",
"use_bias_ffn": true,
"use_bias_out": false,
"use_bias_qkv": false,
"vocab_size": 16384
}
|