| name: mosaicfm | |
| d_model: 2560 | |
| n_layers: 32 | |
| init_device: cpu | |
| expansion_ratio: 4 | |
| standard_scale_outputs: false | |
| transformer_activation: gelu | |
| n_heads: 20 | |
| norm_scheme: pre | |
| use_generative_training: false | |
| use_cell_conditioned_generation: false | |
| use_glu: false | |
| cell_emb_style: cls | |
| attn_config: | |
| attn_impl: flash | |
| use_attn_mask: false | |
| attn_type: grouped_query_attention | |
| kv_nheads: 20 | |
| attn_pdrop: 0.0 | |
| norm_config: | |
| norm_type: layernorm | |
| eps: 1.0e-05 | |
| expression_encoder: | |
| input_emb_style: continuous | |
| dropout: 0.1 | |
| max_value: 512 | |
| activation: gelu | |
| use_norm: true | |
| gene_encoder: | |
| use_norm: true | |
| mvc: | |
| arch_style: inner product | |
| query_activation: sigmoid | |
| scaled_dot_product: true | |
| expression_decoder: | |
| n_outputs: 1 | |
| n_layers: 1 | |
| activation: gelu | |
| chemical_encoder: | |
| num_drugs: 378 | |
| fp_dim: 2048 | |
| activation: gelu | |
| padding_idx: 0 | |
| freeze: false | |
| vocab_size: 62721 | |
| precision: amp_bf16 | |
| wandb_id: vevotx/vevo-MFM-v2/cdk744ih | |