mvc:
  arch_style: inner product
  query_activation: sigmoid
  scaled_dot_product: true
name: vevo_scgpt
d_model: 2048
n_heads: 16
use_glu: false
n_layers: 24
attn_config:
  attn_impl: flash
  use_attn_mask: false
  attn_type: grouped_query_attention
  kv_nheads: 16
  attn_pdrop: 0
init_device: cpu
norm_config:
  eps: 1.0e-05
  norm_type: layernorm
norm_scheme: pre
gene_encoder:
  use_norm: true
cell_emb_style: cls
expansion_ratio: 4
expression_decoder:
  n_layers: 1
  n_outputs: 1
  activation: leaky_relu
expression_encoder:
  dropout: 0.1
  use_norm: true
  max_value: 512
  activation: relu
  input_emb_style: continuous
standard_scale_outputs: false
transformer_activation: relu
use_generative_training: false
use_cell_conditioned_generation: false
vocab_size: 62720
precision: amp_bf16
wandb_id: vevotx/vevo-scgpt/26iormxc