checkpoint-1000/rvq_encoder_config.json

{
  "codebook_vocab_sizes": [
    16384,
    1024,
    1024,
    1024,
    1024,
    1024,
    1024,
    1024
  ],
  "conv_dilations": [
    1,
    3,
    9
  ],
  "d_model": 1088,
  "depth_decoder": true,
  "depth_decoder_dim": 512,
  "depth_decoder_dropout": 0.1,
  "depth_decoder_ff_mult": 4,
  "depth_decoder_heads": 8,
  "depth_decoder_layers": 2,
  "dropout": 0.1,
  "ff_mult": 4,
  "latent_channels": 128,
  "max_position_embeddings": 128,
  "mup": true,
  "mup_attention_multiplier": 8.0,
  "mup_output_mult": 1.0,
  "mup_readout_zero_init": true,
  "num_heads": 17,
  "num_layers": 8
}