Add RVQ encoder checkpoint 10500

checkpoint-10500/mup_base_shapes.bsh

@@ -0,0 +1,517 @@
+# This is a base shape file encoded in yaml
+# - `null` indicates a dimension is "finite", i.e. a non-"width" dimension
+# - a number indicates the base dimension of an "infinite" dimension, i.e. some notion of "width"
+blocks.0.conv1.bias:
+- 128
+blocks.0.conv1.weight:
+- 128
+- 128
+- null
+blocks.0.conv2.bias:
+- 128
+blocks.0.conv2.weight:
+- 128
+- 128
+- null
+blocks.0.norm.bias:
+- 128
+blocks.0.norm.weight:
+- 128
+blocks.1.conv1.bias:
+- 128
+blocks.1.conv1.weight:
+- 128
+- 128
+- null
+blocks.1.conv2.bias:
+- 128
+blocks.1.conv2.weight:
+- 128
+- 128
+- null
+blocks.1.norm.bias:
+- 128
+blocks.1.norm.weight:
+- 128
+blocks.2.conv1.bias:
+- 128
+blocks.2.conv1.weight:
+- 128
+- 128
+- null
+blocks.2.conv2.bias:
+- 128
+blocks.2.conv2.weight:
+- 128
+- 128
+- null
+blocks.2.norm.bias:
+- 128
+blocks.2.norm.weight:
+- 128
+conv_in.bias:
+- 128
+conv_in.weight:
+- 128
+- null
+- null
+depth_decoder.context_projection.weight:
+- null
+- 128
+depth_decoder.heads.0.bias:
+- null
+depth_decoder.heads.0.weight:
+- null
+- null
+depth_decoder.heads.1.bias:
+- null
+depth_decoder.heads.1.weight:
+- null
+- null
+depth_decoder.heads.2.bias:
+- null
+depth_decoder.heads.2.weight:
+- null
+- null
+depth_decoder.heads.3.bias:
+- null
+depth_decoder.heads.3.weight:
+- null
+- null
+depth_decoder.heads.4.bias:
+- null
+depth_decoder.heads.4.weight:
+- null
+- null
+depth_decoder.heads.5.bias:
+- null
+depth_decoder.heads.5.weight:
+- null
+- null
+depth_decoder.heads.6.bias:
+- null
+depth_decoder.heads.6.weight:
+- null
+- null
+depth_decoder.layers.0.k_proj.bias:
+- null
+depth_decoder.layers.0.k_proj.weight:
+- null
+- null
+depth_decoder.layers.0.linear1.bias:
+- null
+depth_decoder.layers.0.linear1.weight:
+- null
+- null
+depth_decoder.layers.0.linear2.bias:
+- null
+depth_decoder.layers.0.linear2.weight:
+- null
+- null
+depth_decoder.layers.0.norm1.bias:
+- null
+depth_decoder.layers.0.norm1.weight:
+- null
+depth_decoder.layers.0.norm2.bias:
+- null
+depth_decoder.layers.0.norm2.weight:
+- null
+depth_decoder.layers.0.out_proj.bias:
+- null
+depth_decoder.layers.0.out_proj.weight:
+- null
+- null
+depth_decoder.layers.0.q_proj.bias:
+- null
+depth_decoder.layers.0.q_proj.weight:
+- null
+- null
+depth_decoder.layers.0.v_proj.bias:
+- null
+depth_decoder.layers.0.v_proj.weight:
+- null
+- null
+depth_decoder.layers.1.k_proj.bias:
+- null
+depth_decoder.layers.1.k_proj.weight:
+- null
+- null
+depth_decoder.layers.1.linear1.bias:
+- null
+depth_decoder.layers.1.linear1.weight:
+- null
+- null
+depth_decoder.layers.1.linear2.bias:
+- null
+depth_decoder.layers.1.linear2.weight:
+- null
+- null
+depth_decoder.layers.1.norm1.bias:
+- null
+depth_decoder.layers.1.norm1.weight:
+- null
+depth_decoder.layers.1.norm2.bias:
+- null
+depth_decoder.layers.1.norm2.weight:
+- null
+depth_decoder.layers.1.out_proj.bias:
+- null
+depth_decoder.layers.1.out_proj.weight:
+- null
+- null
+depth_decoder.layers.1.q_proj.bias:
+- null
+depth_decoder.layers.1.q_proj.weight:
+- null
+- null
+depth_decoder.layers.1.v_proj.bias:
+- null
+depth_decoder.layers.1.v_proj.weight:
+- null
+- null
+depth_decoder.norm.bias:
+- null
+depth_decoder.norm.weight:
+- null
+depth_decoder.position:
+- null
+- null
+- null
+depth_decoder.prior_embeddings.0.weight:
+- null
+- null
+depth_decoder.prior_embeddings.1.weight:
+- null
+- null
+depth_decoder.prior_embeddings.2.weight:
+- null
+- null
+depth_decoder.prior_embeddings.3.weight:
+- null
+- null
+depth_decoder.prior_embeddings.4.weight:
+- null
+- null
+depth_decoder.prior_embeddings.5.weight:
+- null
+- null
+depth_decoder.prior_embeddings.6.weight:
+- null
+- null
+heads.0.bias:
+- null
+heads.0.weight:
+- null
+- 128
+norm_out.bias:
+- 128
+norm_out.weight:
+- 128
+position:
+- null
+- null
+- 128
+transformer.0.k_proj.bias:
+- 128
+transformer.0.k_proj.weight:
+- 128
+- 128
+transformer.0.linear1.bias:
+- 512
+transformer.0.linear1.weight:
+- 512
+- 128
+transformer.0.linear2.bias:
+- 128
+transformer.0.linear2.weight:
+- 128
+- 512
+transformer.0.norm1.bias:
+- 128
+transformer.0.norm1.weight:
+- 128
+transformer.0.norm2.bias:
+- 128
+transformer.0.norm2.weight:
+- 128
+transformer.0.out_proj.bias:
+- 128
+transformer.0.out_proj.weight:
+- 128
+- 128
+transformer.0.q_proj.bias:
+- 128
+transformer.0.q_proj.weight:
+- 128
+- 128
+transformer.0.v_proj.bias:
+- 128
+transformer.0.v_proj.weight:
+- 128
+- 128
+transformer.1.k_proj.bias:
+- 128
+transformer.1.k_proj.weight:
+- 128
+- 128
+transformer.1.linear1.bias:
+- 512
+transformer.1.linear1.weight:
+- 512
+- 128
+transformer.1.linear2.bias:
+- 128
+transformer.1.linear2.weight:
+- 128
+- 512
+transformer.1.norm1.bias:
+- 128
+transformer.1.norm1.weight:
+- 128
+transformer.1.norm2.bias:
+- 128
+transformer.1.norm2.weight:
+- 128
+transformer.1.out_proj.bias:
+- 128
+transformer.1.out_proj.weight:
+- 128
+- 128
+transformer.1.q_proj.bias:
+- 128
+transformer.1.q_proj.weight:
+- 128
+- 128
+transformer.1.v_proj.bias:
+- 128
+transformer.1.v_proj.weight:
+- 128
+- 128
+transformer.2.k_proj.bias:
+- 128
+transformer.2.k_proj.weight:
+- 128
+- 128
+transformer.2.linear1.bias:
+- 512
+transformer.2.linear1.weight:
+- 512
+- 128
+transformer.2.linear2.bias:
+- 128
+transformer.2.linear2.weight:
+- 128
+- 512
+transformer.2.norm1.bias:
+- 128
+transformer.2.norm1.weight:
+- 128
+transformer.2.norm2.bias:
+- 128
+transformer.2.norm2.weight:
+- 128
+transformer.2.out_proj.bias:
+- 128
+transformer.2.out_proj.weight:
+- 128
+- 128
+transformer.2.q_proj.bias:
+- 128
+transformer.2.q_proj.weight:
+- 128
+- 128
+transformer.2.v_proj.bias:
+- 128
+transformer.2.v_proj.weight:
+- 128
+- 128
+transformer.3.k_proj.bias:
+- 128
+transformer.3.k_proj.weight:
+- 128
+- 128
+transformer.3.linear1.bias:
+- 512
+transformer.3.linear1.weight:
+- 512
+- 128
+transformer.3.linear2.bias:
+- 128
+transformer.3.linear2.weight:
+- 128
+- 512
+transformer.3.norm1.bias:
+- 128
+transformer.3.norm1.weight:
+- 128
+transformer.3.norm2.bias:
+- 128
+transformer.3.norm2.weight:
+- 128
+transformer.3.out_proj.bias:
+- 128
+transformer.3.out_proj.weight:
+- 128
+- 128
+transformer.3.q_proj.bias:
+- 128
+transformer.3.q_proj.weight:
+- 128
+- 128
+transformer.3.v_proj.bias:
+- 128
+transformer.3.v_proj.weight:
+- 128
+- 128
+transformer.4.k_proj.bias:
+- 128
+transformer.4.k_proj.weight:
+- 128
+- 128
+transformer.4.linear1.bias:
+- 512
+transformer.4.linear1.weight:
+- 512
+- 128
+transformer.4.linear2.bias:
+- 128
+transformer.4.linear2.weight:
+- 128
+- 512
+transformer.4.norm1.bias:
+- 128
+transformer.4.norm1.weight:
+- 128
+transformer.4.norm2.bias:
+- 128
+transformer.4.norm2.weight:
+- 128
+transformer.4.out_proj.bias:
+- 128
+transformer.4.out_proj.weight:
+- 128
+- 128
+transformer.4.q_proj.bias:
+- 128
+transformer.4.q_proj.weight:
+- 128
+- 128
+transformer.4.v_proj.bias:
+- 128
+transformer.4.v_proj.weight:
+- 128
+- 128
+transformer.5.k_proj.bias:
+- 128
+transformer.5.k_proj.weight:
+- 128
+- 128
+transformer.5.linear1.bias:
+- 512
+transformer.5.linear1.weight:
+- 512
+- 128
+transformer.5.linear2.bias:
+- 128
+transformer.5.linear2.weight:
+- 128
+- 512
+transformer.5.norm1.bias:
+- 128
+transformer.5.norm1.weight:
+- 128
+transformer.5.norm2.bias:
+- 128
+transformer.5.norm2.weight:
+- 128
+transformer.5.out_proj.bias:
+- 128
+transformer.5.out_proj.weight:
+- 128
+- 128
+transformer.5.q_proj.bias:
+- 128
+transformer.5.q_proj.weight:
+- 128
+- 128
+transformer.5.v_proj.bias:
+- 128
+transformer.5.v_proj.weight:
+- 128
+- 128
+transformer.6.k_proj.bias:
+- 128
+transformer.6.k_proj.weight:
+- 128
+- 128
+transformer.6.linear1.bias:
+- 512
+transformer.6.linear1.weight:
+- 512
+- 128
+transformer.6.linear2.bias:
+- 128
+transformer.6.linear2.weight:
+- 128
+- 512
+transformer.6.norm1.bias:
+- 128
+transformer.6.norm1.weight:
+- 128
+transformer.6.norm2.bias:
+- 128
+transformer.6.norm2.weight:
+- 128
+transformer.6.out_proj.bias:
+- 128
+transformer.6.out_proj.weight:
+- 128
+- 128
+transformer.6.q_proj.bias:
+- 128
+transformer.6.q_proj.weight:
+- 128
+- 128
+transformer.6.v_proj.bias:
+- 128
+transformer.6.v_proj.weight:
+- 128
+- 128
+transformer.7.k_proj.bias:
+- 128
+transformer.7.k_proj.weight:
+- 128
+- 128
+transformer.7.linear1.bias:
+- 512
+transformer.7.linear1.weight:
+- 512
+- 128
+transformer.7.linear2.bias:
+- 128
+transformer.7.linear2.weight:
+- 128
+- 512
+transformer.7.norm1.bias:
+- 128
+transformer.7.norm1.weight:
+- 128
+transformer.7.norm2.bias:
+- 128
+transformer.7.norm2.weight:
+- 128
+transformer.7.out_proj.bias:
+- 128
+transformer.7.out_proj.weight:
+- 128
+- 128
+transformer.7.q_proj.bias:
+- 128
+transformer.7.q_proj.weight:
+- 128
+- 128
+transformer.7.v_proj.bias:
+- 128
+transformer.7.v_proj.weight:
+- 128
+- 128

checkpoint-10500/mup_base_shapes.bsh.meta.json

@@ -0,0 +1,7 @@
+{
+  "depth_decoder_dim": 512,
+  "depth_decoder_ff_mult": 4,
+  "depth_decoder_heads": 8,
+  "depth_decoder_layers": 2,
+  "scope": "rvq-depth-decoder-v1"
+}

checkpoint-10500/rvq_encoder.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2ed5fbbe8649384e8ac45f8133500664bd38a9043712897b653cbd40760edda7
+size 676055232

checkpoint-10500/rvq_encoder_config.json

@@ -0,0 +1,34 @@
+{
+  "codebook_vocab_sizes": [
+    16384,
+    1024,
+    1024,
+    1024,
+    1024,
+    1024,
+    1024,
+    1024
+  ],
+  "conv_dilations": [
+    1,
+    3,
+    9
+  ],
+  "d_model": 1088,
+  "depth_decoder": true,
+  "depth_decoder_dim": 512,
+  "depth_decoder_dropout": 0.1,
+  "depth_decoder_ff_mult": 4,
+  "depth_decoder_heads": 8,
+  "depth_decoder_layers": 2,
+  "dropout": 0.1,
+  "ff_mult": 4,
+  "latent_channels": 128,
+  "max_position_embeddings": 128,
+  "mup": true,
+  "mup_attention_multiplier": 8.0,
+  "mup_output_mult": 1.0,
+  "mup_readout_zero_init": true,
+  "num_heads": 17,
+  "num_layers": 8
+}