Add RVQ encoder checkpoint 2000

best/checkpoint-2000/mup_base_shapes.bsh

@@ -0,0 +1,409 @@
+# This is a base shape file encoded in yaml
+# - `null` indicates a dimension is "finite", i.e. a non-"width" dimension
+# - a number indicates the base dimension of an "infinite" dimension, i.e. some notion of "width"
+blocks.0.conv1.bias:
+- 128
+blocks.0.conv1.weight:
+- 128
+- 128
+- null
+blocks.0.conv2.bias:
+- 128
+blocks.0.conv2.weight:
+- 128
+- 128
+- null
+blocks.0.norm.bias:
+- 128
+blocks.0.norm.weight:
+- 128
+blocks.1.conv1.bias:
+- 128
+blocks.1.conv1.weight:
+- 128
+- 128
+- null
+blocks.1.conv2.bias:
+- 128
+blocks.1.conv2.weight:
+- 128
+- 128
+- null
+blocks.1.norm.bias:
+- 128
+blocks.1.norm.weight:
+- 128
+blocks.2.conv1.bias:
+- 128
+blocks.2.conv1.weight:
+- 128
+- 128
+- null
+blocks.2.conv2.bias:
+- 128
+blocks.2.conv2.weight:
+- 128
+- 128
+- null
+blocks.2.norm.bias:
+- 128
+blocks.2.norm.weight:
+- 128
+conv_in.bias:
+- 128
+conv_in.weight:
+- 128
+- null
+- null
+heads.0.bias:
+- null
+heads.0.weight:
+- null
+- 128
+heads.1.bias:
+- null
+heads.1.weight:
+- null
+- 128
+heads.2.bias:
+- null
+heads.2.weight:
+- null
+- 128
+heads.3.bias:
+- null
+heads.3.weight:
+- null
+- 128
+heads.4.bias:
+- null
+heads.4.weight:
+- null
+- 128
+heads.5.bias:
+- null
+heads.5.weight:
+- null
+- 128
+heads.6.bias:
+- null
+heads.6.weight:
+- null
+- 128
+heads.7.bias:
+- null
+heads.7.weight:
+- null
+- 128
+norm_out.bias:
+- 128
+norm_out.weight:
+- 128
+position:
+- null
+- null
+- 128
+transformer.0.k_proj.bias:
+- 128
+transformer.0.k_proj.weight:
+- 128
+- 128
+transformer.0.linear1.bias:
+- 512
+transformer.0.linear1.weight:
+- 512
+- 128
+transformer.0.linear2.bias:
+- 128
+transformer.0.linear2.weight:
+- 128
+- 512
+transformer.0.norm1.bias:
+- 128
+transformer.0.norm1.weight:
+- 128
+transformer.0.norm2.bias:
+- 128
+transformer.0.norm2.weight:
+- 128
+transformer.0.out_proj.bias:
+- 128
+transformer.0.out_proj.weight:
+- 128
+- 128
+transformer.0.q_proj.bias:
+- 128
+transformer.0.q_proj.weight:
+- 128
+- 128
+transformer.0.v_proj.bias:
+- 128
+transformer.0.v_proj.weight:
+- 128
+- 128
+transformer.1.k_proj.bias:
+- 128
+transformer.1.k_proj.weight:
+- 128
+- 128
+transformer.1.linear1.bias:
+- 512
+transformer.1.linear1.weight:
+- 512
+- 128
+transformer.1.linear2.bias:
+- 128
+transformer.1.linear2.weight:
+- 128
+- 512
+transformer.1.norm1.bias:
+- 128
+transformer.1.norm1.weight:
+- 128
+transformer.1.norm2.bias:
+- 128
+transformer.1.norm2.weight:
+- 128
+transformer.1.out_proj.bias:
+- 128
+transformer.1.out_proj.weight:
+- 128
+- 128
+transformer.1.q_proj.bias:
+- 128
+transformer.1.q_proj.weight:
+- 128
+- 128
+transformer.1.v_proj.bias:
+- 128
+transformer.1.v_proj.weight:
+- 128
+- 128
+transformer.2.k_proj.bias:
+- 128
+transformer.2.k_proj.weight:
+- 128
+- 128
+transformer.2.linear1.bias:
+- 512
+transformer.2.linear1.weight:
+- 512
+- 128
+transformer.2.linear2.bias:
+- 128
+transformer.2.linear2.weight:
+- 128
+- 512
+transformer.2.norm1.bias:
+- 128
+transformer.2.norm1.weight:
+- 128
+transformer.2.norm2.bias:
+- 128
+transformer.2.norm2.weight:
+- 128
+transformer.2.out_proj.bias:
+- 128
+transformer.2.out_proj.weight:
+- 128
+- 128
+transformer.2.q_proj.bias:
+- 128
+transformer.2.q_proj.weight:
+- 128
+- 128
+transformer.2.v_proj.bias:
+- 128
+transformer.2.v_proj.weight:
+- 128
+- 128
+transformer.3.k_proj.bias:
+- 128
+transformer.3.k_proj.weight:
+- 128
+- 128
+transformer.3.linear1.bias:
+- 512
+transformer.3.linear1.weight:
+- 512
+- 128
+transformer.3.linear2.bias:
+- 128
+transformer.3.linear2.weight:
+- 128
+- 512
+transformer.3.norm1.bias:
+- 128
+transformer.3.norm1.weight:
+- 128
+transformer.3.norm2.bias:
+- 128
+transformer.3.norm2.weight:
+- 128
+transformer.3.out_proj.bias:
+- 128
+transformer.3.out_proj.weight:
+- 128
+- 128
+transformer.3.q_proj.bias:
+- 128
+transformer.3.q_proj.weight:
+- 128
+- 128
+transformer.3.v_proj.bias:
+- 128
+transformer.3.v_proj.weight:
+- 128
+- 128
+transformer.4.k_proj.bias:
+- 128
+transformer.4.k_proj.weight:
+- 128
+- 128
+transformer.4.linear1.bias:
+- 512
+transformer.4.linear1.weight:
+- 512
+- 128
+transformer.4.linear2.bias:
+- 128
+transformer.4.linear2.weight:
+- 128
+- 512
+transformer.4.norm1.bias:
+- 128
+transformer.4.norm1.weight:
+- 128
+transformer.4.norm2.bias:
+- 128
+transformer.4.norm2.weight:
+- 128
+transformer.4.out_proj.bias:
+- 128
+transformer.4.out_proj.weight:
+- 128
+- 128
+transformer.4.q_proj.bias:
+- 128
+transformer.4.q_proj.weight:
+- 128
+- 128
+transformer.4.v_proj.bias:
+- 128
+transformer.4.v_proj.weight:
+- 128
+- 128
+transformer.5.k_proj.bias:
+- 128
+transformer.5.k_proj.weight:
+- 128
+- 128
+transformer.5.linear1.bias:
+- 512
+transformer.5.linear1.weight:
+- 512
+- 128
+transformer.5.linear2.bias:
+- 128
+transformer.5.linear2.weight:
+- 128
+- 512
+transformer.5.norm1.bias:
+- 128
+transformer.5.norm1.weight:
+- 128
+transformer.5.norm2.bias:
+- 128
+transformer.5.norm2.weight:
+- 128
+transformer.5.out_proj.bias:
+- 128
+transformer.5.out_proj.weight:
+- 128
+- 128
+transformer.5.q_proj.bias:
+- 128
+transformer.5.q_proj.weight:
+- 128
+- 128
+transformer.5.v_proj.bias:
+- 128
+transformer.5.v_proj.weight:
+- 128
+- 128
+transformer.6.k_proj.bias:
+- 128
+transformer.6.k_proj.weight:
+- 128
+- 128
+transformer.6.linear1.bias:
+- 512
+transformer.6.linear1.weight:
+- 512
+- 128
+transformer.6.linear2.bias:
+- 128
+transformer.6.linear2.weight:
+- 128
+- 512
+transformer.6.norm1.bias:
+- 128
+transformer.6.norm1.weight:
+- 128
+transformer.6.norm2.bias:
+- 128
+transformer.6.norm2.weight:
+- 128
+transformer.6.out_proj.bias:
+- 128
+transformer.6.out_proj.weight:
+- 128
+- 128
+transformer.6.q_proj.bias:
+- 128
+transformer.6.q_proj.weight:
+- 128
+- 128
+transformer.6.v_proj.bias:
+- 128
+transformer.6.v_proj.weight:
+- 128
+- 128
+transformer.7.k_proj.bias:
+- 128
+transformer.7.k_proj.weight:
+- 128
+- 128
+transformer.7.linear1.bias:
+- 512
+transformer.7.linear1.weight:
+- 512
+- 128
+transformer.7.linear2.bias:
+- 128
+transformer.7.linear2.weight:
+- 128
+- 512
+transformer.7.norm1.bias:
+- 128
+transformer.7.norm1.weight:
+- 128
+transformer.7.norm2.bias:
+- 128
+transformer.7.norm2.weight:
+- 128
+transformer.7.out_proj.bias:
+- 128
+transformer.7.out_proj.weight:
+- 128
+- 128
+transformer.7.q_proj.bias:
+- 128
+transformer.7.q_proj.weight:
+- 128
+- 128
+transformer.7.v_proj.bias:
+- 128
+transformer.7.v_proj.weight:
+- 128
+- 128

best/checkpoint-2000/rvq_encoder.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f1ce7e6dc63694998a3300de71bed3315594db6b7ee3e2290e90935b911afb04
+size 618960456

best/checkpoint-2000/rvq_encoder_config.json

@@ -0,0 +1,28 @@
+{
+  "codebook_vocab_sizes": [
+    16384,
+    1024,
+    1024,
+    1024,
+    1024,
+    1024,
+    1024,
+    1024
+  ],
+  "conv_dilations": [
+    1,
+    3,
+    9
+  ],
+  "d_model": 1088,
+  "dropout": 0.1,
+  "ff_mult": 4,
+  "latent_channels": 128,
+  "max_position_embeddings": 128,
+  "mup": true,
+  "mup_attention_multiplier": 8.0,
+  "mup_output_mult": 1.0,
+  "mup_readout_zero_init": true,
+  "num_heads": 17,
+  "num_layers": 8
+}