Add files using upload-large-folder tool

checkpoints/interrupted_8156/checkpoint.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d02caffb1317cdce768f9eb85e1c36b78cffb2f4af264c33f3b367be1e6a7305
+size 813577067

checkpoints/interrupted_8156/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:448d7e136e95a788259d774733fc6d2658e258503bf542840238f142c8b10ad3
+size 813515200

checkpoints/interrupted_8156/optimizer_rank0.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5ca6d614da79d50e94b111fe737dcfdc88ee12f1e77e53c033aacc979077b564
+size 1609755607

checkpoints/interrupted_8156/optimizer_rank1.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ef4ebbfb4aa8144794c68b7a25c6a123971f9233b3b6d8a4352871bd96dbd158
+size 1609755607

checkpoints/interrupted_8156/optimizer_rank2.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2aae73303bc8ab802fbb19edd851b421ad09841f10787928cd0ba61b06b9de3b
+size 1609755607

checkpoints/interrupted_8156/optimizer_rank3.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0fd8fcaab104d46ed115d6e7c0c2ca6d1a04836b7a25352bda3c56736c88d7e6
+size 1609755607

checkpoints/interrupted_8156/optimizer_rank4.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b10fd7490e9d95e1e8ad348aaa5f8e8462e2c57adf5aaaaa458dd11240cbdb2d
+size 1609755607

checkpoints/interrupted_8156/optimizer_rank5.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5a0d806b77e901d948f86b2b3a8aa2bc1a2670ec0306f851acbf724e57e58bae
+size 1609755607

checkpoints/interrupted_8156/optimizer_rank6.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0616dac15f34b56335cd27a182aa052510e98e3eaf552a8a830ff7c7c30f37c6
+size 1609755607

checkpoints/interrupted_8156/optimizer_rank7.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6edf94da5635ae4279eae58ddede712567f9ac6e3585c98c21e27b05a09295bd
+size 1609755607

config.json

@@ -1,3 +1,101 @@
 {
-  "view_time": 0
+  "hidden_size": 896,
+  "num_hidden_layers": 16,
+  "intermediate_size": 256,
+  "vocab_size": 32000,
+  "num_attention_heads": 14,
+  "num_key_value_heads": 2,
+  "attention_type": "gqa",
+  "attention_dropout": 0.0,
+  "use_qk_norm": true,
+  "sliding_window": null,
+  "causal_conv_kernel_size": 4,
+  "causal_conv_dilation_cycle": 8,
+  "causal_state_rank": 16,
+  "causal_context_gate_init": 1.0,
+  "causal_contextual_mix_init": 0.0,
+  "causal_context_fusion_size": 0,
+  "causal_stable_delta": false,
+  "causal_delta_chunk_size": 512,
+  "causal_delta_timescales": 1,
+  "causal_delta_collision_normalized": false,
+  "causal_delta_lexical_values": false,
+  "causal_delta_lexical_forge": false,
+  "causal_delta_occurrence_address": false,
+  "tr_mha_num_experts": 4,
+  "tr_mha_adapter_rank": 8,
+  "tr_mha_top_k": 2,
+  "tr_mha_adapter_gate_init": 0.1,
+  "tr_mha_id_primary_logit": 2.0,
+  "tr_mha_id_secondary_logit": 1.0,
+  "tr_mha_id_other_logit": -2.0,
+  "tr_mha_verifier_gate_init": 0.1,
+  "tr_mha_verifier_temperature": 1.0,
+  "tr_mha_targets": "qv",
+  "max_position_embeddings": 2048,
+  "rope_theta": 10000.0,
+  "rope_type": "standard",
+  "rope_fraction": 1.0,
+  "mlp_type": "tr_hash_engine",
+  "hidden_act": "silu",
+  "num_experts": 4,
+  "expert_initialization": "gpt_normal",
+  "token_frequencies": null,
+  "routing_strategy": "token_id_multi_hash",
+  "route_hash_count": 2,
+  "lsh_routing": false,
+  "lsh_bits": 0,
+  "lsh_from_layer": 0,
+  "lsh_threshold_mode": "zero",
+  "shared_expert": true,
+  "shared_intermediate_size": 3072,
+  "shared_expert_chunk_tokens": 0,
+  "use_shared_routed_gates": false,
+  "shared_gate_init": 1.0,
+  "routed_gate_init": 1.0,
+  "shared_output_scale": 1.0,
+  "routed_output_scale": 2.0,
+  "shared_output_scale_first_layer": null,
+  "shared_output_scale_last_layer": null,
+  "routed_output_scale_first_layer": null,
+  "routed_output_scale_last_layer": null,
+  "top_k": 2,
+  "top_k_primary_weight": 0.5,
+  "learn_hash_pair_gates": false,
+  "hash_pair_gate_init": 0.5,
+  "learn_hash_channel_modulation": false,
+  "hash_channel_scale_init": 0.0,
+  "static_expert_capacity": false,
+  "use_custom_kernels": "auto",
+  "collect_moe_telemetry": false,
+  "use_cggr": "auto",
+  "active_num_experts": null,
+  "active_expert_width": null,
+  "lexical_object_rank": 16,
+  "lexical_object_gate_init": 0.1,
+  "tie_lexical_object_embeddings": false,
+  "micro_num_experts": 4,
+  "micro_expert_width": 16,
+  "micro_expert_gate_init": 0.1,
+  "lexical_gqa_rank": 16,
+  "lexical_gqa_gate_init": 0.0,
+  "lexical_gqa_use_token_code": true,
+  "lexical_key_gate_init": 0.05,
+  "lexical_zipf_path": null,
+  "lexical_zipf_mode": "uniform",
+  "lexical_zipf_alpha": 0.25,
+  "lexical_zipf_floor": 0.1,
+  "lexical_zipf_permutation_seed": 1729,
+  "norm_type": "rmsnorm",
+  "norm_eps": 1e-06,
+  "tie_word_embeddings": true,
+  "use_sdpa": true,
+  "use_cache": true,
+  "is_causal": true,
+  "initializer_range": 0.02,
+  "use_mup_init": false,
+  "use_mup_attn_scale": false,
+  "use_mup_output_mult": false,
+  "mup_base_width": 256,
+  "extra_config": {}
 }

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:448d7e136e95a788259d774733fc6d2658e258503bf542840238f142c8b10ad3
+size 813515200

model_config.yaml

@@ -0,0 +1,99 @@
+active_expert_width: null
+active_num_experts: null
+attention_dropout: 0.0
+attention_type: gqa
+causal_context_fusion_size: 0
+causal_context_gate_init: 1.0
+causal_contextual_mix_init: 0.0
+causal_conv_dilation_cycle: 8
+causal_conv_kernel_size: 4
+causal_delta_chunk_size: 512
+causal_delta_collision_normalized: false
+causal_delta_lexical_forge: false
+causal_delta_lexical_values: false
+causal_delta_occurrence_address: false
+causal_delta_timescales: 1
+causal_stable_delta: false
+causal_state_rank: 16
+collect_moe_telemetry: false
+expert_initialization: gpt_normal
+extra_config: {}
+hash_channel_scale_init: 0.0
+hash_pair_gate_init: 0.5
+hidden_act: silu
+hidden_size: 896
+initializer_range: 0.02
+intermediate_size: 256
+is_causal: true
+learn_hash_channel_modulation: false
+learn_hash_pair_gates: false
+lexical_gqa_gate_init: 0.0
+lexical_gqa_rank: 16
+lexical_gqa_use_token_code: true
+lexical_key_gate_init: 0.05
+lexical_object_gate_init: 0.1
+lexical_object_rank: 16
+lexical_zipf_alpha: 0.25
+lexical_zipf_floor: 0.1
+lexical_zipf_mode: uniform
+lexical_zipf_path: null
+lexical_zipf_permutation_seed: 1729
+lsh_bits: 0
+lsh_from_layer: 0
+lsh_routing: false
+lsh_threshold_mode: zero
+max_position_embeddings: 2048
+micro_expert_gate_init: 0.1
+micro_expert_width: 16
+micro_num_experts: 4
+mlp_type: tr_hash_engine
+mup_base_width: 256
+norm_eps: 1.0e-06
+norm_type: rmsnorm
+num_attention_heads: 14
+num_experts: 4
+num_hidden_layers: 16
+num_key_value_heads: 2
+rope_fraction: 1.0
+rope_theta: 10000.0
+rope_type: standard
+route_hash_count: 2
+routed_gate_init: 1.0
+routed_output_scale: 2.0
+routed_output_scale_first_layer: null
+routed_output_scale_last_layer: null
+routing_strategy: token_id_multi_hash
+shared_expert: true
+shared_expert_chunk_tokens: 0
+shared_gate_init: 1.0
+shared_intermediate_size: 3072
+shared_output_scale: 1.0
+shared_output_scale_first_layer: null
+shared_output_scale_last_layer: null
+sliding_window: null
+static_expert_capacity: false
+tie_lexical_object_embeddings: false
+tie_word_embeddings: true
+token_frequencies: null
+top_k: 2
+top_k_primary_weight: 0.5
+tr_mha_adapter_gate_init: 0.1
+tr_mha_adapter_rank: 8
+tr_mha_id_other_logit: -2.0
+tr_mha_id_primary_logit: 2.0
+tr_mha_id_secondary_logit: 1.0
+tr_mha_num_experts: 4
+tr_mha_targets: qv
+tr_mha_top_k: 2
+tr_mha_verifier_gate_init: 0.1
+tr_mha_verifier_temperature: 1.0
+use_cache: true
+use_cggr: auto
+use_custom_kernels: auto
+use_mup_attn_scale: false
+use_mup_init: false
+use_mup_output_mult: false
+use_qk_norm: true
+use_sdpa: true
+use_shared_routed_gates: false
+vocab_size: 32000

training/tensorboard/events.out.tfevents.1787249377.9c6c244bb4b9.2747.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6525ab558b8b684d20c112ca441b9cf9fb027e0c79aabf0b50b0d3f9ecc0359a
+size 1125235

training/tensorboard/events.out.tfevents.1787296464.9c6c244bb4b9.5004.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:65c9bc0a357f486b4ef38ce1caa210ac2c7225e42e1a887f6a08afaaea141302
+size 88