| { |
| "model_type": "qwen3_gated_moe_qwen3_next", |
| "hf_model_type": "qwen3_next", |
| "architecture": "qwen3_next", |
| "parameter_count": 2945914, |
| "tokenizer_size": 1003, |
| "final_eval_loss": 1.4319972917437553, |
| "final_eval_ppl": 4.187053613650411, |
| "final_train_lm_loss": 1.4144856333732605, |
| "router_usage": { |
| "model.layers.0.mlp.gate": [ |
| 0.1257, |
| 0.098, |
| 0.1444, |
| 0.1294, |
| 0.157, |
| 0.1163, |
| 0.1017, |
| 0.1274 |
| ], |
| "model.layers.1.mlp.gate": [ |
| 0.1737, |
| 0.0915, |
| 0.0998, |
| 0.1762, |
| 0.1692, |
| 0.0784, |
| 0.0892, |
| 0.1219 |
| ], |
| "model.layers.2.mlp.gate": [ |
| 0.1078, |
| 0.1311, |
| 0.2018, |
| 0.091, |
| 0.1046, |
| 0.1521, |
| 0.1415, |
| 0.07 |
| ], |
| "model.layers.3.mlp.gate": [ |
| 0.0827, |
| 0.1803, |
| 0.0596, |
| 0.1023, |
| 0.066, |
| 0.1445, |
| 0.1774, |
| 0.1873 |
| ] |
| }, |
| "args": { |
| "train_files": [ |
| "data/train-00000-of-00004-2d5a1467fff1081b.parquet", |
| "data/train-00001-of-00004-5852b56a2bd28fd9.parquet", |
| "data/train-00002-of-00004-a26307300439e943.parquet", |
| "data/train-00003-of-00004-d243063613e5a057.parquet" |
| ], |
| "output_dir": "runs/qwen3_next_3m_fix2_100p_1epoch", |
| "max_rows": 0, |
| "data_fraction": 1.0, |
| "vocab_size": 1024, |
| "base_vocab_size": 1000, |
| "min_frequency": 2, |
| "legacy_tokenizer": true, |
| "legacy_special_token_ids": true, |
| "architecture": "qwen3_next", |
| "hidden_size": 216, |
| "intermediate_size": 540, |
| "moe_intermediate_size": 54, |
| "shared_expert_intermediate_size": 54, |
| "num_hidden_layers": 4, |
| "num_attention_heads": 8, |
| "num_key_value_heads": 1, |
| "head_dim": 56, |
| "max_position_embeddings": 1024, |
| "attention_bias": false, |
| "freeze_random_qkv_bias": false, |
| "qkv_bias_init_range": 0.2, |
| "attention_dropout": 0.0, |
| "rms_norm_eps": 1e-06, |
| "rope_theta": 10000000.0, |
| "partial_rotary_factor": 0.25, |
| "use_cache": false, |
| "tie_word_embeddings": true, |
| "decoder_sparse_step": 1, |
| "mlp_only_layers": "", |
| "num_local_experts": 8, |
| "num_experts_per_tok": 2, |
| "norm_topk_prob": true, |
| "output_router_logits": true, |
| "router_aux_loss_coef": 0.01, |
| "layer_types": "", |
| "full_attention_interval": 4, |
| "linear_conv_kernel_dim": 4, |
| "linear_key_head_dim": 54, |
| "linear_value_head_dim": 54, |
| "linear_num_key_heads": 4, |
| "linear_num_value_heads": 8, |
| "block_size": 256, |
| "batch_size": 16, |
| "max_steps": 152568, |
| "num_epochs": 1.0, |
| "learning_rate": 0.0003, |
| "weight_decay": 0.0, |
| "grad_clip": 1.0, |
| "warmup_steps": 1000, |
| "min_lr_ratio": 0.1, |
| "validation_fraction": 0.01, |
| "log_steps": 100, |
| "eval_steps": 5000, |
| "eval_batches": 16, |
| "save_steps": 10000, |
| "resume_from_checkpoint": "", |
| "seed": 1234, |
| "device": "cuda", |
| "dtype": "float32", |
| "num_workers": 0, |
| "generate_new_tokens": 100, |
| "generate_prompts": [ |
| "Once upon", |
| "There was a little", |
| "One day" |
| ], |
| "generation_temperature": 0.8, |
| "generation_top_p": 0.95, |
| "generation_top_k": 40, |
| "generation_repetition_penalty": 1.1, |
| "do_sample": true |
| }, |
| "config": { |
| "transformers_version": "5.14.1", |
| "architectures": [ |
| "Qwen3NextForCausalLM" |
| ], |
| "output_hidden_states": false, |
| "return_dict": true, |
| "dtype": "float32", |
| "chunk_size_feed_forward": 0, |
| "is_encoder_decoder": false, |
| "id2label": { |
| "0": "LABEL_0", |
| "1": "LABEL_1" |
| }, |
| "label2id": { |
| "LABEL_0": 0, |
| "LABEL_1": 1 |
| }, |
| "problem_type": null, |
| "vocab_size": 1024, |
| "hidden_size": 216, |
| "intermediate_size": 540, |
| "num_hidden_layers": 4, |
| "num_attention_heads": 8, |
| "num_key_value_heads": 1, |
| "hidden_act": "silu", |
| "max_position_embeddings": 1024, |
| "initializer_range": 0.02, |
| "rms_norm_eps": 1e-06, |
| "use_cache": false, |
| "tie_word_embeddings": true, |
| "rope_parameters": { |
| "rope_type": "default", |
| "rope_theta": 10000000.0, |
| "partial_rotary_factor": 0.25 |
| }, |
| "attention_bias": false, |
| "attention_dropout": 0.0, |
| "head_dim": 56, |
| "linear_conv_kernel_dim": 4, |
| "linear_key_head_dim": 54, |
| "linear_value_head_dim": 54, |
| "linear_num_key_heads": 4, |
| "linear_num_value_heads": 8, |
| "decoder_sparse_step": 1, |
| "moe_intermediate_size": 54, |
| "shared_expert_intermediate_size": 54, |
| "num_experts_per_tok": 2, |
| "num_experts": 8, |
| "norm_topk_prob": true, |
| "output_router_logits": true, |
| "router_aux_loss_coef": 0.01, |
| "mlp_only_layers": [], |
| "layer_types": [ |
| "linear_attention", |
| "linear_attention", |
| "linear_attention", |
| "full_attention" |
| ], |
| "pad_token_id": 1000, |
| "bos_token_id": 1000, |
| "eos_token_id": 1001, |
| "_name_or_path": "", |
| "use_sliding_window": false, |
| "sliding_window": null, |
| "partial_rotary_factor": 0.25, |
| "model_type": "qwen3_next", |
| "output_attentions": false |
| }, |
| "generations": { |
| "Once upon": "Once upon a time, there was a mighty man. He liked to count things and shout out loud. One day he decided to go on an adventure in the forest. \n\nHe went into a dark forest and saw many trees that were all around him. The first man was very excited. He wanted to find someone to give himself something special. So he ran off to find someone who could help him. \n\nThe s", |
| "There was a little": "There was a little girl named Lucy who lived in a small house. One day, she heard a whistle coming from the ceiling. She was so excited and wanted to see what it could do.\n\nLucy went up to the sound and suddenly she saw a big tree. It had lots of junk inside! The tree looked like it was going to perform for her friend and her friend. They climbed the tree together and L", |
| "One day": "One day, a boy named Tim went for a walk. He saw his friend, Sam, who was also tall and strong.\n\n\"Hi Sam!\" said Tim. \"I want to be my friend.\"\n\nSam looked at Tim. \"You can share the ball with me,\" he said. \"Thank you for sharing the ball with me.\"\n\nTim and Sam decided to share the ball. They took turns pulling and sharing" |
| } |
| } |
