{
  "schema_version": "ax.native_model.v1",
  "model_family": "qwen3_5",
  "tensor_format": "safetensors",
  "layer_count": 40,
  "hidden_size": 2048,
  "intermediate_size": 0,
  "attention_head_count": 16,
  "attention_head_dim": 256,
  "kv_head_count": 2,
  "vocab_size": 248320,
  "tie_word_embeddings": false,
  "rope_theta": 10000000,
  "attn_output_gate": true,
  "partial_rotary_factor": 0.25,
  "rms_norm_eps": 0.000001,
  "moe_norm_topk_prob": false,
  "hidden_size_per_layer_input": 0,
  "linear_attention": {
    "full_attention_interval": 4,
    "num_value_heads": 32,
    "num_key_heads": 16,
    "key_head_dim": 128,
    "value_head_dim": 128,
    "conv_kernel_dim": 4
  },
  "moe": {
    "expert_count": 256,
    "experts_per_token": 8,
    "expert_intermediate_size": 512
  },
  "tensors": [
    {
      "name": "language_model.model.norm.weight",
      "role": "final_norm",
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 270222437,
      "length_bytes": 4096
    },
    {
      "name": "language_model.lm_head.weight",
      "role": "lm_head",
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        248320,
        256
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 15942757,
      "length_bytes": 254279680
    },
    {
      "name": "language_model.model.embed_tokens.weight",
      "role": "token_embedding",
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        248320,
        256
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 3084125746,
      "length_bytes": 254279680
    },
    {
      "name": "language_model.model.layers.0.input_layernorm.weight",
      "role": "attention_norm",
      "layer_index": 0,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 2597705650,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.0.post_attention_layernorm.weight",
      "role": "attention_post_norm",
      "layer_index": 0,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 3512201906,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.0.mlp.switch_mlp.down_proj.weight",
      "role": "ffn_down_exps",
      "layer_index": 0,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        2048,
        64
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 1425797554,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.0.mlp.switch_mlp.gate_proj.weight",
      "role": "ffn_gate_exps",
      "layer_index": 0,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 1136725106,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.0.mlp.gate.weight",
      "role": "ffn_gate_inp",
      "layer_index": 0,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        256,
        512
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 2207333042,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.0.mlp.shared_expert.down_proj.weight",
      "role": "ffn_shared_expert_down",
      "layer_index": 0,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        2048,
        64
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 3818579378,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.0.mlp.shared_expert.gate_proj.weight",
      "role": "ffn_shared_expert_gate",
      "layer_index": 0,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        512,
        256
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 3817479346,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.0.mlp.shared_expert_gate.weight",
      "role": "ffn_shared_expert_gate_inp",
      "layer_index": 0,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        1,
        512
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 3820800178,
      "length_bytes": 2048
    },
    {
      "name": "language_model.model.layers.0.mlp.shared_expert.up_proj.weight",
      "role": "ffn_shared_expert_up",
      "layer_index": 0,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        512,
        256
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 3819718834,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.0.mlp.switch_mlp.up_proj.weight",
      "role": "ffn_up_exps",
      "layer_index": 0,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 3520596914,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.0.linear_attn.A_log",
      "role": "linear_attention_a_log",
      "layer_index": 0,
      "dtype": "bf16",
      "shape": [
        32
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 3502285170,
      "length_bytes": 64
    },
    {
      "name": "language_model.model.layers.0.linear_attn.conv1d.weight",
      "role": "linear_attention_conv1d",
      "layer_index": 0,
      "dtype": "bf16",
      "shape": [
        8192,
        4,
        1
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 1849196082,
      "length_bytes": 65536
    },
    {
      "name": "language_model.model.layers.0.linear_attn.dt_bias",
      "role": "linear_attention_dt_bias",
      "layer_index": 0,
      "dtype": "bf16",
      "shape": [
        32
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 3338405426,
      "length_bytes": 64
    },
    {
      "name": "language_model.model.layers.0.linear_attn.in_proj_a.weight",
      "role": "linear_attention_in_proj_a",
      "layer_index": 0,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        32,
        256
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 3367735154,
      "length_bytes": 32768
    },
    {
      "name": "language_model.model.layers.0.linear_attn.in_proj_b.weight",
      "role": "linear_attention_in_proj_b",
      "layer_index": 0,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        32,
        256
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 3359313778,
      "length_bytes": 32768
    },
    {
      "name": "language_model.model.layers.0.linear_attn.in_proj_qkv.weight",
      "role": "linear_attention_in_proj_qkv",
      "layer_index": 0,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        8192,
        256
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 2556800306,
      "length_bytes": 8388608
    },
    {
      "name": "language_model.model.layers.0.linear_attn.in_proj_z.weight",
      "role": "linear_attention_in_proj_z",
      "layer_index": 0,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        4096,
        256
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 3354822258,
      "length_bytes": 4194304
    },
    {
      "name": "language_model.model.layers.0.linear_attn.norm.weight",
      "role": "linear_attention_norm",
      "layer_index": 0,
      "dtype": "bf16",
      "shape": [
        128
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 3512201650,
      "length_bytes": 256
    },
    {
      "name": "language_model.model.layers.0.linear_attn.out_proj.weight",
      "role": "linear_attention_out_proj",
      "layer_index": 0,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        2048,
        512
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 2363135986,
      "length_bytes": 4194304
    },
    {
      "name": "language_model.model.layers.1.input_layernorm.weight",
      "role": "attention_norm",
      "layer_index": 1,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 3989315346,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.1.post_attention_layernorm.weight",
      "role": "attention_post_norm",
      "layer_index": 1,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 1101569330,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.1.mlp.switch_mlp.down_proj.weight",
      "role": "ffn_down_exps",
      "layer_index": 1,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        2048,
        64
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 4274564626,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.1.mlp.switch_mlp.gate_proj.weight",
      "role": "ffn_gate_exps",
      "layer_index": 1,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 3989349650,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.1.mlp.gate.weight",
      "role": "ffn_gate_inp",
      "layer_index": 1,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        256,
        512
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 1384018738,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.1.mlp.shared_expert.down_proj.weight",
      "role": "ffn_shared_expert_down",
      "layer_index": 1,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        2048,
        64
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 4411502098,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.1.mlp.shared_expert.gate_proj.weight",
      "role": "ffn_shared_expert_gate",
      "layer_index": 1,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        512,
        256
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 3820243122,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.1.mlp.shared_expert_gate.weight",
      "role": "ffn_shared_expert_gate_inp",
      "layer_index": 1,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        1,
        512
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 1849266226,
      "length_bytes": 2048
    },
    {
      "name": "language_model.model.layers.1.mlp.shared_expert.up_proj.weight",
      "role": "ffn_shared_expert_up",
      "layer_index": 1,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        512,
        256
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 5224630706,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.1.mlp.switch_mlp.up_proj.weight",
      "role": "ffn_up_exps",
      "layer_index": 1,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 3820802226,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.1.linear_attn.A_log",
      "role": "linear_attention_a_log",
      "layer_index": 1,
      "dtype": "bf16",
      "shape": [
        32
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 1101569266,
      "length_bytes": 64
    },
    {
      "name": "language_model.model.layers.1.linear_attn.conv1d.weight",
      "role": "linear_attention_conv1d",
      "layer_index": 1,
      "dtype": "bf16",
      "shape": [
        8192,
        4,
        1
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 3989249810,
      "length_bytes": 65536
    },
    {
      "name": "language_model.model.layers.1.linear_attn.dt_bias",
      "role": "linear_attention_dt_bias",
      "layer_index": 1,
      "dtype": "bf16",
      "shape": [
        32
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 3988987602,
      "length_bytes": 64
    },
    {
      "name": "language_model.model.layers.1.linear_attn.in_proj_a.weight",
      "role": "linear_attention_in_proj_a",
      "layer_index": 1,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        32,
        256
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 3988954834,
      "length_bytes": 32768
    },
    {
      "name": "language_model.model.layers.1.linear_attn.in_proj_b.weight",
      "role": "linear_attention_in_proj_b",
      "layer_index": 1,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        32,
        256
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 3980522738,
      "length_bytes": 32768
    },
    {
      "name": "language_model.model.layers.1.linear_attn.in_proj_qkv.weight",
      "role": "linear_attention_in_proj_qkv",
      "layer_index": 1,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        8192,
        256
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 3980557554,
      "length_bytes": 8388608
    },
    {
      "name": "language_model.model.layers.1.linear_attn.in_proj_z.weight",
      "role": "linear_attention_in_proj_z",
      "layer_index": 1,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        4096,
        256
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 3976000754,
      "length_bytes": 4194304
    },
    {
      "name": "language_model.model.layers.1.linear_attn.norm.weight",
      "role": "linear_attention_norm",
      "layer_index": 1,
      "dtype": "bf16",
      "shape": [
        128
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 1384014130,
      "length_bytes": 256
    },
    {
      "name": "language_model.model.layers.1.linear_attn.out_proj.weight",
      "role": "linear_attention_out_proj",
      "layer_index": 1,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        2048,
        512
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 3971806450,
      "length_bytes": 4194304
    },
    {
      "name": "language_model.model.layers.2.input_layernorm.weight",
      "role": "attention_norm",
      "layer_index": 2,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 4578389842,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.2.post_attention_layernorm.weight",
      "role": "attention_post_norm",
      "layer_index": 2,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 4578393938,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.2.mlp.switch_mlp.down_proj.weight",
      "role": "ffn_down_exps",
      "layer_index": 2,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        2048,
        64
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 2817561842,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.2.mlp.switch_mlp.gate_proj.weight",
      "role": "ffn_gate_exps",
      "layer_index": 2,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 4578398034,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.2.mlp.gate.weight",
      "role": "ffn_gate_inp",
      "layer_index": 2,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        256,
        512
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 2210513842,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.2.mlp.shared_expert.down_proj.weight",
      "role": "ffn_shared_expert_down",
      "layer_index": 2,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        2048,
        64
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 2791151602,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.2.mlp.shared_expert.gate_proj.weight",
      "role": "ffn_shared_expert_gate",
      "layer_index": 2,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        512,
        256
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 1300474642,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.2.mlp.shared_expert_gate.weight",
      "role": "ffn_shared_expert_gate_inp",
      "layer_index": 2,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        1,
        512
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 4897821778,
      "length_bytes": 2048
    },
    {
      "name": "language_model.model.layers.2.mlp.shared_expert.up_proj.weight",
      "role": "ffn_shared_expert_up",
      "layer_index": 2,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        512,
        256
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 4887381074,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.2.mlp.switch_mlp.up_proj.weight",
      "role": "ffn_up_exps",
      "layer_index": 2,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 4712618066,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.2.linear_attn.A_log",
      "role": "linear_attention_a_log",
      "layer_index": 2,
      "dtype": "bf16",
      "shape": [
        32
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 1384014066,
      "length_bytes": 64
    },
    {
      "name": "language_model.model.layers.2.linear_attn.conv1d.weight",
      "role": "linear_attention_conv1d",
      "layer_index": 2,
      "dtype": "bf16",
      "shape": [
        8192,
        4,
        1
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 1271471730,
      "length_bytes": 65536
    },
    {
      "name": "language_model.model.layers.2.linear_attn.dt_bias",
      "role": "linear_attention_dt_bias",
      "layer_index": 2,
      "dtype": "bf16",
      "shape": [
        32
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 1081196818,
      "length_bytes": 64
    },
    {
      "name": "language_model.model.layers.2.linear_attn.in_proj_a.weight",
      "role": "linear_attention_in_proj_a",
      "layer_index": 2,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        32,
        256
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 1809883474,
      "length_bytes": 32768
    },
    {
      "name": "language_model.model.layers.2.linear_attn.in_proj_b.weight",
      "role": "linear_attention_in_proj_b",
      "layer_index": 2,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        32,
        256
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 1849268274,
      "length_bytes": 32768
    },
    {
      "name": "language_model.model.layers.2.linear_attn.in_proj_qkv.weight",
      "role": "linear_attention_in_proj_qkv",
      "layer_index": 2,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        8192,
        256
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 1281453682,
      "length_bytes": 8388608
    },
    {
      "name": "language_model.model.layers.2.linear_attn.in_proj_z.weight",
      "role": "linear_attention_in_proj_z",
      "layer_index": 2,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        4096,
        256
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 4421758482,
      "length_bytes": 4194304
    },
    {
      "name": "language_model.model.layers.2.linear_attn.norm.weight",
      "role": "linear_attention_norm",
      "layer_index": 2,
      "dtype": "bf16",
      "shape": [
        128
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 4570951186,
      "length_bytes": 256
    },
    {
      "name": "language_model.model.layers.2.linear_attn.out_proj.weight",
      "role": "linear_attention_out_proj",
      "layer_index": 2,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        2048,
        512
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 4570951442,
      "length_bytes": 4194304
    },
    {
      "name": "language_model.model.layers.3.self_attn.k_proj.weight",
      "role": "attention_k",
      "layer_index": 3,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        512,
        256
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 4906741138,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.3.self_attn.k_norm.weight",
      "role": "attention_k_norm",
      "layer_index": 3,
      "dtype": "bf16",
      "shape": [
        256
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 4920023154,
      "length_bytes": 512
    },
    {
      "name": "language_model.model.layers.3.input_layernorm.weight",
      "role": "attention_norm",
      "layer_index": 3,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 4920023666,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.3.self_attn.o_proj.weight",
      "role": "attention_o",
      "layer_index": 3,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        2048,
        512
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 4907333266,
      "length_bytes": 4194304
    },
    {
      "name": "language_model.model.layers.3.post_attention_layernorm.weight",
      "role": "attention_post_norm",
      "layer_index": 3,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 4929951090,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.3.self_attn.q_proj.weight",
      "role": "attention_q",
      "layer_index": 3,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        8192,
        256
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 4897828242,
      "length_bytes": 8388608
    },
    {
      "name": "language_model.model.layers.3.self_attn.q_norm.weight",
      "role": "attention_q_norm",
      "layer_index": 3,
      "dtype": "bf16",
      "shape": [
        256
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 1993476306,
      "length_bytes": 512
    },
    {
      "name": "language_model.model.layers.3.self_attn.v_proj.weight",
      "role": "attention_v",
      "layer_index": 3,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        512,
        256
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 2517770482,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.3.mlp.switch_mlp.down_proj.weight",
      "role": "ffn_down_exps",
      "layer_index": 3,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        2048,
        64
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 5080975730,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.3.mlp.switch_mlp.gate_proj.weight",
      "role": "ffn_gate_exps",
      "layer_index": 3,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 1599017394,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.3.mlp.gate.weight",
      "role": "ffn_gate_inp",
      "layer_index": 3,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        256,
        512
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 2778909266,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.3.mlp.shared_expert.down_proj.weight",
      "role": "ffn_shared_expert_down",
      "layer_index": 3,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        2048,
        64
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 5224106418,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.3.mlp.shared_expert.gate_proj.weight",
      "role": "ffn_shared_expert_gate",
      "layer_index": 3,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        512,
        256
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 5223582130,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.3.mlp.shared_expert_gate.weight",
      "role": "ffn_shared_expert_gate_inp",
      "layer_index": 3,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        1,
        512
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 4897823890,
      "length_bytes": 2048
    },
    {
      "name": "language_model.model.layers.3.mlp.shared_expert.up_proj.weight",
      "role": "ffn_shared_expert_up",
      "layer_index": 3,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        512,
        256
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 2219009362,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.3.mlp.switch_mlp.up_proj.weight",
      "role": "ffn_up_exps",
      "layer_index": 3,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 4938369394,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.4.input_layernorm.weight",
      "role": "attention_norm",
      "layer_index": 4,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 4426286610,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.4.post_attention_layernorm.weight",
      "role": "attention_post_norm",
      "layer_index": 4,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 5244718706,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.4.mlp.switch_mlp.down_proj.weight",
      "role": "ffn_down_exps",
      "layer_index": 4,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        2048,
        64
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 860394130,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.4.mlp.switch_mlp.gate_proj.weight",
      "role": "ffn_gate_exps",
      "layer_index": 4,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 4426290706,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.4.mlp.gate.weight",
      "role": "ffn_gate_inp",
      "layer_index": 4,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        256,
        512
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 2196361138,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.4.mlp.shared_expert.down_proj.weight",
      "role": "ffn_shared_expert_down",
      "layer_index": 4,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        2048,
        64
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 4420709906,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.4.mlp.shared_expert.gate_proj.weight",
      "role": "ffn_shared_expert_gate",
      "layer_index": 4,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        512,
        256
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 2196890034,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.4.mlp.shared_expert_gate.weight",
      "role": "ffn_shared_expert_gate_inp",
      "layer_index": 4,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        1,
        512
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 859699858,
      "length_bytes": 2048
    },
    {
      "name": "language_model.model.layers.4.mlp.shared_expert.up_proj.weight",
      "role": "ffn_shared_expert_up",
      "layer_index": 4,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        512,
        256
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 859771538,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.4.mlp.switch_mlp.up_proj.weight",
      "role": "ffn_up_exps",
      "layer_index": 4,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 2375325682,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.4.linear_attn.A_log",
      "role": "linear_attention_a_log",
      "layer_index": 4,
      "dtype": "bf16",
      "shape": [
        32
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 5240262194,
      "length_bytes": 64
    },
    {
      "name": "language_model.model.layers.4.linear_attn.conv1d.weight",
      "role": "linear_attention_conv1d",
      "layer_index": 4,
      "dtype": "bf16",
      "shape": [
        8192,
        4,
        1
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 4874655826,
      "length_bytes": 65536
    },
    {
      "name": "language_model.model.layers.4.linear_attn.dt_bias",
      "role": "linear_attention_dt_bias",
      "layer_index": 4,
      "dtype": "bf16",
      "shape": [
        32
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 2575200722,
      "length_bytes": 64
    },
    {
      "name": "language_model.model.layers.4.linear_attn.in_proj_a.weight",
      "role": "linear_attention_in_proj_a",
      "layer_index": 4,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        32,
        256
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 5235211506,
      "length_bytes": 32768
    },
    {
      "name": "language_model.model.layers.4.linear_attn.in_proj_b.weight",
      "role": "linear_attention_in_proj_b",
      "layer_index": 4,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        32,
        256
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 5240227378,
      "length_bytes": 32768
    },
    {
      "name": "language_model.model.layers.4.linear_attn.in_proj_qkv.weight",
      "role": "linear_attention_in_proj_qkv",
      "layer_index": 4,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        8192,
        256
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 4412026386,
      "length_bytes": 8388608
    },
    {
      "name": "language_model.model.layers.4.linear_attn.in_proj_z.weight",
      "role": "linear_attention_in_proj_z",
      "layer_index": 4,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        4096,
        256
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 5235244338,
      "length_bytes": 4194304
    },
    {
      "name": "language_model.model.layers.4.linear_attn.norm.weight",
      "role": "linear_attention_norm",
      "layer_index": 4,
      "dtype": "bf16",
      "shape": [
        128
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 3502284914,
      "length_bytes": 256
    },
    {
      "name": "language_model.model.layers.4.linear_attn.out_proj.weight",
      "role": "linear_attention_out_proj",
      "layer_index": 4,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        2048,
        512
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 5240262258,
      "length_bytes": 4194304
    },
    {
      "name": "language_model.model.layers.5.input_layernorm.weight",
      "role": "attention_norm",
      "layer_index": 5,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 844482834,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.5.post_attention_layernorm.weight",
      "role": "attention_post_norm",
      "layer_index": 5,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 844478738,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.5.mlp.switch_mlp.down_proj.weight",
      "role": "ffn_down_exps",
      "layer_index": 5,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        2048,
        64
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 4140346898,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.5.mlp.switch_mlp.gate_proj.weight",
      "role": "ffn_gate_exps",
      "layer_index": 5,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 710244626,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.5.mlp.gate.weight",
      "role": "ffn_gate_inp",
      "layer_index": 5,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        256,
        512
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 2790594546,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.5.mlp.shared_expert.down_proj.weight",
      "role": "ffn_shared_expert_down",
      "layer_index": 5,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        2048,
        64
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 675637458,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.5.mlp.shared_expert.gate_proj.weight",
      "role": "ffn_shared_expert_gate",
      "layer_index": 5,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        512,
        256
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 676161746,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.5.mlp.shared_expert_gate.weight",
      "role": "ffn_shared_expert_gate_inp",
      "layer_index": 5,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        1,
        512
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 675045586,
      "length_bytes": 2048
    },
    {
      "name": "language_model.model.layers.5.mlp.shared_expert.up_proj.weight",
      "role": "ffn_shared_expert_up",
      "layer_index": 5,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        512,
        256
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 675080402,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.5.mlp.switch_mlp.up_proj.weight",
      "role": "ffn_up_exps",
      "layer_index": 5,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 3674870706,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.5.linear_attn.A_log",
      "role": "linear_attention_a_log",
      "layer_index": 5,
      "dtype": "bf16",
      "shape": [
        32
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 1392655730,
      "length_bytes": 64
    },
    {
      "name": "language_model.model.layers.5.linear_attn.conv1d.weight",
      "role": "linear_attention_conv1d",
      "layer_index": 5,
      "dtype": "bf16",
      "shape": [
        8192,
        4,
        1
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 4575407954,
      "length_bytes": 65536
    },
    {
      "name": "language_model.model.layers.5.linear_attn.dt_bias",
      "role": "linear_attention_dt_bias",
      "layer_index": 5,
      "dtype": "bf16",
      "shape": [
        32
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 845011474,
      "length_bytes": 64
    },
    {
      "name": "language_model.model.layers.5.linear_attn.in_proj_a.weight",
      "role": "linear_attention_in_proj_a",
      "layer_index": 5,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        32,
        256
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 2554113266,
      "length_bytes": 32768
    },
    {
      "name": "language_model.model.layers.5.linear_attn.in_proj_b.weight",
      "role": "linear_attention_in_proj_b",
      "layer_index": 5,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        32,
        256
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 3674278834,
      "length_bytes": 32768
    },
    {
      "name": "language_model.model.layers.5.linear_attn.in_proj_qkv.weight",
      "role": "linear_attention_in_proj_qkv",
      "layer_index": 5,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        8192,
        256
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 851044946,
      "length_bytes": 8388608
    },
    {
      "name": "language_model.model.layers.5.linear_attn.in_proj_z.weight",
      "role": "linear_attention_in_proj_z",
      "layer_index": 5,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        4096,
        256
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 845802066,
      "length_bytes": 4194304
    },
    {
      "name": "language_model.model.layers.5.linear_attn.norm.weight",
      "role": "linear_attention_norm",
      "layer_index": 5,
      "dtype": "bf16",
      "shape": [
        128
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 845011218,
      "length_bytes": 256
    },
    {
      "name": "language_model.model.layers.5.linear_attn.out_proj.weight",
      "role": "linear_attention_out_proj",
      "layer_index": 5,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        2048,
        512
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 1421598642,
      "length_bytes": 4194304
    },
    {
      "name": "language_model.model.layers.6.input_layernorm.weight",
      "role": "attention_norm",
      "layer_index": 6,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 1780055634,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.6.post_attention_layernorm.weight",
      "role": "attention_post_norm",
      "layer_index": 6,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 1040125298,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.6.mlp.switch_mlp.down_proj.weight",
      "role": "ffn_down_exps",
      "layer_index": 6,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        2048,
        64
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 3368064882,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.6.mlp.switch_mlp.gate_proj.weight",
      "role": "ffn_gate_exps",
      "layer_index": 6,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 1993556178,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.6.mlp.gate.weight",
      "role": "ffn_gate_inp",
      "layer_index": 6,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        256,
        512
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 647412946,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.6.mlp.shared_expert.down_proj.weight",
      "role": "ffn_shared_expert_down",
      "layer_index": 6,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        2048,
        64
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 478543058,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.6.mlp.shared_expert.gate_proj.weight",
      "role": "ffn_shared_expert_gate",
      "layer_index": 6,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        512,
        256
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 479067346,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.6.mlp.shared_expert_gate.weight",
      "role": "ffn_shared_expert_gate_inp",
      "layer_index": 6,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        1,
        512
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 478475474,
      "length_bytes": 2048
    },
    {
      "name": "language_model.model.layers.6.mlp.shared_expert.up_proj.weight",
      "role": "ffn_shared_expert_up",
      "layer_index": 6,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        512,
        256
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 1101575730,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.6.mlp.switch_mlp.up_proj.weight",
      "role": "ffn_up_exps",
      "layer_index": 6,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 504757458,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.6.linear_attn.A_log",
      "role": "linear_attention_a_log",
      "layer_index": 6,
      "dtype": "bf16",
      "shape": [
        32
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 661044690,
      "length_bytes": 64
    },
    {
      "name": "language_model.model.layers.6.linear_attn.conv1d.weight",
      "role": "linear_attention_conv1d",
      "layer_index": 6,
      "dtype": "bf16",
      "shape": [
        8192,
        4,
        1
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 674979410,
      "length_bytes": 65536
    },
    {
      "name": "language_model.model.layers.6.linear_attn.dt_bias",
      "role": "linear_attention_dt_bias",
      "layer_index": 6,
      "dtype": "bf16",
      "shape": [
        32
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 661044754,
      "length_bytes": 64
    },
    {
      "name": "language_model.model.layers.6.linear_attn.in_proj_a.weight",
      "role": "linear_attention_in_proj_a",
      "layer_index": 6,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        32,
        256
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 2298384114,
      "length_bytes": 32768
    },
    {
      "name": "language_model.model.layers.6.linear_attn.in_proj_b.weight",
      "role": "linear_attention_in_proj_b",
      "layer_index": 6,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        32,
        256
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 661053010,
      "length_bytes": 32768
    },
    {
      "name": "language_model.model.layers.6.linear_attn.in_proj_qkv.weight",
      "role": "linear_attention_in_proj_qkv",
      "layer_index": 6,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        8192,
        256
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 666590802,
      "length_bytes": 8388608
    },
    {
      "name": "language_model.model.layers.6.linear_attn.in_proj_z.weight",
      "role": "linear_attention_in_proj_z",
      "layer_index": 6,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        4096,
        256
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 661347922,
      "length_bytes": 4194304
    },
    {
      "name": "language_model.model.layers.6.linear_attn.norm.weight",
      "role": "linear_attention_norm",
      "layer_index": 6,
      "dtype": "bf16",
      "shape": [
        128
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 661044434,
      "length_bytes": 256
    },
    {
      "name": "language_model.model.layers.6.linear_attn.out_proj.weight",
      "role": "linear_attention_out_proj",
      "layer_index": 6,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        2048,
        512
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 656850130,
      "length_bytes": 4194304
    },
    {
      "name": "language_model.model.layers.7.self_attn.k_proj.weight",
      "role": "attention_k",
      "layer_index": 7,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        512,
        256
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 469038226,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.7.self_attn.k_norm.weight",
      "role": "attention_k_norm",
      "layer_index": 7,
      "dtype": "bf16",
      "shape": [
        256
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 2565199826,
      "length_bytes": 512
    },
    {
      "name": "language_model.model.layers.7.input_layernorm.weight",
      "role": "attention_norm",
      "layer_index": 7,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 685074642,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.7.self_attn.o_proj.weight",
      "role": "attention_o",
      "layer_index": 7,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        2048,
        512
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 1352005874,
      "length_bytes": 4194304
    },
    {
      "name": "language_model.model.layers.7.post_attention_layernorm.weight",
      "role": "attention_post_norm",
      "layer_index": 7,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 859695762,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.7.self_attn.q_proj.weight",
      "role": "attention_q",
      "layer_index": 7,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        8192,
        256
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 470086802,
      "length_bytes": 8388608
    },
    {
      "name": "language_model.model.layers.7.self_attn.q_norm.weight",
      "role": "attention_q_norm",
      "layer_index": 7,
      "dtype": "bf16",
      "shape": [
        256
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 675045010,
      "length_bytes": 512
    },
    {
      "name": "language_model.model.layers.7.self_attn.v_proj.weight",
      "role": "attention_v",
      "layer_index": 7,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        512,
        256
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 468448402,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.7.mlp.switch_mlp.down_proj.weight",
      "role": "ffn_down_exps",
      "layer_index": 7,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        2048,
        64
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 174765202,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.7.mlp.switch_mlp.gate_proj.weight",
      "role": "ffn_gate_exps",
      "layer_index": 7,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 325760146,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.7.mlp.gate.weight",
      "role": "ffn_gate_inp",
      "layer_index": 7,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        256,
        512
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 3818052786,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.7.mlp.shared_expert.down_proj.weight",
      "role": "ffn_shared_expert_down",
      "layer_index": 7,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        2048,
        64
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 165819538,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.7.mlp.shared_expert.gate_proj.weight",
      "role": "ffn_shared_expert_gate",
      "layer_index": 7,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        512,
        256
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 1270942834,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.7.mlp.shared_expert_gate.weight",
      "role": "ffn_shared_expert_gate_inp",
      "layer_index": 7,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        1,
        512
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 165260434,
      "length_bytes": 2048
    },
    {
      "name": "language_model.model.layers.7.mlp.shared_expert.up_proj.weight",
      "role": "ffn_shared_expert_up",
      "layer_index": 7,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        512,
        256
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 165262482,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.7.mlp.switch_mlp.up_proj.weight",
      "role": "ffn_up_exps",
      "layer_index": 7,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 1849303346,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.8.input_layernorm.weight",
      "role": "attention_norm",
      "layer_index": 8,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 859767442,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.8.post_attention_layernorm.weight",
      "role": "attention_post_norm",
      "layer_index": 8,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 1774738514,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.8.mlp.switch_mlp.down_proj.weight",
      "role": "ffn_down_exps",
      "layer_index": 8,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        2048,
        64
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 5225530157,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.8.mlp.switch_mlp.gate_proj.weight",
      "role": "ffn_gate_exps",
      "layer_index": 8,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 25254738,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.8.mlp.gate.weight",
      "role": "ffn_gate_inp",
      "layer_index": 8,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        256,
        512
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 159488850,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.8.mlp.shared_expert.down_proj.weight",
      "role": "ffn_shared_expert_down",
      "layer_index": 8,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        2048,
        64
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 3073447469,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.8.mlp.shared_expert.gate_proj.weight",
      "role": "ffn_shared_expert_gate",
      "layer_index": 8,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        512,
        256
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 2734853293,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.8.mlp.shared_expert_gate.weight",
      "role": "ffn_shared_expert_gate_inp",
      "layer_index": 8,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        1,
        512
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 2878094637,
      "length_bytes": 2048
    },
    {
      "name": "language_model.model.layers.8.mlp.shared_expert.up_proj.weight",
      "role": "ffn_shared_expert_up",
      "layer_index": 8,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        512,
        256
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 3261277997,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.8.mlp.switch_mlp.up_proj.weight",
      "role": "ffn_up_exps",
      "layer_index": 8,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 2634229842,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.8.linear_attn.A_log",
      "role": "linear_attention_a_log",
      "layer_index": 8,
      "dtype": "bf16",
      "shape": [
        32
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 701855954,
      "length_bytes": 64
    },
    {
      "name": "language_model.model.layers.8.linear_attn.conv1d.weight",
      "role": "linear_attention_conv1d",
      "layer_index": 8,
      "dtype": "bf16",
      "shape": [
        8192,
        4,
        1
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 3980195058,
      "length_bytes": 65536
    },
    {
      "name": "language_model.model.layers.8.linear_attn.dt_bias",
      "role": "linear_attention_dt_bias",
      "layer_index": 8,
      "dtype": "bf16",
      "shape": [
        32
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 160275538,
      "length_bytes": 64
    },
    {
      "name": "language_model.model.layers.8.linear_attn.in_proj_a.weight",
      "role": "linear_attention_in_proj_a",
      "layer_index": 8,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        32,
        256
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 1319342770,
      "length_bytes": 32768
    },
    {
      "name": "language_model.model.layers.8.linear_attn.in_proj_b.weight",
      "role": "linear_attention_in_proj_b",
      "layer_index": 8,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        32,
        256
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 1809850706,
      "length_bytes": 32768
    },
    {
      "name": "language_model.model.layers.8.linear_attn.in_proj_qkv.weight",
      "role": "linear_attention_in_proj_qkv",
      "layer_index": 8,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        8192,
        256
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 459977874,
      "length_bytes": 8388608
    },
    {
      "name": "language_model.model.layers.8.linear_attn.in_proj_z.weight",
      "role": "linear_attention_in_proj_z",
      "layer_index": 8,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        4096,
        256
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 160541842,
      "length_bytes": 4194304
    },
    {
      "name": "language_model.model.layers.8.linear_attn.norm.weight",
      "role": "linear_attention_norm",
      "layer_index": 8,
      "dtype": "bf16",
      "shape": [
        128
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 160275282,
      "length_bytes": 256
    },
    {
      "name": "language_model.model.layers.8.linear_attn.out_proj.weight",
      "role": "linear_attention_out_proj",
      "layer_index": 8,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        2048,
        512
      ],
      "file": "model-00001-of-00004.safetensors",
      "offset_bytes": 4883151698,
      "length_bytes": 4194304
    },
    {
      "name": "language_model.model.layers.9.input_layernorm.weight",
      "role": "attention_norm",
      "layer_index": 9,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 2878090541,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.9.post_attention_layernorm.weight",
      "role": "attention_post_norm",
      "layer_index": 9,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 2686116845,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.9.mlp.switch_mlp.down_proj.weight",
      "role": "ffn_down_exps",
      "layer_index": 9,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        2048,
        64
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 2252212333,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.9.mlp.switch_mlp.gate_proj.weight",
      "role": "ffn_gate_exps",
      "layer_index": 9,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 2391754861,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.9.mlp.gate.weight",
      "role": "ffn_gate_inp",
      "layer_index": 9,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        256,
        512
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 2899150125,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.9.mlp.shared_expert.down_proj.weight",
      "role": "ffn_shared_expert_down",
      "layer_index": 9,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        2048,
        64
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 3260753709,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.9.mlp.shared_expert.gate_proj.weight",
      "role": "ffn_shared_expert_gate",
      "layer_index": 9,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        512,
        256
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 2049965997,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.9.mlp.shared_expert_gate.weight",
      "role": "ffn_shared_expert_gate_inp",
      "layer_index": 9,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        1,
        512
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 3265996589,
      "length_bytes": 2048
    },
    {
      "name": "language_model.model.layers.9.mlp.shared_expert.up_proj.weight",
      "role": "ffn_shared_expert_up",
      "layer_index": 9,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        512,
        256
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 3291361133,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.9.mlp.switch_mlp.up_proj.weight",
      "role": "ffn_up_exps",
      "layer_index": 9,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 2117339181,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.9.linear_attn.A_log",
      "role": "linear_attention_a_log",
      "layer_index": 9,
      "dtype": "bf16",
      "shape": [
        32
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 5091171053,
      "length_bytes": 64
    },
    {
      "name": "language_model.model.layers.9.linear_attn.conv1d.weight",
      "role": "linear_attention_conv1d",
      "layer_index": 9,
      "dtype": "bf16",
      "shape": [
        8192,
        4,
        1
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 2251606061,
      "length_bytes": 65536
    },
    {
      "name": "language_model.model.layers.9.linear_attn.dt_bias",
      "role": "linear_attention_dt_bias",
      "layer_index": 9,
      "dtype": "bf16",
      "shape": [
        32
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 2069358573,
      "length_bytes": 64
    },
    {
      "name": "language_model.model.layers.9.linear_attn.in_proj_a.weight",
      "role": "linear_attention_in_proj_a",
      "layer_index": 9,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        32,
        256
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 1765461741,
      "length_bytes": 32768
    },
    {
      "name": "language_model.model.layers.9.linear_attn.in_proj_b.weight",
      "role": "linear_attention_in_proj_b",
      "layer_index": 9,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        32,
        256
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 2708501613,
      "length_bytes": 32768
    },
    {
      "name": "language_model.model.layers.9.linear_attn.in_proj_qkv.weight",
      "role": "linear_attention_in_proj_qkv",
      "layer_index": 9,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        8192,
        256
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 2660714605,
      "length_bytes": 8388608
    },
    {
      "name": "language_model.model.layers.9.linear_attn.in_proj_z.weight",
      "role": "linear_attention_in_proj_z",
      "layer_index": 9,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        4096,
        256
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 2386430061,
      "length_bytes": 4194304
    },
    {
      "name": "language_model.model.layers.9.linear_attn.norm.weight",
      "role": "linear_attention_norm",
      "layer_index": 9,
      "dtype": "bf16",
      "shape": [
        128
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 2908689709,
      "length_bytes": 256
    },
    {
      "name": "language_model.model.layers.9.linear_attn.out_proj.weight",
      "role": "linear_attention_out_proj",
      "layer_index": 9,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        2048,
        512
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 4917222061,
      "length_bytes": 4194304
    },
    {
      "name": "language_model.model.layers.10.input_layernorm.weight",
      "role": "attention_norm",
      "layer_index": 10,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 2082990125,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.10.post_attention_layernorm.weight",
      "role": "attention_post_norm",
      "layer_index": 10,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 2044649965,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.10.mlp.switch_mlp.down_proj.weight",
      "role": "ffn_down_exps",
      "layer_index": 10,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        2048,
        64
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 1773985773,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.10.mlp.switch_mlp.gate_proj.weight",
      "role": "ffn_gate_exps",
      "layer_index": 10,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 2743872813,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.10.mlp.gate.weight",
      "role": "ffn_gate_inp",
      "layer_index": 10,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        256,
        512
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 2044092909,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.10.mlp.shared_expert.down_proj.weight",
      "role": "ffn_shared_expert_down",
      "layer_index": 10,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        2048,
        64
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 2108424237,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.10.mlp.shared_expert.gate_proj.weight",
      "role": "ffn_shared_expert_gate",
      "layer_index": 10,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        512,
        256
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 2525972589,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.10.mlp.shared_expert_gate.weight",
      "role": "ffn_shared_expert_gate_inp",
      "layer_index": 10,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        1,
        512
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 2049438829,
      "length_bytes": 2048
    },
    {
      "name": "language_model.model.layers.10.mlp.shared_expert.up_proj.weight",
      "role": "ffn_shared_expert_up",
      "layer_index": 10,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        512,
        256
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 2908132653,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.10.mlp.switch_mlp.up_proj.weight",
      "role": "ffn_up_exps",
      "layer_index": 10,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 2526496877,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.10.linear_attn.A_log",
      "role": "linear_attention_a_log",
      "layer_index": 10,
      "dtype": "bf16",
      "shape": [
        32
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 2045211117,
      "length_bytes": 64
    },
    {
      "name": "language_model.model.layers.10.linear_attn.conv1d.weight",
      "role": "linear_attention_conv1d",
      "layer_index": 10,
      "dtype": "bf16",
      "shape": [
        8192,
        4,
        1
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 2069293037,
      "length_bytes": 65536
    },
    {
      "name": "language_model.model.layers.10.linear_attn.dt_bias",
      "role": "linear_attention_dt_bias",
      "layer_index": 10,
      "dtype": "bf16",
      "shape": [
        32
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 2049441133,
      "length_bytes": 64
    },
    {
      "name": "language_model.model.layers.10.linear_attn.in_proj_a.weight",
      "role": "linear_attention_in_proj_a",
      "layer_index": 10,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        32,
        256
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 3060766253,
      "length_bytes": 32768
    },
    {
      "name": "language_model.model.layers.10.linear_attn.in_proj_b.weight",
      "role": "linear_attention_in_proj_b",
      "layer_index": 10,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        32,
        256
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 2925991469,
      "length_bytes": 32768
    },
    {
      "name": "language_model.model.layers.10.linear_attn.in_proj_qkv.weight",
      "role": "linear_attention_in_proj_qkv",
      "layer_index": 10,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        8192,
        256
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 1734178477,
      "length_bytes": 8388608
    },
    {
      "name": "language_model.model.layers.10.linear_attn.in_proj_z.weight",
      "role": "linear_attention_in_proj_z",
      "layer_index": 10,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        4096,
        256
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 4893436397,
      "length_bytes": 4194304
    },
    {
      "name": "language_model.model.layers.10.linear_attn.norm.weight",
      "role": "linear_attention_norm",
      "layer_index": 10,
      "dtype": "bf16",
      "shape": [
        128
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 2049440877,
      "length_bytes": 256
    },
    {
      "name": "language_model.model.layers.10.linear_attn.out_proj.weight",
      "role": "linear_attention_out_proj",
      "layer_index": 10,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        2048,
        512
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 2045211181,
      "length_bytes": 4194304
    },
    {
      "name": "language_model.model.layers.11.self_attn.k_proj.weight",
      "role": "attention_k",
      "layer_index": 11,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        512,
        256
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 3099989549,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.11.self_attn.k_norm.weight",
      "role": "attention_k_norm",
      "layer_index": 11,
      "dtype": "bf16",
      "shape": [
        256
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 2685980845,
      "length_bytes": 512
    },
    {
      "name": "language_model.model.layers.11.input_layernorm.weight",
      "role": "attention_norm",
      "layer_index": 11,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 2686014125,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.11.self_attn.o_proj.weight",
      "role": "attention_o",
      "layer_index": 11,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        2048,
        512
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 1752876781,
      "length_bytes": 4194304
    },
    {
      "name": "language_model.model.layers.11.post_attention_layernorm.weight",
      "role": "attention_post_norm",
      "layer_index": 11,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 1752577709,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.11.self_attn.q_proj.weight",
      "role": "attention_q",
      "layer_index": 11,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        8192,
        256
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 2725903533,
      "length_bytes": 8388608
    },
    {
      "name": "language_model.model.layers.11.self_attn.q_norm.weight",
      "role": "attention_q_norm",
      "layer_index": 11,
      "dtype": "bf16",
      "shape": [
        256
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 2049438317,
      "length_bytes": 512
    },
    {
      "name": "language_model.model.layers.11.self_attn.v_proj.weight",
      "role": "attention_v",
      "layer_index": 11,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        512,
        256
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 2043469805,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.11.mlp.switch_mlp.down_proj.weight",
      "role": "ffn_down_exps",
      "layer_index": 11,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        2048,
        64
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 4347586989,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.11.mlp.switch_mlp.gate_proj.weight",
      "role": "ffn_gate_exps",
      "layer_index": 11,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 1599960749,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.11.mlp.gate.weight",
      "role": "ffn_gate_inp",
      "layer_index": 11,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        256,
        512
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 1752053421,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.11.mlp.shared_expert.down_proj.weight",
      "role": "ffn_shared_expert_down",
      "layer_index": 11,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        2048,
        64
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 1908727789,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.11.mlp.shared_expert.gate_proj.weight",
      "role": "ffn_shared_expert_gate",
      "layer_index": 11,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        512,
        256
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 3452417965,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.11.mlp.shared_expert_gate.weight",
      "role": "ffn_shared_expert_gate_inp",
      "layer_index": 11,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        1,
        512
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 1765459693,
      "length_bytes": 2048
    },
    {
      "name": "language_model.model.layers.11.mlp.shared_expert.up_proj.weight",
      "role": "ffn_shared_expert_up",
      "layer_index": 11,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        512,
        256
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 3457398701,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.11.mlp.switch_mlp.up_proj.weight",
      "role": "ffn_up_exps",
      "layer_index": 11,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 5091275565,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.12.input_layernorm.weight",
      "role": "attention_norm",
      "layer_index": 12,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 2051096493,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.12.post_attention_layernorm.weight",
      "role": "attention_post_norm",
      "layer_index": 12,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 3901116589,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.12.mlp.switch_mlp.down_proj.weight",
      "role": "ffn_down_exps",
      "layer_index": 12,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        2048,
        64
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 4204980653,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.12.mlp.switch_mlp.gate_proj.weight",
      "role": "ffn_gate_exps",
      "layer_index": 12,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 4036946349,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.12.mlp.gate.weight",
      "role": "ffn_gate_inp",
      "layer_index": 12,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        256,
        512
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 3901120685,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.12.mlp.shared_expert.down_proj.weight",
      "role": "ffn_shared_expert_down",
      "layer_index": 12,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        2048,
        64
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 4897630701,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.12.mlp.shared_expert.gate_proj.weight",
      "role": "ffn_shared_expert_gate",
      "layer_index": 12,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        512,
        256
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 4481804717,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.12.mlp.shared_expert_gate.weight",
      "role": "ffn_shared_expert_gate_inp",
      "layer_index": 12,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        1,
        512
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 4898187757,
      "length_bytes": 2048
    },
    {
      "name": "language_model.model.layers.12.mlp.shared_expert.up_proj.weight",
      "role": "ffn_shared_expert_up",
      "layer_index": 12,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        512,
        256
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 1743091373,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.12.mlp.switch_mlp.up_proj.weight",
      "role": "ffn_up_exps",
      "layer_index": 12,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 3601418221,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.12.linear_attn.A_log",
      "role": "linear_attention_a_log",
      "layer_index": 12,
      "dtype": "bf16",
      "shape": [
        32
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 3900559341,
      "length_bytes": 64
    },
    {
      "name": "language_model.model.layers.12.linear_attn.conv1d.weight",
      "role": "linear_attention_conv1d",
      "layer_index": 12,
      "dtype": "bf16",
      "shape": [
        8192,
        4,
        1
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 3601352685,
      "length_bytes": 65536
    },
    {
      "name": "language_model.model.layers.12.linear_attn.dt_bias",
      "role": "linear_attention_dt_bias",
      "layer_index": 12,
      "dtype": "bf16",
      "shape": [
        32
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 2708501549,
      "length_bytes": 64
    },
    {
      "name": "language_model.model.layers.12.linear_attn.in_proj_a.weight",
      "role": "linear_attention_in_proj_a",
      "layer_index": 12,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        32,
        256
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 3892137965,
      "length_bytes": 32768
    },
    {
      "name": "language_model.model.layers.12.linear_attn.in_proj_b.weight",
      "role": "linear_attention_in_proj_b",
      "layer_index": 12,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        32,
        256
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 3308664685,
      "length_bytes": 32768
    },
    {
      "name": "language_model.model.layers.12.linear_attn.in_proj_qkv.weight",
      "role": "linear_attention_in_proj_qkv",
      "layer_index": 12,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        8192,
        256
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 3869853677,
      "length_bytes": 8388608
    },
    {
      "name": "language_model.model.layers.12.linear_attn.in_proj_z.weight",
      "role": "linear_attention_in_proj_z",
      "layer_index": 12,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        4096,
        256
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 3878766573,
      "length_bytes": 4194304
    },
    {
      "name": "language_model.model.layers.12.linear_attn.norm.weight",
      "role": "linear_attention_norm",
      "layer_index": 12,
      "dtype": "bf16",
      "shape": [
        128
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 4036143277,
      "length_bytes": 256
    },
    {
      "name": "language_model.model.layers.12.linear_attn.out_proj.weight",
      "role": "linear_attention_out_proj",
      "layer_index": 12,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        2048,
        512
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 3453204397,
      "length_bytes": 4194304
    },
    {
      "name": "language_model.model.layers.13.input_layernorm.weight",
      "role": "attention_norm",
      "layer_index": 13,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 5368202349,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.13.post_attention_layernorm.weight",
      "role": "attention_post_norm",
      "layer_index": 13,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 4916693613,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.13.mlp.switch_mlp.down_proj.weight",
      "role": "ffn_down_exps",
      "layer_index": 13,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        2048,
        64
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 3458185133,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.13.mlp.switch_mlp.gate_proj.weight",
      "role": "ffn_gate_exps",
      "layer_index": 13,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 3109443117,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.13.mlp.gate.weight",
      "role": "ffn_gate_inp",
      "layer_index": 13,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        256,
        512
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 4916697709,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.13.mlp.shared_expert.down_proj.weight",
      "role": "ffn_shared_expert_down",
      "layer_index": 13,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        2048,
        64
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 4898847277,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.13.mlp.shared_expert.gate_proj.weight",
      "role": "ffn_shared_expert_gate",
      "layer_index": 13,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        512,
        256
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 3600828333,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.13.mlp.shared_expert_gate.weight",
      "role": "ffn_shared_expert_gate_inp",
      "layer_index": 13,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        1,
        512
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 2743868717,
      "length_bytes": 2048
    },
    {
      "name": "language_model.model.layers.13.mlp.shared_expert.up_proj.weight",
      "role": "ffn_shared_expert_up",
      "layer_index": 13,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        512,
        256
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 3100513837,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.13.mlp.switch_mlp.up_proj.weight",
      "role": "ffn_up_exps",
      "layer_index": 13,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 4938734253,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.13.linear_attn.A_log",
      "role": "linear_attention_a_log",
      "layer_index": 13,
      "dtype": "bf16",
      "shape": [
        32
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 1752876717,
      "length_bytes": 64
    },
    {
      "name": "language_model.model.layers.13.linear_attn.conv1d.weight",
      "role": "linear_attention_conv1d",
      "layer_index": 13,
      "dtype": "bf16",
      "shape": [
        8192,
        4,
        1
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 2050490285,
      "length_bytes": 65536
    },
    {
      "name": "language_model.model.layers.13.linear_attn.dt_bias",
      "role": "linear_attention_dt_bias",
      "layer_index": 13,
      "dtype": "bf16",
      "shape": [
        32
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 4916431405,
      "length_bytes": 64
    },
    {
      "name": "language_model.model.layers.13.linear_attn.in_proj_a.weight",
      "role": "linear_attention_in_proj_a",
      "layer_index": 13,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        32,
        256
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 5091171117,
      "length_bytes": 32768
    },
    {
      "name": "language_model.model.layers.13.linear_attn.in_proj_b.weight",
      "role": "linear_attention_in_proj_b",
      "layer_index": 13,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        32,
        256
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 1765564397,
      "length_bytes": 32768
    },
    {
      "name": "language_model.model.layers.13.linear_attn.in_proj_qkv.weight",
      "role": "linear_attention_in_proj_qkv",
      "layer_index": 13,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        8192,
        256
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 3592402861,
      "length_bytes": 8388608
    },
    {
      "name": "language_model.model.layers.13.linear_attn.in_proj_z.weight",
      "role": "linear_attention_in_proj_z",
      "layer_index": 13,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        4096,
        256
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 2078271533,
      "length_bytes": 4194304
    },
    {
      "name": "language_model.model.layers.13.linear_attn.norm.weight",
      "role": "linear_attention_norm",
      "layer_index": 13,
      "dtype": "bf16",
      "shape": [
        128
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 3260458541,
      "length_bytes": 256
    },
    {
      "name": "language_model.model.layers.13.linear_attn.out_proj.weight",
      "role": "linear_attention_out_proj",
      "layer_index": 13,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        2048,
        512
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 3261802285,
      "length_bytes": 4194304
    },
    {
      "name": "language_model.model.layers.14.input_layernorm.weight",
      "role": "attention_norm",
      "layer_index": 14,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 5368468653,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.14.post_attention_layernorm.weight",
      "role": "attention_post_norm",
      "layer_index": 14,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 2899739949,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.14.mlp.switch_mlp.down_proj.weight",
      "role": "ffn_down_exps",
      "layer_index": 14,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        2048,
        64
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 1305812653,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.14.mlp.switch_mlp.gate_proj.weight",
      "role": "ffn_gate_exps",
      "layer_index": 14,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 4750797293,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.14.mlp.gate.weight",
      "role": "ffn_gate_inp",
      "layer_index": 14,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        256,
        512
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 4036422061,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.14.mlp.shared_expert.down_proj.weight",
      "role": "ffn_shared_expert_down",
      "layer_index": 14,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        2048,
        64
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 2908689965,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.14.mlp.shared_expert.gate_proj.weight",
      "role": "ffn_shared_expert_gate",
      "layer_index": 14,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        512,
        256
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 4929804973,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.14.mlp.shared_expert_gate.weight",
      "role": "ffn_shared_expert_gate_inp",
      "layer_index": 14,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        1,
        512
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 1288343213,
      "length_bytes": 2048
    },
    {
      "name": "language_model.model.layers.14.mlp.shared_expert.up_proj.weight",
      "role": "ffn_shared_expert_up",
      "layer_index": 14,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        512,
        256
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 1288410797,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.14.mlp.switch_mlp.up_proj.weight",
      "role": "ffn_up_exps",
      "layer_index": 14,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 1456811693,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.14.linear_attn.A_log",
      "role": "linear_attention_a_log",
      "layer_index": 14,
      "dtype": "bf16",
      "shape": [
        32
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 3900559405,
      "length_bytes": 64
    },
    {
      "name": "language_model.model.layers.14.linear_attn.conv1d.weight",
      "role": "linear_attention_conv1d",
      "layer_index": 14,
      "dtype": "bf16",
      "shape": [
        8192,
        4,
        1
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 5081930413,
      "length_bytes": 65536
    },
    {
      "name": "language_model.model.layers.14.linear_attn.dt_bias",
      "role": "linear_attention_dt_bias",
      "layer_index": 14,
      "dtype": "bf16",
      "shape": [
        32
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 5359747885,
      "length_bytes": 64
    },
    {
      "name": "language_model.model.layers.14.linear_attn.in_proj_a.weight",
      "role": "linear_attention_in_proj_a",
      "layer_index": 14,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        32,
        256
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 5091242797,
      "length_bytes": 32768
    },
    {
      "name": "language_model.model.layers.14.linear_attn.in_proj_b.weight",
      "role": "linear_attention_in_proj_b",
      "layer_index": 14,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        32,
        256
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 5091203885,
      "length_bytes": 32768
    },
    {
      "name": "language_model.model.layers.14.linear_attn.in_proj_qkv.weight",
      "role": "linear_attention_in_proj_qkv",
      "layer_index": 14,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        8192,
        256
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 5081995949,
      "length_bytes": 8388608
    },
    {
      "name": "language_model.model.layers.14.linear_attn.in_proj_z.weight",
      "role": "linear_attention_in_proj_z",
      "layer_index": 14,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        4096,
        256
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 2894923053,
      "length_bytes": 4194304
    },
    {
      "name": "language_model.model.layers.14.linear_attn.norm.weight",
      "role": "linear_attention_norm",
      "layer_index": 14,
      "dtype": "bf16",
      "shape": [
        128
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 5368169325,
      "length_bytes": 256
    },
    {
      "name": "language_model.model.layers.14.linear_attn.out_proj.weight",
      "role": "linear_attention_out_proj",
      "layer_index": 14,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        2048,
        512
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 3060799021,
      "length_bytes": 4194304
    },
    {
      "name": "language_model.model.layers.15.self_attn.k_proj.weight",
      "role": "attention_k",
      "layer_index": 15,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        512,
        256
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 1278836269,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.15.self_attn.k_norm.weight",
      "role": "attention_k_norm",
      "layer_index": 15,
      "dtype": "bf16",
      "shape": [
        256
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 2049965485,
      "length_bytes": 512
    },
    {
      "name": "language_model.model.layers.15.input_layernorm.weight",
      "role": "attention_norm",
      "layer_index": 15,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 1440030381,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.15.self_attn.o_proj.weight",
      "role": "attention_o",
      "layer_index": 15,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        2048,
        512
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 1273490989,
      "length_bytes": 4194304
    },
    {
      "name": "language_model.model.layers.15.post_attention_layernorm.weight",
      "role": "attention_post_norm",
      "layer_index": 15,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 3600824237,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.15.self_attn.q_proj.weight",
      "role": "attention_q",
      "layer_index": 15,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        8192,
        256
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 1279884845,
      "length_bytes": 8388608
    },
    {
      "name": "language_model.model.layers.15.self_attn.q_norm.weight",
      "role": "attention_q_norm",
      "layer_index": 15,
      "dtype": "bf16",
      "shape": [
        256
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 2044026861,
      "length_bytes": 512
    },
    {
      "name": "language_model.model.layers.15.self_attn.v_proj.weight",
      "role": "attention_v",
      "layer_index": 15,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        512,
        256
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 1278279213,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.15.mlp.switch_mlp.down_proj.weight",
      "role": "ffn_down_exps",
      "layer_index": 15,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        2048,
        64
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 4616546733,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.15.mlp.switch_mlp.gate_proj.weight",
      "role": "ffn_gate_exps",
      "layer_index": 15,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 1909252077,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.15.mlp.gate.weight",
      "role": "ffn_gate_inp",
      "layer_index": 15,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        256,
        512
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 1272966701,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.15.mlp.shared_expert.down_proj.weight",
      "role": "ffn_shared_expert_down",
      "layer_index": 15,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        2048,
        64
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 1229368877,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.15.mlp.shared_expert.gate_proj.weight",
      "role": "ffn_shared_expert_gate",
      "layer_index": 15,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        512,
        256
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 1230450221,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.15.mlp.shared_expert_gate.weight",
      "role": "ffn_shared_expert_gate_inp",
      "layer_index": 15,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        1,
        512
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 1599958701,
      "length_bytes": 2048
    },
    {
      "name": "language_model.model.layers.15.mlp.shared_expert.up_proj.weight",
      "role": "ffn_shared_expert_up",
      "layer_index": 15,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        512,
        256
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 1742567085,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.15.mlp.switch_mlp.up_proj.weight",
      "role": "ffn_up_exps",
      "layer_index": 15,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 3901644973,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.16.input_layernorm.weight",
      "role": "attention_norm",
      "layer_index": 16,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 1067150701,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.16.post_attention_layernorm.weight",
      "role": "attention_post_norm",
      "layer_index": 16,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 1278275117,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.16.mlp.switch_mlp.down_proj.weight",
      "role": "ffn_down_exps",
      "layer_index": 16,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        2048,
        64
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 1086565869,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.16.mlp.switch_mlp.gate_proj.weight",
      "role": "ffn_gate_exps",
      "layer_index": 16,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 924509549,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.16.mlp.gate.weight",
      "role": "ffn_gate_inp",
      "layer_index": 16,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        256,
        512
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 2725379245,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.16.mlp.shared_expert.down_proj.weight",
      "role": "ffn_shared_expert_down",
      "layer_index": 16,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        2048,
        64
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 1591029421,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.16.mlp.shared_expert.gate_proj.weight",
      "role": "ffn_shared_expert_gate",
      "layer_index": 16,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        512,
        256
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 764597549,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.16.mlp.shared_expert_gate.weight",
      "role": "ffn_shared_expert_gate_inp",
      "layer_index": 16,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        1,
        512
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 764005677,
      "length_bytes": 2048
    },
    {
      "name": "language_model.model.layers.16.mlp.shared_expert.up_proj.weight",
      "role": "ffn_shared_expert_up",
      "layer_index": 16,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        512,
        256
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 764040493,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.16.mlp.switch_mlp.up_proj.weight",
      "role": "ffn_up_exps",
      "layer_index": 16,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 790291757,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.16.linear_attn.A_log",
      "role": "linear_attention_a_log",
      "layer_index": 16,
      "dtype": "bf16",
      "shape": [
        32
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 1072397677,
      "length_bytes": 64
    },
    {
      "name": "language_model.model.layers.16.linear_attn.conv1d.weight",
      "role": "linear_attention_conv1d",
      "layer_index": 16,
      "dtype": "bf16",
      "shape": [
        8192,
        4,
        1
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 1220783597,
      "length_bytes": 65536
    },
    {
      "name": "language_model.model.layers.16.linear_attn.dt_bias",
      "role": "linear_attention_dt_bias",
      "layer_index": 16,
      "dtype": "bf16",
      "shape": [
        32
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 1072401837,
      "length_bytes": 64
    },
    {
      "name": "language_model.model.layers.16.linear_attn.in_proj_a.weight",
      "role": "linear_attention_in_proj_a",
      "layer_index": 16,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        32,
        256
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 1288310445,
      "length_bytes": 32768
    },
    {
      "name": "language_model.model.layers.16.linear_attn.in_proj_b.weight",
      "role": "linear_attention_in_proj_b",
      "layer_index": 16,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        32,
        256
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 2669135981,
      "length_bytes": 32768
    },
    {
      "name": "language_model.model.layers.16.linear_attn.in_proj_qkv.weight",
      "role": "linear_attention_in_proj_qkv",
      "layer_index": 16,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        8192,
        256
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 1077915117,
      "length_bytes": 8388608
    },
    {
      "name": "language_model.model.layers.16.linear_attn.in_proj_z.weight",
      "role": "linear_attention_in_proj_z",
      "layer_index": 16,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        4096,
        256
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 1072408045,
      "length_bytes": 4194304
    },
    {
      "name": "language_model.model.layers.16.linear_attn.norm.weight",
      "role": "linear_attention_norm",
      "layer_index": 16,
      "dtype": "bf16",
      "shape": [
        128
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 1765564141,
      "length_bytes": 256
    },
    {
      "name": "language_model.model.layers.16.linear_attn.out_proj.weight",
      "role": "linear_attention_out_proj",
      "layer_index": 16,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        2048,
        512
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 1067679085,
      "length_bytes": 4194304
    },
    {
      "name": "language_model.model.layers.17.input_layernorm.weight",
      "role": "attention_norm",
      "layer_index": 17,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 1288306285,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.17.post_attention_layernorm.weight",
      "role": "attention_post_norm",
      "layer_index": 17,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 758670189,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.17.mlp.switch_mlp.down_proj.weight",
      "role": "ffn_down_exps",
      "layer_index": 17,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        2048,
        64
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 473441133,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.17.mlp.switch_mlp.gate_proj.weight",
      "role": "ffn_gate_exps",
      "layer_index": 17,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 3735635949,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.17.mlp.gate.weight",
      "role": "ffn_gate_inp",
      "layer_index": 17,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        256,
        512
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 5081406125,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.17.mlp.shared_expert.down_proj.weight",
      "role": "ffn_shared_expert_down",
      "layer_index": 17,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        2048,
        64
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 472392557,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.17.mlp.shared_expert.gate_proj.weight",
      "role": "ffn_shared_expert_gate",
      "layer_index": 17,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        512,
        256
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 472916845,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.17.mlp.shared_expert_gate.weight",
      "role": "ffn_shared_expert_gate_inp",
      "layer_index": 17,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        1,
        512
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 1077650925,
      "length_bytes": 2048
    },
    {
      "name": "language_model.model.layers.17.mlp.shared_expert.up_proj.weight",
      "role": "ffn_shared_expert_up",
      "layer_index": 17,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        512,
        256
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 3309287277,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.17.mlp.switch_mlp.up_proj.weight",
      "role": "ffn_up_exps",
      "layer_index": 17,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 616047469,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.17.linear_attn.A_log",
      "role": "linear_attention_a_log",
      "layer_index": 17,
      "dtype": "bf16",
      "shape": [
        32
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 2743835821,
      "length_bytes": 64
    },
    {
      "name": "language_model.model.layers.17.linear_attn.conv1d.weight",
      "role": "linear_attention_conv1d",
      "layer_index": 17,
      "dtype": "bf16",
      "shape": [
        8192,
        4,
        1
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 4898189805,
      "length_bytes": 65536
    },
    {
      "name": "language_model.model.layers.17.linear_attn.dt_bias",
      "role": "linear_attention_dt_bias",
      "layer_index": 17,
      "dtype": "bf16",
      "shape": [
        32
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 763671661,
      "length_bytes": 64
    },
    {
      "name": "language_model.model.layers.17.linear_attn.in_proj_a.weight",
      "role": "linear_attention_in_proj_a",
      "layer_index": 17,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        32,
        256
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 763673773,
      "length_bytes": 32768
    },
    {
      "name": "language_model.model.layers.17.linear_attn.in_proj_b.weight",
      "role": "linear_attention_in_proj_b",
      "layer_index": 17,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        32,
        256
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 763710637,
      "length_bytes": 32768
    },
    {
      "name": "language_model.model.layers.17.linear_attn.in_proj_qkv.weight",
      "role": "linear_attention_in_proj_qkv",
      "layer_index": 17,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        8192,
        256
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 3883225069,
      "length_bytes": 8388608
    },
    {
      "name": "language_model.model.layers.17.linear_attn.in_proj_z.weight",
      "role": "linear_attention_in_proj_z",
      "layer_index": 17,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        4096,
        256
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 2695623725,
      "length_bytes": 4194304
    },
    {
      "name": "language_model.model.layers.17.linear_attn.norm.weight",
      "role": "linear_attention_norm",
      "layer_index": 17,
      "dtype": "bf16",
      "shape": [
        128
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 763671405,
      "length_bytes": 256
    },
    {
      "name": "language_model.model.layers.17.linear_attn.out_proj.weight",
      "role": "linear_attention_out_proj",
      "layer_index": 17,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        2048,
        512
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 759477101,
      "length_bytes": 4194304
    },
    {
      "name": "language_model.model.layers.18.input_layernorm.weight",
      "role": "attention_norm",
      "layer_index": 18,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 5091236653,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.18.post_attention_layernorm.weight",
      "role": "attention_post_norm",
      "layer_index": 18,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 790287661,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.18.mlp.switch_mlp.down_proj.weight",
      "role": "ffn_down_exps",
      "layer_index": 18,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        2048,
        64
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 168350445,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.18.mlp.switch_mlp.gate_proj.weight",
      "role": "ffn_gate_exps",
      "layer_index": 18,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 2926548525,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.18.mlp.gate.weight",
      "role": "ffn_gate_inp",
      "layer_index": 18,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        256,
        512
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 445190893,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.18.mlp.shared_expert.down_proj.weight",
      "role": "ffn_shared_expert_down",
      "layer_index": 18,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        2048,
        64
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 150524589,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.18.mlp.shared_expert.gate_proj.weight",
      "role": "ffn_shared_expert_gate",
      "layer_index": 18,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        512,
        256
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 151048877,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.18.mlp.shared_expert_gate.weight",
      "role": "ffn_shared_expert_gate_inp",
      "layer_index": 18,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        1,
        512
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 149932717,
      "length_bytes": 2048
    },
    {
      "name": "language_model.model.layers.18.mlp.shared_expert.up_proj.weight",
      "role": "ffn_shared_expert_up",
      "layer_index": 18,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        512,
        256
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 149967533,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.18.mlp.switch_mlp.up_proj.weight",
      "role": "ffn_up_exps",
      "layer_index": 18,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 302568173,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.18.linear_attn.A_log",
      "role": "linear_attention_a_log",
      "layer_index": 18,
      "dtype": "bf16",
      "shape": [
        32
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 450171629,
      "length_bytes": 64
    },
    {
      "name": "language_model.model.layers.18.linear_attn.conv1d.weight",
      "role": "linear_attention_conv1d",
      "layer_index": 18,
      "dtype": "bf16",
      "shape": [
        8192,
        4,
        1
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 463872813,
      "length_bytes": 65536
    },
    {
      "name": "language_model.model.layers.18.linear_attn.dt_bias",
      "role": "linear_attention_dt_bias",
      "layer_index": 18,
      "dtype": "bf16",
      "shape": [
        32
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 2049405485,
      "length_bytes": 64
    },
    {
      "name": "language_model.model.layers.18.linear_attn.in_proj_a.weight",
      "role": "linear_attention_in_proj_a",
      "layer_index": 18,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        32,
        256
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 450173741,
      "length_bytes": 32768
    },
    {
      "name": "language_model.model.layers.18.linear_attn.in_proj_b.weight",
      "role": "linear_attention_in_proj_b",
      "layer_index": 18,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        32,
        256
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 458597165,
      "length_bytes": 32768
    },
    {
      "name": "language_model.model.layers.18.linear_attn.in_proj_qkv.weight",
      "role": "linear_attention_in_proj_qkv",
      "layer_index": 18,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        8192,
        256
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 3266096941,
      "length_bytes": 8388608
    },
    {
      "name": "language_model.model.layers.18.linear_attn.in_proj_z.weight",
      "role": "linear_attention_in_proj_z",
      "layer_index": 18,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        4096,
        256
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 459154221,
      "length_bytes": 4194304
    },
    {
      "name": "language_model.model.layers.18.linear_attn.norm.weight",
      "role": "linear_attention_norm",
      "layer_index": 18,
      "dtype": "bf16",
      "shape": [
        128
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 2686083821,
      "length_bytes": 256
    },
    {
      "name": "language_model.model.layers.18.linear_attn.out_proj.weight",
      "role": "linear_attention_out_proj",
      "layer_index": 18,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        2048,
        512
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 445977325,
      "length_bytes": 4194304
    },
    {
      "name": "language_model.model.layers.19.self_attn.k_proj.weight",
      "role": "attention_k",
      "layer_index": 19,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        512,
        256
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 758674285,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.19.self_attn.k_norm.weight",
      "role": "attention_k_norm",
      "layer_index": 19,
      "dtype": "bf16",
      "shape": [
        256
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 144361133,
      "length_bytes": 512
    },
    {
      "name": "language_model.model.layers.19.input_layernorm.weight",
      "role": "attention_norm",
      "layer_index": 19,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 1072397741,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.19.self_attn.o_proj.weight",
      "role": "attention_o",
      "layer_index": 19,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        2048,
        512
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 144624301,
      "length_bytes": 4194304
    },
    {
      "name": "language_model.model.layers.19.post_attention_layernorm.weight",
      "role": "attention_post_norm",
      "layer_index": 19,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 1765560045,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.19.self_attn.q_proj.weight",
      "role": "attention_q",
      "layer_index": 19,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        8192,
        256
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 450206509,
      "length_bytes": 8388608
    },
    {
      "name": "language_model.model.layers.19.self_attn.q_norm.weight",
      "role": "attention_q_norm",
      "layer_index": 19,
      "dtype": "bf16",
      "shape": [
        256
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 144361645,
      "length_bytes": 512
    },
    {
      "name": "language_model.model.layers.19.self_attn.v_proj.weight",
      "role": "attention_v",
      "layer_index": 19,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        512,
        256
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 1277718061,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.19.mlp.switch_mlp.down_proj.weight",
      "role": "ffn_down_exps",
      "layer_index": 19,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        2048,
        64
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 4482329005,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.19.mlp.switch_mlp.gate_proj.weight",
      "role": "ffn_gate_exps",
      "layer_index": 19,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 3318200237,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.19.mlp.gate.weight",
      "role": "ffn_gate_inp",
      "layer_index": 19,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        256,
        512
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 2050555821,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.19.mlp.shared_expert.down_proj.weight",
      "role": "ffn_shared_expert_down",
      "layer_index": 19,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        2048,
        64
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 689837,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.19.mlp.shared_expert.gate_proj.weight",
      "role": "ffn_shared_expert_gate",
      "layer_index": 19,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        512,
        256
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 1214125,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.19.mlp.shared_expert_gate.weight",
      "role": "ffn_shared_expert_gate_inp",
      "layer_index": 19,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        1,
        512
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 4035879085,
      "length_bytes": 2048
    },
    {
      "name": "language_model.model.layers.19.mlp.shared_expert.up_proj.weight",
      "role": "ffn_shared_expert_up",
      "layer_index": 19,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        512,
        256
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 1229893165,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.19.mlp.switch_mlp.up_proj.weight",
      "role": "ffn_up_exps",
      "layer_index": 19,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 1738413,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.20.input_layernorm.weight",
      "role": "attention_norm",
      "layer_index": 20,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 4938202411,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.20.post_attention_layernorm.weight",
      "role": "attention_post_norm",
      "layer_index": 20,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 2376592875,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.20.mlp.switch_mlp.down_proj.weight",
      "role": "ffn_down_exps",
      "layer_index": 20,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        2048,
        64
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 2545719147,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.20.mlp.switch_mlp.gate_proj.weight",
      "role": "ffn_gate_exps",
      "layer_index": 20,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 2814154603,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.20.mlp.gate.weight",
      "role": "ffn_gate_inp",
      "layer_index": 20,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        256,
        512
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 2222875883,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.20.mlp.shared_expert.down_proj.weight",
      "role": "ffn_shared_expert_down",
      "layer_index": 20,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        2048,
        64
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 3613096235,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.20.mlp.shared_expert.gate_proj.weight",
      "role": "ffn_shared_expert_gate",
      "layer_index": 20,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        512,
        256
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 3595712811,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.20.mlp.shared_expert_gate.weight",
      "role": "ffn_shared_expert_gate_inp",
      "layer_index": 20,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        1,
        512
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 3418190123,
      "length_bytes": 2048
    },
    {
      "name": "language_model.model.layers.20.mlp.shared_expert.up_proj.weight",
      "role": "ffn_shared_expert_up",
      "layer_index": 20,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        512,
        256
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 3396630443,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.20.mlp.switch_mlp.up_proj.weight",
      "role": "ffn_up_exps",
      "layer_index": 20,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 3113942891,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.20.linear_attn.A_log",
      "role": "linear_attention_a_log",
      "layer_index": 20,
      "dtype": "bf16",
      "shape": [
        32
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 4884687915,
      "length_bytes": 64
    },
    {
      "name": "language_model.model.layers.20.linear_attn.conv1d.weight",
      "role": "linear_attention_conv1d",
      "layer_index": 20,
      "dtype": "bf16",
      "shape": [
        8192,
        4,
        1
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 591533,
      "length_bytes": 65536
    },
    {
      "name": "language_model.model.layers.20.linear_attn.dt_bias",
      "role": "linear_attention_dt_bias",
      "layer_index": 20,
      "dtype": "bf16",
      "shape": [
        32
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 3405547691,
      "length_bytes": 64
    },
    {
      "name": "language_model.model.layers.20.linear_attn.in_proj_a.weight",
      "role": "linear_attention_in_proj_a",
      "layer_index": 20,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        32,
        256
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 3604642091,
      "length_bytes": 32768
    },
    {
      "name": "language_model.model.layers.20.linear_attn.in_proj_b.weight",
      "role": "linear_attention_in_proj_b",
      "layer_index": 20,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        32,
        256
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 4683158379,
      "length_bytes": 32768
    },
    {
      "name": "language_model.model.layers.20.linear_attn.in_proj_qkv.weight",
      "role": "linear_attention_in_proj_qkv",
      "layer_index": 20,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        8192,
        256
      ],
      "file": "model-00002-of-00004.safetensors",
      "offset_bytes": 2886534445,
      "length_bytes": 8388608
    },
    {
      "name": "language_model.model.layers.20.linear_attn.in_proj_z.weight",
      "role": "linear_attention_in_proj_z",
      "layer_index": 20,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        4096,
        256
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 3249264555,
      "length_bytes": 4194304
    },
    {
      "name": "language_model.model.layers.20.linear_attn.norm.weight",
      "role": "linear_attention_norm",
      "layer_index": 20,
      "dtype": "bf16",
      "shape": [
        128
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 3405547435,
      "length_bytes": 256
    },
    {
      "name": "language_model.model.layers.20.linear_attn.out_proj.weight",
      "role": "linear_attention_out_proj",
      "layer_index": 20,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        2048,
        512
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 4728892459,
      "length_bytes": 4194304
    },
    {
      "name": "language_model.model.layers.21.input_layernorm.weight",
      "role": "attention_norm",
      "layer_index": 21,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 2213696235,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.21.post_attention_layernorm.weight",
      "role": "attention_post_norm",
      "layer_index": 21,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 3397154731,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.21.mlp.switch_mlp.down_proj.weight",
      "role": "ffn_down_exps",
      "layer_index": 21,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        2048,
        64
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 1772929707,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.21.mlp.switch_mlp.gate_proj.weight",
      "role": "ffn_gate_exps",
      "layer_index": 21,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 3624769899,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.21.mlp.gate.weight",
      "role": "ffn_gate_inp",
      "layer_index": 21,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        256,
        512
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 4885902507,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.21.mlp.shared_expert.down_proj.weight",
      "role": "ffn_shared_expert_down",
      "layer_index": 21,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        2048,
        64
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 2240736491,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.21.mlp.shared_expert.gate_proj.weight",
      "role": "ffn_shared_expert_gate",
      "layer_index": 21,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        512,
        256
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 3763444331,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.21.mlp.shared_expert_gate.weight",
      "role": "ffn_shared_expert_gate_inp",
      "layer_index": 21,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        1,
        512
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 1746510315,
      "length_bytes": 2048
    },
    {
      "name": "language_model.model.layers.21.mlp.shared_expert.up_proj.weight",
      "role": "ffn_shared_expert_up",
      "layer_index": 21,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        512,
        256
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 1746547243,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.21.mlp.switch_mlp.up_proj.weight",
      "role": "ffn_up_exps",
      "layer_index": 21,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 1932378795,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.21.linear_attn.A_log",
      "role": "linear_attention_a_log",
      "layer_index": 21,
      "dtype": "bf16",
      "shape": [
        32
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 3614998827,
      "length_bytes": 64
    },
    {
      "name": "language_model.model.layers.21.linear_attn.conv1d.weight",
      "role": "linear_attention_conv1d",
      "layer_index": 21,
      "dtype": "bf16",
      "shape": [
        8192,
        4,
        1
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 3624175979,
      "length_bytes": 65536
    },
    {
      "name": "language_model.model.layers.21.linear_attn.dt_bias",
      "role": "linear_attention_dt_bias",
      "layer_index": 21,
      "dtype": "bf16",
      "shape": [
        32
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 3249260395,
      "length_bytes": 64
    },
    {
      "name": "language_model.model.layers.21.linear_attn.in_proj_a.weight",
      "role": "linear_attention_in_proj_a",
      "layer_index": 21,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        32,
        256
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 4534585835,
      "length_bytes": 32768
    },
    {
      "name": "language_model.model.layers.21.linear_attn.in_proj_b.weight",
      "role": "linear_attention_in_proj_b",
      "layer_index": 21,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        32,
        256
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 2241260779,
      "length_bytes": 32768
    },
    {
      "name": "language_model.model.layers.21.linear_attn.in_proj_qkv.weight",
      "role": "linear_attention_in_proj_qkv",
      "layer_index": 21,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        8192,
        256
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 2214487275,
      "length_bytes": 8388608
    },
    {
      "name": "language_model.model.layers.21.linear_attn.in_proj_z.weight",
      "role": "linear_attention_in_proj_z",
      "layer_index": 21,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        4096,
        256
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 3758987883,
      "length_bytes": 4194304
    },
    {
      "name": "language_model.model.layers.21.linear_attn.norm.weight",
      "role": "linear_attention_norm",
      "layer_index": 21,
      "dtype": "bf16",
      "shape": [
        128
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 2523790955,
      "length_bytes": 256
    },
    {
      "name": "language_model.model.layers.21.linear_attn.out_proj.weight",
      "role": "linear_attention_out_proj",
      "layer_index": 21,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        2048,
        512
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 3578935595,
      "length_bytes": 4194304
    },
    {
      "name": "language_model.model.layers.22.input_layernorm.weight",
      "role": "attention_norm",
      "layer_index": 22,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 3087112043,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.22.post_attention_layernorm.weight",
      "role": "attention_post_norm",
      "layer_index": 22,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 3249260459,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.22.mlp.switch_mlp.down_proj.weight",
      "role": "ffn_down_exps",
      "layer_index": 22,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        2048,
        64
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 2241850859,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.22.mlp.switch_mlp.gate_proj.weight",
      "role": "ffn_gate_exps",
      "layer_index": 22,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 3436329259,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.22.mlp.gate.weight",
      "role": "ffn_gate_inp",
      "layer_index": 22,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        256,
        512
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 3418208555,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.22.mlp.shared_expert.down_proj.weight",
      "role": "ffn_shared_expert_down",
      "layer_index": 22,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        2048,
        64
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 4884753579,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.22.mlp.shared_expert.gate_proj.weight",
      "role": "ffn_shared_expert_gate",
      "layer_index": 22,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        512,
        256
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 1429735019,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.22.mlp.shared_expert_gate.weight",
      "role": "ffn_shared_expert_gate_inp",
      "layer_index": 22,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        1,
        512
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 3614964011,
      "length_bytes": 2048
    },
    {
      "name": "language_model.model.layers.22.mlp.shared_expert.up_proj.weight",
      "role": "ffn_shared_expert_up",
      "layer_index": 22,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        512,
        256
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 1420494443,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.22.mlp.switch_mlp.up_proj.weight",
      "role": "ffn_up_exps",
      "layer_index": 22,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 1459900011,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.22.linear_attn.A_log",
      "role": "linear_attention_a_log",
      "layer_index": 22,
      "dtype": "bf16",
      "shape": [
        32
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 1603055467,
      "length_bytes": 64
    },
    {
      "name": "language_model.model.layers.22.linear_attn.conv1d.weight",
      "role": "linear_attention_conv1d",
      "layer_index": 22,
      "dtype": "bf16",
      "shape": [
        8192,
        4,
        1
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 4885836971,
      "length_bytes": 65536
    },
    {
      "name": "language_model.model.layers.22.linear_attn.dt_bias",
      "role": "linear_attention_dt_bias",
      "layer_index": 22,
      "dtype": "bf16",
      "shape": [
        32
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 4884720747,
      "length_bytes": 64
    },
    {
      "name": "language_model.model.layers.22.linear_attn.in_proj_a.weight",
      "role": "linear_attention_in_proj_a",
      "layer_index": 22,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        32,
        256
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 2201076395,
      "length_bytes": 32768
    },
    {
      "name": "language_model.model.layers.22.linear_attn.in_proj_b.weight",
      "role": "linear_attention_in_proj_b",
      "layer_index": 22,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        32,
        256
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 1612255659,
      "length_bytes": 32768
    },
    {
      "name": "language_model.model.layers.22.linear_attn.in_proj_qkv.weight",
      "role": "linear_attention_in_proj_qkv",
      "layer_index": 22,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        8192,
        256
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 1603342763,
      "length_bytes": 8388608
    },
    {
      "name": "language_model.model.layers.22.linear_attn.in_proj_z.weight",
      "role": "linear_attention_in_proj_z",
      "layer_index": 22,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        4096,
        256
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 2519596651,
      "length_bytes": 4194304
    },
    {
      "name": "language_model.model.layers.22.linear_attn.norm.weight",
      "role": "linear_attention_norm",
      "layer_index": 22,
      "dtype": "bf16",
      "shape": [
        128
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 1603053163,
      "length_bytes": 256
    },
    {
      "name": "language_model.model.layers.22.linear_attn.out_proj.weight",
      "role": "linear_attention_out_proj",
      "layer_index": 22,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        2048,
        512
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 1455687275,
      "length_bytes": 4194304
    },
    {
      "name": "language_model.model.layers.23.self_attn.k_proj.weight",
      "role": "attention_k",
      "layer_index": 23,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        512,
        256
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 3774016299,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.23.self_attn.k_norm.weight",
      "role": "attention_k_norm",
      "layer_index": 23,
      "dtype": "bf16",
      "shape": [
        256
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 1603342251,
      "length_bytes": 512
    },
    {
      "name": "language_model.model.layers.23.input_layernorm.weight",
      "role": "attention_norm",
      "layer_index": 23,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 3387971499,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.23.self_attn.o_proj.weight",
      "role": "attention_o",
      "layer_index": 23,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        2048,
        512
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 2201113323,
      "length_bytes": 4194304
    },
    {
      "name": "language_model.model.layers.23.post_attention_layernorm.weight",
      "role": "attention_post_norm",
      "layer_index": 23,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 3387975595,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.23.self_attn.q_proj.weight",
      "role": "attention_q",
      "layer_index": 23,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        8192,
        256
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 4912969003,
      "length_bytes": 8388608
    },
    {
      "name": "language_model.model.layers.23.self_attn.q_norm.weight",
      "role": "attention_q_norm",
      "layer_index": 23,
      "dtype": "bf16",
      "shape": [
        256
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 4078269291,
      "length_bytes": 512
    },
    {
      "name": "language_model.model.layers.23.self_attn.v_proj.weight",
      "role": "attention_v",
      "layer_index": 23,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        512,
        256
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 3808654187,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.23.mlp.switch_mlp.down_proj.weight",
      "role": "ffn_down_exps",
      "layer_index": 23,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        2048,
        64
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 4381951915,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.23.mlp.switch_mlp.gate_proj.weight",
      "role": "ffn_gate_exps",
      "layer_index": 23,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 4087823787,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.23.mlp.gate.weight",
      "role": "ffn_gate_inp",
      "layer_index": 23,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        256,
        512
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 4086725995,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.23.mlp.shared_expert.down_proj.weight",
      "role": "ffn_shared_expert_down",
      "layer_index": 23,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        2048,
        64
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 4533537259,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.23.mlp.shared_expert.gate_proj.weight",
      "role": "ffn_shared_expert_gate",
      "layer_index": 23,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        512,
        256
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 4711590955,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.23.mlp.shared_expert_gate.weight",
      "role": "ffn_shared_expert_gate_inp",
      "layer_index": 23,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        1,
        512
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 1747071531,
      "length_bytes": 2048
    },
    {
      "name": "language_model.model.layers.23.mlp.shared_expert.up_proj.weight",
      "role": "ffn_shared_expert_up",
      "layer_index": 23,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        512,
        256
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 3614998891,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.23.mlp.switch_mlp.up_proj.weight",
      "role": "ffn_up_exps",
      "layer_index": 23,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 4238818731,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.24.input_layernorm.weight",
      "role": "attention_norm",
      "layer_index": 24,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 5368254507,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.24.post_attention_layernorm.weight",
      "role": "attention_post_norm",
      "layer_index": 24,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 3087148907,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.24.mlp.switch_mlp.down_proj.weight",
      "role": "ffn_down_exps",
      "layer_index": 24,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        2048,
        64
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 4741508139,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.24.mlp.switch_mlp.gate_proj.weight",
      "role": "ffn_gate_exps",
      "layer_index": 24,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 3809178475,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.24.mlp.gate.weight",
      "role": "ffn_gate_inp",
      "layer_index": 24,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        256,
        512
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 4710523947,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.24.mlp.shared_expert.down_proj.weight",
      "role": "ffn_shared_expert_down",
      "layer_index": 24,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        2048,
        64
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 4885277867,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.24.mlp.shared_expert.gate_proj.weight",
      "role": "ffn_shared_expert_gate",
      "layer_index": 24,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        512,
        256
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 4884147243,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.24.mlp.shared_expert_gate.weight",
      "role": "ffn_shared_expert_gate_inp",
      "layer_index": 24,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        1,
        512
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 2532609899,
      "length_bytes": 2048
    },
    {
      "name": "language_model.model.layers.24.mlp.shared_expert.up_proj.weight",
      "role": "ffn_shared_expert_up",
      "layer_index": 24,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        512,
        256
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 4087250283,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.24.mlp.switch_mlp.up_proj.weight",
      "role": "ffn_up_exps",
      "layer_index": 24,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 3253753771,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.24.linear_attn.A_log",
      "role": "linear_attention_a_log",
      "layer_index": 24,
      "dtype": "bf16",
      "shape": [
        32
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 4701086699,
      "length_bytes": 64
    },
    {
      "name": "language_model.model.layers.24.linear_attn.conv1d.weight",
      "role": "linear_attention_conv1d",
      "layer_index": 24,
      "dtype": "bf16",
      "shape": [
        8192,
        4,
        1
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 5208280619,
      "length_bytes": 65536
    },
    {
      "name": "language_model.model.layers.24.linear_attn.dt_bias",
      "role": "linear_attention_dt_bias",
      "layer_index": 24,
      "dtype": "bf16",
      "shape": [
        32
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 4692698027,
      "length_bytes": 64
    },
    {
      "name": "language_model.model.layers.24.linear_attn.in_proj_a.weight",
      "role": "linear_attention_in_proj_a",
      "layer_index": 24,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        32,
        256
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 4683750315,
      "length_bytes": 32768
    },
    {
      "name": "language_model.model.layers.24.linear_attn.in_proj_b.weight",
      "role": "linear_attention_in_proj_b",
      "layer_index": 24,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        32,
        256
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 4682812267,
      "length_bytes": 32768
    },
    {
      "name": "language_model.model.layers.24.linear_attn.in_proj_qkv.weight",
      "role": "linear_attention_in_proj_qkv",
      "layer_index": 24,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        8192,
        256
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 4895077611,
      "length_bytes": 8388608
    },
    {
      "name": "language_model.model.layers.24.linear_attn.in_proj_z.weight",
      "role": "linear_attention_in_proj_z",
      "layer_index": 24,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        4096,
        256
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 4678323051,
      "length_bytes": 4194304
    },
    {
      "name": "language_model.model.layers.24.linear_attn.norm.weight",
      "role": "linear_attention_norm",
      "layer_index": 24,
      "dtype": "bf16",
      "shape": [
        128
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 2241293547,
      "length_bytes": 256
    },
    {
      "name": "language_model.model.layers.24.linear_attn.out_proj.weight",
      "role": "linear_attention_out_proj",
      "layer_index": 24,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        2048,
        512
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 3387979691,
      "length_bytes": 4194304
    },
    {
      "name": "language_model.model.layers.25.input_layernorm.weight",
      "role": "attention_norm",
      "layer_index": 25,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 3624241515,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.25.post_attention_layernorm.weight",
      "role": "attention_post_norm",
      "layer_index": 25,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 1746506219,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.25.mlp.switch_mlp.down_proj.weight",
      "role": "ffn_down_exps",
      "layer_index": 25,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        2048,
        64
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 5225123883,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.25.mlp.switch_mlp.gate_proj.weight",
      "role": "ffn_gate_exps",
      "layer_index": 25,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 5074062891,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.25.mlp.gate.weight",
      "role": "ffn_gate_inp",
      "layer_index": 25,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        256,
        512
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 4692173739,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.25.mlp.shared_expert.down_proj.weight",
      "role": "ffn_shared_expert_down",
      "layer_index": 25,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        2048,
        64
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 2376068587,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.25.mlp.shared_expert.gate_proj.weight",
      "role": "ffn_shared_expert_gate",
      "layer_index": 25,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        512,
        256
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 3624245611,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.25.mlp.shared_expert_gate.weight",
      "role": "ffn_shared_expert_gate_inp",
      "layer_index": 25,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        1,
        512
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 1419935339,
      "length_bytes": 2048
    },
    {
      "name": "language_model.model.layers.25.mlp.shared_expert.up_proj.weight",
      "role": "ffn_shared_expert_up",
      "layer_index": 25,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        512,
        256
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 4683191147,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.25.mlp.switch_mlp.up_proj.weight",
      "role": "ffn_up_exps",
      "layer_index": 25,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 4939517483,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.25.linear_attn.A_log",
      "role": "linear_attention_a_log",
      "layer_index": 25,
      "dtype": "bf16",
      "shape": [
        32
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 4533537195,
      "length_bytes": 64
    },
    {
      "name": "language_model.model.layers.25.linear_attn.conv1d.weight",
      "role": "linear_attention_conv1d",
      "layer_index": 25,
      "dtype": "bf16",
      "shape": [
        8192,
        4,
        1
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 1755462315,
      "length_bytes": 65536
    },
    {
      "name": "language_model.model.layers.25.linear_attn.dt_bias",
      "role": "linear_attention_dt_bias",
      "layer_index": 25,
      "dtype": "bf16",
      "shape": [
        32
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 4087807339,
      "length_bytes": 64
    },
    {
      "name": "language_model.model.layers.25.linear_attn.in_proj_a.weight",
      "role": "linear_attention_in_proj_a",
      "layer_index": 25,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        32,
        256
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 4884687979,
      "length_bytes": 32768
    },
    {
      "name": "language_model.model.layers.25.linear_attn.in_proj_b.weight",
      "role": "linear_attention_in_proj_b",
      "layer_index": 25,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        32,
        256
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 4925584683,
      "length_bytes": 32768
    },
    {
      "name": "language_model.model.layers.25.linear_attn.in_proj_qkv.weight",
      "role": "linear_attention_in_proj_qkv",
      "layer_index": 25,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        8192,
        256
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 4525115819,
      "length_bytes": 8388608
    },
    {
      "name": "language_model.model.layers.25.linear_attn.in_proj_z.weight",
      "role": "linear_attention_in_proj_z",
      "layer_index": 25,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        4096,
        256
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 4921357611,
      "length_bytes": 4194304
    },
    {
      "name": "language_model.model.layers.25.linear_attn.norm.weight",
      "role": "linear_attention_norm",
      "layer_index": 25,
      "dtype": "bf16",
      "shape": [
        128
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 4938206507,
      "length_bytes": 256
    },
    {
      "name": "language_model.model.layers.25.linear_attn.out_proj.weight",
      "role": "linear_attention_out_proj",
      "layer_index": 25,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        2048,
        512
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 3392436139,
      "length_bytes": 4194304
    },
    {
      "name": "language_model.model.layers.26.input_layernorm.weight",
      "role": "attention_norm",
      "layer_index": 26,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 1276184043,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.26.post_attention_layernorm.weight",
      "role": "attention_post_norm",
      "layer_index": 26,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 1276179947,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.26.mlp.switch_mlp.down_proj.weight",
      "role": "ffn_down_exps",
      "layer_index": 26,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        2048,
        64
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 2948667243,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.26.mlp.switch_mlp.gate_proj.weight",
      "role": "ffn_gate_exps",
      "layer_index": 26,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 1141913003,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.26.mlp.gate.weight",
      "role": "ffn_gate_inp",
      "layer_index": 26,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        256,
        512
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 2536806251,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.26.mlp.shared_expert.down_proj.weight",
      "role": "ffn_shared_expert_down",
      "layer_index": 26,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        2048,
        64
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 4709737515,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.26.mlp.shared_expert.gate_proj.weight",
      "role": "ffn_shared_expert_gate",
      "layer_index": 26,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        512,
        256
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 3112877931,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.26.mlp.shared_expert_gate.weight",
      "role": "ffn_shared_expert_gate_inp",
      "layer_index": 26,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        1,
        512
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 838477163,
      "length_bytes": 2048
    },
    {
      "name": "language_model.model.layers.26.mlp.shared_expert.up_proj.weight",
      "role": "ffn_shared_expert_up",
      "layer_index": 26,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        512,
        256
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 972696939,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.26.mlp.switch_mlp.up_proj.weight",
      "role": "ffn_up_exps",
      "layer_index": 26,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 999273899,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.26.linear_attn.A_log",
      "role": "linear_attention_a_log",
      "layer_index": 26,
      "dtype": "bf16",
      "shape": [
        32
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 2510880235,
      "length_bytes": 64
    },
    {
      "name": "language_model.model.layers.26.linear_attn.conv1d.weight",
      "role": "linear_attention_conv1d",
      "layer_index": 26,
      "dtype": "bf16",
      "shape": [
        8192,
        4,
        1
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 2510880299,
      "length_bytes": 65536
    },
    {
      "name": "language_model.model.layers.26.linear_attn.dt_bias",
      "role": "linear_attention_dt_bias",
      "layer_index": 26,
      "dtype": "bf16",
      "shape": [
        32
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 1755462251,
      "length_bytes": 64
    },
    {
      "name": "language_model.model.layers.26.linear_attn.in_proj_a.weight",
      "role": "linear_attention_in_proj_a",
      "layer_index": 26,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        32,
        256
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 1419322859,
      "length_bytes": 32768
    },
    {
      "name": "language_model.model.layers.26.linear_attn.in_proj_b.weight",
      "role": "linear_attention_in_proj_b",
      "layer_index": 26,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        32,
        256
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 1419376107,
      "length_bytes": 32768
    },
    {
      "name": "language_model.model.layers.26.linear_attn.in_proj_qkv.weight",
      "role": "linear_attention_in_proj_qkv",
      "layer_index": 26,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        8192,
        256
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 2537330539,
      "length_bytes": 8388608
    },
    {
      "name": "language_model.model.layers.26.linear_attn.in_proj_z.weight",
      "role": "linear_attention_in_proj_z",
      "layer_index": 26,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        4096,
        256
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 4934008107,
      "length_bytes": 4194304
    },
    {
      "name": "language_model.model.layers.26.linear_attn.norm.weight",
      "role": "linear_attention_norm",
      "layer_index": 26,
      "dtype": "bf16",
      "shape": [
        128
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 4668852715,
      "length_bytes": 256
    },
    {
      "name": "language_model.model.layers.26.linear_attn.out_proj.weight",
      "role": "linear_attention_out_proj",
      "layer_index": 26,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        2048,
        512
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 3082884971,
      "length_bytes": 4194304
    },
    {
      "name": "language_model.model.layers.27.self_attn.k_proj.weight",
      "role": "attention_k",
      "layer_index": 27,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        512,
        256
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 998749611,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.27.self_attn.k_norm.weight",
      "role": "attention_k_norm",
      "layer_index": 27,
      "dtype": "bf16",
      "shape": [
        256
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 4516169643,
      "length_bytes": 512
    },
    {
      "name": "language_model.model.layers.27.input_layernorm.weight",
      "role": "attention_norm",
      "layer_index": 27,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 1602770539,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.27.self_attn.o_proj.weight",
      "role": "attention_o",
      "layer_index": 27,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        2048,
        512
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 2532611947,
      "length_bytes": 4194304
    },
    {
      "name": "language_model.model.layers.27.post_attention_layernorm.weight",
      "role": "attention_post_norm",
      "layer_index": 27,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 828640619,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.27.self_attn.q_proj.weight",
      "role": "attention_q",
      "layer_index": 27,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        8192,
        256
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 830088555,
      "length_bytes": 8388608
    },
    {
      "name": "language_model.model.layers.27.self_attn.q_norm.weight",
      "role": "attention_q_norm",
      "layer_index": 27,
      "dtype": "bf16",
      "shape": [
        256
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 5225123371,
      "length_bytes": 512
    },
    {
      "name": "language_model.model.layers.27.self_attn.v_proj.weight",
      "role": "attention_v",
      "layer_index": 27,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        512,
        256
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 829007211,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.27.mlp.switch_mlp.down_proj.weight",
      "role": "ffn_down_exps",
      "layer_index": 27,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        2048,
        64
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 551226731,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.27.mlp.switch_mlp.gate_proj.weight",
      "role": "ffn_gate_exps",
      "layer_index": 27,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 693865835,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.27.mlp.gate.weight",
      "role": "ffn_gate_inp",
      "layer_index": 27,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        256,
        512
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 828116331,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.27.mlp.shared_expert.down_proj.weight",
      "role": "ffn_shared_expert_down",
      "layer_index": 27,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        2048,
        64
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 533593451,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.27.mlp.shared_expert.gate_proj.weight",
      "role": "ffn_shared_expert_gate",
      "layer_index": 27,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        512,
        256
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 3419027755,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.27.mlp.shared_expert_gate.weight",
      "role": "ffn_shared_expert_gate_inp",
      "layer_index": 27,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        1,
        512
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 533525867,
      "length_bytes": 2048
    },
    {
      "name": "language_model.model.layers.27.mlp.shared_expert.up_proj.weight",
      "role": "ffn_shared_expert_up",
      "layer_index": 27,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        512,
        256
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 4903498987,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.27.mlp.switch_mlp.up_proj.weight",
      "role": "ffn_up_exps",
      "layer_index": 27,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 838479211,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.28.input_layernorm.weight",
      "role": "attention_norm",
      "layer_index": 28,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 534150507,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.28.post_attention_layernorm.weight",
      "role": "attention_post_norm",
      "layer_index": 28,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 3418186027,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.28.mlp.switch_mlp.down_proj.weight",
      "role": "ffn_down_exps",
      "layer_index": 28,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        2048,
        64
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 1285101035,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.28.mlp.switch_mlp.gate_proj.weight",
      "role": "ffn_gate_exps",
      "layer_index": 28,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 4534634987,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.28.mlp.gate.weight",
      "role": "ffn_gate_inp",
      "layer_index": 28,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        256,
        512
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 519007147,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.28.mlp.shared_expert.down_proj.weight",
      "role": "ffn_shared_expert_down",
      "layer_index": 28,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        2048,
        64
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 492661675,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.28.mlp.shared_expert.gate_proj.weight",
      "role": "ffn_shared_expert_gate",
      "layer_index": 28,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        512,
        256
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 493284267,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.28.mlp.shared_expert_gate.weight",
      "role": "ffn_shared_expert_gate_inp",
      "layer_index": 28,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        1,
        512
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 1603336107,
      "length_bytes": 2048
    },
    {
      "name": "language_model.model.layers.28.mlp.shared_expert.up_proj.weight",
      "role": "ffn_shared_expert_up",
      "layer_index": 28,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        512,
        256
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 357854123,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.28.mlp.switch_mlp.up_proj.weight",
      "role": "ffn_up_exps",
      "layer_index": 28,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 2376662507,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.28.linear_attn.A_log",
      "role": "linear_attention_a_log",
      "layer_index": 28,
      "dtype": "bf16",
      "shape": [
        32
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 524512427,
      "length_bytes": 64
    },
    {
      "name": "language_model.model.layers.28.linear_attn.conv1d.weight",
      "role": "linear_attention_conv1d",
      "layer_index": 28,
      "dtype": "bf16",
      "shape": [
        8192,
        4,
        1
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 533460203,
      "length_bytes": 65536
    },
    {
      "name": "language_model.model.layers.28.linear_attn.dt_bias",
      "role": "linear_attention_dt_bias",
      "layer_index": 28,
      "dtype": "bf16",
      "shape": [
        32
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 990360939,
      "length_bytes": 64
    },
    {
      "name": "language_model.model.layers.28.linear_attn.in_proj_a.weight",
      "role": "linear_attention_in_proj_a",
      "layer_index": 28,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        32,
        256
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 1429407339,
      "length_bytes": 32768
    },
    {
      "name": "language_model.model.layers.28.linear_attn.in_proj_b.weight",
      "role": "linear_attention_in_proj_b",
      "layer_index": 28,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        32,
        256
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 524514539,
      "length_bytes": 32768
    },
    {
      "name": "language_model.model.layers.28.linear_attn.in_proj_qkv.weight",
      "role": "linear_attention_in_proj_qkv",
      "layer_index": 28,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        8192,
        256
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 525071595,
      "length_bytes": 8388608
    },
    {
      "name": "language_model.model.layers.28.linear_attn.in_proj_z.weight",
      "role": "linear_attention_in_proj_z",
      "layer_index": 28,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        4096,
        256
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 3591518507,
      "length_bytes": 4194304
    },
    {
      "name": "language_model.model.layers.28.linear_attn.norm.weight",
      "role": "linear_attention_norm",
      "layer_index": 28,
      "dtype": "bf16",
      "shape": [
        128
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 524250027,
      "length_bytes": 256
    },
    {
      "name": "language_model.model.layers.28.linear_attn.out_proj.weight",
      "role": "linear_attention_out_proj",
      "layer_index": 28,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        2048,
        512
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 520055723,
      "length_bytes": 4194304
    },
    {
      "name": "language_model.model.layers.29.input_layernorm.weight",
      "role": "attention_norm",
      "layer_index": 29,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 340153067,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.29.post_attention_layernorm.weight",
      "role": "attention_post_norm",
      "layer_index": 29,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 340148971,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.29.mlp.switch_mlp.down_proj.weight",
      "role": "ffn_down_exps",
      "layer_index": 29,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        2048,
        64
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 3943756651,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.29.mlp.switch_mlp.gate_proj.weight",
      "role": "ffn_gate_exps",
      "layer_index": 29,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 358443947,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.29.mlp.gate.weight",
      "role": "ffn_gate_inp",
      "layer_index": 29,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        256,
        512
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 1611731371,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.29.mlp.shared_expert.down_proj.weight",
      "role": "ffn_shared_expert_down",
      "layer_index": 29,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        2048,
        64
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 322241259,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.29.mlp.shared_expert.gate_proj.weight",
      "role": "ffn_shared_expert_gate",
      "layer_index": 29,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        512,
        256
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 322765547,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.29.mlp.shared_expert_gate.weight",
      "role": "ffn_shared_expert_gate_inp",
      "layer_index": 29,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        1,
        512
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 1603053419,
      "length_bytes": 2048
    },
    {
      "name": "language_model.model.layers.29.mlp.shared_expert.up_proj.weight",
      "role": "ffn_shared_expert_up",
      "layer_index": 29,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        512,
        256
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 4677798763,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.29.mlp.switch_mlp.up_proj.weight",
      "role": "ffn_up_exps",
      "layer_index": 29,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 2066596523,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.29.linear_attn.A_log",
      "role": "linear_attention_a_log",
      "layer_index": 29,
      "dtype": "bf16",
      "shape": [
        32
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 344351467,
      "length_bytes": 64
    },
    {
      "name": "language_model.model.layers.29.linear_attn.conv1d.weight",
      "role": "linear_attention_conv1d",
      "layer_index": 29,
      "dtype": "bf16",
      "shape": [
        8192,
        4,
        1
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 357788523,
      "length_bytes": 65536
    },
    {
      "name": "language_model.model.layers.29.linear_attn.dt_bias",
      "role": "linear_attention_dt_bias",
      "layer_index": 29,
      "dtype": "bf16",
      "shape": [
        32
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 344351531,
      "length_bytes": 64
    },
    {
      "name": "language_model.model.layers.29.linear_attn.in_proj_a.weight",
      "role": "linear_attention_in_proj_a",
      "layer_index": 29,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        32,
        256
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 344351595,
      "length_bytes": 32768
    },
    {
      "name": "language_model.model.layers.29.linear_attn.in_proj_b.weight",
      "role": "linear_attention_in_proj_b",
      "layer_index": 29,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        32,
        256
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 344386411,
      "length_bytes": 32768
    },
    {
      "name": "language_model.model.layers.29.linear_attn.in_proj_qkv.weight",
      "role": "linear_attention_in_proj_qkv",
      "layer_index": 29,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        8192,
        256
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 349399915,
      "length_bytes": 8388608
    },
    {
      "name": "language_model.model.layers.29.linear_attn.in_proj_z.weight",
      "role": "linear_attention_in_proj_z",
      "layer_index": 29,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        4096,
        256
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 344681323,
      "length_bytes": 4194304
    },
    {
      "name": "language_model.model.layers.29.linear_attn.norm.weight",
      "role": "linear_attention_norm",
      "layer_index": 29,
      "dtype": "bf16",
      "shape": [
        128
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 3758987627,
      "length_bytes": 256
    },
    {
      "name": "language_model.model.layers.29.linear_attn.out_proj.weight",
      "role": "linear_attention_out_proj",
      "layer_index": 29,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        2048,
        512
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 340157163,
      "length_bytes": 4194304
    },
    {
      "name": "language_model.model.layers.30.input_layernorm.weight",
      "role": "attention_norm",
      "layer_index": 30,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 2532556651,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.30.post_attention_layernorm.weight",
      "role": "attention_post_norm",
      "layer_index": 30,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 312203627,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.30.mlp.switch_mlp.down_proj.weight",
      "role": "ffn_down_exps",
      "layer_index": 30,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        2048,
        64
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 2679936875,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.30.mlp.switch_mlp.gate_proj.weight",
      "role": "ffn_gate_exps",
      "layer_index": 30,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 177953131,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.30.mlp.gate.weight",
      "role": "ffn_gate_inp",
      "layer_index": 30,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        256,
        512
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 1755593387,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.30.mlp.shared_expert.down_proj.weight",
      "role": "ffn_shared_expert_down",
      "layer_index": 30,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        2048,
        64
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 3103965035,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.30.mlp.shared_expert.gate_proj.weight",
      "role": "ffn_shared_expert_gate",
      "layer_index": 30,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        512,
        256
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 5367730219,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.30.mlp.shared_expert_gate.weight",
      "role": "ffn_shared_expert_gate_inp",
      "layer_index": 30,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        1,
        512
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 828644715,
      "length_bytes": 2048
    },
    {
      "name": "language_model.model.layers.30.mlp.shared_expert.up_proj.weight",
      "role": "ffn_shared_expert_up",
      "layer_index": 30,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        512,
        256
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 3248736107,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.30.mlp.switch_mlp.up_proj.weight",
      "role": "ffn_up_exps",
      "layer_index": 30,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 1612288427,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.30.linear_attn.A_log",
      "role": "linear_attention_a_log",
      "layer_index": 30,
      "dtype": "bf16",
      "shape": [
        32
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 320858731,
      "length_bytes": 64
    },
    {
      "name": "language_model.model.layers.30.linear_attn.conv1d.weight",
      "role": "linear_attention_conv1d",
      "layer_index": 30,
      "dtype": "bf16",
      "shape": [
        8192,
        4,
        1
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 3943396203,
      "length_bytes": 65536
    },
    {
      "name": "language_model.model.layers.30.linear_attn.dt_bias",
      "role": "linear_attention_dt_bias",
      "layer_index": 30,
      "dtype": "bf16",
      "shape": [
        32
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 2519334443,
      "length_bytes": 64
    },
    {
      "name": "language_model.model.layers.30.linear_attn.in_proj_a.weight",
      "role": "linear_attention_in_proj_a",
      "layer_index": 30,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        32,
        256
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 320860843,
      "length_bytes": 32768
    },
    {
      "name": "language_model.model.layers.30.linear_attn.in_proj_b.weight",
      "role": "linear_attention_in_proj_b",
      "layer_index": 30,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        32,
        256
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 685444459,
      "length_bytes": 32768
    },
    {
      "name": "language_model.model.layers.30.linear_attn.in_proj_qkv.weight",
      "role": "linear_attention_in_proj_qkv",
      "layer_index": 30,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        8192,
        256
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 4875758635,
      "length_bytes": 8388608
    },
    {
      "name": "language_model.model.layers.30.linear_attn.in_proj_z.weight",
      "role": "linear_attention_in_proj_z",
      "layer_index": 30,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        4096,
        256
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 1451492971,
      "length_bytes": 4194304
    },
    {
      "name": "language_model.model.layers.30.linear_attn.norm.weight",
      "role": "linear_attention_norm",
      "layer_index": 30,
      "dtype": "bf16",
      "shape": [
        128
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 320858475,
      "length_bytes": 256
    },
    {
      "name": "language_model.model.layers.30.linear_attn.out_proj.weight",
      "role": "linear_attention_out_proj",
      "layer_index": 30,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        2048,
        512
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 3413987627,
      "length_bytes": 4194304
    },
    {
      "name": "language_model.model.layers.31.self_attn.k_proj.weight",
      "role": "attention_k",
      "layer_index": 31,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        512,
        256
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 9591083,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.31.self_attn.k_norm.weight",
      "role": "attention_k_norm",
      "layer_index": 31,
      "dtype": "bf16",
      "shape": [
        256
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 2213700331,
      "length_bytes": 512
    },
    {
      "name": "language_model.model.layers.31.input_layernorm.weight",
      "role": "attention_norm",
      "layer_index": 31,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 2201109163,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.31.self_attn.o_proj.weight",
      "role": "attention_o",
      "layer_index": 31,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        2048,
        512
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 1447298667,
      "length_bytes": 4194304
    },
    {
      "name": "language_model.model.layers.31.post_attention_layernorm.weight",
      "role": "attention_post_norm",
      "layer_index": 31,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 8997163,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.31.self_attn.q_proj.weight",
      "role": "attention_q",
      "layer_index": 31,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        8192,
        256
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 10115371,
      "length_bytes": 8388608
    },
    {
      "name": "language_model.model.layers.31.self_attn.q_norm.weight",
      "role": "attention_q_norm",
      "layer_index": 31,
      "dtype": "bf16",
      "shape": [
        256
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 4373036459,
      "length_bytes": 512
    },
    {
      "name": "language_model.model.layers.31.self_attn.v_proj.weight",
      "role": "attention_v",
      "layer_index": 31,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        512,
        256
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 4938993195,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.31.mlp.switch_mlp.down_proj.weight",
      "role": "ffn_down_exps",
      "layer_index": 31,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        2048,
        64
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 3778792229,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.31.mlp.switch_mlp.gate_proj.weight",
      "role": "ffn_gate_exps",
      "layer_index": 31,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 43735403,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.31.mlp.gate.weight",
      "role": "ffn_gate_inp",
      "layer_index": 31,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        256,
        512
      ],
      "file": "model-00003-of-00004.safetensors",
      "offset_bytes": 8472875,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.31.mlp.shared_expert.down_proj.weight",
      "role": "ffn_shared_expert_down",
      "layer_index": 31,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        2048,
        64
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 1773732837,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.31.mlp.shared_expert.gate_proj.weight",
      "role": "ffn_shared_expert_gate",
      "layer_index": 31,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        512,
        256
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 1943694565,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.31.mlp.shared_expert_gate.weight",
      "role": "ffn_shared_expert_gate_inp",
      "layer_index": 31,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        1,
        512
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 2682656485,
      "length_bytes": 2048
    },
    {
      "name": "language_model.model.layers.31.mlp.shared_expert.up_proj.weight",
      "role": "ffn_shared_expert_up",
      "layer_index": 31,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        512,
        256
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 2825266917,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.31.mlp.switch_mlp.up_proj.weight",
      "role": "ffn_up_exps",
      "layer_index": 31,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 2682660581,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.32.input_layernorm.weight",
      "role": "attention_norm",
      "layer_index": 32,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 2087759141,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.32.post_attention_layernorm.weight",
      "role": "attention_post_norm",
      "layer_index": 32,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 1468793125,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.32.mlp.switch_mlp.down_proj.weight",
      "role": "ffn_down_exps",
      "layer_index": 32,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        2048,
        64
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 2413434597,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.32.mlp.switch_mlp.gate_proj.weight",
      "role": "ffn_gate_exps",
      "layer_index": 32,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 2981504741,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.32.mlp.gate.weight",
      "role": "ffn_gate_inp",
      "layer_index": 32,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        256,
        512
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 3145690917,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.32.mlp.shared_expert.down_proj.weight",
      "role": "ffn_shared_expert_down",
      "layer_index": 32,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        2048,
        64
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 2972591845,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.32.mlp.shared_expert.gate_proj.weight",
      "role": "ffn_shared_expert_gate",
      "layer_index": 32,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        512,
        256
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 3146215205,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.32.mlp.shared_expert_gate.weight",
      "role": "ffn_shared_expert_gate_inp",
      "layer_index": 32,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        1,
        512
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 1536502117,
      "length_bytes": 2048
    },
    {
      "name": "language_model.model.layers.32.mlp.shared_expert.up_proj.weight",
      "role": "ffn_shared_expert_up",
      "layer_index": 32,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        512,
        256
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 3146840101,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.32.mlp.switch_mlp.up_proj.weight",
      "role": "ffn_up_exps",
      "layer_index": 32,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 2547914469,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.32.linear_attn.A_log",
      "role": "linear_attention_a_log",
      "layer_index": 32,
      "dtype": "bf16",
      "shape": [
        32
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 2252656229,
      "length_bytes": 64
    },
    {
      "name": "language_model.model.layers.32.linear_attn.conv1d.weight",
      "role": "linear_attention_conv1d",
      "layer_index": 32,
      "dtype": "bf16",
      "shape": [
        8192,
        4,
        1
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 2252590693,
      "length_bytes": 65536
    },
    {
      "name": "language_model.model.layers.32.linear_attn.dt_bias",
      "role": "linear_attention_dt_bias",
      "layer_index": 32,
      "dtype": "bf16",
      "shape": [
        32
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 2248396325,
      "length_bytes": 64
    },
    {
      "name": "language_model.model.layers.32.linear_attn.in_proj_a.weight",
      "role": "linear_attention_in_proj_a",
      "layer_index": 32,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        32,
        256
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 1783466981,
      "length_bytes": 32768
    },
    {
      "name": "language_model.model.layers.32.linear_attn.in_proj_b.weight",
      "role": "linear_attention_in_proj_b",
      "layer_index": 32,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        32,
        256
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 2105097509,
      "length_bytes": 32768
    },
    {
      "name": "language_model.model.layers.32.linear_attn.in_proj_qkv.weight",
      "role": "linear_attention_in_proj_qkv",
      "layer_index": 32,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        8192,
        256
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 1752417125,
      "length_bytes": 8388608
    },
    {
      "name": "language_model.model.layers.32.linear_attn.in_proj_z.weight",
      "role": "linear_attention_in_proj_z",
      "layer_index": 32,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        4096,
        256
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 2105130277,
      "length_bytes": 4194304
    },
    {
      "name": "language_model.model.layers.32.linear_attn.norm.weight",
      "role": "linear_attention_norm",
      "layer_index": 32,
      "dtype": "bf16",
      "shape": [
        128
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 3146741541,
      "length_bytes": 256
    },
    {
      "name": "language_model.model.layers.32.linear_attn.out_proj.weight",
      "role": "linear_attention_out_proj",
      "layer_index": 32,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        2048,
        512
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 2400065253,
      "length_bytes": 4194304
    },
    {
      "name": "language_model.model.layers.33.input_layernorm.weight",
      "role": "attention_norm",
      "layer_index": 33,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 1536504165,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.33.post_attention_layernorm.weight",
      "role": "attention_post_norm",
      "layer_index": 33,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 1536465253,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.33.mlp.switch_mlp.down_proj.weight",
      "role": "ffn_down_exps",
      "layer_index": 33,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        2048,
        64
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 4057011557,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.33.mlp.switch_mlp.gate_proj.weight",
      "role": "ffn_gate_exps",
      "layer_index": 33,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 2257440421,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.33.mlp.gate.weight",
      "role": "ffn_gate_inp",
      "layer_index": 33,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        256,
        512
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 3635656997,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.33.mlp.shared_expert.down_proj.weight",
      "role": "ffn_shared_expert_down",
      "layer_index": 33,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        2048,
        64
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 3164178469,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.33.mlp.shared_expert.gate_proj.weight",
      "role": "ffn_shared_expert_gate",
      "layer_index": 33,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        512,
        256
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 4375308517,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.33.mlp.shared_expert_gate.weight",
      "role": "ffn_shared_expert_gate_inp",
      "layer_index": 33,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        1,
        512
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 1485689189,
      "length_bytes": 2048
    },
    {
      "name": "language_model.model.layers.33.mlp.shared_expert.up_proj.weight",
      "role": "ffn_shared_expert_up",
      "layer_index": 33,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        512,
        256
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 1712529957,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.33.mlp.switch_mlp.up_proj.weight",
      "role": "ffn_up_exps",
      "layer_index": 33,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 3492524325,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.33.linear_attn.A_log",
      "role": "linear_attention_a_log",
      "layer_index": 33,
      "dtype": "bf16",
      "shape": [
        32
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 2113586597,
      "length_bytes": 64
    },
    {
      "name": "language_model.model.layers.33.linear_attn.conv1d.weight",
      "role": "linear_attention_conv1d",
      "layer_index": 33,
      "dtype": "bf16",
      "shape": [
        8192,
        4,
        1
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 1703205285,
      "length_bytes": 65536
    },
    {
      "name": "language_model.model.layers.33.linear_attn.dt_bias",
      "role": "linear_attention_dt_bias",
      "layer_index": 33,
      "dtype": "bf16",
      "shape": [
        32
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 2109390117,
      "length_bytes": 64
    },
    {
      "name": "language_model.model.layers.33.linear_attn.in_proj_a.weight",
      "role": "linear_attention_in_proj_a",
      "layer_index": 33,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        32,
        256
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 1545226597,
      "length_bytes": 32768
    },
    {
      "name": "language_model.model.layers.33.linear_attn.in_proj_b.weight",
      "role": "linear_attention_in_proj_b",
      "layer_index": 33,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        32,
        256
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 1536469349,
      "length_bytes": 32768
    },
    {
      "name": "language_model.model.layers.33.linear_attn.in_proj_qkv.weight",
      "role": "linear_attention_in_proj_qkv",
      "layer_index": 33,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        8192,
        256
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 1560498597,
      "length_bytes": 8388608
    },
    {
      "name": "language_model.model.layers.33.linear_attn.in_proj_z.weight",
      "role": "linear_attention_in_proj_z",
      "layer_index": 33,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        4096,
        256
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 1769505765,
      "length_bytes": 4194304
    },
    {
      "name": "language_model.model.layers.33.linear_attn.norm.weight",
      "role": "linear_attention_norm",
      "layer_index": 33,
      "dtype": "bf16",
      "shape": [
        128
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 1791908837,
      "length_bytes": 256
    },
    {
      "name": "language_model.model.layers.33.linear_attn.out_proj.weight",
      "role": "linear_attention_out_proj",
      "layer_index": 33,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        2048,
        512
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 3140953893,
      "length_bytes": 4194304
    },
    {
      "name": "language_model.model.layers.34.input_layernorm.weight",
      "role": "attention_norm",
      "layer_index": 34,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 3164174373,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.34.post_attention_layernorm.weight",
      "role": "attention_post_norm",
      "layer_index": 34,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 1743219493,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.34.mlp.switch_mlp.down_proj.weight",
      "role": "ffn_down_exps",
      "layer_index": 34,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        2048,
        64
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 3307833381,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.34.mlp.switch_mlp.gate_proj.weight",
      "role": "ffn_gate_exps",
      "layer_index": 34,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 1809214693,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.34.mlp.gate.weight",
      "role": "ffn_gate_inp",
      "layer_index": 34,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        256,
        512
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 1545259429,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.34.mlp.shared_expert.down_proj.weight",
      "role": "ffn_shared_expert_down",
      "layer_index": 34,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        2048,
        64
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 3459647525,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.34.mlp.shared_expert.gate_proj.weight",
      "role": "ffn_shared_expert_gate",
      "layer_index": 34,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        512,
        256
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 4374227173,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.34.mlp.shared_expert_gate.weight",
      "role": "ffn_shared_expert_gate_inp",
      "layer_index": 34,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        1,
        512
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 3482935397,
      "length_bytes": 2048
    },
    {
      "name": "language_model.model.layers.34.mlp.shared_expert.up_proj.weight",
      "role": "ffn_shared_expert_up",
      "layer_index": 34,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        512,
        256
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 3481772133,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.34.mlp.switch_mlp.up_proj.weight",
      "role": "ffn_up_exps",
      "layer_index": 34,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 3173615653,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.34.linear_attn.A_log",
      "role": "linear_attention_a_log",
      "layer_index": 34,
      "dtype": "bf16",
      "shape": [
        32
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 1760805797,
      "length_bytes": 64
    },
    {
      "name": "language_model.model.layers.34.linear_attn.conv1d.weight",
      "role": "linear_attention_conv1d",
      "layer_index": 34,
      "dtype": "bf16",
      "shape": [
        8192,
        4,
        1
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 1477202213,
      "length_bytes": 65536
    },
    {
      "name": "language_model.model.layers.34.linear_attn.dt_bias",
      "role": "linear_attention_dt_bias",
      "layer_index": 34,
      "dtype": "bf16",
      "shape": [
        32
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 1454258405,
      "length_bytes": 64
    },
    {
      "name": "language_model.model.layers.34.linear_attn.in_proj_a.weight",
      "role": "linear_attention_in_proj_a",
      "layer_index": 34,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        32,
        256
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 3146774565,
      "length_bytes": 32768
    },
    {
      "name": "language_model.model.layers.34.linear_attn.in_proj_b.weight",
      "role": "linear_attention_in_proj_b",
      "layer_index": 34,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        32,
        256
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 3164141605,
      "length_bytes": 32768
    },
    {
      "name": "language_model.model.layers.34.linear_attn.in_proj_qkv.weight",
      "role": "linear_attention_in_proj_qkv",
      "layer_index": 34,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        8192,
        256
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 4196341093,
      "length_bytes": 8388608
    },
    {
      "name": "language_model.model.layers.34.linear_attn.in_proj_z.weight",
      "role": "linear_attention_in_proj_z",
      "layer_index": 34,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        4096,
        256
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 2968397541,
      "length_bytes": 4194304
    },
    {
      "name": "language_model.model.layers.34.linear_attn.norm.weight",
      "role": "linear_attention_norm",
      "layer_index": 34,
      "dtype": "bf16",
      "shape": [
        128
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 1725637157,
      "length_bytes": 256
    },
    {
      "name": "language_model.model.layers.34.linear_attn.out_proj.weight",
      "role": "linear_attention_out_proj",
      "layer_index": 34,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        2048,
        512
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 2248396389,
      "length_bytes": 4194304
    },
    {
      "name": "language_model.model.layers.35.self_attn.k_proj.weight",
      "role": "attention_k",
      "layer_index": 35,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        512,
        256
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 3491885349,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.35.self_attn.k_norm.weight",
      "role": "attention_k_norm",
      "layer_index": 35,
      "dtype": "bf16",
      "shape": [
        256
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 3644569893,
      "length_bytes": 512
    },
    {
      "name": "language_model.model.layers.35.input_layernorm.weight",
      "role": "attention_norm",
      "layer_index": 35,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 3778788133,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.35.self_attn.o_proj.weight",
      "role": "attention_o",
      "layer_index": 35,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        2048,
        512
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 1713054245,
      "length_bytes": 4194304
    },
    {
      "name": "language_model.model.layers.35.post_attention_layernorm.weight",
      "role": "attention_post_norm",
      "layer_index": 35,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 3913009957,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.35.self_attn.q_proj.weight",
      "role": "attention_q",
      "layer_index": 35,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        8192,
        256
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 3482970405,
      "length_bytes": 8388608
    },
    {
      "name": "language_model.model.layers.35.self_attn.q_norm.weight",
      "role": "attention_q_norm",
      "layer_index": 35,
      "dtype": "bf16",
      "shape": [
        256
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 4374161125,
      "length_bytes": 512
    },
    {
      "name": "language_model.model.layers.35.self_attn.v_proj.weight",
      "role": "attention_v",
      "layer_index": 35,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        512,
        256
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 1751892837,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.35.mlp.switch_mlp.down_proj.weight",
      "role": "ffn_down_exps",
      "layer_index": 35,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        2048,
        64
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 2825791205,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.35.mlp.switch_mlp.gate_proj.weight",
      "role": "ffn_gate_exps",
      "layer_index": 35,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 3913807141,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.35.mlp.gate.weight",
      "role": "ffn_gate_inp",
      "layer_index": 35,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        256,
        512
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 3913282341,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.35.mlp.shared_expert.down_proj.weight",
      "role": "ffn_shared_expert_down",
      "layer_index": 35,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        2048,
        64
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 1711989285,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.35.mlp.shared_expert.gate_proj.weight",
      "role": "ffn_shared_expert_gate",
      "layer_index": 35,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        512,
        256
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 4056417573,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.35.mlp.shared_expert_gate.weight",
      "role": "ffn_shared_expert_gate_inp",
      "layer_index": 35,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        1,
        512
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 1703270821,
      "length_bytes": 2048
    },
    {
      "name": "language_model.model.layers.35.mlp.shared_expert.up_proj.weight",
      "role": "ffn_shared_expert_up",
      "layer_index": 35,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        512,
        256
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 1725655845,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.35.mlp.switch_mlp.up_proj.weight",
      "role": "ffn_up_exps",
      "layer_index": 35,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 3644570405,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.36.input_layernorm.weight",
      "role": "attention_norm",
      "layer_index": 36,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 3913016101,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.36.post_attention_layernorm.weight",
      "role": "attention_post_norm",
      "layer_index": 36,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 3473313829,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.36.mlp.switch_mlp.down_proj.weight",
      "role": "ffn_down_exps",
      "layer_index": 36,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        2048,
        64
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 1568954789,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.36.mlp.switch_mlp.gate_proj.weight",
      "role": "ffn_gate_exps",
      "layer_index": 36,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 1952705765,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.36.mlp.gate.weight",
      "role": "ffn_gate_inp",
      "layer_index": 36,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        256,
        512
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 2682132197,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.36.mlp.shared_expert.down_proj.weight",
      "role": "ffn_shared_expert_down",
      "layer_index": 36,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        2048,
        64
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 4373047013,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.36.mlp.shared_expert.gate_proj.weight",
      "role": "ffn_shared_expert_gate",
      "layer_index": 36,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        512,
        256
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 4376424933,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.36.mlp.shared_expert_gate.weight",
      "role": "ffn_shared_expert_gate_inp",
      "layer_index": 36,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        1,
        512
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 3145148197,
      "length_bytes": 2048
    },
    {
      "name": "language_model.model.layers.36.mlp.shared_expert.up_proj.weight",
      "role": "ffn_shared_expert_up",
      "layer_index": 36,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        512,
        256
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 2086923557,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.36.mlp.switch_mlp.up_proj.weight",
      "role": "ffn_up_exps",
      "layer_index": 36,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 4221787877,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.36.linear_attn.A_log",
      "role": "linear_attention_a_log",
      "layer_index": 36,
      "dtype": "bf16",
      "shape": [
        32
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 4204731813,
      "length_bytes": 64
    },
    {
      "name": "language_model.model.layers.36.linear_attn.conv1d.weight",
      "role": "linear_attention_conv1d",
      "layer_index": 36,
      "dtype": "bf16",
      "shape": [
        8192,
        4,
        1
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 1568887205,
      "length_bytes": 65536
    },
    {
      "name": "language_model.model.layers.36.linear_attn.dt_bias",
      "role": "linear_attention_dt_bias",
      "layer_index": 36,
      "dtype": "bf16",
      "shape": [
        32
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 4204731749,
      "length_bytes": 64
    },
    {
      "name": "language_model.model.layers.36.linear_attn.in_proj_a.weight",
      "role": "linear_attention_in_proj_a",
      "layer_index": 36,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        32,
        256
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 1773700069,
      "length_bytes": 32768
    },
    {
      "name": "language_model.model.layers.36.linear_attn.in_proj_b.weight",
      "role": "linear_attention_in_proj_b",
      "layer_index": 36,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        32,
        256
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 4196242789,
      "length_bytes": 32768
    },
    {
      "name": "language_model.model.layers.36.linear_attn.in_proj_qkv.weight",
      "role": "linear_attention_in_proj_qkv",
      "layer_index": 36,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        8192,
        256
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 3636181285,
      "length_bytes": 8388608
    },
    {
      "name": "language_model.model.layers.36.linear_attn.in_proj_z.weight",
      "role": "linear_attention_in_proj_z",
      "layer_index": 36,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        4096,
        256
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 4191786341,
      "length_bytes": 4194304
    },
    {
      "name": "language_model.model.layers.36.linear_attn.norm.weight",
      "role": "linear_attention_norm",
      "layer_index": 36,
      "dtype": "bf16",
      "shape": [
        128
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 4204748261,
      "length_bytes": 256
    },
    {
      "name": "language_model.model.layers.36.linear_attn.out_proj.weight",
      "role": "linear_attention_out_proj",
      "layer_index": 36,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        2048,
        512
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 2253246117,
      "length_bytes": 4194304
    },
    {
      "name": "language_model.model.layers.37.input_layernorm.weight",
      "role": "attention_norm",
      "layer_index": 37,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 1454254309,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.37.post_attention_layernorm.weight",
      "role": "attention_post_norm",
      "layer_index": 37,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 1454250213,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.37.mlp.switch_mlp.down_proj.weight",
      "role": "ffn_down_exps",
      "layer_index": 37,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        2048,
        64
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 1025857765,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.37.mlp.switch_mlp.gate_proj.weight",
      "role": "ffn_gate_exps",
      "layer_index": 37,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 1319459045,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.37.mlp.gate.weight",
      "role": "ffn_gate_inp",
      "layer_index": 37,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        256,
        512
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 1453725925,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.37.mlp.shared_expert.down_proj.weight",
      "role": "ffn_shared_expert_down",
      "layer_index": 37,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        2048,
        64
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 1024809189,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.37.mlp.shared_expert.gate_proj.weight",
      "role": "ffn_shared_expert_gate",
      "layer_index": 37,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        512,
        256
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 1025333477,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.37.mlp.shared_expert_gate.weight",
      "role": "ffn_shared_expert_gate_inp",
      "layer_index": 37,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        1,
        512
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 3460171813,
      "length_bytes": 2048
    },
    {
      "name": "language_model.model.layers.37.mlp.shared_expert.up_proj.weight",
      "role": "ffn_shared_expert_up",
      "layer_index": 37,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        512,
        256
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 1024252133,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.37.mlp.switch_mlp.up_proj.weight",
      "role": "ffn_up_exps",
      "layer_index": 37,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 1168464101,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.37.linear_attn.A_log",
      "role": "linear_attention_a_log",
      "layer_index": 37,
      "dtype": "bf16",
      "shape": [
        32
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 2086923493,
      "length_bytes": 64
    },
    {
      "name": "language_model.model.layers.37.linear_attn.conv1d.weight",
      "role": "linear_attention_conv1d",
      "layer_index": 37,
      "dtype": "bf16",
      "shape": [
        8192,
        4,
        1
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 4374161637,
      "length_bytes": 65536
    },
    {
      "name": "language_model.model.layers.37.linear_attn.dt_bias",
      "role": "linear_attention_dt_bias",
      "layer_index": 37,
      "dtype": "bf16",
      "shape": [
        32
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 3482970277,
      "length_bytes": 64
    },
    {
      "name": "language_model.model.layers.37.linear_attn.in_proj_a.weight",
      "role": "linear_attention_in_proj_a",
      "layer_index": 37,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        32,
        256
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 1485756773,
      "length_bytes": 32768
    },
    {
      "name": "language_model.model.layers.37.linear_attn.in_proj_b.weight",
      "role": "linear_attention_in_proj_b",
      "layer_index": 37,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        32,
        256
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 4376127717,
      "length_bytes": 32768
    },
    {
      "name": "language_model.model.layers.37.linear_attn.in_proj_qkv.weight",
      "role": "linear_attention_in_proj_qkv",
      "layer_index": 37,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        8192,
        256
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 1519638885,
      "length_bytes": 8388608
    },
    {
      "name": "language_model.model.layers.37.linear_attn.in_proj_z.weight",
      "role": "linear_attention_in_proj_z",
      "layer_index": 37,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        4096,
        256
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 3460173861,
      "length_bytes": 4194304
    },
    {
      "name": "language_model.model.layers.37.linear_attn.norm.weight",
      "role": "linear_attention_norm",
      "layer_index": 37,
      "dtype": "bf16",
      "shape": [
        128
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 4376162533,
      "length_bytes": 256
    },
    {
      "name": "language_model.model.layers.37.linear_attn.out_proj.weight",
      "role": "linear_attention_out_proj",
      "layer_index": 37,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        2048,
        512
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 2109390181,
      "length_bytes": 4194304
    },
    {
      "name": "language_model.model.layers.38.input_layernorm.weight",
      "role": "attention_norm",
      "layer_index": 38,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 1010478821,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.38.post_attention_layernorm.weight",
      "role": "attention_post_norm",
      "layer_index": 38,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 1791904741,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.38.mlp.switch_mlp.down_proj.weight",
      "role": "ffn_down_exps",
      "layer_index": 38,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        2048,
        64
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 598912677,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.38.mlp.switch_mlp.gate_proj.weight",
      "role": "ffn_gate_exps",
      "layer_index": 38,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 875736805,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.38.mlp.gate.weight",
      "role": "ffn_gate_inp",
      "layer_index": 38,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        256,
        512
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 1009954533,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.38.mlp.shared_expert.down_proj.weight",
      "role": "ffn_shared_expert_down",
      "layer_index": 38,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        2048,
        64
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 581054117,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.38.mlp.shared_expert.gate_proj.weight",
      "role": "ffn_shared_expert_gate",
      "layer_index": 38,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        512,
        256
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 581611173,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.38.mlp.shared_expert_gate.weight",
      "role": "ffn_shared_expert_gate_inp",
      "layer_index": 38,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        1,
        512
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 1024184549,
      "length_bytes": 2048
    },
    {
      "name": "language_model.model.layers.38.mlp.shared_expert.up_proj.weight",
      "role": "ffn_shared_expert_up",
      "layer_index": 38,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        512,
        256
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 1782942693,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.38.mlp.switch_mlp.up_proj.weight",
      "role": "ffn_up_exps",
      "layer_index": 38,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 741519077,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.38.linear_attn.A_log",
      "role": "linear_attention_a_log",
      "layer_index": 38,
      "dtype": "bf16",
      "shape": [
        32
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 1014939685,
      "length_bytes": 64
    },
    {
      "name": "language_model.model.layers.38.linear_attn.conv1d.weight",
      "role": "linear_attention_conv1d",
      "layer_index": 38,
      "dtype": "bf16",
      "shape": [
        8192,
        4,
        1
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 1024118885,
      "length_bytes": 65536
    },
    {
      "name": "language_model.model.layers.38.linear_attn.dt_bias",
      "role": "linear_attention_dt_bias",
      "layer_index": 38,
      "dtype": "bf16",
      "shape": [
        32
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 1743223589,
      "length_bytes": 64
    },
    {
      "name": "language_model.model.layers.38.linear_attn.in_proj_a.weight",
      "role": "linear_attention_in_proj_a",
      "layer_index": 38,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        32,
        256
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 1703172517,
      "length_bytes": 32768
    },
    {
      "name": "language_model.model.layers.38.linear_attn.in_proj_b.weight",
      "role": "linear_attention_in_proj_b",
      "layer_index": 38,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        32,
        256
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 2253213349,
      "length_bytes": 32768
    },
    {
      "name": "language_model.model.layers.38.linear_attn.in_proj_qkv.weight",
      "role": "linear_attention_in_proj_qkv",
      "layer_index": 38,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        8192,
        256
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 1015730277,
      "length_bytes": 8388608
    },
    {
      "name": "language_model.model.layers.38.linear_attn.in_proj_z.weight",
      "role": "linear_attention_in_proj_z",
      "layer_index": 38,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        4096,
        256
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 1546604965,
      "length_bytes": 4194304
    },
    {
      "name": "language_model.model.layers.38.linear_attn.norm.weight",
      "role": "linear_attention_norm",
      "layer_index": 38,
      "dtype": "bf16",
      "shape": [
        128
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 1014939429,
      "length_bytes": 256
    },
    {
      "name": "language_model.model.layers.38.linear_attn.out_proj.weight",
      "role": "linear_attention_out_proj",
      "layer_index": 38,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        2048,
        512
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 1010745061,
      "length_bytes": 4194304
    },
    {
      "name": "language_model.model.layers.39.self_attn.k_proj.weight",
      "role": "attention_k",
      "layer_index": 39,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        512,
        256
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 3442051109,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.39.self_attn.k_norm.weight",
      "role": "attention_k_norm",
      "layer_index": 39,
      "dtype": "bf16",
      "shape": [
        256
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 580332645,
      "length_bytes": 512
    },
    {
      "name": "language_model.model.layers.39.input_layernorm.weight",
      "role": "attention_norm",
      "layer_index": 39,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 4056413477,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.39.self_attn.o_proj.weight",
      "role": "attention_o",
      "layer_index": 39,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        2048,
        512
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 1464008997,
      "length_bytes": 4194304
    },
    {
      "name": "language_model.model.layers.39.post_attention_layernorm.weight",
      "role": "attention_post_norm",
      "layer_index": 39,
      "dtype": "bf16",
      "shape": [
        2048
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 580328549,
      "length_bytes": 4096
    },
    {
      "name": "language_model.model.layers.39.self_attn.q_proj.weight",
      "role": "attention_q",
      "layer_index": 39,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        8192,
        256
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 1551323557,
      "length_bytes": 8388608
    },
    {
      "name": "language_model.model.layers.39.self_attn.q_norm.weight",
      "role": "attention_q_norm",
      "layer_index": 39,
      "dtype": "bf16",
      "shape": [
        256
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 3913806629,
      "length_bytes": 512
    },
    {
      "name": "language_model.model.layers.39.self_attn.v_proj.weight",
      "role": "attention_v",
      "layer_index": 39,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        512,
        256
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 2412910309,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.39.mlp.switch_mlp.down_proj.weight",
      "role": "ffn_down_exps",
      "layer_index": 39,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        2048,
        64
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 2113621477,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.39.mlp.switch_mlp.gate_proj.weight",
      "role": "ffn_gate_exps",
      "layer_index": 39,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 445570149,
      "length_bytes": 134217728
    },
    {
      "name": "language_model.model.layers.39.mlp.gate.weight",
      "role": "ffn_gate_inp",
      "layer_index": 39,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        256,
        512
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 579804261,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.39.mlp.shared_expert.down_proj.weight",
      "role": "ffn_shared_expert_down",
      "layer_index": 39,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        2048,
        64
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 2247871973,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.39.mlp.shared_expert.gate_proj.weight",
      "role": "ffn_shared_expert_gate",
      "layer_index": 39,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        512,
        256
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 580464293,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.39.mlp.shared_expert_gate.weight",
      "role": "ffn_shared_expert_gate_inp",
      "layer_index": 39,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 8
      },
      "shape": [
        1,
        512
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 270226533,
      "length_bytes": 2048
    },
    {
      "name": "language_model.model.layers.39.mlp.shared_expert.up_proj.weight",
      "role": "ffn_shared_expert_up",
      "layer_index": 39,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        512,
        256
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 2252689061,
      "length_bytes": 524288
    },
    {
      "name": "language_model.model.layers.39.mlp.switch_mlp.up_proj.weight",
      "role": "ffn_up_exps",
      "layer_index": 39,
      "dtype": "u32",
      "source_quantized": true,
      "quantization": {
        "mode": "affine",
        "group_size": 64,
        "bits": 4
      },
      "shape": [
        256,
        512,
        256
      ],
      "file": "model-00004-of-00004.safetensors",
      "offset_bytes": 311352421,
      "length_bytes": 134217728
    }
  ]
}